shahar1 commented on code in PR #76527:
URL: https://github.com/apache/airflow/pull/76527#discussion_r4640306782


##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,746 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its 
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import copy
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+    TRIGGER_ID_KEY,
+    TRIGGER_LOG_FILE_NAME,
+    TRIGGER_LOG_HELD_FILE_NAME,
+    TRIGGER_LOG_ROOT,
+    TRIGGER_LOG_ROTATED_DIR,
+    TRIGGERER_JOB_ID_KEY,
+    get_trigger_log_held_path,
+    get_trigger_log_part_path,
+    get_trigger_log_rotated_path,
+    list_trigger_log_files,
+    parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+    from structlog.typing import FilteringBoundLogger
+
+    from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+    """
+    ``BytesLogger`` appending to ``path``, moved to the rotated directory once 
it reaches ``max_bytes``.
+
+    ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one; 
``max_bytes <= 0`` is no
+    cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on 
a shared log folder another
+    triggerer's orphan scan) is reopened by the next line.
+    """
+
+    __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate", 
"_on_rotate", "_path", "_size")
+
+    def __init__(
+        self,
+        path: Path,
+        max_bytes: int,
+        may_rotate: Callable[[], bool] | None = None,
+        on_rotate: Callable[[], None] | None = None,
+    ):
+        self._path, self._max_bytes = path, max_bytes
+        self._may_rotate, self._on_rotate = may_rotate, on_rotate
+        self._folder_mode, self._file_mode = get_log_permissions()
+        super().__init__(path.open("ab"))
+        self._size = self._file.tell()
+
+    @property
+    def rotated_path(self) -> Path:
+        return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+    def ensure_open(self) -> None:
+        if self._file.closed:
+            self._path.parent.mkdir(mode=self._folder_mode, parents=True, 
exist_ok=True)
+            self._path.touch(mode=self._file_mode)
+            self._file = self._path.open("ab")
+            self._write = self._file.write
+            self._flush = self._file.flush
+            self._size = self._file.tell()
+
+    def _unlinked(self) -> bool:
+        try:
+            return os.fstat(self._file.fileno()).st_nlink == 0
+        except (OSError, ValueError):
+            return False
+
+    def msg(self, message: bytes) -> None:
+        with self._lock:
+            try:
+                if not self._file.closed and self._unlinked():
+                    # Lines written to a removed file are lost.
+                    with suppress(OSError):
+                        self._file.close()
+                self.ensure_open()
+                self._write(message + b"\n")
+                self._flush()
+                self._size += len(message) + 1
+                if 0 < self._max_bytes <= self._size and (self._may_rotate is 
None or self._may_rotate()):
+                    self._file.close()
+                    self.rotated_path.parent.mkdir(mode=self._folder_mode, 
parents=True, exist_ok=True)
+                    with suppress(FileNotFoundError):
+                        os.replace(self._path, self.rotated_path)
+                    if self._on_rotate is not None:
+                        try:
+                            self._on_rotate()
+                        except Exception:
+                            # The line is written: the caller, reading the 
runner's logs, must not see this.
+                            log.exception(
+                                "Failed to upload a rotated trigger log", 
path=str(self.rotated_path)
+                            )
+                    self.ensure_open()  # last: if this fails, the rotated 
lines are sent all the same
+            except OSError:
+                # Reopened by the next line: a file removed meanwhile (to free 
a full disk) frees its space.
+                with suppress(OSError):
+                    self._file.close()
+                raise
+
+    def close(self) -> None:
+        # Closing flushes; on a full disk those lines are lost either way.
+        with suppress(OSError):
+            self._file.close()
+
+    log = debug = info = warn = warning = msg
+    fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see 
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it 
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO", 
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed 
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO", 
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file: 
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines 
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in 
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was 
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver 
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO", 
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+    _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+    """Return the modes of new log folders and files, as ``init_log_file`` 
gives them."""
+    return (
+        int(conf.get("logging", "file_task_handler_new_folder_permissions", 
fallback="0o775"), 8),
+        int(conf.get("logging", "file_task_handler_new_file_permissions", 
fallback="0o664"), 8),
+    )
+
+
+def _has_lines(path: Path) -> bool:
+    try:
+        return path.stat().st_size > 0
+    except OSError:
+        return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+    """
+    Whether a file the handler leaves as it was after ``upload`` is no sign of 
a failed upload.
+
+    True for a handler that streams each line as it is written (it has 
processors, like CloudWatch) or that takes
+    only task logs (Elasticsearch, OpenSearch; by class name, as nothing else 
tells them apart). Any other handler
+    that leaves the file as it was failed to upload it (S3, GCS and WASB 
report that only in their log).
+    """
+    if getattr(handler, "processors", None):
+        return True
+    return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+_handler_keeping_local_copy: tuple[object, object] | None = None
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+    """
+    Upload one of a watcher's files, at ``path`` locally; with 
delete_local_logs, remove it once stored.
+
+    Not by the handler, which removes the whole directory of the file it 
stored: the watcher's other files are
+    in it or below it, and may not be stored yet.
+    """
+    delete = getattr(handler, "delete_local_copy", False) and not 
handler_keeps_no_copy(handler)
+    if _handler_is(handler, _APPENDING_HANDLERS):
+        # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS 
reports a failed write only
+        # through it. The file is emptied once stored, or removed if it can't 
be (sent again, it would be
+        # stored twice), and kept as it was otherwise.
+        if path.is_file() and handler.oss_write(path.read_text(), 
relative_path):
+            try:
+                os.truncate(path, 0)
+            except OSError:
+                path.unlink(missing_ok=True)
+    else:
+        (_get_handler_keeping_local_copy(handler) if delete else 
handler).upload(relative_path, None)
+    if delete and not _is_left_unstored(handler, path, raised=False):
+        with suppress(FileNotFoundError):
+            path.unlink()
+        with suppress(OSError):
+            path.parent.rmdir()  # once its last file is stored
+
+
+def _get_handler_keeping_local_copy(handler):
+    """Return a copy of the handler with ``delete_local_copy`` off, made once: 
it has its own hook."""
+    global _handler_keeping_local_copy
+    if _handler_keeping_local_copy is None or _handler_keeping_local_copy[0] 
is not handler:
+        keeping = copy.copy(handler)
+        keeping.delete_local_copy = False
+        _handler_keeping_local_copy = (handler, keeping)
+    return _handler_keeping_local_copy[1]
+
+
+def _append_file(lines: Path, kept: Path) -> None:
+    """Append ``lines`` to ``kept``; cut back if that fails part way (a full 
disk), as ``lines`` still has them."""
+    size = kept.stat().st_size
+    try:
+        with lines.open("rb") as source, kept.open("ab") as target:
+            shutil.copyfileobj(source, target)
+    except OSError:
+        with suppress(OSError):
+            os.truncate(kept, size)
+        raise
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+    rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True, 
exist_ok=True)
+    os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+    """
+    Whether to send a watcher's file again after an upload.
+
+    True if it has lines and the upload raised, or S3, GCS, WASB or OSS left 
them (others may keep them).
+    """
+    if not _has_lines(path):
+        return False
+    return raised or _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+    """Whether the handler's class, or a base class, has one of these names 
(as for the above)."""
+    classes = type(handler).__mro__
+    if type(handler) not in _checked_handler_classes:
+        _checked_handler_classes.add(type(handler))
+        if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+            log.info(
+                "Remote log handler not known by name: asset watcher log files 
are sent to it as to one "
+                "that stores them, and sent again only if the upload raises",
+                
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+            )
+    return any(cls.__name__ in names for cls in classes)
+
+
+def _get_existing_trigger_ids(session, trigger_ids: set[int]) -> set[int]:
+    return {
+        trigger_id
+        for ids in chunks(sorted(trigger_ids), _TRIGGER_ID_QUERY_CHUNK_SIZE)
+        for trigger_id in 
session.scalars(select(Trigger.id).where(Trigger.id.in_(ids)))
+    }
+
+
+def get_watcher_log_max_open_files() -> int:
+    """
+    Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a 
quarter of ``ulimit -n``.
+
+    The task and callback trigger logs, the database and the sockets need 
descriptors too.
+    """
+    setting = conf.getint("triggerer", "watcher_log_max_open_files")
+    try:
+        import resource  # Unix only
+
+        soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ImportError, OSError, ValueError):
+        return setting
+    if soft == resource.RLIM_INFINITY:
+        return setting
+    limit = max(1, soft // 4)
+    return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+    """The upload of a released watcher's files."""
+
+    factory: TriggerLoggingFactory
+    # The upload of the same watcher's files released before, if it was still 
pending
+    previous: _Release | None
+    upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+    """
+    The asset watcher log files of one triggerer's supervisor, other than 
writing to them.
+
+    Keeps at most ``watcher_log_max_open_files`` of them open, uploads their 
rotated files and those of
+    released watchers off the main loop, and uploads or removes the files 
triggerers that died left here.
+    """
+
+    # The supervisor's trigger log factories, by trigger id
+    logger_cache: dict[int, TriggerLoggingFactory]
+    # Told when closing a trigger's log file fails
+    on_error: Callable[[int | None, OSError], None]
+    # Asset watcher files with an open descriptor, least recently written first
+    _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+    watcher_log_max_open_files: int = 
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+    # When to look for watcher files left by triggerers that died, and the 
thread doing it
+    _orphan_scans_at: list[float] | None = attrs.field(default=None, 
init=False)
+    _orphan_scan_thread: threading.Thread | None = attrs.field(default=None, 
init=False)
+    # Uploads of released watchers' files, and of rotated files in their own 
pool (release uploads wait)
+    _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None, 
init=False)
+    _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+    # Held to store a release, and to forget it on its upload's thread
+    _releases_lock: threading.Lock = attrs.field(factory=threading.Lock, 
init=False)
+    # Log paths of watchers that left while their trigger existed, removed 
here once it no longer does
+    _left: dict[int, str] = attrs.field(factory=dict, init=False)
+    _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None, 
init=False)
+    _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict, 
init=False)
+    # The last number given to each watcher's rotated files here (see 
upload_rotated)
+    _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+    # Each watcher's last rotated file whose upload the handler did not 
confirm, sent again before it is replaced
+    _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict, 
init=False)
+    # Held while a released watcher's file is added to its held file, or the 
held file is taken for upload
+    _held_lock: threading.Lock = attrs.field(factory=threading.Lock, 
init=False)
+
+    def note_watcher_log_write(self, trigger_id: int) -> None:
+        """Keep at most ``watcher_log_max_open_files`` watcher files open, 
closing the least recent."""
+        self._open_watcher_logs.pop(trigger_id, None)
+        self._open_watcher_logs[trigger_id] = None
+        while 0 < self.watcher_log_max_open_files < 
len(self._open_watcher_logs):
+            idle = next(iter(self._open_watcher_logs))
+            del self._open_watcher_logs[idle]
+            if (factory := self.logger_cache.get(idle)) is not None:
+                factory.close()
+
+    def forget_open_file(self, trigger_id: int) -> None:
+        """Stop counting a trigger's file as open, as it is about to be 
closed."""
+        self._open_watcher_logs.pop(trigger_id, None)
+
+    def release(self, trigger_id: int, factory: TriggerLoggingFactory) -> None:
+        """
+        Upload and close the file of a watcher no longer assigned here, off 
the main loop.
+
+        The watcher may come back here at once, on a new live file: this one 
is held next to it until an upload
+        of the watcher's files here takes it (see _hold).
+        """
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        # In the background: a triggerer that lost its watchers (a DB 
partition) releases them all at once.
+        if self._upload_pool is None:
+            self._upload_pool = ThreadPoolExecutor(max_workers=4, 
thread_name_prefix="trigger-log-release")
+        handler = load_remote_log_handler()
+        if handler is not None and not _handler_is(handler, 
_TASK_LOGS_ONLY_HANDLERS):
+            if hasattr(factory, "bound_logger"):
+                factory.close()
+                self._hold(factory.bound_logger._logger._path)
+        with self._releases_lock:
+            release = _Release(factory=factory, 
previous=self._releases.get(trigger_id))
+            release.upload = self._upload_pool.submit(self._upload_released, 
trigger_id, release)
+            self._releases[trigger_id] = release
+
+        def forget(done: Future) -> None:
+            with self._releases_lock:
+                if self._releases.get(trigger_id) is release:
+                    self._releases.pop(trigger_id, None)
+
+        release.upload.add_done_callback(forget)
+
+    def _hold(self, live: Path) -> None:
+        """Move a released watcher's lines out of the way of a watcher back 
here (see release)."""
+        held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME)
+        # Else left in the live file: a watcher back here appends to it, and 
its next upload sends them.
+        with self._held_lock, suppress(OSError):
+            if not _has_lines(live):
+                return
+            if not held.exists():
+                os.replace(live, held)
+                return
+            # Released again before the held file was taken for upload: one 
file, sent once.
+            _append_file(live, held)
+            live.unlink()
+
+    def _take_held(self, trigger_id: int, log_path: str) -> str | None:
+        """Move the held file into the rotated directory for upload; return 
its path there, if there was one."""
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        base = Path(conf.get("logging", "base_log_folder"))
+        held = base / get_trigger_log_held_path(log_path)
+        if (handler := load_remote_log_handler()) is None or not 
held.is_file():
+            return None
+        if handler_keeps_no_copy(handler):
+            path = get_trigger_log_rotated_path(log_path)
+        else:
+            # Numbered only now, as the watcher's rotations wait for this 
upload (see prepare_rotation).
+            part = self._rotated_parts[trigger_id] = 
self._rotated_parts.get(trigger_id, 0) + 1
+            path = get_trigger_log_part_path(log_path, part)
+        with self._held_lock:
+            _move_into_rotated_dir(held, base / path)
+        return path
+
+    def _upload_released(self, trigger_id: int, release: _Release) -> None:
+        if release.previous is not None:
+            # Its files are in the same directory.
+            with suppress(Exception):
+                release.previous.upload.result()
+            release.previous = None
+        factory = release.factory
+        failed = self._upload_and_close(trigger_id, factory, keep_live=True)
+        if failed:
+            self._left.pop(trigger_id, None)  # kept for the next start's 
orphan scan
+        else:
+            self._left[trigger_id] = factory.log_path
+
+    @staticmethod
+    def _remove_released_files(log_path: str) -> None:
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        producer_dir = (Path(conf.get("logging", "base_log_folder")) / 
log_path).parent
+        handler = load_remote_log_handler()
+        if handler is not None and _handler_is(handler, 
_LINES_LEFT_MEAN_FAILED_HANDLERS):
+            if any(_has_lines(path) for path in 
list_trigger_log_files(producer_dir)):
+                return  # a failed upload: sent again by the next start's 
orphan scan
+        shutil.rmtree(producer_dir, ignore_errors=True)
+        with suppress(OSError):
+            producer_dir.parent.rmdir()  # not while another triggerer still 
has a directory in it
+
+    def remove_left(self) -> None:
+        """Remove the files of watchers that left here and whose trigger no 
longer exists."""
+        if not (left := set(self._left)):
+            return
+        try:
+            with create_session() as session:
+                gone = left - _get_existing_trigger_ids(session, left)
+        except Exception:
+            log.exception("Failed to look for triggers that no longer exist")
+            return
+        for trigger_id in gone:
+            if (release := self._releases.get(trigger_id)) is not None and not 
release.upload.done():
+                continue  # its files are still uploading
+            log_path = self._left.pop(trigger_id, None)
+            if log_path is None or trigger_id in self.logger_cache:
+                continue  # back here: tracked again once it leaves
+            try:
+                self._remove_released_files(log_path)
+            except Exception:
+                log.exception("Failed to remove the log files of a trigger 
that no longer exists")
+
+    def prepare_rotation(self, trigger_id: int) -> bool:
+        """Return whether the watcher's file may rotate now; until then, a 
part left unsent is sent again."""
+        if (release := self._releases.get(trigger_id)) is not None and not 
release.upload.done():
+            return False  # its released files are uploading from the rotated 
files' directory
+        upload = self._pending_rotated_uploads.get(trigger_id)
+        if upload is not None and not upload.done():
+            return False
+        if path := self._rotated_unconfirmed.pop(trigger_id, None):
+            # Before the next rotation, which replaces it whether this goes 
through or not.
+            self._pending_rotated_uploads[trigger_id] = 
self._get_rotated_pool().submit(
+                self._upload_rotated_again, trigger_id, path
+            )
+            return False
+        return True
+
+    def _get_rotated_pool(self) -> ThreadPoolExecutor:
+        if self._rotated_pool is None:
+            self._rotated_pool = ThreadPoolExecutor(max_workers=4, 
thread_name_prefix="trigger-log-rotated")
+        return self._rotated_pool
+
+    @staticmethod
+    def _upload_rotated_again(trigger_id: int, path: str) -> None:
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        try:
+            if (handler := load_remote_log_handler()) is not None:
+                upload_watcher_log_file(handler, path, 
Path(conf.get("logging", "base_log_folder")) / path)
+        except Exception:
+            log.exception("Failed to upload a rotated trigger log", 
trigger_id=trigger_id, path=path)
+
+    def upload_rotated(self, trigger_id: int, log_path: str) -> None:
+        # Called by the line that rotated the file: the rest runs in the pool, 
not to hold up the runner's logs.
+        pool = self._get_rotated_pool()
+        rotated = get_trigger_log_rotated_path(log_path)
+        # Numbered here: a watcher's next rotation waits for this upload (see 
prepare_rotation).
+        part = self._rotated_parts.get(trigger_id, 0) + 1
+
+        def upload() -> None:
+            from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+            path = rotated
+            base = Path(conf.get("logging", "base_log_folder"))
+            try:
+                if (handler := load_remote_log_handler()) is None:
+                    return
+                if not handler_keeps_no_copy(handler):
+                    # Each rotated file is its own object (see 
get_trigger_log_part_path), numbered by this
+                    # triggerer, the only one writing there: counted, as 
earlier parts may be gone locally.
+                    for done in (base / rotated).parent.glob("*.log"):
+                        if done.name != TRIGGER_LOG_FILE_NAME:
+                            with suppress(OSError):
+                                done.unlink()  # the previous one, uploaded or 
tried twice: one at a time
+                    path = get_trigger_log_part_path(log_path, part)
+                    # Named only now: until then the triggerer's log server 
serves it as the rotated file.
+                    os.replace(base / rotated, base / path)
+                upload_watcher_log_file(handler, path, base / path)
+                raised = False
+            except Exception:
+                log.exception("Failed to upload a rotated trigger log", 
trigger_id=trigger_id, path=path)
+                raised = True
+            if path != rotated and _is_left_unstored(handler, base / path, 
raised=raised):
+                self._rotated_unconfirmed[trigger_id] = path
+
+        self._pending_rotated_uploads[trigger_id] = pool.submit(upload)
+        self._rotated_parts[trigger_id] = part
+
+    def upload_and_close(self, trigger_id: int, factory: 
TriggerLoggingFactory) -> None:
+        if (release := self._releases.get(trigger_id)) is not None:
+            # A watcher back here: its released files are uploading from the 
same directory.
+            with suppress(Exception):
+                release.upload.result()
+        self._upload_and_close(trigger_id, factory)
+
+    def _upload_and_close(self, trigger_id: int, factory: 
TriggerLoggingFactory, **upload_kwargs) -> bool:
+        """Upload and close a trigger's file; return whether the upload 
raised."""
+        from airflow.jobs.triggerer_job_runner import WatcherLoggingFactory
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        if (rotated_upload := self._pending_rotated_uploads.pop(trigger_id, 
None)) is not None:
+            rotated_upload.result()  # else the rotated file could be read 
again before it is emptied
+        self._rotated_unconfirmed.pop(trigger_id, None)  # sent again below, 
with the live file
+        base = Path(conf.get("logging", "base_log_folder"))
+        failed, held = False, None
+        try:
+            # Not by a factory that never wrote: its upload sends nothing, so 
the held file waits.
+            if isinstance(factory, WatcherLoggingFactory) and hasattr(factory, 
"bound_logger"):
+                held = self._take_held(trigger_id, factory.log_path)
+            factory.upload_to_remote(**upload_kwargs)
+        except Exception:
+            log.exception("Failed to upload trigger logs to remote", 
trigger_id=trigger_id)
+            failed = True
+        finally:
+            # Close even if the upload failed, not to leak the descriptor. 
Closing flushes, which can fail.
+            try:
+                factory.close()
+            except OSError as e:
+                self.on_error(trigger_id, e)
+        if held and _is_left_unstored(load_remote_log_handler(), base / held, 
raised=failed):
+            # Under a rotated file's name now: the watcher's next rotation 
here would remove it unsent.
+            self._rotated_unconfirmed[trigger_id] = held
+        return failed
+
+    def upload_and_close_all(self, factories: list[tuple[int, 
TriggerLoggingFactory]]) -> None:
+        """Upload and close these watchers' files, and wait for the released 
ones' uploads."""
+        self._open_watcher_logs.clear()
+        # Side by side: one after the other, thousands of files take longer 
than a container's stop timeout.
+        with ThreadPoolExecutor(max_workers=16, 
thread_name_prefix="trigger-log-upload") as pool:
+            list(pool.map(lambda item: self.upload_and_close(*item), 
factories))
+        if self._upload_pool is not None:
+            self._upload_pool.shutdown(wait=True)
+
+    def handle_orphaned_trigger_logs(self, own_job_id: int | None) -> None:
+        """
+        Upload, or remove, the asset watcher files left on this disk by 
triggerers that are no longer alive.
+
+        A killed triggerer uploads nothing, so its files are uploaded now, 
then their directories removed.
+        Files remote logging doesn't take (it is off, or the handler keeps no 
copy, like Elasticsearch) are
+        removed instead: all but the most recently written for each watcher, 
and all for triggers that no
+        longer exist. Files of live triggerers, this one included, of 
triggerers without a job (no job id
+        directory), and those a failed upload left stay.
+        """
+        from airflow.jobs.job import Job
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        handler = load_remote_log_handler()
+        base = Path(conf.get("logging", "base_log_folder"))
+        dirs_by_job: dict[int, list[Path]] = {}
+        dirs_by_trigger: dict[Path, int] = {}
+        for producer_dir in 
base.glob(f"{TRIGGER_LOG_ROOT}/{TRIGGER_ID_KEY}=*/{TRIGGERER_JOB_ID_KEY}=*"):
+            # Only directories this module names are ever removed: both ids 
must parse.
+            trigger_id = parse_trigger_log_dir_id(TRIGGER_ID_KEY, 
producer_dir.parent.name)
+            job_id = parse_trigger_log_dir_id(TRIGGERER_JOB_ID_KEY, 
producer_dir.name)
+            if (
+                trigger_id is not None
+                and job_id is not None
+                and job_id != own_job_id
+                and producer_dir.is_dir()
+            ):
+                dirs_by_job.setdefault(job_id, []).append(producer_dir)
+                dirs_by_trigger[producer_dir.parent] = trigger_id
+        if not dirs_by_job:
+            return
+        with create_session() as session:
+            alive = session.scalars(
+                select(Job.id).where(Job.id.in_(dirs_by_job), 
Job.is_alive_clause(["TriggererJob"]))
+            )
+            dead = dirs_by_job.keys() - set(alive)
+            # A watcher whose kwargs changed got a new trigger; the old id's 
files have nothing left to keep.
+            existing = _get_existing_trigger_ids(session, 
set(dirs_by_trigger.values()))
+        dead_dirs = {job_id: dirs_by_job[job_id] for job_id in dead}
+        if handler is not None:
+            kept = {
+                producer_dir: self._upload_orphaned_trigger_log_dir(handler, 
base, producer_dir, job_id)
+                for job_id, dirs in dead_dirs.items()
+                for producer_dir in dirs
+            }
+            # What the handler left as it was is then removed as without 
remote logging.
+            dead_dirs = {job_id: [d for d in dirs if kept[d]] for job_id, dirs 
in dead_dirs.items()}
+            if None in kept.values() and self._orphan_scans_at is not None:
+                # Claimed by a triggerer that may have stopped mid-scan (a 
container restarted in place, so no
+                # later start may come): looked at again once the claim is 
stale.
+                self._orphan_scans_at.append(time.monotonic() + 
_ORPHAN_UPLOAD_CLAIM_MAX_AGE)
+        gone = {
+            trigger_dir for trigger_dir, trigger_id in dirs_by_trigger.items() 
if trigger_id not in existing
+        }
+        self._remove_old_trigger_log_dirs(dead_dirs, gone=gone)
+
+    @staticmethod
+    def _upload_orphaned_trigger_log_dir(handler, base: Path, producer_dir: 
Path, job_id: int) -> bool | None:
+        """Upload a dead triggerer's files: True if the handler left any, None 
if they are claimed."""
+        # Claimed first: triggerers starting together on a shared log folder 
see the same files, and remote
+        # handlers append, so a file uploaded twice would be in the remote 
copy twice.
+        claim = producer_dir / _ORPHAN_UPLOAD_CLAIM
+        with suppress(OSError):
+            if time.time() - claim.stat().st_mtime >= 
_ORPHAN_UPLOAD_CLAIM_MAX_AGE:

Review Comment:
   **minor**: two scans can both take over a stale claim. Both `stat()` the old 
`.uploading` file and see it as stale. The first unlinks it and creates its 
own. The second has already passed the age check, so it unlinks that fresh 
claim, and its `O_EXCL` create succeeds too. Both then upload the same files, 
which is what the claim is there to prevent.
   
   It needs a claim left by a triggerer that died mid-scan and two triggerers 
scanning the same directory at the same moment on a shared log folder, so it is 
narrow. The age check and the removal would have to be one atomic step to close 
it.
   
   *Flagged by copilot (adversarial review); cross-checked.*



##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,746 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its 
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import copy
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+    TRIGGER_ID_KEY,
+    TRIGGER_LOG_FILE_NAME,
+    TRIGGER_LOG_HELD_FILE_NAME,
+    TRIGGER_LOG_ROOT,
+    TRIGGER_LOG_ROTATED_DIR,
+    TRIGGERER_JOB_ID_KEY,
+    get_trigger_log_held_path,
+    get_trigger_log_part_path,
+    get_trigger_log_rotated_path,
+    list_trigger_log_files,
+    parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+    from structlog.typing import FilteringBoundLogger
+
+    from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+    """
+    ``BytesLogger`` appending to ``path``, moved to the rotated directory once 
it reaches ``max_bytes``.
+
+    ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one; 
``max_bytes <= 0`` is no
+    cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on 
a shared log folder another
+    triggerer's orphan scan) is reopened by the next line.
+    """
+
+    __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate", 
"_on_rotate", "_path", "_size")
+
+    def __init__(
+        self,
+        path: Path,
+        max_bytes: int,
+        may_rotate: Callable[[], bool] | None = None,
+        on_rotate: Callable[[], None] | None = None,
+    ):
+        self._path, self._max_bytes = path, max_bytes
+        self._may_rotate, self._on_rotate = may_rotate, on_rotate
+        self._folder_mode, self._file_mode = get_log_permissions()
+        super().__init__(path.open("ab"))
+        self._size = self._file.tell()
+
+    @property
+    def rotated_path(self) -> Path:
+        return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+    def ensure_open(self) -> None:
+        if self._file.closed:
+            self._path.parent.mkdir(mode=self._folder_mode, parents=True, 
exist_ok=True)
+            self._path.touch(mode=self._file_mode)
+            self._file = self._path.open("ab")
+            self._write = self._file.write
+            self._flush = self._file.flush
+            self._size = self._file.tell()
+
+    def _unlinked(self) -> bool:
+        try:
+            return os.fstat(self._file.fileno()).st_nlink == 0
+        except (OSError, ValueError):
+            return False
+
+    def msg(self, message: bytes) -> None:
+        with self._lock:
+            try:
+                if not self._file.closed and self._unlinked():
+                    # Lines written to a removed file are lost.
+                    with suppress(OSError):
+                        self._file.close()
+                self.ensure_open()
+                self._write(message + b"\n")
+                self._flush()
+                self._size += len(message) + 1
+                if 0 < self._max_bytes <= self._size and (self._may_rotate is 
None or self._may_rotate()):
+                    self._file.close()
+                    self.rotated_path.parent.mkdir(mode=self._folder_mode, 
parents=True, exist_ok=True)
+                    with suppress(FileNotFoundError):
+                        os.replace(self._path, self.rotated_path)
+                    if self._on_rotate is not None:
+                        try:
+                            self._on_rotate()
+                        except Exception:
+                            # The line is written: the caller, reading the 
runner's logs, must not see this.
+                            log.exception(
+                                "Failed to upload a rotated trigger log", 
path=str(self.rotated_path)
+                            )
+                    self.ensure_open()  # last: if this fails, the rotated 
lines are sent all the same
+            except OSError:
+                # Reopened by the next line: a file removed meanwhile (to free 
a full disk) frees its space.
+                with suppress(OSError):
+                    self._file.close()
+                raise
+
+    def close(self) -> None:
+        # Closing flushes; on a full disk those lines are lost either way.
+        with suppress(OSError):
+            self._file.close()
+
+    log = debug = info = warn = warning = msg
+    fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see 
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it 
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO", 
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed 
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO", 
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file: 
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines 
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in 
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was 
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver 
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO", 
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+    _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+    """Return the modes of new log folders and files, as ``init_log_file`` 
gives them."""
+    return (
+        int(conf.get("logging", "file_task_handler_new_folder_permissions", 
fallback="0o775"), 8),
+        int(conf.get("logging", "file_task_handler_new_file_permissions", 
fallback="0o664"), 8),
+    )
+
+
+def _has_lines(path: Path) -> bool:
+    try:
+        return path.stat().st_size > 0
+    except OSError:
+        return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+    """
+    Whether a file the handler leaves as it was after ``upload`` is no sign of 
a failed upload.
+
+    True for a handler that streams each line as it is written (it has 
processors, like CloudWatch) or that takes
+    only task logs (Elasticsearch, OpenSearch; by class name, as nothing else 
tells them apart). Any other handler
+    that leaves the file as it was failed to upload it (S3, GCS and WASB 
report that only in their log).
+    """
+    if getattr(handler, "processors", None):
+        return True
+    return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+_handler_keeping_local_copy: tuple[object, object] | None = None
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+    """
+    Upload one of a watcher's files, at ``path`` locally; with 
delete_local_logs, remove it once stored.
+
+    Not by the handler, which removes the whole directory of the file it 
stored: the watcher's other files are
+    in it or below it, and may not be stored yet.
+    """
+    delete = getattr(handler, "delete_local_copy", False) and not 
handler_keeps_no_copy(handler)
+    if _handler_is(handler, _APPENDING_HANDLERS):
+        # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS 
reports a failed write only
+        # through it. The file is emptied once stored, or removed if it can't 
be (sent again, it would be
+        # stored twice), and kept as it was otherwise.
+        if path.is_file() and handler.oss_write(path.read_text(), 
relative_path):
+            try:
+                os.truncate(path, 0)
+            except OSError:
+                path.unlink(missing_ok=True)
+    else:
+        (_get_handler_keeping_local_copy(handler) if delete else 
handler).upload(relative_path, None)
+    if delete and not _is_left_unstored(handler, path, raised=False):
+        with suppress(FileNotFoundError):
+            path.unlink()
+        with suppress(OSError):
+            path.parent.rmdir()  # once its last file is stored
+
+
+def _get_handler_keeping_local_copy(handler):
+    """Return a copy of the handler with ``delete_local_copy`` off, made once: 
it has its own hook."""
+    global _handler_keeping_local_copy
+    if _handler_keeping_local_copy is None or _handler_keeping_local_copy[0] 
is not handler:
+        keeping = copy.copy(handler)
+        keeping.delete_local_copy = False
+        _handler_keeping_local_copy = (handler, keeping)
+    return _handler_keeping_local_copy[1]
+
+
+def _append_file(lines: Path, kept: Path) -> None:
+    """Append ``lines`` to ``kept``; cut back if that fails part way (a full 
disk), as ``lines`` still has them."""
+    size = kept.stat().st_size
+    try:
+        with lines.open("rb") as source, kept.open("ab") as target:
+            shutil.copyfileobj(source, target)
+    except OSError:
+        with suppress(OSError):
+            os.truncate(kept, size)
+        raise
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+    rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True, 
exist_ok=True)
+    os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+    """
+    Whether to send a watcher's file again after an upload.
+
+    True if it has lines and the upload raised, or S3, GCS, WASB or OSS left 
them (others may keep them).
+    """
+    if not _has_lines(path):
+        return False
+    return raised or _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+    """Whether the handler's class, or a base class, has one of these names 
(as for the above)."""
+    classes = type(handler).__mro__
+    if type(handler) not in _checked_handler_classes:
+        _checked_handler_classes.add(type(handler))
+        if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+            log.info(
+                "Remote log handler not known by name: asset watcher log files 
are sent to it as to one "
+                "that stores them, and sent again only if the upload raises",
+                
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+            )
+    return any(cls.__name__ in names for cls in classes)
+
+
+def _get_existing_trigger_ids(session, trigger_ids: set[int]) -> set[int]:
+    return {
+        trigger_id
+        for ids in chunks(sorted(trigger_ids), _TRIGGER_ID_QUERY_CHUNK_SIZE)
+        for trigger_id in 
session.scalars(select(Trigger.id).where(Trigger.id.in_(ids)))
+    }
+
+
+def get_watcher_log_max_open_files() -> int:
+    """
+    Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a 
quarter of ``ulimit -n``.
+
+    The task and callback trigger logs, the database and the sockets need 
descriptors too.
+    """
+    setting = conf.getint("triggerer", "watcher_log_max_open_files")
+    try:
+        import resource  # Unix only
+
+        soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ImportError, OSError, ValueError):
+        return setting
+    if soft == resource.RLIM_INFINITY:
+        return setting
+    limit = max(1, soft // 4)
+    return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+    """The upload of a released watcher's files."""
+
+    factory: TriggerLoggingFactory
+    # The upload of the same watcher's files released before, if it was still 
pending
+    previous: _Release | None
+    upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+    """
+    The asset watcher log files of one triggerer's supervisor, other than 
writing to them.
+
+    Keeps at most ``watcher_log_max_open_files`` of them open, uploads their 
rotated files and those of
+    released watchers off the main loop, and uploads or removes the files 
triggerers that died left here.
+    """
+
+    # The supervisor's trigger log factories, by trigger id
+    logger_cache: dict[int, TriggerLoggingFactory]
+    # Told when closing a trigger's log file fails
+    on_error: Callable[[int | None, OSError], None]
+    # Asset watcher files with an open descriptor, least recently written first
+    _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+    watcher_log_max_open_files: int = 
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+    # When to look for watcher files left by triggerers that died, and the 
thread doing it
+    _orphan_scans_at: list[float] | None = attrs.field(default=None, 
init=False)
+    _orphan_scan_thread: threading.Thread | None = attrs.field(default=None, 
init=False)
+    # Uploads of released watchers' files, and of rotated files in their own 
pool (release uploads wait)
+    _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None, 
init=False)
+    _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+    # Held to store a release, and to forget it on its upload's thread
+    _releases_lock: threading.Lock = attrs.field(factory=threading.Lock, 
init=False)
+    # Log paths of watchers that left while their trigger existed, removed 
here once it no longer does
+    _left: dict[int, str] = attrs.field(factory=dict, init=False)
+    _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None, 
init=False)
+    _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict, 
init=False)
+    # The last number given to each watcher's rotated files here (see 
upload_rotated)
+    _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+    # Each watcher's last rotated file whose upload the handler did not 
confirm, sent again before it is replaced
+    _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict, 
init=False)
+    # Held while a released watcher's file is added to its held file, or the 
held file is taken for upload
+    _held_lock: threading.Lock = attrs.field(factory=threading.Lock, 
init=False)
+
+    def note_watcher_log_write(self, trigger_id: int) -> None:
+        """Keep at most ``watcher_log_max_open_files`` watcher files open, 
closing the least recent."""
+        self._open_watcher_logs.pop(trigger_id, None)
+        self._open_watcher_logs[trigger_id] = None
+        while 0 < self.watcher_log_max_open_files < 
len(self._open_watcher_logs):
+            idle = next(iter(self._open_watcher_logs))
+            del self._open_watcher_logs[idle]
+            if (factory := self.logger_cache.get(idle)) is not None:
+                factory.close()
+
+    def forget_open_file(self, trigger_id: int) -> None:
+        """Stop counting a trigger's file as open, as it is about to be 
closed."""
+        self._open_watcher_logs.pop(trigger_id, None)
+
+    def release(self, trigger_id: int, factory: TriggerLoggingFactory) -> None:
+        """
+        Upload and close the file of a watcher no longer assigned here, off 
the main loop.
+
+        The watcher may come back here at once, on a new live file: this one 
is held next to it until an upload
+        of the watcher's files here takes it (see _hold).
+        """
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        # In the background: a triggerer that lost its watchers (a DB 
partition) releases them all at once.
+        if self._upload_pool is None:
+            self._upload_pool = ThreadPoolExecutor(max_workers=4, 
thread_name_prefix="trigger-log-release")
+        handler = load_remote_log_handler()
+        if handler is not None and not _handler_is(handler, 
_TASK_LOGS_ONLY_HANDLERS):
+            if hasattr(factory, "bound_logger"):
+                factory.close()
+                self._hold(factory.bound_logger._logger._path)
+        with self._releases_lock:
+            release = _Release(factory=factory, 
previous=self._releases.get(trigger_id))
+            release.upload = self._upload_pool.submit(self._upload_released, 
trigger_id, release)
+            self._releases[trigger_id] = release
+
+        def forget(done: Future) -> None:
+            with self._releases_lock:
+                if self._releases.get(trigger_id) is release:
+                    self._releases.pop(trigger_id, None)
+
+        release.upload.add_done_callback(forget)
+
+    def _hold(self, live: Path) -> None:
+        """Move a released watcher's lines out of the way of a watcher back 
here (see release)."""
+        held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME)
+        # Else left in the live file: a watcher back here appends to it, and 
its next upload sends them.
+        with self._held_lock, suppress(OSError):
+            if not _has_lines(live):
+                return
+            if not held.exists():
+                os.replace(live, held)
+                return
+            # Released again before the held file was taken for upload: one 
file, sent once.
+            _append_file(live, held)
+            live.unlink()
+
+    def _take_held(self, trigger_id: int, log_path: str) -> str | None:
+        """Move the held file into the rotated directory for upload; return 
its path there, if there was one."""
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        base = Path(conf.get("logging", "base_log_folder"))
+        held = base / get_trigger_log_held_path(log_path)
+        if (handler := load_remote_log_handler()) is None or not 
held.is_file():
+            return None
+        if handler_keeps_no_copy(handler):
+            path = get_trigger_log_rotated_path(log_path)
+        else:
+            # Numbered only now, as the watcher's rotations wait for this 
upload (see prepare_rotation).
+            part = self._rotated_parts[trigger_id] = 
self._rotated_parts.get(trigger_id, 0) + 1
+            path = get_trigger_log_part_path(log_path, part)
+        with self._held_lock:
+            _move_into_rotated_dir(held, base / path)
+        return path
+
+    def _upload_released(self, trigger_id: int, release: _Release) -> None:
+        if release.previous is not None:
+            # Its files are in the same directory.
+            with suppress(Exception):
+                release.previous.upload.result()
+            release.previous = None
+        factory = release.factory
+        failed = self._upload_and_close(trigger_id, factory, keep_live=True)
+        if failed:
+            self._left.pop(trigger_id, None)  # kept for the next start's 
orphan scan
+        else:
+            self._left[trigger_id] = factory.log_path
+
+    @staticmethod
+    def _remove_released_files(log_path: str) -> None:
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        producer_dir = (Path(conf.get("logging", "base_log_folder")) / 
log_path).parent
+        handler = load_remote_log_handler()
+        if handler is not None and _handler_is(handler, 
_LINES_LEFT_MEAN_FAILED_HANDLERS):
+            if any(_has_lines(path) for path in 
list_trigger_log_files(producer_dir)):
+                return  # a failed upload: sent again by the next start's 
orphan scan
+        shutil.rmtree(producer_dir, ignore_errors=True)
+        with suppress(OSError):
+            producer_dir.parent.rmdir()  # not while another triggerer still 
has a directory in it
+
+    def remove_left(self) -> None:
+        """Remove the files of watchers that left here and whose trigger no 
longer exists."""
+        if not (left := set(self._left)):
+            return
+        try:
+            with create_session() as session:
+                gone = left - _get_existing_trigger_ids(session, left)
+        except Exception:
+            log.exception("Failed to look for triggers that no longer exist")
+            return
+        for trigger_id in gone:
+            if (release := self._releases.get(trigger_id)) is not None and not 
release.upload.done():
+                continue  # its files are still uploading
+            log_path = self._left.pop(trigger_id, None)
+            if log_path is None or trigger_id in self.logger_cache:
+                continue  # back here: tracked again once it leaves
+            try:
+                self._remove_released_files(log_path)
+            except Exception:
+                log.exception("Failed to remove the log files of a trigger 
that no longer exists")
+
+    def prepare_rotation(self, trigger_id: int) -> bool:
+        """Return whether the watcher's file may rotate now; until then, a 
part left unsent is sent again."""
+        if (release := self._releases.get(trigger_id)) is not None and not 
release.upload.done():
+            return False  # its released files are uploading from the rotated 
files' directory
+        upload = self._pending_rotated_uploads.get(trigger_id)
+        if upload is not None and not upload.done():
+            return False
+        if path := self._rotated_unconfirmed.pop(trigger_id, None):
+            # Before the next rotation, which replaces it whether this goes 
through or not.
+            self._pending_rotated_uploads[trigger_id] = 
self._get_rotated_pool().submit(
+                self._upload_rotated_again, trigger_id, path
+            )
+            return False
+        return True
+
+    def _get_rotated_pool(self) -> ThreadPoolExecutor:
+        if self._rotated_pool is None:
+            self._rotated_pool = ThreadPoolExecutor(max_workers=4, 
thread_name_prefix="trigger-log-rotated")
+        return self._rotated_pool
+
+    @staticmethod
+    def _upload_rotated_again(trigger_id: int, path: str) -> None:
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        try:
+            if (handler := load_remote_log_handler()) is not None:
+                upload_watcher_log_file(handler, path, 
Path(conf.get("logging", "base_log_folder")) / path)
+        except Exception:
+            log.exception("Failed to upload a rotated trigger log", 
trigger_id=trigger_id, path=path)
+
+    def upload_rotated(self, trigger_id: int, log_path: str) -> None:
+        # Called by the line that rotated the file: the rest runs in the pool, 
not to hold up the runner's logs.
+        pool = self._get_rotated_pool()
+        rotated = get_trigger_log_rotated_path(log_path)
+        # Numbered here: a watcher's next rotation waits for this upload (see 
prepare_rotation).
+        part = self._rotated_parts.get(trigger_id, 0) + 1
+
+        def upload() -> None:
+            from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+            path = rotated
+            base = Path(conf.get("logging", "base_log_folder"))
+            try:
+                if (handler := load_remote_log_handler()) is None:
+                    return
+                if not handler_keeps_no_copy(handler):
+                    # Each rotated file is its own object (see 
get_trigger_log_part_path), numbered by this
+                    # triggerer, the only one writing there: counted, as 
earlier parts may be gone locally.
+                    for done in (base / rotated).parent.glob("*.log"):
+                        if done.name != TRIGGER_LOG_FILE_NAME:
+                            with suppress(OSError):
+                                done.unlink()  # the previous one, uploaded or 
tried twice: one at a time
+                    path = get_trigger_log_part_path(log_path, part)
+                    # Named only now: until then the triggerer's log server 
serves it as the rotated file.
+                    os.replace(base / rotated, base / path)
+                upload_watcher_log_file(handler, path, base / path)
+                raised = False
+            except Exception:
+                log.exception("Failed to upload a rotated trigger log", 
trigger_id=trigger_id, path=path)
+                raised = True
+            if path != rotated and _is_left_unstored(handler, base / path, 
raised=raised):
+                self._rotated_unconfirmed[trigger_id] = path
+
+        self._pending_rotated_uploads[trigger_id] = pool.submit(upload)
+        self._rotated_parts[trigger_id] = part
+
+    def upload_and_close(self, trigger_id: int, factory: 
TriggerLoggingFactory) -> None:
+        if (release := self._releases.get(trigger_id)) is not None:
+            # A watcher back here: its released files are uploading from the 
same directory.
+            with suppress(Exception):
+                release.upload.result()
+        self._upload_and_close(trigger_id, factory)
+
+    def _upload_and_close(self, trigger_id: int, factory: 
TriggerLoggingFactory, **upload_kwargs) -> bool:
+        """Upload and close a trigger's file; return whether the upload 
raised."""
+        from airflow.jobs.triggerer_job_runner import WatcherLoggingFactory
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        if (rotated_upload := self._pending_rotated_uploads.pop(trigger_id, 
None)) is not None:
+            rotated_upload.result()  # else the rotated file could be read 
again before it is emptied
+        self._rotated_unconfirmed.pop(trigger_id, None)  # sent again below, 
with the live file
+        base = Path(conf.get("logging", "base_log_folder"))
+        failed, held = False, None
+        try:
+            # Not by a factory that never wrote: its upload sends nothing, so 
the held file waits.
+            if isinstance(factory, WatcherLoggingFactory) and hasattr(factory, 
"bound_logger"):
+                held = self._take_held(trigger_id, factory.log_path)
+            factory.upload_to_remote(**upload_kwargs)
+        except Exception:
+            log.exception("Failed to upload trigger logs to remote", 
trigger_id=trigger_id)
+            failed = True
+        finally:
+            # Close even if the upload failed, not to leak the descriptor. 
Closing flushes, which can fail.
+            try:
+                factory.close()
+            except OSError as e:
+                self.on_error(trigger_id, e)
+        if held and _is_left_unstored(load_remote_log_handler(), base / held, 
raised=failed):
+            # Under a rotated file's name now: the watcher's next rotation 
here would remove it unsent.
+            self._rotated_unconfirmed[trigger_id] = held
+        return failed
+
+    def upload_and_close_all(self, factories: list[tuple[int, 
TriggerLoggingFactory]]) -> None:
+        """Upload and close these watchers' files, and wait for the released 
ones' uploads."""
+        self._open_watcher_logs.clear()
+        # Side by side: one after the other, thousands of files take longer 
than a container's stop timeout.
+        with ThreadPoolExecutor(max_workers=16, 
thread_name_prefix="trigger-log-upload") as pool:
+            list(pool.map(lambda item: self.upload_and_close(*item), 
factories))
+        if self._upload_pool is not None:
+            self._upload_pool.shutdown(wait=True)
+
+    def handle_orphaned_trigger_logs(self, own_job_id: int | None) -> None:
+        """
+        Upload, or remove, the asset watcher files left on this disk by 
triggerers that are no longer alive.
+
+        A killed triggerer uploads nothing, so its files are uploaded now, 
then their directories removed.
+        Files remote logging doesn't take (it is off, or the handler keeps no 
copy, like Elasticsearch) are
+        removed instead: all but the most recently written for each watcher, 
and all for triggers that no
+        longer exist. Files of live triggerers, this one included, of 
triggerers without a job (no job id
+        directory), and those a failed upload left stay.
+        """
+        from airflow.jobs.job import Job

Review Comment:
   **nit**: does `from airflow.jobs.job import Job` need to be inside the 
function? If it is there to avoid a circular import, a short comment saying so 
would help.
   
   > Flag any `from` or `import` statement inside a function or method body. 
Imports must be at the top of the file. The only valid exceptions are: (1) 
circular import avoidance, (2) lazy loading for worker isolation, (3) 
`TYPE_CHECKING` blocks. If the import is inside a function, ask the author to 
justify why it cannot be at module level.
   
   ([code-review 
instructions](https://github.com/apache/airflow/blob/main/.github/instructions/code-review.instructions.md#code-quality-rules))



##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,746 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its 
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import copy
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+    TRIGGER_ID_KEY,
+    TRIGGER_LOG_FILE_NAME,
+    TRIGGER_LOG_HELD_FILE_NAME,
+    TRIGGER_LOG_ROOT,
+    TRIGGER_LOG_ROTATED_DIR,
+    TRIGGERER_JOB_ID_KEY,
+    get_trigger_log_held_path,
+    get_trigger_log_part_path,
+    get_trigger_log_rotated_path,
+    list_trigger_log_files,
+    parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+    from structlog.typing import FilteringBoundLogger
+
+    from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+    """
+    ``BytesLogger`` appending to ``path``, moved to the rotated directory once 
it reaches ``max_bytes``.
+
+    ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one; 
``max_bytes <= 0`` is no
+    cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on 
a shared log folder another
+    triggerer's orphan scan) is reopened by the next line.
+    """
+
+    __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate", 
"_on_rotate", "_path", "_size")
+
+    def __init__(
+        self,
+        path: Path,
+        max_bytes: int,
+        may_rotate: Callable[[], bool] | None = None,
+        on_rotate: Callable[[], None] | None = None,
+    ):
+        self._path, self._max_bytes = path, max_bytes
+        self._may_rotate, self._on_rotate = may_rotate, on_rotate
+        self._folder_mode, self._file_mode = get_log_permissions()
+        super().__init__(path.open("ab"))
+        self._size = self._file.tell()
+
+    @property
+    def rotated_path(self) -> Path:
+        return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+    def ensure_open(self) -> None:
+        if self._file.closed:
+            self._path.parent.mkdir(mode=self._folder_mode, parents=True, 
exist_ok=True)
+            self._path.touch(mode=self._file_mode)
+            self._file = self._path.open("ab")
+            self._write = self._file.write
+            self._flush = self._file.flush
+            self._size = self._file.tell()
+
+    def _unlinked(self) -> bool:
+        try:
+            return os.fstat(self._file.fileno()).st_nlink == 0
+        except (OSError, ValueError):
+            return False
+
+    def msg(self, message: bytes) -> None:
+        with self._lock:
+            try:
+                if not self._file.closed and self._unlinked():
+                    # Lines written to a removed file are lost.
+                    with suppress(OSError):
+                        self._file.close()
+                self.ensure_open()
+                self._write(message + b"\n")
+                self._flush()
+                self._size += len(message) + 1
+                if 0 < self._max_bytes <= self._size and (self._may_rotate is 
None or self._may_rotate()):
+                    self._file.close()
+                    self.rotated_path.parent.mkdir(mode=self._folder_mode, 
parents=True, exist_ok=True)
+                    with suppress(FileNotFoundError):
+                        os.replace(self._path, self.rotated_path)
+                    if self._on_rotate is not None:
+                        try:
+                            self._on_rotate()
+                        except Exception:
+                            # The line is written: the caller, reading the 
runner's logs, must not see this.
+                            log.exception(
+                                "Failed to upload a rotated trigger log", 
path=str(self.rotated_path)
+                            )
+                    self.ensure_open()  # last: if this fails, the rotated 
lines are sent all the same
+            except OSError:
+                # Reopened by the next line: a file removed meanwhile (to free 
a full disk) frees its space.
+                with suppress(OSError):
+                    self._file.close()
+                raise
+
+    def close(self) -> None:
+        # Closing flushes; on a full disk those lines are lost either way.
+        with suppress(OSError):
+            self._file.close()
+
+    log = debug = info = warn = warning = msg
+    fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see 
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it 
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO", 
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed 
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO", 
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file: 
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines 
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in 
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was 
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver 
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO", 
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+    _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+    """Return the modes of new log folders and files, as ``init_log_file`` 
gives them."""
+    return (
+        int(conf.get("logging", "file_task_handler_new_folder_permissions", 
fallback="0o775"), 8),
+        int(conf.get("logging", "file_task_handler_new_file_permissions", 
fallback="0o664"), 8),
+    )
+
+
+def _has_lines(path: Path) -> bool:
+    try:
+        return path.stat().st_size > 0
+    except OSError:
+        return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+    """
+    Whether a file the handler leaves as it was after ``upload`` is no sign of 
a failed upload.
+
+    True for a handler that streams each line as it is written (it has 
processors, like CloudWatch) or that takes
+    only task logs (Elasticsearch, OpenSearch; by class name, as nothing else 
tells them apart). Any other handler
+    that leaves the file as it was failed to upload it (S3, GCS and WASB 
report that only in their log).
+    """
+    if getattr(handler, "processors", None):
+        return True
+    return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+_handler_keeping_local_copy: tuple[object, object] | None = None
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+    """
+    Upload one of a watcher's files, at ``path`` locally; with 
delete_local_logs, remove it once stored.
+
+    Not by the handler, which removes the whole directory of the file it 
stored: the watcher's other files are
+    in it or below it, and may not be stored yet.
+    """
+    delete = getattr(handler, "delete_local_copy", False) and not 
handler_keeps_no_copy(handler)
+    if _handler_is(handler, _APPENDING_HANDLERS):
+        # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS 
reports a failed write only
+        # through it. The file is emptied once stored, or removed if it can't 
be (sent again, it would be
+        # stored twice), and kept as it was otherwise.
+        if path.is_file() and handler.oss_write(path.read_text(), 
relative_path):
+            try:
+                os.truncate(path, 0)
+            except OSError:
+                path.unlink(missing_ok=True)
+    else:
+        (_get_handler_keeping_local_copy(handler) if delete else 
handler).upload(relative_path, None)
+    if delete and not _is_left_unstored(handler, path, raised=False):
+        with suppress(FileNotFoundError):
+            path.unlink()
+        with suppress(OSError):
+            path.parent.rmdir()  # once its last file is stored
+
+
+def _get_handler_keeping_local_copy(handler):
+    """Return a copy of the handler with ``delete_local_copy`` off, made once: 
it has its own hook."""
+    global _handler_keeping_local_copy
+    if _handler_keeping_local_copy is None or _handler_keeping_local_copy[0] 
is not handler:
+        keeping = copy.copy(handler)
+        keeping.delete_local_copy = False
+        _handler_keeping_local_copy = (handler, keeping)
+    return _handler_keeping_local_copy[1]
+
+
+def _append_file(lines: Path, kept: Path) -> None:
+    """Append ``lines`` to ``kept``; cut back if that fails part way (a full 
disk), as ``lines`` still has them."""
+    size = kept.stat().st_size
+    try:
+        with lines.open("rb") as source, kept.open("ab") as target:
+            shutil.copyfileobj(source, target)
+    except OSError:
+        with suppress(OSError):
+            os.truncate(kept, size)
+        raise
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+    rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True, 
exist_ok=True)
+    os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+    """
+    Whether to send a watcher's file again after an upload.
+
+    True if it has lines and the upload raised, or S3, GCS, WASB or OSS left 
them (others may keep them).
+    """
+    if not _has_lines(path):
+        return False
+    return raised or _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+    """Whether the handler's class, or a base class, has one of these names 
(as for the above)."""
+    classes = type(handler).__mro__
+    if type(handler) not in _checked_handler_classes:
+        _checked_handler_classes.add(type(handler))
+        if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+            log.info(
+                "Remote log handler not known by name: asset watcher log files 
are sent to it as to one "
+                "that stores them, and sent again only if the upload raises",
+                
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+            )
+    return any(cls.__name__ in names for cls in classes)
+
+
+def _get_existing_trigger_ids(session, trigger_ids: set[int]) -> set[int]:
+    return {
+        trigger_id
+        for ids in chunks(sorted(trigger_ids), _TRIGGER_ID_QUERY_CHUNK_SIZE)
+        for trigger_id in 
session.scalars(select(Trigger.id).where(Trigger.id.in_(ids)))
+    }
+
+
+def get_watcher_log_max_open_files() -> int:
+    """
+    Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a 
quarter of ``ulimit -n``.
+
+    The task and callback trigger logs, the database and the sockets need 
descriptors too.
+    """
+    setting = conf.getint("triggerer", "watcher_log_max_open_files")
+    try:
+        import resource  # Unix only
+
+        soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ImportError, OSError, ValueError):
+        return setting
+    if soft == resource.RLIM_INFINITY:
+        return setting
+    limit = max(1, soft // 4)
+    return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+    """The upload of a released watcher's files."""
+
+    factory: TriggerLoggingFactory
+    # The upload of the same watcher's files released before, if it was still 
pending
+    previous: _Release | None
+    upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+    """
+    The asset watcher log files of one triggerer's supervisor, other than 
writing to them.
+
+    Keeps at most ``watcher_log_max_open_files`` of them open, uploads their 
rotated files and those of
+    released watchers off the main loop, and uploads or removes the files 
triggerers that died left here.
+    """
+
+    # The supervisor's trigger log factories, by trigger id
+    logger_cache: dict[int, TriggerLoggingFactory]
+    # Told when closing a trigger's log file fails
+    on_error: Callable[[int | None, OSError], None]
+    # Asset watcher files with an open descriptor, least recently written first
+    _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+    watcher_log_max_open_files: int = 
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+    # When to look for watcher files left by triggerers that died, and the 
thread doing it
+    _orphan_scans_at: list[float] | None = attrs.field(default=None, 
init=False)
+    _orphan_scan_thread: threading.Thread | None = attrs.field(default=None, 
init=False)
+    # Uploads of released watchers' files, and of rotated files in their own 
pool (release uploads wait)
+    _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None, 
init=False)
+    _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+    # Held to store a release, and to forget it on its upload's thread
+    _releases_lock: threading.Lock = attrs.field(factory=threading.Lock, 
init=False)
+    # Log paths of watchers that left while their trigger existed, removed 
here once it no longer does
+    _left: dict[int, str] = attrs.field(factory=dict, init=False)
+    _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None, 
init=False)
+    _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict, 
init=False)
+    # The last number given to each watcher's rotated files here (see 
upload_rotated)
+    _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+    # Each watcher's last rotated file whose upload the handler did not 
confirm, sent again before it is replaced
+    _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict, 
init=False)
+    # Held while a released watcher's file is added to its held file, or the 
held file is taken for upload
+    _held_lock: threading.Lock = attrs.field(factory=threading.Lock, 
init=False)
+
+    def note_watcher_log_write(self, trigger_id: int) -> None:
+        """Keep at most ``watcher_log_max_open_files`` watcher files open, 
closing the least recent."""
+        self._open_watcher_logs.pop(trigger_id, None)
+        self._open_watcher_logs[trigger_id] = None
+        while 0 < self.watcher_log_max_open_files < 
len(self._open_watcher_logs):
+            idle = next(iter(self._open_watcher_logs))
+            del self._open_watcher_logs[idle]
+            if (factory := self.logger_cache.get(idle)) is not None:
+                factory.close()
+
+    def forget_open_file(self, trigger_id: int) -> None:
+        """Stop counting a trigger's file as open, as it is about to be 
closed."""
+        self._open_watcher_logs.pop(trigger_id, None)
+
+    def release(self, trigger_id: int, factory: TriggerLoggingFactory) -> None:
+        """
+        Upload and close the file of a watcher no longer assigned here, off 
the main loop.
+
+        The watcher may come back here at once, on a new live file: this one 
is held next to it until an upload
+        of the watcher's files here takes it (see _hold).
+        """
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        # In the background: a triggerer that lost its watchers (a DB 
partition) releases them all at once.
+        if self._upload_pool is None:
+            self._upload_pool = ThreadPoolExecutor(max_workers=4, 
thread_name_prefix="trigger-log-release")
+        handler = load_remote_log_handler()
+        if handler is not None and not _handler_is(handler, 
_TASK_LOGS_ONLY_HANDLERS):
+            if hasattr(factory, "bound_logger"):
+                factory.close()
+                self._hold(factory.bound_logger._logger._path)
+        with self._releases_lock:
+            release = _Release(factory=factory, 
previous=self._releases.get(trigger_id))
+            release.upload = self._upload_pool.submit(self._upload_released, 
trigger_id, release)
+            self._releases[trigger_id] = release
+
+        def forget(done: Future) -> None:
+            with self._releases_lock:
+                if self._releases.get(trigger_id) is release:
+                    self._releases.pop(trigger_id, None)
+
+        release.upload.add_done_callback(forget)
+
+    def _hold(self, live: Path) -> None:
+        """Move a released watcher's lines out of the way of a watcher back 
here (see release)."""
+        held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME)
+        # Else left in the live file: a watcher back here appends to it, and 
its next upload sends them.
+        with self._held_lock, suppress(OSError):
+            if not _has_lines(live):
+                return
+            if not held.exists():
+                os.replace(live, held)
+                return
+            # Released again before the held file was taken for upload: one 
file, sent once.
+            _append_file(live, held)
+            live.unlink()
+
+    def _take_held(self, trigger_id: int, log_path: str) -> str | None:
+        """Move the held file into the rotated directory for upload; return 
its path there, if there was one."""
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        base = Path(conf.get("logging", "base_log_folder"))
+        held = base / get_trigger_log_held_path(log_path)
+        if (handler := load_remote_log_handler()) is None or not 
held.is_file():
+            return None
+        if handler_keeps_no_copy(handler):
+            path = get_trigger_log_rotated_path(log_path)
+        else:
+            # Numbered only now, as the watcher's rotations wait for this 
upload (see prepare_rotation).
+            part = self._rotated_parts[trigger_id] = 
self._rotated_parts.get(trigger_id, 0) + 1

Review Comment:
   **minor**: with a store that keeps the local file after a successful upload, 
each release adds one more numbered part here and nothing removes the earlier 
ones until a size rotation runs. HDFS without `delete_local_logs` is such a 
store: `HdfsRemoteLogIO.upload` only calls `load_file`. The only place that 
prunes `rotated/` is the `upload` closure in `upload_rotated`.
   
   So a watcher that is released several times before its file reaches 
`watcher_log_max_bytes` keeps every part on disk. 
`WatcherLoggingFactory.upload_to_remote` also uploads all of them again on each 
release, because they still have lines. Would it be worth removing the parts 
already stored when a release uploads, as a rotation does?
   
   *Flagged by copilot (adversarial review); cross-checked.*



##########
airflow-core/tests/unit/jobs/test_triggerer_job.py:
##########
@@ -1545,6 +1566,1779 @@ def 
test_trigger_logger_fd_closed_when_upload_to_remote_raises(jobless_superviso
     assert 42 not in jobless_supervisor.running_triggers
 
 
+class TestWatcherLogFile:
+    @staticmethod
+    def _watcher_trigger(mocker, trigger_id):
+        asset = mocker.Mock(uri="s3://bucket/orders")
+        asset.name = "orders"
+        watcher = mocker.Mock(asset=asset)
+        watcher.name = "new_files"
+        return mocker.Mock(
+            spec=Trigger,
+            id=trigger_id,
+            classpath="some.path.Trigger",
+            encrypted_kwargs="",
+            task_instance=None,
+            callback=None,
+            assets=[asset],
+            asset_watchers=[watcher],
+        )
+
+    @staticmethod
+    def _create(supervisor, trigger, mocker):
+        supervisor._create_workload(
+            trigger=trigger, dag_bag=mocker.Mock(), 
render_log_fname=mocker.Mock(), session=mocker.Mock()

Review Comment:
   **nit**: these three `mocker.Mock()` have no `spec`, and the same goes for 
`task_logger = mocker.Mock()` (line 2486) and `ti=mocker.Mock()` (line 2736).
   
   > Flag any `mock.Mock()` or `mock.MagicMock()` without `spec` or `autospec`. 
Unspec'd mocks silently accept any attribute access, hiding real bugs.
   
   ([code-review 
instructions](https://github.com/apache/airflow/blob/main/.github/instructions/code-review.instructions.md#testing-requirements))



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to