vincbeck commented on code in PR #76527:
URL: https://github.com/apache/airflow/pull/76527#discussion_r4615838190
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -564,6 +601,76 @@ def upload_to_remote(self):
upload_to_remote(self.bound_logger, self.ti)
[email protected](kw_only=True)
+class WatcherLoggingFactory(TriggerLoggingFactory):
+ """
+ An asset watcher's own log file on this triggerer, as its owner has no log.
+
+ It is kept while the trigger is assigned here, across restarts of the
watcher (SQS-style ones restart
+ after every event), so it is appended to, and capped on local disk. It is
uploaded with the remote log
+ handler task logs use, and every line keeps its ``trigger_id``.
+ """
+
+ max_bytes: int = 0
+ to_stdout: bool = True
+ # Told about every line, to cap how many of these files are open at once.
+ on_write: Callable[[int], None] | None = attrs.field(default=None,
repr=False)
+ may_rotate: Callable[[], bool] | None = attrs.field(default=None,
repr=False)
+ on_rotate: Callable[[], None] | None = attrs.field(default=None,
repr=False)
+
+ def _open(self, processors: Iterable[structlog.typing.Processor]) -> None:
+ if hasattr(self, "bound_logger"):
+ # Before the processors run: a handler streaming each line
(CloudWatch) names its stream after the
+ # open file, and would drop the line of a file the open-files cap
closed.
+ self.bound_logger._logger.ensure_open()
+ else:
+ from airflow.sdk.log import init_log_file # noqa: SDK001
+
+ underlying_logger = CappedBytesLogger(
+ init_log_file(self.log_path), self.max_bytes, self.may_rotate,
self.on_rotate
+ )
+ self.bound_logger = structlog.wrap_logger(underlying_logger,
processors=processors).bind(
+ trigger_id=self.trigger_id
+ )
+ if self.on_write is not None and self.trigger_id is not None:
+ self.on_write(self.trigger_id)
+
+ def close(self):
+ if hasattr(self, "bound_logger"):
+ self.bound_logger._logger.close()
+
+ def upload_to_remote(
+ self, *, keep_live: bool = False, take_held: Callable[[], Path | None]
| None = None
+ ):
+ """
+ Upload the watcher's files on this triggerer.
+
+ ``take_held`` moves the held file (see ``WatcherLogs.release``) into
the rotated directory and returns
+ where; without it, the held file is not sent. With ``keep_live``, the
live file is not sent either: a
+ watcher back here writes it.
+ """
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ if not hasattr(self, "bound_logger") or (handler :=
load_remote_log_handler()) is None:
+ return
+ # Never upload an open file: remote handlers empty or delete it
afterwards. The rotated files first, as
+ # remote handlers append in upload order. Nor an empty one to a
handler that stores files: it would
+ # download and upload the remote copy again for nothing.
+ self.close()
+ live = self.bound_logger._logger._path
+ stores_files = not handler_keeps_no_copy(handler)
+ for local in list_trigger_log_files(live.parent):
+ path: Path | None = local
+ if local.name == TRIGGER_LOG_HELD_FILE_NAME:
+ # Only now: with delete_local_logs, the uploads of the rotated
files before it remove their directory.
+ path = take_held() if take_held is not None else None
+ if path is None or (keep_live and local == live):
+ continue
+ relative_path = (Path(self.log_path).parent /
path.relative_to(live.parent)).as_posix()
+ if not stores_files or (path.is_file() and path.stat().st_size):
+ upload_watcher_log_file(handler, relative_path, path)
Review Comment:
With `delete_local_logs`, a rotated part whose upload failed is deleted by
the upload of the live file. The loop sends `rotated/` first and `trigger.log`
last, and `S3RemoteLogIO.upload` then runs `shutil.rmtree` on the live file's
directory, which contains `rotated/`. Example: at stop, the S3 write of
`rotated/trigger.000003.log` fails, `trigger.log` goes through, and part 3 is
gone with nothing left to retry it. The orphan scan has the same issue
(`watcher_log.py:618`). Fix: don't upload the live file when an earlier file in
the loop was left unstored, so the whole directory stays for the next orphan
scan.
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,699 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_HELD_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_held_path,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate",
"_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ self._folder_mode, self._file_mode = get_log_permissions()
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(mode=self._folder_mode, parents=True,
exist_ok=True)
+ self._path.touch(mode=self._file_mode)
+ self._file = self._path.open("ab")
+ self._write = self._file.write
+ self._flush = self._file.flush
+ self._size = self._file.tell()
+
+ def _unlinked(self) -> bool:
+ try:
+ return os.fstat(self._file.fileno()).st_nlink == 0
+ except (OSError, ValueError):
+ return False
+
+ def msg(self, message: bytes) -> None:
+ with self._lock:
+ try:
+ self.ensure_open()
+ self._write(message + b"\n")
+ self._flush()
+ self._size += len(message) + 1
+ if 0 < self._max_bytes <= self._size and (self._may_rotate is
None or self._may_rotate()):
+ self._file.close()
+ self.rotated_path.parent.mkdir(mode=self._folder_mode,
parents=True, exist_ok=True)
+ with suppress(FileNotFoundError):
+ os.replace(self._path, self.rotated_path)
+ self.ensure_open()
+ if self._on_rotate is not None:
+ try:
+ self._on_rotate()
+ except Exception:
+ # The line is written: the caller, reading the
runner's logs, must not see this.
+ log.exception(
+ "Failed to upload a rotated trigger log",
path=str(self.rotated_path)
+ )
+ except OSError:
+ # Reopened by the next line: a file removed meanwhile (to free
a full disk) frees its space.
+ with suppress(OSError):
+ self._file.close()
+ raise
+
+ def close(self) -> None:
+ # Closing flushes; on a full disk those lines are lost either way.
+ with suppress(OSError):
+ self._file.close()
+
+ log = debug = info = warn = warning = msg
+ fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO",
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO",
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file:
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO",
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+ _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+ """Return the modes of new log folders and files, as ``init_log_file``
gives them."""
+ return (
+ int(conf.get("logging", "file_task_handler_new_folder_permissions",
fallback="0o775"), 8),
+ int(conf.get("logging", "file_task_handler_new_file_permissions",
fallback="0o664"), 8),
+ )
+
+
+def _has_lines(path: Path) -> bool:
+ try:
+ return path.stat().st_size > 0
+ except OSError:
+ return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+ """
+ Whether a file the handler leaves as it was after ``upload`` is no sign of
a failed upload.
+
+ True for a handler that streams each line as it is written (it has
processors, like CloudWatch) or that takes
+ only task logs (Elasticsearch, OpenSearch; by class name, as nothing else
tells them apart). Any other handler
+ that leaves the file as it was failed to upload it (S3, GCS and WASB
report that only in their log).
+ """
+ if getattr(handler, "processors", None):
+ return True
+ return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+ """Upload one of a watcher's files, at ``path`` locally; one sent again
would be stored twice by OSS."""
+ if not _handler_is(handler, _APPENDING_HANDLERS):
+ handler.upload(relative_path, None)
+ return
+ # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS reports a
failed write only through it.
+ # The file is emptied once stored, and kept as it was otherwise, for the
next upload to send again.
+ if path.is_file() and handler.oss_write(path.read_text(), relative_path):
+ if handler.delete_local_copy:
+ shutil.rmtree(path.parent)
+ else:
+ with suppress(FileNotFoundError):
+ os.truncate(path, 0)
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+ # Uploaded from there, as the files before it are: with delete_local_logs,
an upload removes the directory
+ # of the file it sent, which must not be the live file's.
+ rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True,
exist_ok=True)
+ os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+ """
+ Whether to send a watcher's file again after an upload.
+
+ True if it has lines and the upload raised, or S3, GCS, WASB or OSS left
them (others may keep them).
+ """
+ if not _has_lines(path):
+ return False
+ if raised:
+ return not _handler_is(handler, _APPENDING_HANDLERS)
+ return _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+ """Whether the handler's class, or a base class, has one of these names
(as for the above)."""
+ classes = type(handler).__mro__
+ if type(handler) not in _checked_handler_classes:
+ _checked_handler_classes.add(type(handler))
+ if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+ log.info(
+ "Remote log handler not known by name: asset watcher log files
are sent to it as to one "
+ "that stores them, and sent again only if the upload raises",
+
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+ )
+ return any(cls.__name__ in names for cls in classes)
+
+
+def get_watcher_log_max_open_files() -> int:
+ """
+ Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a
quarter of ``ulimit -n``.
+
+ The task and callback trigger logs, the database and the sockets need
descriptors too.
+ """
+ setting = conf.getint("triggerer", "watcher_log_max_open_files")
+ try:
+ import resource # Unix only
+
+ soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+ except (ImportError, OSError, ValueError):
+ return setting
+ if soft == resource.RLIM_INFINITY:
+ return setting
+ limit = max(1, soft // 4)
+ return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+ """The upload of a released watcher's files."""
+
+ factory: WatcherLoggingFactory
+ # The upload of the same watcher's files released before, if it was still
pending
+ previous: _Release | None
+ upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+ """
+ The asset watcher log files of one triggerer's supervisor, other than
writing to them.
+
+ Keeps at most ``watcher_log_max_open_files`` of them open, uploads their
rotated files and those of
+ released watchers off the main loop, and uploads or removes the files
triggerers that died left here.
+ """
+
+ # The supervisor's trigger log factories, by trigger id
+ logger_cache: dict[int, TriggerLoggingFactory]
+ # Told when closing a trigger's log file fails
+ on_error: Callable[[int | None, OSError], None]
+ # Asset watcher files with an open descriptor, least recently written first
+ _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+ watcher_log_max_open_files: int =
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+ # When to look for watcher files left by triggerers that died, and the
thread doing it
+ _orphan_scans_at: list[float] | None = attrs.field(default=None,
init=False)
+ _orphan_scan_thread: threading.Thread | None = attrs.field(default=None,
init=False)
+ # Uploads of released watchers' files, and of rotated files in their own
pool (release uploads wait)
+ _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+ # Log paths of watchers that left while their trigger existed, removed
here once it no longer does
+ _left: dict[int, str] = attrs.field(factory=dict, init=False)
+ _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict,
init=False)
+ # The last number given to each watcher's rotated files here (see
upload_rotated)
+ _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+ # Each watcher's last rotated file whose upload the handler did not
confirm, sent again before it is replaced
+ _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict,
init=False)
+ # Held while a released watcher's file is added to its held file, or the
held file is taken for upload
+ _held_lock: threading.Lock = attrs.field(factory=threading.Lock,
init=False)
+
+ def note_watcher_log_write(self, trigger_id: int) -> None:
+ """Keep at most ``watcher_log_max_open_files`` watcher files open,
closing the least recent."""
+ self._open_watcher_logs.pop(trigger_id, None)
+ self._open_watcher_logs[trigger_id] = None
+ while 0 < self.watcher_log_max_open_files <
len(self._open_watcher_logs):
+ idle = next(iter(self._open_watcher_logs))
+ del self._open_watcher_logs[idle]
+ if (factory := self.logger_cache.get(idle)) is not None:
+ factory.close()
+
+ def forget_open_file(self, trigger_id: int) -> None:
+ """Stop counting a trigger's file as open, as it is about to be
closed."""
+ self._open_watcher_logs.pop(trigger_id, None)
+
+ def release(self, trigger_id: int, factory: WatcherLoggingFactory) -> None:
+ """
+ Upload and close the file of a watcher no longer assigned here, off
the main loop.
+
+ The watcher may come back here at once, on a new live file: this one
is held next to it until an upload
+ of the watcher's files here takes it (see _hold).
+ """
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ # In the background: a triggerer that lost its watchers (a DB
partition) releases them all at once.
+ if self._upload_pool is None:
+ self._upload_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-release")
+ handler = load_remote_log_handler()
+ if handler is not None and not _handler_is(handler,
_TASK_LOGS_ONLY_HANDLERS):
+ if hasattr(factory, "bound_logger"):
+ factory.close()
+ self._hold(factory.bound_logger._logger._path)
+ release = _Release(factory=factory,
previous=self._releases.get(trigger_id))
+ release.upload = self._upload_pool.submit(self._upload_released,
trigger_id, release)
+ self._releases[trigger_id] = release
+
+ def forget(done: Future) -> None:
+ if self._releases.get(trigger_id) is release:
Review Comment:
`forget` runs on the pool thread, so the main thread can store a newer
`_Release` for the same trigger between the `get` and the `pop`, and this pops
that one. `rotated_upload_done` and `remove_left` then think nothing is
uploading for that trigger while it still is. The window is small, but a lock
around this check and pop, and around the store in `release`, would close it.
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,699 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_HELD_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_held_path,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate",
"_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ self._folder_mode, self._file_mode = get_log_permissions()
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(mode=self._folder_mode, parents=True,
exist_ok=True)
+ self._path.touch(mode=self._file_mode)
+ self._file = self._path.open("ab")
+ self._write = self._file.write
+ self._flush = self._file.flush
+ self._size = self._file.tell()
+
+ def _unlinked(self) -> bool:
+ try:
+ return os.fstat(self._file.fileno()).st_nlink == 0
+ except (OSError, ValueError):
+ return False
+
+ def msg(self, message: bytes) -> None:
+ with self._lock:
+ try:
+ self.ensure_open()
+ self._write(message + b"\n")
+ self._flush()
+ self._size += len(message) + 1
+ if 0 < self._max_bytes <= self._size and (self._may_rotate is
None or self._may_rotate()):
+ self._file.close()
+ self.rotated_path.parent.mkdir(mode=self._folder_mode,
parents=True, exist_ok=True)
+ with suppress(FileNotFoundError):
+ os.replace(self._path, self.rotated_path)
+ self.ensure_open()
Review Comment:
If opening the new live file fails here (disk full, no file descriptors
left), the `OSError` skips `_on_rotate`, so the lines just moved to
`rotated/trigger.log` are never uploaded. The next rotation's `os.replace` then
overwrites them. Fix: call `_on_rotate` right after the `os.replace`, before
reopening.
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,699 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_HELD_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_held_path,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate",
"_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ self._folder_mode, self._file_mode = get_log_permissions()
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(mode=self._folder_mode, parents=True,
exist_ok=True)
+ self._path.touch(mode=self._file_mode)
+ self._file = self._path.open("ab")
+ self._write = self._file.write
+ self._flush = self._file.flush
+ self._size = self._file.tell()
+
+ def _unlinked(self) -> bool:
+ try:
+ return os.fstat(self._file.fileno()).st_nlink == 0
+ except (OSError, ValueError):
+ return False
+
+ def msg(self, message: bytes) -> None:
+ with self._lock:
+ try:
+ self.ensure_open()
+ self._write(message + b"\n")
+ self._flush()
+ self._size += len(message) + 1
+ if 0 < self._max_bytes <= self._size and (self._may_rotate is
None or self._may_rotate()):
+ self._file.close()
+ self.rotated_path.parent.mkdir(mode=self._folder_mode,
parents=True, exist_ok=True)
+ with suppress(FileNotFoundError):
+ os.replace(self._path, self.rotated_path)
+ self.ensure_open()
+ if self._on_rotate is not None:
+ try:
+ self._on_rotate()
+ except Exception:
+ # The line is written: the caller, reading the
runner's logs, must not see this.
+ log.exception(
+ "Failed to upload a rotated trigger log",
path=str(self.rotated_path)
+ )
+ except OSError:
+ # Reopened by the next line: a file removed meanwhile (to free
a full disk) frees its space.
+ with suppress(OSError):
+ self._file.close()
+ raise
+
+ def close(self) -> None:
+ # Closing flushes; on a full disk those lines are lost either way.
+ with suppress(OSError):
+ self._file.close()
+
+ log = debug = info = warn = warning = msg
+ fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO",
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO",
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file:
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO",
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+ _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+ """Return the modes of new log folders and files, as ``init_log_file``
gives them."""
+ return (
+ int(conf.get("logging", "file_task_handler_new_folder_permissions",
fallback="0o775"), 8),
+ int(conf.get("logging", "file_task_handler_new_file_permissions",
fallback="0o664"), 8),
+ )
+
+
+def _has_lines(path: Path) -> bool:
+ try:
+ return path.stat().st_size > 0
+ except OSError:
+ return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+ """
+ Whether a file the handler leaves as it was after ``upload`` is no sign of
a failed upload.
+
+ True for a handler that streams each line as it is written (it has
processors, like CloudWatch) or that takes
+ only task logs (Elasticsearch, OpenSearch; by class name, as nothing else
tells them apart). Any other handler
+ that leaves the file as it was failed to upload it (S3, GCS and WASB
report that only in their log).
+ """
+ if getattr(handler, "processors", None):
+ return True
+ return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+ """Upload one of a watcher's files, at ``path`` locally; one sent again
would be stored twice by OSS."""
+ if not _handler_is(handler, _APPENDING_HANDLERS):
+ handler.upload(relative_path, None)
+ return
+ # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS reports a
failed write only through it.
+ # The file is emptied once stored, and kept as it was otherwise, for the
next upload to send again.
+ if path.is_file() and handler.oss_write(path.read_text(), relative_path):
+ if handler.delete_local_copy:
+ shutil.rmtree(path.parent)
+ else:
+ with suppress(FileNotFoundError):
+ os.truncate(path, 0)
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+ # Uploaded from there, as the files before it are: with delete_local_logs,
an upload removes the directory
+ # of the file it sent, which must not be the live file's.
+ rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True,
exist_ok=True)
+ os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+ """
+ Whether to send a watcher's file again after an upload.
+
+ True if it has lines and the upload raised, or S3, GCS, WASB or OSS left
them (others may keep them).
+ """
+ if not _has_lines(path):
+ return False
+ if raised:
+ return not _handler_is(handler, _APPENDING_HANDLERS)
+ return _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+ """Whether the handler's class, or a base class, has one of these names
(as for the above)."""
+ classes = type(handler).__mro__
+ if type(handler) not in _checked_handler_classes:
+ _checked_handler_classes.add(type(handler))
+ if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+ log.info(
+ "Remote log handler not known by name: asset watcher log files
are sent to it as to one "
+ "that stores them, and sent again only if the upload raises",
+
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+ )
+ return any(cls.__name__ in names for cls in classes)
+
+
+def get_watcher_log_max_open_files() -> int:
+ """
+ Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a
quarter of ``ulimit -n``.
+
+ The task and callback trigger logs, the database and the sockets need
descriptors too.
+ """
+ setting = conf.getint("triggerer", "watcher_log_max_open_files")
+ try:
+ import resource # Unix only
+
+ soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+ except (ImportError, OSError, ValueError):
+ return setting
+ if soft == resource.RLIM_INFINITY:
+ return setting
+ limit = max(1, soft // 4)
+ return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+ """The upload of a released watcher's files."""
+
+ factory: WatcherLoggingFactory
+ # The upload of the same watcher's files released before, if it was still
pending
+ previous: _Release | None
+ upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+ """
+ The asset watcher log files of one triggerer's supervisor, other than
writing to them.
+
+ Keeps at most ``watcher_log_max_open_files`` of them open, uploads their
rotated files and those of
+ released watchers off the main loop, and uploads or removes the files
triggerers that died left here.
+ """
+
+ # The supervisor's trigger log factories, by trigger id
+ logger_cache: dict[int, TriggerLoggingFactory]
+ # Told when closing a trigger's log file fails
+ on_error: Callable[[int | None, OSError], None]
+ # Asset watcher files with an open descriptor, least recently written first
+ _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+ watcher_log_max_open_files: int =
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+ # When to look for watcher files left by triggerers that died, and the
thread doing it
+ _orphan_scans_at: list[float] | None = attrs.field(default=None,
init=False)
+ _orphan_scan_thread: threading.Thread | None = attrs.field(default=None,
init=False)
+ # Uploads of released watchers' files, and of rotated files in their own
pool (release uploads wait)
+ _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+ # Log paths of watchers that left while their trigger existed, removed
here once it no longer does
+ _left: dict[int, str] = attrs.field(factory=dict, init=False)
+ _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict,
init=False)
+ # The last number given to each watcher's rotated files here (see
upload_rotated)
+ _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+ # Each watcher's last rotated file whose upload the handler did not
confirm, sent again before it is replaced
+ _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict,
init=False)
+ # Held while a released watcher's file is added to its held file, or the
held file is taken for upload
+ _held_lock: threading.Lock = attrs.field(factory=threading.Lock,
init=False)
+
+ def note_watcher_log_write(self, trigger_id: int) -> None:
+ """Keep at most ``watcher_log_max_open_files`` watcher files open,
closing the least recent."""
+ self._open_watcher_logs.pop(trigger_id, None)
+ self._open_watcher_logs[trigger_id] = None
+ while 0 < self.watcher_log_max_open_files <
len(self._open_watcher_logs):
+ idle = next(iter(self._open_watcher_logs))
+ del self._open_watcher_logs[idle]
+ if (factory := self.logger_cache.get(idle)) is not None:
+ factory.close()
+
+ def forget_open_file(self, trigger_id: int) -> None:
+ """Stop counting a trigger's file as open, as it is about to be
closed."""
+ self._open_watcher_logs.pop(trigger_id, None)
+
+ def release(self, trigger_id: int, factory: WatcherLoggingFactory) -> None:
+ """
+ Upload and close the file of a watcher no longer assigned here, off
the main loop.
+
+ The watcher may come back here at once, on a new live file: this one
is held next to it until an upload
+ of the watcher's files here takes it (see _hold).
+ """
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ # In the background: a triggerer that lost its watchers (a DB
partition) releases them all at once.
+ if self._upload_pool is None:
+ self._upload_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-release")
+ handler = load_remote_log_handler()
+ if handler is not None and not _handler_is(handler,
_TASK_LOGS_ONLY_HANDLERS):
+ if hasattr(factory, "bound_logger"):
+ factory.close()
+ self._hold(factory.bound_logger._logger._path)
+ release = _Release(factory=factory,
previous=self._releases.get(trigger_id))
+ release.upload = self._upload_pool.submit(self._upload_released,
trigger_id, release)
+ self._releases[trigger_id] = release
+
+ def forget(done: Future) -> None:
+ if self._releases.get(trigger_id) is release:
+ self._releases.pop(trigger_id, None)
+
+ release.upload.add_done_callback(forget)
+
+ def _hold(self, live: Path) -> None:
+ """Move a released watcher's lines out of the way of a watcher back
here (see release)."""
+ held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME)
+ # Else left in the live file: a watcher back here appends to it, and
its next upload sends them.
+ with self._held_lock, suppress(OSError):
+ if not _has_lines(live):
+ return
+ if not held.exists():
+ os.replace(live, held)
+ return
+ # Released again before the held file was taken for upload: one
file, sent once.
+ with live.open("rb") as lines, held.open("ab") as kept:
+ shutil.copyfileobj(lines, kept)
+ live.unlink()
+
+ def _take_held(self, trigger_id: int, log_path: str) -> str | None:
+ """Move the held file into the rotated directory for upload; return
its path there, if there was one."""
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ base = Path(conf.get("logging", "base_log_folder"))
+ held = base / get_trigger_log_held_path(log_path)
+ if (handler := load_remote_log_handler()) is None or not
held.is_file():
+ return None
+ if handler_keeps_no_copy(handler):
+ path = get_trigger_log_rotated_path(log_path)
+ else:
+ # Numbered only now, as the watcher's rotations wait for this
upload (see rotated_upload_done).
+ part = self._rotated_parts[trigger_id] =
self._rotated_parts.get(trigger_id, 0) + 1
+ path = get_trigger_log_part_path(log_path, part)
+ with self._held_lock:
+ _move_into_rotated_dir(held, base / path)
+ return path
+
+ def _upload_released(self, trigger_id: int, release: _Release) -> None:
+ if release.previous is not None:
+ # Its files are in the same directory.
+ with suppress(Exception):
+ release.previous.upload.result()
+ release.previous = None
+ factory = release.factory
+ failed = self._upload_and_close(trigger_id, factory, keep_live=True)
+ if failed:
+ self._left.pop(trigger_id, None) # kept for the next start's
orphan scan
+ else:
+ self._left[trigger_id] = factory.log_path
+
+ @staticmethod
+ def _remove_released_files(log_path: str) -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ producer_dir = (Path(conf.get("logging", "base_log_folder")) /
log_path).parent
+ handler = load_remote_log_handler()
+ if handler is not None and _handler_is(handler,
_LINES_LEFT_MEAN_FAILED_HANDLERS):
+ if any(_has_lines(path) for path in
list_trigger_log_files(producer_dir)):
+ return # a failed upload: sent again by the next start's
orphan scan
+ shutil.rmtree(producer_dir, ignore_errors=True)
+ with suppress(OSError):
+ producer_dir.parent.rmdir() # not while another triggerer still
has a directory in it
+
+ def left_trigger_ids(self) -> set[int]:
+ return set(self._left)
+
+ def remove_left(self, gone: set[int]) -> None:
+ """Remove the files of watchers that left here and whose trigger no
longer exists."""
+ for trigger_id in gone:
+ if (release := self._releases.get(trigger_id)) is not None and not
release.upload.done():
+ continue # its files are still uploading
+ log_path = self._left.pop(trigger_id, None)
+ if log_path is None or trigger_id in self.logger_cache:
+ continue # back here: tracked again once it leaves
+ try:
+ self._remove_released_files(log_path)
+ except Exception:
+ log.exception("Failed to remove the log files of a trigger
that no longer exists")
+
+ def rotated_upload_done(self, trigger_id: int) -> bool:
+ if (release := self._releases.get(trigger_id)) is not None and not
release.upload.done():
+ return False # its released files are uploading from the rotated
files' directory
+ upload = self._pending_rotated_uploads.get(trigger_id)
+ if upload is not None and not upload.done():
+ return False
+ if path := self._rotated_unconfirmed.pop(trigger_id, None):
+ # Now, while it is the only file in its directory: with
delete_local_logs, a handler deletes the
+ # directory of the file it uploaded. Replaced by the next rotation
whether it goes through or not.
+ self._pending_rotated_uploads[trigger_id] =
self._rotated_executor().submit(
+ self._upload_rotated_again, trigger_id, path
+ )
+ return False
+ return True
+
+ def _rotated_executor(self) -> ThreadPoolExecutor:
+ if self._rotated_pool is None:
+ self._rotated_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-rotated")
+ return self._rotated_pool
+
+ @staticmethod
+ def _upload_rotated_again(trigger_id: int, path: str) -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ try:
+ if (handler := load_remote_log_handler()) is not None:
+ upload_watcher_log_file(handler, path,
Path(conf.get("logging", "base_log_folder")) / path)
+ except Exception:
+ log.exception("Failed to upload a rotated trigger log",
trigger_id=trigger_id, path=path)
+
+ def upload_rotated(self, trigger_id: int, log_path: str) -> None:
+ # Called by the line that rotated the file: the rest runs in the pool,
not to hold up the runner's logs.
+ pool = self._rotated_executor()
+ rotated = get_trigger_log_rotated_path(log_path)
+ # Numbered here: a watcher's next rotation waits for this upload (see
rotated_upload_done).
+ part = self._rotated_parts.get(trigger_id, 0) + 1
+
+ def upload() -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ path = rotated
+ base = Path(conf.get("logging", "base_log_folder"))
+ try:
+ if (handler := load_remote_log_handler()) is None:
+ return
+ if not handler_keeps_no_copy(handler):
+ # Each rotated file is its own object (see
get_trigger_log_part_path), numbered by this
+ # triggerer, the only one writing there: counted, as
earlier parts may be gone locally.
+ for done in (base / rotated).parent.glob("*.log"):
+ if done.name != TRIGGER_LOG_FILE_NAME:
+ with suppress(OSError):
+ done.unlink() # the previous one, uploaded or
tried twice: one at a time
+ path = get_trigger_log_part_path(log_path, part)
+ # Named only now: until then the triggerer's log server
serves it as the rotated file.
+ os.replace(base / rotated, base / path)
+ upload_watcher_log_file(handler, path, base / path)
+ raised = False
+ except Exception:
+ log.exception("Failed to upload a rotated trigger log",
trigger_id=trigger_id, path=path)
+ raised = True
+ if path != rotated and _is_left_unstored(handler, base / path,
raised=raised):
+ self._rotated_unconfirmed[trigger_id] = path
+
+ self._pending_rotated_uploads[trigger_id] = pool.submit(upload)
+ self._rotated_parts[trigger_id] = part
+
+ def upload_and_close(self, trigger_id: int, factory:
TriggerLoggingFactory) -> None:
+ if (release := self._releases.get(trigger_id)) is not None:
+ # A watcher back here: its released files are uploading from the
same directory.
+ with suppress(Exception):
+ release.upload.result()
+ self._upload_and_close(trigger_id, factory)
+
+ def _upload_and_close(self, trigger_id: int, factory:
TriggerLoggingFactory, **upload_kwargs) -> bool:
+ """Upload and close a trigger's file; return whether the upload
raised."""
+ from airflow.jobs.triggerer_job_runner import WatcherLoggingFactory
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ if (rotated_upload := self._pending_rotated_uploads.pop(trigger_id,
None)) is not None:
+ rotated_upload.result() # else the rotated file could be read
again before it is emptied
+ self._rotated_unconfirmed.pop(trigger_id, None) # sent again below,
with the live file
+ base = Path(conf.get("logging", "base_log_folder"))
+ failed, held = False, None
+
+ def take_held() -> Path | None:
+ nonlocal held
+ held = self._take_held(trigger_id, factory.log_path)
+ return None if held is None else base / held
+
+ if isinstance(factory, WatcherLoggingFactory):
+ upload_kwargs["take_held"] = take_held
+ try:
+ factory.upload_to_remote(**upload_kwargs)
+ except Exception:
+ log.exception("Failed to upload trigger logs to remote",
trigger_id=trigger_id)
+ failed = True
+ finally:
+ # Close even if the upload failed, not to leak the descriptor.
Closing flushes, which can fail.
+ try:
+ factory.close()
+ except OSError as e:
+ self.on_error(trigger_id, e)
+ if held and _is_left_unstored(load_remote_log_handler(), base / held,
raised=failed):
+ # Under a rotated file's name now: the watcher's next rotation
here would remove it unsent.
+ self._rotated_unconfirmed[trigger_id] = held
+ return failed
+
+ def upload_and_close_all(self, factories: list[tuple[int,
TriggerLoggingFactory]]) -> None:
+ """Upload and close these watchers' files, and wait for the released
ones' uploads."""
+ self._open_watcher_logs.clear()
+ # Side by side: one after the other, thousands of files take longer
than a container's stop timeout.
+ with ThreadPoolExecutor(max_workers=16,
thread_name_prefix="trigger-log-upload") as pool:
+ list(pool.map(lambda item: self.upload_and_close(*item),
factories))
+ if self._upload_pool is not None:
+ self._upload_pool.shutdown(wait=True)
+
+ def handle_orphaned_trigger_logs(self, own_job_id: int | None) -> None:
+ """
+ Upload, or remove, the asset watcher files left on this disk by
triggerers that are no longer alive.
+
+ A killed triggerer uploads nothing, so its files are uploaded now,
then their directories removed.
+ Files remote logging doesn't take (it is off, or the handler keeps no
copy, like Elasticsearch) are
+ removed instead: all but the most recently written for each watcher,
and all for triggers that no
+ longer exist. Files of live triggerers, this one included, of
triggerers without a job (no job id
+ directory), and those a failed upload left stay.
+ """
+ from airflow.jobs.job import Job
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ handler = load_remote_log_handler()
+ base = Path(conf.get("logging", "base_log_folder"))
+ dirs_by_job: dict[int, list[Path]] = {}
+ dirs_by_trigger: dict[Path, int] = {}
+ for producer_dir in
base.glob(f"{TRIGGER_LOG_ROOT}/{TRIGGER_ID_KEY}=*/{TRIGGERER_JOB_ID_KEY}=*"):
+ # Only directories this module names are ever removed: both ids
must parse.
+ trigger_id = parse_trigger_log_dir_id(TRIGGER_ID_KEY,
producer_dir.parent.name)
+ job_id = parse_trigger_log_dir_id(TRIGGERER_JOB_ID_KEY,
producer_dir.name)
+ if (
+ trigger_id is not None
+ and job_id is not None
+ and job_id != own_job_id
+ and producer_dir.is_dir()
+ ):
+ dirs_by_job.setdefault(job_id, []).append(producer_dir)
+ dirs_by_trigger[producer_dir.parent] = trigger_id
+ if not dirs_by_job:
+ return
+ trigger_ids = sorted(set(dirs_by_trigger.values()))
+ with create_session() as session:
+ jobs = session.scalars(select(Job).where(Job.id.in_(dirs_by_job)))
+ dead = dirs_by_job.keys() - {job.id for job in jobs if
job.is_alive()}
+ # A watcher whose kwargs changed got a new trigger; the old id's
files have nothing left to keep.
+ existing = {
+ trigger_id
+ for ids in chunks(trigger_ids, _TRIGGER_ID_QUERY_CHUNK_SIZE)
+ for trigger_id in
session.scalars(select(Trigger.id).where(Trigger.id.in_(ids)))
+ }
+ dead_dirs = {job_id: dirs_by_job[job_id] for job_id in dead}
+ if handler is not None:
+ # What the handler left as it was is then removed as without
remote logging.
+ dead_dirs = {
+ job_id: [d for d in dirs if
self._upload_orphaned_trigger_log_dir(handler, base, d, job_id)]
+ for job_id, dirs in dead_dirs.items()
+ }
+ gone = {
+ trigger_dir for trigger_dir, trigger_id in dirs_by_trigger.items()
if trigger_id not in existing
+ }
+ self._remove_old_trigger_log_dirs(dead_dirs, gone=gone)
+
+ @staticmethod
+ def _upload_orphaned_trigger_log_dir(handler, base: Path, producer_dir:
Path, job_id: int) -> bool:
+ """Upload a dead triggerer's files unless another triggerer is; True
if the handler left any."""
+ # Claimed first: triggerers starting together on a shared log folder
see the same files, and remote
+ # handlers append, so a file uploaded twice would be in the remote
copy twice.
+ claim = producer_dir / _ORPHAN_UPLOAD_CLAIM
+ with suppress(OSError):
+ if time.time() - claim.stat().st_mtime >=
_ORPHAN_UPLOAD_CLAIM_MAX_AGE:
Review Comment:
A claim younger than 10 minutes makes both orphan scans skip the directory,
and there is no third scan. Example: a triggerer stops in the middle of a scan
(the scan thread is a daemon and isn't joined), so its `.uploading` claim
stays, and the container restarts within seconds. Both scans of the new
triggerer skip that directory, and its files wait for a restart at least 10
minutes later. Should stop remove the claim, or a later scan retry claimed
directories?
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -564,6 +601,76 @@ def upload_to_remote(self):
upload_to_remote(self.bound_logger, self.ti)
[email protected](kw_only=True)
+class WatcherLoggingFactory(TriggerLoggingFactory):
+ """
+ An asset watcher's own log file on this triggerer, as its owner has no log.
+
+ It is kept while the trigger is assigned here, across restarts of the
watcher (SQS-style ones restart
+ after every event), so it is appended to, and capped on local disk. It is
uploaded with the remote log
+ handler task logs use, and every line keeps its ``trigger_id``.
+ """
+
+ max_bytes: int = 0
+ to_stdout: bool = True
+ # Told about every line, to cap how many of these files are open at once.
+ on_write: Callable[[int], None] | None = attrs.field(default=None,
repr=False)
+ may_rotate: Callable[[], bool] | None = attrs.field(default=None,
repr=False)
+ on_rotate: Callable[[], None] | None = attrs.field(default=None,
repr=False)
+
+ def _open(self, processors: Iterable[structlog.typing.Processor]) -> None:
+ if hasattr(self, "bound_logger"):
+ # Before the processors run: a handler streaming each line
(CloudWatch) names its stream after the
+ # open file, and would drop the line of a file the open-files cap
closed.
+ self.bound_logger._logger.ensure_open()
Review Comment:
nit: `ensure_open` runs here and again in `CappedBytesLogger.msg`, so every
watcher line does two `fstat` calls on the supervisor's main loop. Is the one
in `msg` needed?
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -885,7 +1030,11 @@ def load_triggers(self) -> None:
queues=self.queues,
team_name=self.team_name,
)
- ids = Trigger.ids_for_triggerer(self.job.id, queues=self.queues,
team_name=self.team_name)
+ left = self.watcher_logs.left_trigger_ids()
+ ids, existing = Trigger.fetch_ids_for_triggerer(
Review Comment:
`_left` only shrinks when a trigger is deleted, so after handing off 5,000
watchers this adds a 5,000-id `IN` to the assignment query every second, for as
long as those triggers run elsewhere. Can this check run on a slower timer, or
in chunks like the orphan scan?
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,699 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_HELD_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_held_path,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate",
"_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ self._folder_mode, self._file_mode = get_log_permissions()
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(mode=self._folder_mode, parents=True,
exist_ok=True)
+ self._path.touch(mode=self._file_mode)
+ self._file = self._path.open("ab")
+ self._write = self._file.write
+ self._flush = self._file.flush
+ self._size = self._file.tell()
+
+ def _unlinked(self) -> bool:
+ try:
+ return os.fstat(self._file.fileno()).st_nlink == 0
+ except (OSError, ValueError):
+ return False
+
+ def msg(self, message: bytes) -> None:
+ with self._lock:
+ try:
+ self.ensure_open()
+ self._write(message + b"\n")
+ self._flush()
+ self._size += len(message) + 1
+ if 0 < self._max_bytes <= self._size and (self._may_rotate is
None or self._may_rotate()):
+ self._file.close()
+ self.rotated_path.parent.mkdir(mode=self._folder_mode,
parents=True, exist_ok=True)
+ with suppress(FileNotFoundError):
+ os.replace(self._path, self.rotated_path)
+ self.ensure_open()
+ if self._on_rotate is not None:
+ try:
+ self._on_rotate()
+ except Exception:
+ # The line is written: the caller, reading the
runner's logs, must not see this.
+ log.exception(
+ "Failed to upload a rotated trigger log",
path=str(self.rotated_path)
+ )
+ except OSError:
+ # Reopened by the next line: a file removed meanwhile (to free
a full disk) frees its space.
+ with suppress(OSError):
+ self._file.close()
+ raise
+
+ def close(self) -> None:
+ # Closing flushes; on a full disk those lines are lost either way.
+ with suppress(OSError):
+ self._file.close()
+
+ log = debug = info = warn = warning = msg
+ fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO",
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO",
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file:
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO",
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+ _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+ """Return the modes of new log folders and files, as ``init_log_file``
gives them."""
+ return (
+ int(conf.get("logging", "file_task_handler_new_folder_permissions",
fallback="0o775"), 8),
+ int(conf.get("logging", "file_task_handler_new_file_permissions",
fallback="0o664"), 8),
+ )
+
+
+def _has_lines(path: Path) -> bool:
+ try:
+ return path.stat().st_size > 0
+ except OSError:
+ return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+ """
+ Whether a file the handler leaves as it was after ``upload`` is no sign of
a failed upload.
+
+ True for a handler that streams each line as it is written (it has
processors, like CloudWatch) or that takes
+ only task logs (Elasticsearch, OpenSearch; by class name, as nothing else
tells them apart). Any other handler
+ that leaves the file as it was failed to upload it (S3, GCS and WASB
report that only in their log).
+ """
+ if getattr(handler, "processors", None):
+ return True
+ return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+ """Upload one of a watcher's files, at ``path`` locally; one sent again
would be stored twice by OSS."""
+ if not _handler_is(handler, _APPENDING_HANDLERS):
+ handler.upload(relative_path, None)
+ return
+ # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS reports a
failed write only through it.
+ # The file is emptied once stored, and kept as it was otherwise, for the
next upload to send again.
+ if path.is_file() and handler.oss_write(path.read_text(), relative_path):
+ if handler.delete_local_copy:
+ shutil.rmtree(path.parent)
+ else:
+ with suppress(FileNotFoundError):
+ os.truncate(path, 0)
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+ # Uploaded from there, as the files before it are: with delete_local_logs,
an upload removes the directory
+ # of the file it sent, which must not be the live file's.
+ rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True,
exist_ok=True)
+ os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+ """
+ Whether to send a watcher's file again after an upload.
+
+ True if it has lines and the upload raised, or S3, GCS, WASB or OSS left
them (others may keep them).
+ """
+ if not _has_lines(path):
+ return False
+ if raised:
+ return not _handler_is(handler, _APPENDING_HANDLERS)
+ return _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+ """Whether the handler's class, or a base class, has one of these names
(as for the above)."""
+ classes = type(handler).__mro__
+ if type(handler) not in _checked_handler_classes:
+ _checked_handler_classes.add(type(handler))
+ if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+ log.info(
+ "Remote log handler not known by name: asset watcher log files
are sent to it as to one "
+ "that stores them, and sent again only if the upload raises",
+
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+ )
+ return any(cls.__name__ in names for cls in classes)
+
+
+def get_watcher_log_max_open_files() -> int:
+ """
+ Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a
quarter of ``ulimit -n``.
+
+ The task and callback trigger logs, the database and the sockets need
descriptors too.
+ """
+ setting = conf.getint("triggerer", "watcher_log_max_open_files")
+ try:
+ import resource # Unix only
+
+ soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+ except (ImportError, OSError, ValueError):
+ return setting
+ if soft == resource.RLIM_INFINITY:
+ return setting
+ limit = max(1, soft // 4)
+ return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+ """The upload of a released watcher's files."""
+
+ factory: WatcherLoggingFactory
+ # The upload of the same watcher's files released before, if it was still
pending
+ previous: _Release | None
+ upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+ """
+ The asset watcher log files of one triggerer's supervisor, other than
writing to them.
+
+ Keeps at most ``watcher_log_max_open_files`` of them open, uploads their
rotated files and those of
+ released watchers off the main loop, and uploads or removes the files
triggerers that died left here.
+ """
+
+ # The supervisor's trigger log factories, by trigger id
+ logger_cache: dict[int, TriggerLoggingFactory]
+ # Told when closing a trigger's log file fails
+ on_error: Callable[[int | None, OSError], None]
+ # Asset watcher files with an open descriptor, least recently written first
+ _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+ watcher_log_max_open_files: int =
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+ # When to look for watcher files left by triggerers that died, and the
thread doing it
+ _orphan_scans_at: list[float] | None = attrs.field(default=None,
init=False)
+ _orphan_scan_thread: threading.Thread | None = attrs.field(default=None,
init=False)
+ # Uploads of released watchers' files, and of rotated files in their own
pool (release uploads wait)
+ _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+ # Log paths of watchers that left while their trigger existed, removed
here once it no longer does
+ _left: dict[int, str] = attrs.field(factory=dict, init=False)
+ _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict,
init=False)
+ # The last number given to each watcher's rotated files here (see
upload_rotated)
+ _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+ # Each watcher's last rotated file whose upload the handler did not
confirm, sent again before it is replaced
+ _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict,
init=False)
+ # Held while a released watcher's file is added to its held file, or the
held file is taken for upload
+ _held_lock: threading.Lock = attrs.field(factory=threading.Lock,
init=False)
+
+ def note_watcher_log_write(self, trigger_id: int) -> None:
+ """Keep at most ``watcher_log_max_open_files`` watcher files open,
closing the least recent."""
+ self._open_watcher_logs.pop(trigger_id, None)
+ self._open_watcher_logs[trigger_id] = None
+ while 0 < self.watcher_log_max_open_files <
len(self._open_watcher_logs):
+ idle = next(iter(self._open_watcher_logs))
+ del self._open_watcher_logs[idle]
+ if (factory := self.logger_cache.get(idle)) is not None:
+ factory.close()
+
+ def forget_open_file(self, trigger_id: int) -> None:
+ """Stop counting a trigger's file as open, as it is about to be
closed."""
+ self._open_watcher_logs.pop(trigger_id, None)
+
+ def release(self, trigger_id: int, factory: WatcherLoggingFactory) -> None:
+ """
+ Upload and close the file of a watcher no longer assigned here, off
the main loop.
+
+ The watcher may come back here at once, on a new live file: this one
is held next to it until an upload
+ of the watcher's files here takes it (see _hold).
+ """
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ # In the background: a triggerer that lost its watchers (a DB
partition) releases them all at once.
+ if self._upload_pool is None:
+ self._upload_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-release")
+ handler = load_remote_log_handler()
+ if handler is not None and not _handler_is(handler,
_TASK_LOGS_ONLY_HANDLERS):
+ if hasattr(factory, "bound_logger"):
+ factory.close()
+ self._hold(factory.bound_logger._logger._path)
+ release = _Release(factory=factory,
previous=self._releases.get(trigger_id))
+ release.upload = self._upload_pool.submit(self._upload_released,
trigger_id, release)
+ self._releases[trigger_id] = release
+
+ def forget(done: Future) -> None:
+ if self._releases.get(trigger_id) is release:
+ self._releases.pop(trigger_id, None)
+
+ release.upload.add_done_callback(forget)
+
+ def _hold(self, live: Path) -> None:
+ """Move a released watcher's lines out of the way of a watcher back
here (see release)."""
+ held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME)
+ # Else left in the live file: a watcher back here appends to it, and
its next upload sends them.
+ with self._held_lock, suppress(OSError):
+ if not _has_lines(live):
+ return
+ if not held.exists():
+ os.replace(live, held)
+ return
+ # Released again before the held file was taken for upload: one
file, sent once.
+ with live.open("rb") as lines, held.open("ab") as kept:
+ shutil.copyfileobj(lines, kept)
+ live.unlink()
+
+ def _take_held(self, trigger_id: int, log_path: str) -> str | None:
+ """Move the held file into the rotated directory for upload; return
its path there, if there was one."""
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ base = Path(conf.get("logging", "base_log_folder"))
+ held = base / get_trigger_log_held_path(log_path)
+ if (handler := load_remote_log_handler()) is None or not
held.is_file():
+ return None
+ if handler_keeps_no_copy(handler):
+ path = get_trigger_log_rotated_path(log_path)
+ else:
+ # Numbered only now, as the watcher's rotations wait for this
upload (see rotated_upload_done).
+ part = self._rotated_parts[trigger_id] =
self._rotated_parts.get(trigger_id, 0) + 1
+ path = get_trigger_log_part_path(log_path, part)
+ with self._held_lock:
+ _move_into_rotated_dir(held, base / path)
+ return path
+
+ def _upload_released(self, trigger_id: int, release: _Release) -> None:
+ if release.previous is not None:
+ # Its files are in the same directory.
+ with suppress(Exception):
+ release.previous.upload.result()
+ release.previous = None
+ factory = release.factory
+ failed = self._upload_and_close(trigger_id, factory, keep_live=True)
+ if failed:
+ self._left.pop(trigger_id, None) # kept for the next start's
orphan scan
+ else:
+ self._left[trigger_id] = factory.log_path
+
+ @staticmethod
+ def _remove_released_files(log_path: str) -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ producer_dir = (Path(conf.get("logging", "base_log_folder")) /
log_path).parent
+ handler = load_remote_log_handler()
+ if handler is not None and _handler_is(handler,
_LINES_LEFT_MEAN_FAILED_HANDLERS):
+ if any(_has_lines(path) for path in
list_trigger_log_files(producer_dir)):
+ return # a failed upload: sent again by the next start's
orphan scan
+ shutil.rmtree(producer_dir, ignore_errors=True)
+ with suppress(OSError):
+ producer_dir.parent.rmdir() # not while another triggerer still
has a directory in it
+
+ def left_trigger_ids(self) -> set[int]:
Review Comment:
nit: `AGENTS.md` asks for verb names, so `get_left_trigger_ids`. Same for
`rotated_upload_done`, maybe `is_rotated_upload_done`, though it also submits
the retry upload, which a predicate name hides.
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,699 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_HELD_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_held_path,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate",
"_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ self._folder_mode, self._file_mode = get_log_permissions()
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(mode=self._folder_mode, parents=True,
exist_ok=True)
+ self._path.touch(mode=self._file_mode)
+ self._file = self._path.open("ab")
+ self._write = self._file.write
+ self._flush = self._file.flush
+ self._size = self._file.tell()
+
+ def _unlinked(self) -> bool:
+ try:
+ return os.fstat(self._file.fileno()).st_nlink == 0
+ except (OSError, ValueError):
+ return False
+
+ def msg(self, message: bytes) -> None:
+ with self._lock:
+ try:
+ self.ensure_open()
+ self._write(message + b"\n")
+ self._flush()
+ self._size += len(message) + 1
+ if 0 < self._max_bytes <= self._size and (self._may_rotate is
None or self._may_rotate()):
+ self._file.close()
+ self.rotated_path.parent.mkdir(mode=self._folder_mode,
parents=True, exist_ok=True)
+ with suppress(FileNotFoundError):
+ os.replace(self._path, self.rotated_path)
+ self.ensure_open()
+ if self._on_rotate is not None:
+ try:
+ self._on_rotate()
+ except Exception:
+ # The line is written: the caller, reading the
runner's logs, must not see this.
+ log.exception(
+ "Failed to upload a rotated trigger log",
path=str(self.rotated_path)
+ )
+ except OSError:
+ # Reopened by the next line: a file removed meanwhile (to free
a full disk) frees its space.
+ with suppress(OSError):
+ self._file.close()
+ raise
+
+ def close(self) -> None:
+ # Closing flushes; on a full disk those lines are lost either way.
+ with suppress(OSError):
+ self._file.close()
+
+ log = debug = info = warn = warning = msg
+ fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO",
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO",
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file:
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO",
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+ _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+ """Return the modes of new log folders and files, as ``init_log_file``
gives them."""
+ return (
+ int(conf.get("logging", "file_task_handler_new_folder_permissions",
fallback="0o775"), 8),
+ int(conf.get("logging", "file_task_handler_new_file_permissions",
fallback="0o664"), 8),
+ )
+
+
+def _has_lines(path: Path) -> bool:
+ try:
+ return path.stat().st_size > 0
+ except OSError:
+ return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+ """
+ Whether a file the handler leaves as it was after ``upload`` is no sign of
a failed upload.
+
+ True for a handler that streams each line as it is written (it has
processors, like CloudWatch) or that takes
+ only task logs (Elasticsearch, OpenSearch; by class name, as nothing else
tells them apart). Any other handler
+ that leaves the file as it was failed to upload it (S3, GCS and WASB
report that only in their log).
+ """
+ if getattr(handler, "processors", None):
+ return True
+ return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+ """Upload one of a watcher's files, at ``path`` locally; one sent again
would be stored twice by OSS."""
+ if not _handler_is(handler, _APPENDING_HANDLERS):
+ handler.upload(relative_path, None)
+ return
+ # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS reports a
failed write only through it.
+ # The file is emptied once stored, and kept as it was otherwise, for the
next upload to send again.
+ if path.is_file() and handler.oss_write(path.read_text(), relative_path):
+ if handler.delete_local_copy:
+ shutil.rmtree(path.parent)
+ else:
+ with suppress(FileNotFoundError):
+ os.truncate(path, 0)
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+ # Uploaded from there, as the files before it are: with delete_local_logs,
an upload removes the directory
+ # of the file it sent, which must not be the live file's.
+ rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True,
exist_ok=True)
+ os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:
+ """
+ Whether to send a watcher's file again after an upload.
+
+ True if it has lines and the upload raised, or S3, GCS, WASB or OSS left
them (others may keep them).
+ """
+ if not _has_lines(path):
+ return False
+ if raised:
+ return not _handler_is(handler, _APPENDING_HANDLERS)
+ return _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+ """Whether the handler's class, or a base class, has one of these names
(as for the above)."""
+ classes = type(handler).__mro__
+ if type(handler) not in _checked_handler_classes:
+ _checked_handler_classes.add(type(handler))
+ if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+ log.info(
+ "Remote log handler not known by name: asset watcher log files
are sent to it as to one "
+ "that stores them, and sent again only if the upload raises",
+
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+ )
+ return any(cls.__name__ in names for cls in classes)
+
+
+def get_watcher_log_max_open_files() -> int:
+ """
+ Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a
quarter of ``ulimit -n``.
+
+ The task and callback trigger logs, the database and the sockets need
descriptors too.
+ """
+ setting = conf.getint("triggerer", "watcher_log_max_open_files")
+ try:
+ import resource # Unix only
+
+ soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+ except (ImportError, OSError, ValueError):
+ return setting
+ if soft == resource.RLIM_INFINITY:
+ return setting
+ limit = max(1, soft // 4)
+ return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True, eq=False)
+class _Release:
+ """The upload of a released watcher's files."""
+
+ factory: WatcherLoggingFactory
+ # The upload of the same watcher's files released before, if it was still
pending
+ previous: _Release | None
+ upload: Future = attrs.field(init=False)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+ """
+ The asset watcher log files of one triggerer's supervisor, other than
writing to them.
+
+ Keeps at most ``watcher_log_max_open_files`` of them open, uploads their
rotated files and those of
+ released watchers off the main loop, and uploads or removes the files
triggerers that died left here.
+ """
+
+ # The supervisor's trigger log factories, by trigger id
+ logger_cache: dict[int, TriggerLoggingFactory]
+ # Told when closing a trigger's log file fails
+ on_error: Callable[[int | None, OSError], None]
+ # Asset watcher files with an open descriptor, least recently written first
+ _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+ watcher_log_max_open_files: int =
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+ # When to look for watcher files left by triggerers that died, and the
thread doing it
+ _orphan_scans_at: list[float] | None = attrs.field(default=None,
init=False)
+ _orphan_scan_thread: threading.Thread | None = attrs.field(default=None,
init=False)
+ # Uploads of released watchers' files, and of rotated files in their own
pool (release uploads wait)
+ _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _releases: dict[int, _Release] = attrs.field(factory=dict, init=False)
+ # Log paths of watchers that left while their trigger existed, removed
here once it no longer does
+ _left: dict[int, str] = attrs.field(factory=dict, init=False)
+ _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict,
init=False)
+ # The last number given to each watcher's rotated files here (see
upload_rotated)
+ _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+ # Each watcher's last rotated file whose upload the handler did not
confirm, sent again before it is replaced
+ _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict,
init=False)
+ # Held while a released watcher's file is added to its held file, or the
held file is taken for upload
+ _held_lock: threading.Lock = attrs.field(factory=threading.Lock,
init=False)
+
+ def note_watcher_log_write(self, trigger_id: int) -> None:
+ """Keep at most ``watcher_log_max_open_files`` watcher files open,
closing the least recent."""
+ self._open_watcher_logs.pop(trigger_id, None)
+ self._open_watcher_logs[trigger_id] = None
+ while 0 < self.watcher_log_max_open_files <
len(self._open_watcher_logs):
+ idle = next(iter(self._open_watcher_logs))
+ del self._open_watcher_logs[idle]
+ if (factory := self.logger_cache.get(idle)) is not None:
+ factory.close()
+
+ def forget_open_file(self, trigger_id: int) -> None:
+ """Stop counting a trigger's file as open, as it is about to be
closed."""
+ self._open_watcher_logs.pop(trigger_id, None)
+
+ def release(self, trigger_id: int, factory: WatcherLoggingFactory) -> None:
+ """
+ Upload and close the file of a watcher no longer assigned here, off
the main loop.
+
+ The watcher may come back here at once, on a new live file: this one
is held next to it until an upload
+ of the watcher's files here takes it (see _hold).
+ """
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ # In the background: a triggerer that lost its watchers (a DB
partition) releases them all at once.
+ if self._upload_pool is None:
+ self._upload_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-release")
+ handler = load_remote_log_handler()
+ if handler is not None and not _handler_is(handler,
_TASK_LOGS_ONLY_HANDLERS):
+ if hasattr(factory, "bound_logger"):
+ factory.close()
+ self._hold(factory.bound_logger._logger._path)
+ release = _Release(factory=factory,
previous=self._releases.get(trigger_id))
+ release.upload = self._upload_pool.submit(self._upload_released,
trigger_id, release)
+ self._releases[trigger_id] = release
+
+ def forget(done: Future) -> None:
+ if self._releases.get(trigger_id) is release:
+ self._releases.pop(trigger_id, None)
+
+ release.upload.add_done_callback(forget)
+
+ def _hold(self, live: Path) -> None:
+ """Move a released watcher's lines out of the way of a watcher back
here (see release)."""
+ held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME)
+ # Else left in the live file: a watcher back here appends to it, and
its next upload sends them.
+ with self._held_lock, suppress(OSError):
+ if not _has_lines(live):
+ return
+ if not held.exists():
+ os.replace(live, held)
+ return
+ # Released again before the held file was taken for upload: one
file, sent once.
+ with live.open("rb") as lines, held.open("ab") as kept:
+ shutil.copyfileobj(lines, kept)
+ live.unlink()
+
+ def _take_held(self, trigger_id: int, log_path: str) -> str | None:
+ """Move the held file into the rotated directory for upload; return
its path there, if there was one."""
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ base = Path(conf.get("logging", "base_log_folder"))
+ held = base / get_trigger_log_held_path(log_path)
+ if (handler := load_remote_log_handler()) is None or not
held.is_file():
+ return None
+ if handler_keeps_no_copy(handler):
+ path = get_trigger_log_rotated_path(log_path)
+ else:
+ # Numbered only now, as the watcher's rotations wait for this
upload (see rotated_upload_done).
+ part = self._rotated_parts[trigger_id] =
self._rotated_parts.get(trigger_id, 0) + 1
+ path = get_trigger_log_part_path(log_path, part)
+ with self._held_lock:
+ _move_into_rotated_dir(held, base / path)
+ return path
+
+ def _upload_released(self, trigger_id: int, release: _Release) -> None:
+ if release.previous is not None:
+ # Its files are in the same directory.
+ with suppress(Exception):
+ release.previous.upload.result()
+ release.previous = None
+ factory = release.factory
+ failed = self._upload_and_close(trigger_id, factory, keep_live=True)
+ if failed:
+ self._left.pop(trigger_id, None) # kept for the next start's
orphan scan
+ else:
+ self._left[trigger_id] = factory.log_path
+
+ @staticmethod
+ def _remove_released_files(log_path: str) -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ producer_dir = (Path(conf.get("logging", "base_log_folder")) /
log_path).parent
+ handler = load_remote_log_handler()
+ if handler is not None and _handler_is(handler,
_LINES_LEFT_MEAN_FAILED_HANDLERS):
+ if any(_has_lines(path) for path in
list_trigger_log_files(producer_dir)):
+ return # a failed upload: sent again by the next start's
orphan scan
+ shutil.rmtree(producer_dir, ignore_errors=True)
+ with suppress(OSError):
+ producer_dir.parent.rmdir() # not while another triggerer still
has a directory in it
+
+ def left_trigger_ids(self) -> set[int]:
+ return set(self._left)
+
+ def remove_left(self, gone: set[int]) -> None:
+ """Remove the files of watchers that left here and whose trigger no
longer exists."""
+ for trigger_id in gone:
+ if (release := self._releases.get(trigger_id)) is not None and not
release.upload.done():
+ continue # its files are still uploading
+ log_path = self._left.pop(trigger_id, None)
+ if log_path is None or trigger_id in self.logger_cache:
+ continue # back here: tracked again once it leaves
+ try:
+ self._remove_released_files(log_path)
+ except Exception:
+ log.exception("Failed to remove the log files of a trigger
that no longer exists")
+
+ def rotated_upload_done(self, trigger_id: int) -> bool:
+ if (release := self._releases.get(trigger_id)) is not None and not
release.upload.done():
+ return False # its released files are uploading from the rotated
files' directory
+ upload = self._pending_rotated_uploads.get(trigger_id)
+ if upload is not None and not upload.done():
+ return False
+ if path := self._rotated_unconfirmed.pop(trigger_id, None):
+ # Now, while it is the only file in its directory: with
delete_local_logs, a handler deletes the
+ # directory of the file it uploaded. Replaced by the next rotation
whether it goes through or not.
+ self._pending_rotated_uploads[trigger_id] =
self._rotated_executor().submit(
+ self._upload_rotated_again, trigger_id, path
+ )
+ return False
+ return True
+
+ def _rotated_executor(self) -> ThreadPoolExecutor:
+ if self._rotated_pool is None:
+ self._rotated_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-rotated")
+ return self._rotated_pool
+
+ @staticmethod
+ def _upload_rotated_again(trigger_id: int, path: str) -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ try:
+ if (handler := load_remote_log_handler()) is not None:
+ upload_watcher_log_file(handler, path,
Path(conf.get("logging", "base_log_folder")) / path)
+ except Exception:
+ log.exception("Failed to upload a rotated trigger log",
trigger_id=trigger_id, path=path)
+
+ def upload_rotated(self, trigger_id: int, log_path: str) -> None:
+ # Called by the line that rotated the file: the rest runs in the pool,
not to hold up the runner's logs.
+ pool = self._rotated_executor()
+ rotated = get_trigger_log_rotated_path(log_path)
+ # Numbered here: a watcher's next rotation waits for this upload (see
rotated_upload_done).
+ part = self._rotated_parts.get(trigger_id, 0) + 1
+
+ def upload() -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ path = rotated
+ base = Path(conf.get("logging", "base_log_folder"))
+ try:
+ if (handler := load_remote_log_handler()) is None:
+ return
+ if not handler_keeps_no_copy(handler):
+ # Each rotated file is its own object (see
get_trigger_log_part_path), numbered by this
+ # triggerer, the only one writing there: counted, as
earlier parts may be gone locally.
+ for done in (base / rotated).parent.glob("*.log"):
+ if done.name != TRIGGER_LOG_FILE_NAME:
+ with suppress(OSError):
+ done.unlink() # the previous one, uploaded or
tried twice: one at a time
+ path = get_trigger_log_part_path(log_path, part)
+ # Named only now: until then the triggerer's log server
serves it as the rotated file.
+ os.replace(base / rotated, base / path)
+ upload_watcher_log_file(handler, path, base / path)
+ raised = False
+ except Exception:
+ log.exception("Failed to upload a rotated trigger log",
trigger_id=trigger_id, path=path)
+ raised = True
+ if path != rotated and _is_left_unstored(handler, base / path,
raised=raised):
+ self._rotated_unconfirmed[trigger_id] = path
+
+ self._pending_rotated_uploads[trigger_id] = pool.submit(upload)
+ self._rotated_parts[trigger_id] = part
+
+ def upload_and_close(self, trigger_id: int, factory:
TriggerLoggingFactory) -> None:
+ if (release := self._releases.get(trigger_id)) is not None:
+ # A watcher back here: its released files are uploading from the
same directory.
+ with suppress(Exception):
+ release.upload.result()
+ self._upload_and_close(trigger_id, factory)
+
+ def _upload_and_close(self, trigger_id: int, factory:
TriggerLoggingFactory, **upload_kwargs) -> bool:
+ """Upload and close a trigger's file; return whether the upload
raised."""
+ from airflow.jobs.triggerer_job_runner import WatcherLoggingFactory
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ if (rotated_upload := self._pending_rotated_uploads.pop(trigger_id,
None)) is not None:
+ rotated_upload.result() # else the rotated file could be read
again before it is emptied
+ self._rotated_unconfirmed.pop(trigger_id, None) # sent again below,
with the live file
+ base = Path(conf.get("logging", "base_log_folder"))
+ failed, held = False, None
+
+ def take_held() -> Path | None:
+ nonlocal held
+ held = self._take_held(trigger_id, factory.log_path)
+ return None if held is None else base / held
+
+ if isinstance(factory, WatcherLoggingFactory):
+ upload_kwargs["take_held"] = take_held
+ try:
+ factory.upload_to_remote(**upload_kwargs)
+ except Exception:
+ log.exception("Failed to upload trigger logs to remote",
trigger_id=trigger_id)
+ failed = True
+ finally:
+ # Close even if the upload failed, not to leak the descriptor.
Closing flushes, which can fail.
+ try:
+ factory.close()
+ except OSError as e:
+ self.on_error(trigger_id, e)
+ if held and _is_left_unstored(load_remote_log_handler(), base / held,
raised=failed):
+ # Under a rotated file's name now: the watcher's next rotation
here would remove it unsent.
+ self._rotated_unconfirmed[trigger_id] = held
+ return failed
+
+ def upload_and_close_all(self, factories: list[tuple[int,
TriggerLoggingFactory]]) -> None:
+ """Upload and close these watchers' files, and wait for the released
ones' uploads."""
+ self._open_watcher_logs.clear()
+ # Side by side: one after the other, thousands of files take longer
than a container's stop timeout.
+ with ThreadPoolExecutor(max_workers=16,
thread_name_prefix="trigger-log-upload") as pool:
+ list(pool.map(lambda item: self.upload_and_close(*item),
factories))
+ if self._upload_pool is not None:
+ self._upload_pool.shutdown(wait=True)
+
+ def handle_orphaned_trigger_logs(self, own_job_id: int | None) -> None:
+ """
+ Upload, or remove, the asset watcher files left on this disk by
triggerers that are no longer alive.
+
+ A killed triggerer uploads nothing, so its files are uploaded now,
then their directories removed.
+ Files remote logging doesn't take (it is off, or the handler keeps no
copy, like Elasticsearch) are
+ removed instead: all but the most recently written for each watcher,
and all for triggers that no
+ longer exist. Files of live triggerers, this one included, of
triggerers without a job (no job id
+ directory), and those a failed upload left stay.
+ """
+ from airflow.jobs.job import Job
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ handler = load_remote_log_handler()
+ base = Path(conf.get("logging", "base_log_folder"))
+ dirs_by_job: dict[int, list[Path]] = {}
+ dirs_by_trigger: dict[Path, int] = {}
+ for producer_dir in
base.glob(f"{TRIGGER_LOG_ROOT}/{TRIGGER_ID_KEY}=*/{TRIGGERER_JOB_ID_KEY}=*"):
+ # Only directories this module names are ever removed: both ids
must parse.
+ trigger_id = parse_trigger_log_dir_id(TRIGGER_ID_KEY,
producer_dir.parent.name)
+ job_id = parse_trigger_log_dir_id(TRIGGERER_JOB_ID_KEY,
producer_dir.name)
+ if (
+ trigger_id is not None
+ and job_id is not None
+ and job_id != own_job_id
+ and producer_dir.is_dir()
+ ):
+ dirs_by_job.setdefault(job_id, []).append(producer_dir)
+ dirs_by_trigger[producer_dir.parent] = trigger_id
+ if not dirs_by_job:
+ return
+ trigger_ids = sorted(set(dirs_by_trigger.values()))
+ with create_session() as session:
+ jobs = session.scalars(select(Job).where(Job.id.in_(dirs_by_job)))
Review Comment:
nit: `Job.is_alive_clause` can do this in the query, instead of loading
every `Job` row and calling `is_alive()` on each.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]