dabla commented on code in PR #76545:
URL: https://github.com/apache/airflow/pull/76545#discussion_r4614250799
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -225,6 +223,34 @@ class _ResponseFrame(_FrameMixin, msgspec.Struct,
array_like=True, frozen=True,
error: dict[str, Any] | None = None
+class _RequestWorker:
+ """
+ The one thread that makes a decoder's ``asend()`` requests, whichever
event loop they come from.
+
+ Requests queue for it in the order they were made, and it needs no loop to
finish one. The
+ first request of a process starts it: threads do not survive a fork, so a
forked child gets
+ a thread of its own.
+ """
+
+ __slots__ = ("_executor", "_lock", "_pid")
+
+ def __init__(self) -> None:
+ self._executor: ThreadPoolExecutor | None = None
+ self._pid: int | None = None
+ self._lock = threading.Lock()
+
+ def executor(self) -> ThreadPoolExecutor:
Review Comment:
Renamed to `get_executor()` in 54a00819cf: a method is named by what it does.
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -245,55 +271,56 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
err_decoder: TypeAdapter[ErrorResponse] = attrs.field(factory=lambda:
TypeAdapter(ToTask), repr=False)
- _thread_lock: threading.Lock = attrs.field(factory=threading.Lock,
repr=False)
- _async_lock: asyncio.Lock = attrs.field(factory=asyncio.Lock, repr=False)
- _loop_thread_id: int | None = attrs.field(default=None, repr=False,
init=False)
+ # Reentrant, so that a send() which interrupts its own thread's (a signal
handler on the main
+ # thread) is let in instead of waiting for itself, and can tell from
_exchanging whether that
+ # thread was in the middle of a request. The RLock records its owner when
it is taken, so no
+ # moment is left where the thread holds the lock but is not yet known as
its holder.
+ _thread_lock: threading.RLock = attrs.field(factory=threading.RLock,
repr=False)
+ _worker: _RequestWorker = attrs.field(factory=lambda: _RequestWorker(),
repr=False, init=False)
Review Comment:
Done in 54a00819cf, `factory=_RequestWorker`: the class is already the
callable the factory needs.
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
assert result is not None
- def
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self,
socket_pair):
+ @staticmethod
+ def _serve(sock, count: int, delay: float = 0.0) -> None:
+ """Answer ``count`` requests with their own key, after ``delay``
seconds each; stop on a closed socket."""
+ import time
+
+ try:
+ for _ in range(count):
+ length = int.from_bytes(sock.recv(4), "big")
+ body = b""
+ while len(body) < length:
+ body += sock.recv(length - len(body))
+ request = msgspec.msgpack.decode(body, type=_RequestFrame)
+ assert request.body is not None
+ time.sleep(delay)
+ resp = {"type": "VariableResult", "key": request.body["key"],
"value": "v"}
+ encoded = msgspec.msgpack.encode(_ResponseFrame(request.id,
resp, None))
+ sock.sendall(len(encoded).to_bytes(4, "big") + encoded)
+ except OSError:
+ pass
+
+ def
test_send_on_a_thread_already_in_a_request_raises_instead_of_waiting(self,
socket_pair):
+ """
+ A call on a thread that holds the channel for its own request would
wait for itself: the
+ one real deadlock left, so it raises at once, and the interrupted
request still completes.
"""
- Regression: send() called from the loop's own thread while the loop is
*paused* (between
- two run_until_complete() calls) and an asend() is parked mid-I/O must
raise
- DeadlockImminentError instead of blocking forever.
+ r, w = socket_pair
+ decoder = CommsDecoder(socket=r, log=structlog.get_logger())
+ threading.Thread(target=self._serve, args=(w, 1), daemon=True).start()
+ raised: list[BaseException] = []
+ original_get_response = CommsDecoder._get_response
+
+ def interrupted(self):
+ # What a signal handler would do while this thread waits for the
response.
+ try:
+ self.send(GetVariable(key="from the handler"))
+ except DeadlockImminentError as exc:
+ raised.append(exc)
+ return original_get_response(self)
+
+ with mock.patch.object(CommsDecoder, "_get_response", interrupted):
+ result = decoder.send(GetVariable(key="outer"))
+
+ assert result.key == "outer"
+ assert len(raised) == 1
+ assert "already in the middle of a request" in str(raised[0])
+ assert not _is_taken(decoder._thread_lock)
- The running-loop check alone does not cover this:
asyncio.get_running_loop() raises while
- the loop is paused, so send() fell through to a blocking acquire of
_thread_lock. The
- holder, an asend() coroutine, can only release it once the loop runs
again, on this very
- thread, which is now blocked. Seen in production as an
IterableOperator freezing with
- every thread idle: AsyncAwareExecutor.map pulled the next sub-task's
input through a sync
- XCom read between two run_until_complete() calls while a sub-task's
asend() was in flight.
+ @pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="needs
SIGALRM timers")
+ def
test_a_signal_handler_calling_send_during_a_send_raises_instead_of_freezing(self,
socket_pair):
+ """
+ The case the check exists for: the runner's SIGTERM handler runs
on_kill() on the main
+ thread, and an on_kill() making a sync SDK call while the main thread
waits for a response
+ would wait for its own thread. A real signal lands mid-request here.
"""
r, w = socket_pair
decoder = CommsDecoder(socket=r, log=structlog.get_logger())
+ # The response comes after the signal has fired.
+ threading.Thread(target=self._serve, args=(w, 1, 0.3),
daemon=True).start()
+ raised: list[BaseException] = []
- def _read_request(sock) -> _RequestFrame:
- length = int.from_bytes(sock.recv(4), "big")
- body = b""
- while len(body) < length:
- body += sock.recv(length - len(body))
- return msgspec.msgpack.decode(body, type=_RequestFrame)
+ def on_alarm(signum, frame):
+ try:
+ decoder.send(GetVariable(key="from on_kill"))
+ except DeadlockImminentError as exc:
+ raised.append(exc)
- def _respond(sock, req: _RequestFrame) -> None:
- assert req.body is not None
- resp = {"type": "VariableResult", "key": req.body["key"], "value":
"v"}
- encoded = msgspec.msgpack.encode(_ResponseFrame(req.id, resp,
None))
- sock.sendall(len(encoded).to_bytes(4, "big") + encoded)
+ previous = signal.signal(signal.SIGALRM, on_alarm)
+ try:
+ signal.setitimer(signal.ITIMER_REAL, 0.05)
+ result = decoder.send(GetVariable(key="interrupted"))
+ finally:
+ signal.setitimer(signal.ITIMER_REAL, 0)
+ signal.signal(signal.SIGALRM, previous)
+
+ assert result.key == "interrupted"
+ assert len(raised) == 1
+
+ def test_a_failed_request_leaves_no_request_in_flight_behind(self,
socket_pair):
+ """
+ A request that fails mid-exchange releases the channel and clears its
in-flight mark, so the
+ thread's next call is served instead of being taken for a call from
inside a request.
+ """
+ r, w = socket_pair
+ decoder = CommsDecoder(socket=r, log=structlog.get_logger())
+ threading.Thread(target=self._serve, args=(w, 2), daemon=True).start()
+
+ with mock.patch.object(
+ CommsDecoder, "_get_response", autospec=True,
side_effect=EOFError("supervisor gone")
+ ):
+ with pytest.raises(EOFError):
+ decoder.send(GetVariable(key="lost"))
+
+ assert decoder._exchanging is False
+ assert not _is_taken(decoder._thread_lock)
+ assert decoder.send(GetVariable(key="next")) is not None
Review Comment:
Changing only the assertion fails: the mock raised before the response was
read, so `"next"` really got the `"lost"` response (that is #77240). In
54a00819cf the test fails the decode instead (`_from_frame` raises after the
frame is read) and checks `.key == "next"`.
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
assert result is not None
- def
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self,
socket_pair):
+ @staticmethod
+ def _serve(sock, count: int, delay: float = 0.0) -> None:
+ """Answer ``count`` requests with their own key, after ``delay``
seconds each; stop on a closed socket."""
+ import time
+
+ try:
+ for _ in range(count):
+ length = int.from_bytes(sock.recv(4), "big")
+ body = b""
+ while len(body) < length:
+ body += sock.recv(length - len(body))
+ request = msgspec.msgpack.decode(body, type=_RequestFrame)
+ assert request.body is not None
+ time.sleep(delay)
+ resp = {"type": "VariableResult", "key": request.body["key"],
"value": "v"}
+ encoded = msgspec.msgpack.encode(_ResponseFrame(request.id,
resp, None))
+ sock.sendall(len(encoded).to_bytes(4, "big") + encoded)
+ except OSError:
+ pass
+
+ def
test_send_on_a_thread_already_in_a_request_raises_instead_of_waiting(self,
socket_pair):
Review Comment:
Added `execution_timeout(10)` to both in 54a00819cf.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]