https://github.com/python/cpython/commit/0749244d13412d7cb5b53d834f586f2198f5b9a6
commit: 0749244d13412d7cb5b53d834f586f2198f5b9a6
branch: main
author: Brett Simmers <[email protected]>
committer: colesbury <[email protected]>
date: 2024-02-20T09:57:48-05:00
summary:
gh-112175: Add `eval_breaker` to `PyThreadState` (#115194)
This change adds an `eval_breaker` field to `PyThreadState`. The primary
motivation is for performance in free-threaded builds: with thread-local eval
breakers, we can stop a specific thread (e.g., for an async exception) without
interrupting other threads.
The source of truth for the global instrumentation version is stored in the
`instrumentation_version` field in PyInterpreterState. Threads usually read the
version from their local `eval_breaker`, where it continues to be colocated
with the eval breaker bits.
files:
A Misc/NEWS.d/next/Core and
Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst
M Include/cpython/pystate.h
M Include/internal/pycore_ceval.h
M Include/internal/pycore_ceval_state.h
M Include/internal/pycore_gc.h
M Include/internal/pycore_runtime.h
M Modules/signalmodule.c
M Python/brc.c
M Python/bytecodes.c
M Python/ceval.c
M Python/ceval_gil.c
M Python/ceval_macros.h
M Python/executor_cases.c.h
M Python/gc.c
M Python/gc_free_threading.c
M Python/generated_cases.c.h
M Python/instrumentation.c
M Python/pylifecycle.c
M Python/pystate.c
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index b99450a8a8d093..ac7ff83748dbfc 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -68,6 +68,11 @@ struct _ts {
PyThreadState *next;
PyInterpreterState *interp;
+ /* The global instrumentation version in high bits, plus flags indicating
+ when to break out of the interpreter loop in lower bits. See details in
+ pycore_ceval.h. */
+ uintptr_t eval_breaker;
+
struct {
/* Has been initialized to a safe state.
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index b158fc9ff5ebc1..bf77526cf75cc1 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -42,7 +42,7 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *);
extern void _Py_FinishPendingCalls(PyThreadState *tstate);
extern void _PyEval_InitState(PyInterpreterState *);
-extern void _PyEval_SignalReceived(PyInterpreterState *interp);
+extern void _PyEval_SignalReceived(void);
// bitwise flags:
#define _Py_PENDING_MAINTHREADONLY 1
@@ -55,7 +55,6 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall(
void *arg,
int flags);
-extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp);
#ifdef HAVE_FORK
extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate);
#endif
@@ -200,40 +199,43 @@ int _PyEval_UnpackIterable(PyThreadState *tstate,
PyObject *v, int argcnt, int a
void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame
*frame);
-#define _PY_GIL_DROP_REQUEST_BIT 0
-#define _PY_SIGNALS_PENDING_BIT 1
-#define _PY_CALLS_TO_DO_BIT 2
-#define _PY_ASYNC_EXCEPTION_BIT 3
-#define _PY_GC_SCHEDULED_BIT 4
-#define _PY_EVAL_PLEASE_STOP_BIT 5
-#define _PY_EVAL_EXPLICIT_MERGE_BIT 6
+/* Bits that can be set in PyThreadState.eval_breaker */
+#define _PY_GIL_DROP_REQUEST_BIT (1U << 0)
+#define _PY_SIGNALS_PENDING_BIT (1U << 1)
+#define _PY_CALLS_TO_DO_BIT (1U << 2)
+#define _PY_ASYNC_EXCEPTION_BIT (1U << 3)
+#define _PY_GC_SCHEDULED_BIT (1U << 4)
+#define _PY_EVAL_PLEASE_STOP_BIT (1U << 5)
+#define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6)
/* Reserve a few bits for future use */
#define _PY_EVAL_EVENTS_BITS 8
#define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1)
static inline void
-_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t
set)
+_Py_set_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit)
{
- assert(set == 0 || set == 1);
- uintptr_t to_set = set << bit;
- uintptr_t mask = ((uintptr_t)1) << bit;
- uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker);
- if ((old & mask) == to_set) {
- return;
- }
- uintptr_t new;
- do {
- new = (old & ~mask) | to_set;
- } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker,
&old, new));
+ _Py_atomic_or_uintptr(&tstate->eval_breaker, bit);
+}
+
+static inline void
+_Py_unset_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit)
+{
+ _Py_atomic_and_uintptr(&tstate->eval_breaker, ~bit);
}
-static inline bool
-_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit)
+static inline int
+_Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit)
{
- return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) &
(((uintptr_t)1) << bit);
+ uintptr_t b = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
+ return (b & bit) != 0;
}
+// Free-threaded builds use these functions to set or unset a bit on all
+// threads in the given interpreter.
+void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
+void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
+
#ifdef __cplusplus
}
diff --git a/Include/internal/pycore_ceval_state.h
b/Include/internal/pycore_ceval_state.h
index 28738980eb49be..b453328f15649e 100644
--- a/Include/internal/pycore_ceval_state.h
+++ b/Include/internal/pycore_ceval_state.h
@@ -78,13 +78,10 @@ struct _ceval_runtime_state {
struct _ceval_state {
- /* This single variable consolidates all requests to break out of
- * the fast path in the eval loop.
- * It is by far the hottest field in this struct and
- * should be placed at the beginning. */
- uintptr_t eval_breaker;
- /* Avoid false sharing */
- int64_t padding[7];
+ /* This variable holds the global instrumentation version. When a thread is
+ running, this value is overlaid onto PyThreadState.eval_breaker so that
+ changes in the instrumentation version will trigger the eval breaker. */
+ uintptr_t instrumentation_version;
int recursion_limit;
struct _gil_runtime_state *gil;
int own_gil;
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index a98864f7431398..40414a868518bb 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -286,7 +286,7 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState
*interp, PyObject *objs);
// Functions to clear types free lists
extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
-extern void _Py_ScheduleGC(PyInterpreterState *interp);
+extern void _Py_ScheduleGC(PyThreadState *tstate);
extern void _Py_RunGC(PyThreadState *tstate);
#ifdef __cplusplus
diff --git a/Include/internal/pycore_runtime.h
b/Include/internal/pycore_runtime.h
index 7c705d1224f915..0c9c59e85b2fcf 100644
--- a/Include/internal/pycore_runtime.h
+++ b/Include/internal/pycore_runtime.h
@@ -191,7 +191,10 @@ typedef struct pyruntimestate {
int64_t next_id;
} interpreters;
+ /* Platform-specific identifier and PyThreadState, respectively, for the
+ main thread in the main interpreter. */
unsigned long main_thread;
+ PyThreadState *main_tstate;
/* ---------- IMPORTANT ---------------------------
The fields above this line are declared as early as
diff --git a/Misc/NEWS.d/next/Core and
Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst b/Misc/NEWS.d/next/Core
and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst
new file mode 100644
index 00000000000000..6d919134bf4d9c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and
Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst
@@ -0,0 +1 @@
+Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific
threads to be interrupted.
diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c
index 394a997b20c06d..652e69b0d28b21 100644
--- a/Modules/signalmodule.c
+++ b/Modules/signalmodule.c
@@ -276,11 +276,7 @@ trip_signal(int sig_num)
cleared in PyErr_CheckSignals() before .tripped. */
_Py_atomic_store_int(&is_tripped, 1);
- /* Signals are always handled by the main interpreter */
- PyInterpreterState *interp = _PyInterpreterState_Main();
-
- /* Notify ceval.c */
- _PyEval_SignalReceived(interp);
+ _PyEval_SignalReceived();
/* And then write to the wakeup fd *after* setting all the globals and
doing the _PyEval_SignalReceived. We used to write to the wakeup fd
@@ -303,6 +299,7 @@ trip_signal(int sig_num)
int fd = wakeup.fd;
if (fd != INVALID_FD) {
+ PyInterpreterState *interp = _PyInterpreterState_Main();
unsigned char byte = (unsigned char)sig_num;
#ifdef MS_WINDOWS
if (wakeup.use_send) {
@@ -1770,8 +1767,8 @@ PyErr_CheckSignals(void)
Python code to ensure signals are handled. Checking for the GC here
allows long running native code to clean cycles created using the C-API
even if it doesn't run the evaluation loop */
- if (_Py_eval_breaker_bit_is_set(tstate->interp, _PY_GC_SCHEDULED_BIT)) {
- _Py_set_eval_breaker_bit(tstate->interp, _PY_GC_SCHEDULED_BIT, 0);
+ if (_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) {
+ _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT);
_Py_RunGC(tstate);
}
diff --git a/Python/brc.c b/Python/brc.c
index f1fd57a2964cf5..b73c721e71aef6 100644
--- a/Python/brc.c
+++ b/Python/brc.c
@@ -94,7 +94,7 @@ _Py_brc_queue_object(PyObject *ob)
}
// Notify owning thread
- _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 1);
+ _Py_set_eval_breaker_bit(&tstate->base, _PY_EVAL_EXPLICIT_MERGE_BIT);
PyMutex_Unlock(&bucket->mutex);
}
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 10bb1525fc1801..9d790a9d3e6577 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -8,7 +8,6 @@
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
-#include "pycore_ceval.h" // _PyEval_SignalAsyncExc()
#include "pycore_code.h"
#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_function.h"
@@ -146,7 +145,7 @@ dummy_func(
TIER_ONE_ONLY
assert(frame == tstate->current_frame);
uintptr_t global_version =
-
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
+ _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((code_version & 255) == 0);
@@ -168,14 +167,14 @@ dummy_func(
DEOPT_IF(_Py_emscripten_signal_clock == 0);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
- uintptr_t eval_breaker =
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker);
+ uintptr_t eval_breaker =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
DEOPT_IF(eval_breaker != version);
}
inst(INSTRUMENTED_RESUME, (--)) {
- uintptr_t global_version =
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
~_PY_EVAL_EVENTS_MASK;
+ uintptr_t global_version =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
diff --git a/Python/ceval.c b/Python/ceval.c
index 6f647cfdd53d83..596d5f449c06fa 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5,7 +5,7 @@
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_call.h" // _PyObject_CallNoArgs()
-#include "pycore_ceval.h" // _PyEval_SignalAsyncExc()
+#include "pycore_ceval.h"
#include "pycore_code.h"
#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_function.h"
diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c
index deb9741291fca7..f5c44307a513f8 100644
--- a/Python/ceval_gil.c
+++ b/Python/ceval_gil.c
@@ -56,60 +56,52 @@
#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL)
_Py_atomic_load_relaxed(ATOMIC_VAL)
#endif
-/* bpo-40010: eval_breaker should be recomputed if there
- is a pending signal: signal received by another thread which cannot
- handle signals.
- Similarly, we set CALLS_TO_DO and ASYNC_EXCEPTION to match the thread.
-*/
+// Atomically copy the bits indicated by mask between two values.
static inline void
-update_eval_breaker_from_thread(PyInterpreterState *interp, PyThreadState
*tstate)
+copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask)
{
- if (tstate == NULL) {
+ uintptr_t from_bits = _Py_atomic_load_uintptr_relaxed(from) & mask;
+ uintptr_t old_value = _Py_atomic_load_uintptr_relaxed(to);
+ uintptr_t to_bits = old_value & mask;
+ if (from_bits == to_bits) {
return;
}
- if (_Py_IsMainThread()) {
- int32_t calls_to_do = _Py_atomic_load_int32_relaxed(
- &_PyRuntime.ceval.pending_mainthread.calls_to_do);
- if (calls_to_do) {
- _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1);
- }
- if (_Py_ThreadCanHandleSignals(interp)) {
- if (_Py_atomic_load_int(&_PyRuntime.signals.is_tripped)) {
- _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1);
- }
- }
- }
- if (tstate->async_exc != NULL) {
- _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1);
- }
+ uintptr_t new_value;
+ do {
+ new_value = (old_value & ~mask) | from_bits;
+ } while (!_Py_atomic_compare_exchange_uintptr(to, &old_value, new_value));
}
+// When attaching a thread, set the global instrumentation version and
+// _PY_CALLS_TO_DO_BIT from the current state of the interpreter.
static inline void
-SET_GIL_DROP_REQUEST(PyInterpreterState *interp)
+update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState
*tstate)
{
- _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 1);
-}
-
-
-static inline void
-RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)
-{
- _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 0);
-}
-
-
-static inline void
-SIGNAL_PENDING_CALLS(PyInterpreterState *interp)
-{
- _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1);
-}
+#ifdef Py_GIL_DISABLED
+ // Free-threaded builds eagerly update the eval_breaker on *all* threads as
+ // needed, so this function doesn't apply.
+ return;
+#endif
+ int32_t calls_to_do = _Py_atomic_load_int32_relaxed(
+ &interp->ceval.pending.calls_to_do);
+ if (calls_to_do) {
+ _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);
+ }
+ else if (_Py_IsMainThread()) {
+ calls_to_do = _Py_atomic_load_int32_relaxed(
+ &_PyRuntime.ceval.pending_mainthread.calls_to_do);
+ if (calls_to_do) {
+ _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);
+ }
+ }
-static inline void
-UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)
-{
- _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 0);
+ // _PY_CALLS_TO_DO_BIT was derived from other state above, so the only bits
+ // we copy from our interpreter's state are the instrumentation version.
+ copy_eval_breaker_bits(&interp->ceval.instrumentation_version,
+ &tstate->eval_breaker,
+ ~_PY_EVAL_EVENTS_MASK);
}
/*
@@ -254,13 +246,14 @@ drop_gil(PyInterpreterState *interp, PyThreadState
*tstate)
the GIL, and that's the only time we might delete the
interpreter, so checking tstate first prevents the crash.
See https://github.com/python/cpython/issues/104341. */
- if (tstate != NULL && _Py_eval_breaker_bit_is_set(interp,
_PY_GIL_DROP_REQUEST_BIT)) {
+ if (tstate != NULL &&
+ _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) {
MUTEX_LOCK(gil->switch_mutex);
/* Not switched yet => wait */
if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder))
== tstate)
{
assert(_PyThreadState_CheckConsistency(tstate));
- RESET_GIL_DROP_REQUEST(tstate->interp);
+ _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT);
/* NOTE: if COND_WAIT does not atomically start waiting when
releasing the mutex, another thread can run through, take
the GIL and drop it again, and reset the condition
@@ -321,6 +314,8 @@ take_gil(PyThreadState *tstate)
_Py_atomic_load_int_relaxed(&gil->locked) &&
gil->switch_number == saved_switchnum)
{
+ PyThreadState *holder_tstate =
+ (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder);
if (_PyThreadState_MustExit(tstate)) {
MUTEX_UNLOCK(gil->mutex);
// gh-96387: If the loop requested a drop request in a previous
@@ -330,13 +325,13 @@ take_gil(PyThreadState *tstate)
// may have to request again a drop request (iterate one more
// time).
if (drop_requested) {
- RESET_GIL_DROP_REQUEST(interp);
+ _Py_unset_eval_breaker_bit(holder_tstate,
_PY_GIL_DROP_REQUEST_BIT);
}
PyThread_exit_thread();
}
assert(_PyThreadState_CheckConsistency(tstate));
- SET_GIL_DROP_REQUEST(interp);
+ _Py_set_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT);
drop_requested = 1;
}
}
@@ -369,13 +364,15 @@ take_gil(PyThreadState *tstate)
in take_gil() while the main thread called
wait_for_thread_shutdown() from Py_Finalize(). */
MUTEX_UNLOCK(gil->mutex);
- drop_gil(interp, tstate);
+ /* Passing NULL to drop_gil() indicates that this thread is about to
+ terminate and will never hold the GIL again. */
+ drop_gil(interp, NULL);
PyThread_exit_thread();
}
assert(_PyThreadState_CheckConsistency(tstate));
- RESET_GIL_DROP_REQUEST(interp);
- update_eval_breaker_from_thread(interp, tstate);
+ _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT);
+ update_eval_breaker_for_thread(interp, tstate);
MUTEX_UNLOCK(gil->mutex);
@@ -590,15 +587,6 @@ _PyEval_ReInitThreads(PyThreadState *tstate)
}
#endif
-/* This function is used to signal that async exceptions are waiting to be
- raised. */
-
-void
-_PyEval_SignalAsyncExc(PyInterpreterState *interp)
-{
- _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1);
-}
-
PyThreadState *
PyEval_SaveThread(void)
{
@@ -646,11 +634,9 @@ PyEval_RestoreThread(PyThreadState *tstate)
*/
void
-_PyEval_SignalReceived(PyInterpreterState *interp)
+_PyEval_SignalReceived(void)
{
- if (_Py_ThreadCanHandleSignals(interp)) {
- _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1);
- }
+ _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_SIGNALS_PENDING_BIT);
}
/* Push one item onto the queue while holding the lock. */
@@ -702,6 +688,26 @@ _pop_pending_call(struct _pending_calls *pending,
}
}
+#ifndef Py_GIL_DISABLED
+static void
+signal_active_thread(PyInterpreterState *interp, uintptr_t bit)
+{
+ struct _gil_runtime_state *gil = interp->ceval.gil;
+
+ // If a thread from the targeted interpreter is holding the GIL, signal
+ // that thread. Otherwise, the next thread to run from the targeted
+ // interpreter will have its bit set as part of taking the GIL.
+ MUTEX_LOCK(gil->mutex);
+ if (_Py_atomic_load_int_relaxed(&gil->locked)) {
+ PyThreadState *holder =
(PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder);
+ if (holder->interp == interp) {
+ _Py_set_eval_breaker_bit(holder, bit);
+ }
+ }
+ MUTEX_UNLOCK(gil->mutex);
+}
+#endif
+
/* This implementation is thread-safe. It allows
scheduling to be made from any thread, and even from an executing
callback.
@@ -711,10 +717,9 @@ int
_PyEval_AddPendingCall(PyInterpreterState *interp,
_Py_pending_call_func func, void *arg, int flags)
{
- assert(!(flags & _Py_PENDING_MAINTHREADONLY)
- || _Py_IsMainInterpreter(interp));
struct _pending_calls *pending = &interp->ceval.pending;
- if (flags & _Py_PENDING_MAINTHREADONLY) {
+ int main_only = (flags & _Py_PENDING_MAINTHREADONLY) != 0;
+ if (main_only) {
/* The main thread only exists in the main interpreter. */
assert(_Py_IsMainInterpreter(interp));
pending = &_PyRuntime.ceval.pending_mainthread;
@@ -724,8 +729,17 @@ _PyEval_AddPendingCall(PyInterpreterState *interp,
int result = _push_pending_call(pending, func, arg, flags);
PyMutex_Unlock(&pending->mutex);
- /* signal main loop */
- SIGNAL_PENDING_CALLS(interp);
+ if (main_only) {
+ _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_CALLS_TO_DO_BIT);
+ }
+ else {
+#ifdef Py_GIL_DISABLED
+ _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT);
+#else
+ signal_active_thread(interp, _PY_CALLS_TO_DO_BIT);
+#endif
+ }
+
return result;
}
@@ -742,13 +756,13 @@ static int
handle_signals(PyThreadState *tstate)
{
assert(_PyThreadState_CheckConsistency(tstate));
- _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 0);
+ _Py_unset_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT);
if (!_Py_ThreadCanHandleSignals(tstate->interp)) {
return 0;
}
if (_PyErr_CheckSignalsTstate(tstate) < 0) {
/* On failure, re-schedule a call to handle_signals(). */
- _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 1);
+ _Py_set_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT);
return -1;
}
return 0;
@@ -783,9 +797,30 @@ _make_pending_calls(struct _pending_calls *pending)
return 0;
}
+static void
+signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp)
+{
+#ifdef Py_GIL_DISABLED
+ _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT);
+#else
+ _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);
+#endif
+}
+
+static void
+unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp)
+{
+#ifdef Py_GIL_DISABLED
+ _Py_unset_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT);
+#else
+ _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);
+#endif
+}
+
static int
-make_pending_calls(PyInterpreterState *interp)
+make_pending_calls(PyThreadState *tstate)
{
+ PyInterpreterState *interp = tstate->interp;
struct _pending_calls *pending = &interp->ceval.pending;
struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread;
@@ -811,12 +846,12 @@ make_pending_calls(PyInterpreterState *interp)
/* unsignal before starting to call callbacks, so that any callback
added in-between re-signals */
- UNSIGNAL_PENDING_CALLS(interp);
+ unsignal_pending_calls(tstate, interp);
if (_make_pending_calls(pending) != 0) {
pending->busy = 0;
/* There might not be more calls to make, but we play it safe. */
- SIGNAL_PENDING_CALLS(interp);
+ signal_pending_calls(tstate, interp);
return -1;
}
@@ -824,7 +859,7 @@ make_pending_calls(PyInterpreterState *interp)
if (_make_pending_calls(pending_main) != 0) {
pending->busy = 0;
/* There might not be more calls to make, but we play it safe. */
- SIGNAL_PENDING_CALLS(interp);
+ signal_pending_calls(tstate, interp);
return -1;
}
}
@@ -833,13 +868,37 @@ make_pending_calls(PyInterpreterState *interp)
return 0;
}
+void
+_Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit)
+{
+ _PyRuntimeState *runtime = &_PyRuntime;
+
+ HEAD_LOCK(runtime);
+ for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate
= tstate->next) {
+ _Py_set_eval_breaker_bit(tstate, bit);
+ }
+ HEAD_UNLOCK(runtime);
+}
+
+void
+_Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit)
+{
+ _PyRuntimeState *runtime = &_PyRuntime;
+
+ HEAD_LOCK(runtime);
+ for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate
= tstate->next) {
+ _Py_unset_eval_breaker_bit(tstate, bit);
+ }
+ HEAD_UNLOCK(runtime);
+}
+
void
_Py_FinishPendingCalls(PyThreadState *tstate)
{
assert(PyGILState_Check());
assert(_PyThreadState_CheckConsistency(tstate));
- if (make_pending_calls(tstate->interp) < 0) {
+ if (make_pending_calls(tstate) < 0) {
PyObject *exc = _PyErr_GetRaisedException(tstate);
PyErr_BadInternalCall();
_PyErr_ChainExceptions1(exc);
@@ -862,7 +921,7 @@ _PyEval_MakePendingCalls(PyThreadState *tstate)
}
}
- res = make_pending_calls(tstate->interp);
+ res = make_pending_calls(tstate);
if (res != 0) {
return res;
}
@@ -955,11 +1014,11 @@ _PyEval_InitState(PyInterpreterState *interp)
int
_Py_HandlePending(PyThreadState *tstate)
{
- PyInterpreterState *interp = tstate->interp;
+ uintptr_t breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
/* Stop-the-world */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_PLEASE_STOP_BIT)) {
- _Py_set_eval_breaker_bit(interp, _PY_EVAL_PLEASE_STOP_BIT, 0);
+ if ((breaker & _PY_EVAL_PLEASE_STOP_BIT) != 0) {
+ _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PLEASE_STOP_BIT);
_PyThreadState_Suspend(tstate);
/* The attach blocks until the stop-the-world event is complete. */
@@ -967,35 +1026,35 @@ _Py_HandlePending(PyThreadState *tstate)
}
/* Pending signals */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) {
+ if ((breaker & _PY_SIGNALS_PENDING_BIT) != 0) {
if (handle_signals(tstate) != 0) {
return -1;
}
}
/* Pending calls */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_CALLS_TO_DO_BIT)) {
- if (make_pending_calls(interp) != 0) {
+ if ((breaker & _PY_CALLS_TO_DO_BIT) != 0) {
+ if (make_pending_calls(tstate) != 0) {
return -1;
}
}
#ifdef Py_GIL_DISABLED
/* Objects with refcounts to merge */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_EXPLICIT_MERGE_BIT)) {
- _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 0);
+ if ((breaker & _PY_EVAL_EXPLICIT_MERGE_BIT) != 0) {
+ _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT);
_Py_brc_merge_refcounts(tstate);
}
#endif
/* GC scheduled to run */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) {
- _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0);
+ if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) {
+ _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT);
_Py_RunGC(tstate);
}
/* GIL drop request */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) {
+ if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) {
/* Give another thread a chance */
_PyThreadState_Detach(tstate);
@@ -1005,11 +1064,10 @@ _Py_HandlePending(PyThreadState *tstate)
}
/* Check for asynchronous exception. */
- if (_Py_eval_breaker_bit_is_set(interp, _PY_ASYNC_EXCEPTION_BIT)) {
- _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 0);
- if (tstate->async_exc != NULL) {
- PyObject *exc = tstate->async_exc;
- tstate->async_exc = NULL;
+ if ((breaker & _PY_ASYNC_EXCEPTION_BIT) != 0) {
+ _Py_unset_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT);
+ PyObject *exc = _Py_atomic_exchange_ptr(&tstate->async_exc, NULL);
+ if (exc != NULL) {
_PyErr_SetNone(tstate, exc);
Py_DECREF(exc);
return -1;
@@ -1017,4 +1075,3 @@ _Py_HandlePending(PyThreadState *tstate)
}
return 0;
}
-
diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index f796b60335612c..01a9b32229d8a5 100644
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -124,7 +124,7 @@
#define CHECK_EVAL_BREAKER() \
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \
QSBR_QUIESCENT_STATE(tstate); \
- if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
_PY_EVAL_EVENTS_MASK) { \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
_PY_EVAL_EVENTS_MASK) { \
if (_Py_HandlePending(tstate) != 0) { \
GOTO_ERROR(error); \
} \
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 445f98b469e978..2ca54b6fe9cd38 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -17,7 +17,7 @@
if (_Py_emscripten_signal_clock == 0) goto deoptimize;
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
- uintptr_t eval_breaker =
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker);
+ uintptr_t eval_breaker =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
if (eval_breaker != version) goto deoptimize;
diff --git a/Python/gc.c b/Python/gc.c
index c6831f4c74bcac..8c2def1017bf2e 100644
--- a/Python/gc.c
+++ b/Python/gc.c
@@ -1771,9 +1771,12 @@ PyObject_IS_GC(PyObject *obj)
}
void
-_Py_ScheduleGC(PyInterpreterState *interp)
+_Py_ScheduleGC(PyThreadState *tstate)
{
- _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1);
+ if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT))
+ {
+ _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT);
+ }
}
void
@@ -1794,7 +1797,7 @@ _PyObject_GC_Link(PyObject *op)
!_Py_atomic_load_int_relaxed(&gcstate->collecting) &&
!_PyErr_Occurred(tstate))
{
- _Py_ScheduleGC(tstate->interp);
+ _Py_ScheduleGC(tstate);
}
}
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
index a758c99285a539..2993ef4ac7818b 100644
--- a/Python/gc_free_threading.c
+++ b/Python/gc_free_threading.c
@@ -981,7 +981,7 @@ record_allocation(PyThreadState *tstate)
if (gc_should_collect(gcstate) &&
!_Py_atomic_load_int_relaxed(&gcstate->collecting))
{
- _Py_ScheduleGC(tstate->interp);
+ _Py_ScheduleGC(tstate);
}
}
}
@@ -1564,9 +1564,12 @@ PyObject_IS_GC(PyObject *obj)
}
void
-_Py_ScheduleGC(PyInterpreterState *interp)
+_Py_ScheduleGC(PyThreadState *tstate)
{
- _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1);
+ if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT))
+ {
+ _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT);
+ }
}
void
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 78991066974df3..01e67acdc5c0e5 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -3139,7 +3139,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
- uintptr_t global_version =
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
~_PY_EVAL_EVENTS_MASK;
+ uintptr_t global_version =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
@@ -4809,7 +4809,7 @@
TIER_ONE_ONLY
assert(frame == tstate->current_frame);
uintptr_t global_version =
-
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
+ _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((code_version & 255) == 0);
@@ -4836,7 +4836,7 @@
DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
- uintptr_t eval_breaker =
_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker);
+ uintptr_t eval_breaker =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version =
_PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
DEOPT_IF(eval_breaker != version, RESUME);
diff --git a/Python/instrumentation.c b/Python/instrumentation.c
index 533aece210202b..878d19f0552bf5 100644
--- a/Python/instrumentation.c
+++ b/Python/instrumentation.c
@@ -891,18 +891,43 @@ static inline int most_significant_bit(uint8_t bits) {
static uint32_t
global_version(PyInterpreterState *interp)
{
- return interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK;
+ return (uint32_t)_Py_atomic_load_uintptr_relaxed(
+ &interp->ceval.instrumentation_version);
}
+/* Atomically set the given version in the given location, without touching
+ anything in _PY_EVAL_EVENTS_MASK. */
static void
-set_global_version(PyInterpreterState *interp, uint32_t version)
+set_version_raw(uintptr_t *ptr, uint32_t version)
{
- assert((version & _PY_EVAL_EVENTS_MASK) == 0);
- uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker);
- intptr_t new;
+ uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr);
+ uintptr_t new;
do {
new = (old & _PY_EVAL_EVENTS_MASK) | version;
- } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker,
&old, new));
+ } while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new));
+}
+
+static void
+set_global_version(PyThreadState *tstate, uint32_t version)
+{
+ assert((version & _PY_EVAL_EVENTS_MASK) == 0);
+ PyInterpreterState *interp = tstate->interp;
+ set_version_raw(&interp->ceval.instrumentation_version, version);
+
+#ifdef Py_GIL_DISABLED
+ // Set the version on all threads in free-threaded builds.
+ _PyRuntimeState *runtime = &_PyRuntime;
+ HEAD_LOCK(runtime);
+ for (tstate = interp->threads.head; tstate;
+ tstate = PyThreadState_Next(tstate)) {
+ set_version_raw(&tstate->eval_breaker, version);
+ };
+ HEAD_UNLOCK(runtime);
+#else
+ // Normal builds take the current version from instrumentation_version when
+ // attaching a thread, so we only have to set the current thread's version.
+ set_version_raw(&tstate->eval_breaker, version);
+#endif
}
static bool
@@ -1566,7 +1591,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState
*interp)
{
if (is_version_up_to_date(code, interp)) {
assert(
- (interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK) == 0 ||
+ interp->ceval.instrumentation_version == 0 ||
instrumentation_cross_checks(interp, code)
);
return 0;
@@ -1778,7 +1803,8 @@ int
_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
- PyInterpreterState *interp = _PyInterpreterState_GET();
+ PyThreadState *tstate = _PyThreadState_GET();
+ PyInterpreterState *interp = tstate->interp;
assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS));
if (check_tool(interp, tool_id)) {
return -1;
@@ -1793,7 +1819,7 @@ _PyMonitoring_SetEvents(int tool_id,
_PyMonitoringEventSet events)
PyErr_Format(PyExc_OverflowError, "events set too many times");
return -1;
}
- set_global_version(interp, new_version);
+ set_global_version(tstate, new_version);
_Py_Executors_InvalidateAll(interp);
return instrument_all_executing_code_objects(interp);
}
@@ -2122,7 +2148,8 @@ monitoring_restart_events_impl(PyObject *module)
* last restart version > instrumented version for all code objects
* last restart version < current version
*/
- PyInterpreterState *interp = _PyInterpreterState_GET();
+ PyThreadState *tstate = _PyThreadState_GET();
+ PyInterpreterState *interp = tstate->interp;
uint32_t restart_version = global_version(interp) +
MONITORING_VERSION_INCREMENT;
uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT;
if (new_version <= MONITORING_VERSION_INCREMENT) {
@@ -2130,7 +2157,7 @@ monitoring_restart_events_impl(PyObject *module)
return NULL;
}
interp->last_restart_version = restart_version;
- set_global_version(interp, new_version);
+ set_global_version(tstate, new_version);
if (instrument_all_executing_code_objects(interp)) {
return NULL;
}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 7b537af8c87697..656d82136d263b 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -663,6 +663,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
if (tstate == NULL) {
return _PyStatus_ERR("can't make first thread");
}
+ runtime->main_tstate = tstate;
_PyThreadState_Bind(tstate);
init_interp_create_gil(tstate, config.gil);
diff --git a/Python/pystate.c b/Python/pystate.c
index 3484beab7aaed4..4cd975a84d1645 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -794,9 +794,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState
*tstate)
Py_CLEAR(interp->audit_hooks);
- // At this time, all the threads should be cleared so we don't need
- // atomic operations for eval_breaker
- interp->ceval.eval_breaker = 0;
+ interp->ceval.instrumentation_version = 0;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
interp->monitors.tools[i] = 0;
@@ -1318,6 +1316,8 @@ init_threadstate(_PyThreadStateImpl *_tstate,
assert(interp != NULL);
tstate->interp = interp;
+ tstate->eval_breaker =
+
_Py_atomic_load_uintptr_relaxed(&interp->ceval.instrumentation_version);
// next/prev are set in add_threadstate().
assert(tstate->next == NULL);
@@ -2021,8 +2021,7 @@ park_detached_threads(struct _stoptheworld_state *stw)
}
}
else if (state == _Py_THREAD_ATTACHED && t != stw->requester) {
- // TODO: set this per-thread, rather than per-interpreter.
- _Py_set_eval_breaker_bit(t->interp, _PY_EVAL_PLEASE_STOP_BIT, 1);
+ _Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT);
}
}
stw->thread_countdown -= num_parked;
@@ -2186,19 +2185,18 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject
*exc)
* deadlock, we need to release head_mutex before
* the decref.
*/
- PyObject *old_exc = tstate->async_exc;
- tstate->async_exc = Py_XNewRef(exc);
+ Py_XINCREF(exc);
+ PyObject *old_exc = _Py_atomic_exchange_ptr(&tstate->async_exc, exc);
HEAD_UNLOCK(runtime);
Py_XDECREF(old_exc);
- _PyEval_SignalAsyncExc(tstate->interp);
+ _Py_set_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT);
return 1;
}
HEAD_UNLOCK(runtime);
return 0;
}
-
//---------------------------------
// API for the current thread state
//---------------------------------
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
