https://github.com/python/cpython/commit/2cf18a44303b6d84faa8ecffaecc427b53ae121e
commit: 2cf18a44303b6d84faa8ecffaecc427b53ae121e
branch: main
author: Mark Shannon <[email protected]>
committer: markshannon <[email protected]>
date: 2024-03-15T10:48:00Z
summary:
GH-116422: Modify a few uops so that they can be supported by tier 2 with
hot/cold splitting (GH-116832)
files:
M Include/internal/pycore_ceval.h
M Python/bytecodes.c
M Python/executor_cases.c.h
M Python/generated_cases.c.h
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 6eab2ba1daedf8..946f82ae3c20e3 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -138,12 +138,12 @@ extern void _PyEval_DeactivateOpCache(void);
/* With USE_STACKCHECK macro defined, trigger stack checks in
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
- return (tstate->c_recursion_remaining-- <= 0
+ return (tstate->c_recursion_remaining-- < 0
|| (tstate->c_recursion_remaining & 63) == 0);
}
#else
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
- return tstate->c_recursion_remaining-- <= 0;
+ return tstate->c_recursion_remaining-- < 0;
}
#endif
@@ -161,6 +161,11 @@ static inline int
_Py_EnterRecursiveCallTstate(PyThreadState *tstate,
return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
}
+static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState
*tstate) {
+ assert(tstate->c_recursion_remaining > 0);
+ tstate->c_recursion_remaining--;
+}
+
static inline int _Py_EnterRecursiveCall(const char *where) {
PyThreadState *tstate = _PyThreadState_GET();
return _Py_EnterRecursiveCallTstate(tstate, where);
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index e8fcb6c6d0067c..bf7782a65bc5ce 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -3376,14 +3376,12 @@ dummy_func(
DEOPT_IF(total_args != 1);
DEOPT_IF(!PyCFunction_CheckExact(callable));
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O);
+ // CPython promises to check all non-vectorcall function calls.
+ DEOPT_IF(tstate->c_recursion_remaining <= 0);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- // This is slower but CPython promises to check all non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python
object")) {
- GOTO_ERROR(error);
- }
PyObject *arg = args[0];
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc,
PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -3480,10 +3478,11 @@ dummy_func(
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
+ if (res == NULL) {
+ GOTO_ERROR(error);
+ }
Py_DECREF(callable);
Py_DECREF(arg);
- ERROR_IF(res == NULL, error);
}
inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null,
args[oparg] -- res)) {
@@ -3505,11 +3504,12 @@ dummy_func(
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
+ if (res == NULL) {
+ GOTO_ERROR(error);
+ }
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
}
// This is secretly a super-instruction
@@ -3543,16 +3543,14 @@ dummy_func(
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O);
+ // CPython promises to check all non-vectorcall function calls.
+ DEOPT_IF(tstate->c_recursion_remaining <= 0);
PyObject *arg = args[1];
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
- // This is slower but CPython promises to check all non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python
object")) {
- GOTO_ERROR(error);
- }
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -3616,13 +3614,11 @@ dummy_func(
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
DEOPT_IF(meth->ml_flags != METH_NOARGS);
+ // CPython promises to check all non-vectorcall function calls.
+ DEOPT_IF(tstate->c_recursion_remaining <= 0);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
- // This is slower but CPython promises to check all non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python
object")) {
- GOTO_ERROR(error);
- }
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 077499ebc01687..bdc9c0b1501c91 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -3189,14 +3189,12 @@
if (total_args != 1) goto deoptimize;
if (!PyCFunction_CheckExact(callable)) goto deoptimize;
if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize;
+ // CPython promises to check all non-vectorcall function calls.
+ if (tstate->c_recursion_remaining <= 0) goto deoptimize;
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- // This is slower but CPython promises to check all non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python
object")) {
- GOTO_ERROR(error);
- }
PyObject *arg = args[0];
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc,
PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -3305,9 +3303,11 @@
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res == NULL) {
+ GOTO_ERROR(error);
+ }
Py_DECREF(callable);
Py_DECREF(arg);
- if (res == NULL) { stack_pointer += -2 - oparg; goto
error_tier_two; }
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
@@ -3340,10 +3340,12 @@
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res == NULL) {
+ GOTO_ERROR(error);
+ }
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto
error_tier_two; }
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
@@ -3368,16 +3370,14 @@
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize;
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != METH_O) goto deoptimize;
+ // CPython promises to check all non-vectorcall function calls.
+ if (tstate->c_recursion_remaining <= 0) goto deoptimize;
PyObject *arg = args[1];
PyObject *self = args[0];
if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize;
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
- // This is slower but CPython promises to check all non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python
object")) {
- GOTO_ERROR(error);
- }
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -3450,13 +3450,11 @@
PyObject *self = args[0];
if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize;
if (meth->ml_flags != METH_NOARGS) goto deoptimize;
+ // CPython promises to check all non-vectorcall function calls.
+ if (tstate->c_recursion_remaining <= 0) goto deoptimize;
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
- // This is slower but CPython promises to check all non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python
object")) {
- GOTO_ERROR(error);
- }
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 645d0fb510e6fb..2fbd9ed403f5d1 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -1174,14 +1174,12 @@
DEOPT_IF(total_args != 1, CALL);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
+ // CPython promises to check all non-vectorcall function calls.
+ DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- // This is slower but CPython promises to check all
non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a
Python object")) {
- GOTO_ERROR(error);
- }
PyObject *arg = args[0];
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc,
PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -1350,10 +1348,12 @@
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res == NULL) {
+ GOTO_ERROR(error);
+ }
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
DISPATCH();
@@ -1481,9 +1481,11 @@
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res == NULL) {
+ GOTO_ERROR(error);
+ }
Py_DECREF(callable);
Py_DECREF(arg);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
DISPATCH();
@@ -1649,13 +1651,11 @@
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
+ // CPython promises to check all non-vectorcall function calls.
+ DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
- // This is slower but CPython promises to check all
non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a
Python object")) {
- GOTO_ERROR(error);
- }
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -1698,16 +1698,14 @@
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
+ // CPython promises to check all non-vectorcall function calls.
+ DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
PyObject *arg = args[1];
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
- // This is slower but CPython promises to check all
non-vectorcall
- // function calls.
- if (_Py_EnterRecursiveCallTstate(tstate, " while calling a
Python object")) {
- GOTO_ERROR(error);
- }
+ _Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
