https://github.com/python/cpython/commit/b05afdd5ec325bdb4cc89bb3be177ed577bea41f
commit: b05afdd5ec325bdb4cc89bb3be177ed577bea41f
branch: main
author: Michael Droettboom <[email protected]>
committer: Fidget-Spinner <[email protected]>
date: 2024-02-26T17:51:47Z
summary:
gh-115168: Add pystats counter for invalidated executors (GH-115169)
files:
M Include/cpython/optimizer.h
M Include/cpython/pystats.h
M Modules/_testinternalcapi.c
M Python/instrumentation.c
M Python/optimizer.c
M Python/optimizer_analysis.c
M Python/pylifecycle.c
M Python/pystate.c
M Python/specialize.c
M Python/sysmodule.c
M Tools/scripts/summarize_stats.py
diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h
index fe54d1ddfe6129..8fc9fb62aebdb4 100644
--- a/Include/cpython/optimizer.h
+++ b/Include/cpython/optimizer.h
@@ -100,8 +100,8 @@ void _Py_ExecutorClear(_PyExecutorObject *);
void _Py_BloomFilter_Init(_PyBloomFilter *);
void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj);
PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void
*obj);
-PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState
*interp, void *obj);
-extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp);
+PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState
*interp, void *obj, int is_invalidation);
+extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int
is_invalidation);
/* For testing */
PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index db9aaedec950e4..887fbbedf88502 100644
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -115,6 +115,7 @@ typedef struct _optimization_stats {
uint64_t inner_loop;
uint64_t recursive_call;
uint64_t low_confidence;
+ uint64_t executors_invalidated;
UOpStats opcode[512];
uint64_t unsupported_opcode[256];
uint64_t trace_length_hist[_Py_UOP_HIST_SIZE];
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 0d23b1899f22e4..5b714ca3f3dc33 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1035,7 +1035,7 @@ static PyObject *
invalidate_executors(PyObject *self, PyObject *obj)
{
PyInterpreterState *interp = PyInterpreterState_Get();
- _Py_Executors_InvalidateDependency(interp, obj);
+ _Py_Executors_InvalidateDependency(interp, obj, 1);
Py_RETURN_NONE;
}
diff --git a/Python/instrumentation.c b/Python/instrumentation.c
index 878d19f0552bf5..6f1bc2e0a107df 100644
--- a/Python/instrumentation.c
+++ b/Python/instrumentation.c
@@ -1599,7 +1599,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState
*interp)
if (code->co_executors != NULL) {
_PyCode_Clear_Executors(code);
}
- _Py_Executors_InvalidateDependency(interp, code);
+ _Py_Executors_InvalidateDependency(interp, code, 1);
int code_len = (int)Py_SIZE(code);
/* Exit early to avoid creating instrumentation
* data for potential statically allocated code
@@ -1820,7 +1820,7 @@ _PyMonitoring_SetEvents(int tool_id,
_PyMonitoringEventSet events)
return -1;
}
set_global_version(tstate, new_version);
- _Py_Executors_InvalidateAll(interp);
+ _Py_Executors_InvalidateAll(interp, 1);
return instrument_all_executing_code_objects(interp);
}
@@ -1850,7 +1850,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int
tool_id, _PyMonitoringEvent
/* Force instrumentation update */
code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT;
}
- _Py_Executors_InvalidateDependency(interp, code);
+ _Py_Executors_InvalidateDependency(interp, code, 1);
if (_Py_Instrument(code, interp)) {
return -1;
}
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 6b2ba3addefc95..c04ee17ee2171d 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1348,7 +1348,7 @@ _Py_Executor_DependsOn(_PyExecutorObject *executor, void
*obj)
* May cause other executors to be invalidated as well
*/
void
-_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj)
+_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int
is_invalidation)
{
_PyBloomFilter obj_filter;
_Py_BloomFilter_Init(&obj_filter);
@@ -1360,6 +1360,9 @@ _Py_Executors_InvalidateDependency(PyInterpreterState
*interp, void *obj)
_PyExecutorObject *next = exec->vm_data.links.next;
if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) {
_Py_ExecutorClear(exec);
+ if (is_invalidation) {
+ OPT_STAT_INC(executors_invalidated);
+ }
}
exec = next;
}
@@ -1367,7 +1370,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState
*interp, void *obj)
/* Invalidate all executors */
void
-_Py_Executors_InvalidateAll(PyInterpreterState *interp)
+_Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation)
{
while (interp->executor_list_head) {
_PyExecutorObject *executor = interp->executor_list_head;
@@ -1378,5 +1381,8 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp)
else {
_Py_ExecutorClear(executor);
}
+ if (is_invalidation) {
+ OPT_STAT_INC(executors_invalidated);
+ }
}
}
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 1a9e82903ffbc0..e9751291e2fd5a 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -405,7 +405,7 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject*
dict,
{
RARE_EVENT_STAT_INC(watched_globals_modification);
assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS);
- _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict);
+ _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict, 1);
increment_mutations(dict);
PyDict_Unwatch(GLOBALS_WATCHER_ID, dict);
return 0;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 04487345f7ec05..3a2c0a450ac9d9 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -612,7 +612,7 @@ builtins_dict_watcher(PyDict_WatchEvent event, PyObject
*dict, PyObject *key, Py
{
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->rare_events.builtin_dict <
_Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
- _Py_Executors_InvalidateAll(interp);
+ _Py_Executors_InvalidateAll(interp, 1);
}
RARE_EVENT_INTERP_INC(interp, builtin_dict);
return 0;
@@ -1628,7 +1628,7 @@ finalize_modules(PyThreadState *tstate)
PyInterpreterState *interp = tstate->interp;
// Invalidate all executors and turn off tier 2 optimizer
- _Py_Executors_InvalidateAll(interp);
+ _Py_Executors_InvalidateAll(interp, 0);
_PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL);
Py_XDECREF(old);
diff --git a/Python/pystate.c b/Python/pystate.c
index bb8e24c1dbe12f..a80c1b7fb9c866 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -2666,7 +2666,7 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState
*interp,
return;
}
if (eval_frame != NULL) {
- _Py_Executors_InvalidateAll(interp);
+ _Py_Executors_InvalidateAll(interp, 1);
}
RARE_EVENT_INC(set_eval_frame_func);
interp->eval_frame = eval_frame;
diff --git a/Python/specialize.c b/Python/specialize.c
index 871979d92298b6..f83d8a9ceb0282 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -236,6 +236,7 @@ print_optimization_stats(FILE *out, OptimizationStats
*stats)
fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop);
fprintf(out, "Optimization recursive call: %" PRIu64 "\n",
stats->recursive_call);
fprintf(out, "Optimization low confidence: %" PRIu64 "\n",
stats->low_confidence);
+ fprintf(out, "Executors invalidated: %" PRIu64 "\n",
stats->executors_invalidated);
print_histogram(out, "Trace length", stats->trace_length_hist);
print_histogram(out, "Trace run length", stats->trace_run_length_hist);
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 69b6d886ccc3e9..1bfd031fdd26b2 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2138,7 +2138,7 @@ sys__clear_internal_caches_impl(PyObject *module)
/*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/
{
PyInterpreterState *interp = _PyInterpreterState_GET();
- _Py_Executors_InvalidateAll(interp);
+ _Py_Executors_InvalidateAll(interp, 0);
PyType_ClearCache();
Py_RETURN_NONE;
}
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 6b60b59b3b0e79..2925e096f4d95e 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -451,6 +451,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int,
int | None]]:
inner_loop = self._data["Optimization inner loop"]
recursive_call = self._data["Optimization recursive call"]
low_confidence = self._data["Optimization low confidence"]
+ executors_invalidated = self._data["Executors invalidated"]
return {
Doc(
@@ -493,11 +494,19 @@ def get_optimization_stats(self) -> dict[str, tuple[int,
int | None]]:
"A trace is abandoned because the likelihood of the jump to
top being taken "
"is too low.",
): (low_confidence, attempts),
+ Doc(
+ "Executors invalidated",
+ "The number of executors that were invalidated due to watched "
+ "dictionary changes.",
+ ): (executors_invalidated, created),
Doc("Traces executed", "The number of traces that were executed"):
(
executed,
None,
),
- Doc("Uops executed", "The total number of uops (micro-operations)
that were executed"): (
+ Doc(
+ "Uops executed",
+ "The total number of uops (micro-operations) that were
executed",
+ ): (
uops,
executed,
),
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
