https://github.com/python/cpython/commit/ea3cd0498c443e93be441736c804258e93d21edd
commit: ea3cd0498c443e93be441736c804258e93d21edd
branch: main
author: Michael Droettboom <[email protected]>
committer: markshannon <[email protected]>
date: 2024-01-25T11:10:51Z
summary:
gh-114312: Collect stats for unlikely events (GH-114493)
files:
A Lib/test/test_optimizer.py
M Include/cpython/pystats.h
M Include/internal/pycore_code.h
M Include/internal/pycore_interp.h
M Modules/_testinternalcapi.c
M Objects/funcobject.c
M Objects/typeobject.c
M Python/pylifecycle.c
M Python/pystate.c
M Python/specialize.c
M Tools/scripts/summarize_stats.py
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index ba67eefef3e37a..bf0cfe4cb695b4 100644
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -122,11 +122,25 @@ typedef struct _optimization_stats {
uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE];
} OptimizationStats;
+typedef struct _rare_event_stats {
+ /* Setting an object's class, obj.__class__ = ... */
+ uint64_t set_class;
+ /* Setting the bases of a class, cls.__bases__ = ... */
+ uint64_t set_bases;
+ /* Setting the PEP 523 frame eval function,
_PyInterpreterState_SetFrameEvalFunc() */
+ uint64_t set_eval_frame_func;
+ /* Modifying the builtins, __builtins__.__dict__[var] = ... */
+ uint64_t builtin_dict;
+ /* Modifying a function, e.g. func.__defaults__ = ..., etc. */
+ uint64_t func_modification;
+} RareEventStats;
+
typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
OptimizationStats optimization_stats;
+ RareEventStats rare_event_stats;
GCStats *gc_stats;
} PyStats;
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 73df6c3568ffe0..fdd5918228455d 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -295,6 +295,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co);
_Py_stats->optimization_stats.name[bucket]++; \
} \
} while (0)
+#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats)
_Py_stats->rare_event_stats.name++; } while (0)
// Export for '_opcode' shared extension
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
@@ -313,6 +314,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
#define UOP_STAT_INC(opname, name) ((void)0)
#define OPT_UNSUPPORTED_OPCODE(opname) ((void)0)
#define OPT_HIST(length, name) ((void)0)
+#define RARE_EVENT_STAT_INC(name) ((void)0)
#endif // !Py_STATS
// Utility functions for reading/writing 32/64-bit values in the inline caches.
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index f953b8426e180a..662a18d93f329d 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -60,6 +60,21 @@ struct _stoptheworld_state {
/* cross-interpreter data registry */
+/* Tracks some rare events per-interpreter, used by the optimizer to turn
on/off
+ specific optimizations. */
+typedef struct _rare_events {
+ /* Setting an object's class, obj.__class__ = ... */
+ uint8_t set_class;
+ /* Setting the bases of a class, cls.__bases__ = ... */
+ uint8_t set_bases;
+ /* Setting the PEP 523 frame eval function,
_PyInterpreterState_SetFrameEvalFunc() */
+ uint8_t set_eval_frame_func;
+ /* Modifying the builtins, __builtins__.__dict__[var] = ... */
+ uint8_t builtin_dict;
+ int builtins_dict_watcher_id;
+ /* Modifying a function, e.g. func.__defaults__ = ..., etc. */
+ uint8_t func_modification;
+} _rare_events;
/* interpreter state */
@@ -217,6 +232,7 @@ struct _is {
uint16_t optimizer_resume_threshold;
uint16_t optimizer_backedge_threshold;
uint32_t next_func_version;
+ _rare_events rare_events;
_Py_GlobalMonitors monitors;
bool sys_profile_initialized;
@@ -347,6 +363,19 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New(
PyInterpreterState **pinterp);
+#define RARE_EVENT_INTERP_INC(interp, name) \
+ do { \
+ /* saturating add */ \
+ if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \
+ RARE_EVENT_STAT_INC(name); \
+ } while (0); \
+
+#define RARE_EVENT_INC(name) \
+ do { \
+ PyInterpreterState *interp = PyInterpreterState_Get(); \
+ RARE_EVENT_INTERP_INC(interp, name); \
+ } while (0); \
+
#ifdef __cplusplus
}
#endif
diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py
new file mode 100644
index 00000000000000..b56bf3cfd9560e
--- /dev/null
+++ b/Lib/test/test_optimizer.py
@@ -0,0 +1,75 @@
+import _testinternalcapi
+import unittest
+import types
+
+
+class TestRareEventCounters(unittest.TestCase):
+ def test_set_class(self):
+ class A:
+ pass
+ class B:
+ pass
+ a = A()
+
+ orig_counter = _testinternalcapi.get_rare_event_counters()["set_class"]
+ a.__class__ = B
+ self.assertEqual(
+ orig_counter + 1,
+ _testinternalcapi.get_rare_event_counters()["set_class"]
+ )
+
+ def test_set_bases(self):
+ class A:
+ pass
+ class B:
+ pass
+ class C(B):
+ pass
+
+ orig_counter = _testinternalcapi.get_rare_event_counters()["set_bases"]
+ C.__bases__ = (A,)
+ self.assertEqual(
+ orig_counter + 1,
+ _testinternalcapi.get_rare_event_counters()["set_bases"]
+ )
+
+ def test_set_eval_frame_func(self):
+ orig_counter =
_testinternalcapi.get_rare_event_counters()["set_eval_frame_func"]
+ _testinternalcapi.set_eval_frame_record([])
+ self.assertEqual(
+ orig_counter + 1,
+ _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"]
+ )
+ _testinternalcapi.set_eval_frame_default()
+
+ def test_builtin_dict(self):
+ orig_counter =
_testinternalcapi.get_rare_event_counters()["builtin_dict"]
+ if isinstance(__builtins__, types.ModuleType):
+ builtins = __builtins__.__dict__
+ else:
+ builtins = __builtins__
+ builtins["FOO"] = 42
+ self.assertEqual(
+ orig_counter + 1,
+ _testinternalcapi.get_rare_event_counters()["builtin_dict"]
+ )
+ del builtins["FOO"]
+
+ def test_func_modification(self):
+ def func(x=0):
+ pass
+
+ for attribute in (
+ "__code__",
+ "__defaults__",
+ "__kwdefaults__"
+ ):
+ orig_counter =
_testinternalcapi.get_rare_event_counters()["func_modification"]
+ setattr(func, attribute, getattr(func, attribute))
+ self.assertEqual(
+ orig_counter + 1,
+
_testinternalcapi.get_rare_event_counters()["func_modification"]
+ )
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 7d277df164d3ec..2c32c691afa583 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1635,6 +1635,21 @@ get_type_module_name(PyObject *self, PyObject *type)
return _PyType_GetModuleName((PyTypeObject *)type);
}
+static PyObject *
+get_rare_event_counters(PyObject *self, PyObject *type)
+{
+ PyInterpreterState *interp = PyInterpreterState_Get();
+
+ return Py_BuildValue(
+ "{sksksksksk}",
+ "set_class", interp->rare_events.set_class,
+ "set_bases", interp->rare_events.set_bases,
+ "set_eval_frame_func", interp->rare_events.set_eval_frame_func,
+ "builtin_dict", interp->rare_events.builtin_dict,
+ "func_modification", interp->rare_events.func_modification
+ );
+}
+
#ifdef Py_GIL_DISABLED
static PyObject *
@@ -1711,6 +1726,7 @@ static PyMethodDef module_functions[] = {
{"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS},
_TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF
{"get_type_module_name", get_type_module_name, METH_O},
+ {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
#ifdef Py_GIL_DISABLED
{"py_thread_id", get_py_thread_id, METH_NOARGS},
#endif
diff --git a/Objects/funcobject.c b/Objects/funcobject.c
index 2620dc69bfd79b..08b2823d8cf024 100644
--- a/Objects/funcobject.c
+++ b/Objects/funcobject.c
@@ -53,6 +53,15 @@ handle_func_event(PyFunction_WatchEvent event,
PyFunctionObject *func,
if (interp->active_func_watchers) {
notify_func_watchers(interp, event, func, new_value);
}
+ switch (event) {
+ case PyFunction_EVENT_MODIFY_CODE:
+ case PyFunction_EVENT_MODIFY_DEFAULTS:
+ case PyFunction_EVENT_MODIFY_KWDEFAULTS:
+ RARE_EVENT_INTERP_INC(interp, func_modification);
+ break;
+ default:
+ break;
+ }
}
int
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 3a35a5b5975898..a8c3b8896d36eb 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1371,6 +1371,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases,
void *context)
res = 0;
}
+ RARE_EVENT_INC(set_bases);
Py_DECREF(old_bases);
Py_DECREF(old_base);
@@ -5842,6 +5843,8 @@ object_set_class(PyObject *self, PyObject *value, void
*closure)
Py_SET_TYPE(self, newto);
if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE)
Py_DECREF(oldto);
+
+ RARE_EVENT_INC(set_class);
return 0;
}
else {
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 0d5eec06e9b458..261622adc4cc77 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -605,6 +605,12 @@ init_interp_create_gil(PyThreadState *tstate, int gil)
_PyEval_InitGIL(tstate, own_gil);
}
+static int
+builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key,
PyObject *new_value)
+{
+ RARE_EVENT_INC(builtin_dict);
+ return 0;
+}
static PyStatus
pycore_create_interpreter(_PyRuntimeState *runtime,
@@ -1266,6 +1272,14 @@ init_interp_main(PyThreadState *tstate)
}
}
+ if ((interp->rare_events.builtins_dict_watcher_id =
PyDict_AddWatcher(&builtins_dict_watcher)) == -1) {
+ return _PyStatus_ERR("failed to add builtin dict watcher");
+ }
+
+ if (PyDict_Watch(interp->rare_events.builtins_dict_watcher_id,
interp->builtins) != 0) {
+ return _PyStatus_ERR("failed to set builtin dict watcher");
+ }
+
assert(!_PyErr_Occurred(tstate));
return _PyStatus_OK();
@@ -1592,6 +1606,10 @@ static void
finalize_modules(PyThreadState *tstate)
{
PyInterpreterState *interp = tstate->interp;
+
+ // Stop collecting stats on __builtin__ modifications during teardown
+ PyDict_Unwatch(interp->rare_events.builtins_dict_watcher_id,
interp->builtins);
+
PyObject *modules = _PyImport_GetModules(interp);
if (modules == NULL) {
// Already done
diff --git a/Python/pystate.c b/Python/pystate.c
index 548c77b7dc7ebb..c9b521351444a7 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -2616,6 +2616,7 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState
*interp,
if (eval_frame != NULL) {
_Py_Executors_InvalidateAll(interp);
}
+ RARE_EVENT_INC(set_eval_frame_func);
interp->eval_frame = eval_frame;
}
diff --git a/Python/specialize.c b/Python/specialize.c
index 13e0440dd9dd0d..a9efbe0453b94e 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -267,6 +267,16 @@ print_optimization_stats(FILE *out, OptimizationStats
*stats)
}
}
+static void
+print_rare_event_stats(FILE *out, RareEventStats *stats)
+{
+ fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class);
+ fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases);
+ fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n",
stats->set_eval_frame_func);
+ fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n",
stats->builtin_dict);
+ fprintf(out, "Rare event (func_modification): %" PRIu64 "\n",
stats->func_modification);
+}
+
static void
print_stats(FILE *out, PyStats *stats)
{
@@ -275,6 +285,7 @@ print_stats(FILE *out, PyStats *stats)
print_object_stats(out, &stats->object_stats);
print_gc_stats(out, stats->gc_stats);
print_optimization_stats(out, &stats->optimization_stats);
+ print_rare_event_stats(out, &stats->rare_event_stats);
}
void
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 1e9dc07bae8981..9b7e7b999ea7c7 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -412,6 +412,14 @@ def get_histogram(self, prefix: str) -> list[tuple[int,
int]]:
rows.sort()
return rows
+ def get_rare_events(self) -> list[tuple[str, int]]:
+ prefix = "Rare event "
+ return [
+ (key[len(prefix) + 1:-1], val)
+ for key, val in self._data.items()
+ if key.startswith(prefix)
+ ]
+
class Count(int):
def markdown(self) -> str:
@@ -1064,6 +1072,17 @@ def iter_optimization_tables(base_stats: Stats,
head_stats: Stats | None = None)
)
+def rare_event_section() -> Section:
+ def calc_rare_event_table(stats: Stats) -> Table:
+ return [(x, Count(y)) for x, y in stats.get_rare_events()]
+
+ return Section(
+ "Rare events",
+ "Counts of rare/unlikely events",
+ [Table(("Event", "Count:"), calc_rare_event_table, JoinMode.CHANGE)],
+ )
+
+
def meta_stats_section() -> Section:
def calc_rows(stats: Stats) -> Rows:
return [("Number of data files", Count(stats.get("__nfiles__")))]
@@ -1085,6 +1104,7 @@ def calc_rows(stats: Stats) -> Rows:
object_stats_section(),
gc_stats_section(),
optimization_section(),
+ rare_event_section(),
meta_stats_section(),
]
@@ -1162,7 +1182,7 @@ def output_stats(inputs: list[Path], json_output=str |
None):
case 1:
data = load_raw_data(Path(inputs[0]))
if json_output is not None:
- with open(json_output, 'w', encoding='utf-8') as f:
+ with open(json_output, "w", encoding="utf-8") as f:
save_raw_data(data, f) # type: ignore
stats = Stats(data)
output_markdown(sys.stdout, LAYOUT, stats)
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
