https://github.com/python/cpython/commit/7dfa048bbb1e8b3047a6e3bae7931147c32afbef commit: 7dfa048bbb1e8b3047a6e3bae7931147c32afbef branch: main author: Petr Viktorin <[email protected]> committer: encukou <[email protected]> date: 2025-08-18T14:25:51+02:00 summary:
gh-135228: Create __dict__ and __weakref__ descriptors for object (GH-136966) This partially reverts #137047, keeping the tests for GC collectability of the original class that dataclass adds `__slots__` to. The reference leaks solved there are instead solved by having the `__dict__` & `__weakref__` descriptors not tied to (and referencing) their class. Instead, they're shared between all classes that need them (within an interpreter). The `__objclass__` ol the descriptors is set to `object`, since these descriptors work with *any* object. (The appropriate checks were already made in the get/set code, so the `__objclass__` check was redundant.) The repr of these descriptors (and any others whose `__objclass__` is `object`) now doesn't mention the objclass. This change required adjustment of introspection code that checks `__objclass__` to determine an object's “own” (i.e. not inherited) `__dict__`. Third-party code that does similar introspection of the internals will also need adjusting. Co-authored-by: Jelle Zijlstra <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-10-22-15.gh-issue-136966.J5lrE0.rst M Doc/whatsnew/3.15.rst M Include/internal/pycore_interp_structs.h M Include/internal/pycore_typeobject.h M Lib/dataclasses.py M Lib/inspect.py M Lib/test/test_descr.py M Objects/descrobject.c M Objects/typeobject.c M Python/clinic/sysmodule.c.h M Python/pylifecycle.c M Python/sysmodule.c M Tools/c-analyzer/cpython/_analyzer.py diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 407606da961c16..81aa12184ed35c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -209,6 +209,13 @@ Other language changes as keyword arguments at construction time. (Contributed by Serhiy Storchaka, Oleg Iarygin, and Yoav Nir in :gh:`74185`.) +* The :attr:`~object.__dict__` and :attr:`!__weakref__` descriptors now use a + single descriptor instance per interpreter, shared across all types that + need them. + This speeds up class creation, and helps avoid reference cycles. + (Contributed by Petr Viktorin in :gh:`135228`.) + + New modules =========== diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e300732e9e58c3..2cb1b104681300 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -691,6 +691,13 @@ struct _Py_interp_cached_objects { PyTypeObject *paramspecargs_type; PyTypeObject *paramspeckwargs_type; PyTypeObject *constevaluator_type; + + /* Descriptors for __dict__ and __weakref__ */ +#ifdef Py_GIL_DISABLED + PyMutex descriptor_mutex; +#endif + PyObject *dict_descriptor; + PyObject *weakref_descriptor; }; struct _Py_interp_static_objects { diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 0ee7d555c56cdd..24df69aa93fda2 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -40,6 +40,7 @@ extern void _PyTypes_FiniTypes(PyInterpreterState *); extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); extern void _PyTypes_AfterFork(void); +extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *); static inline PyObject ** _PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index d29f1615f276d2..b98f21dcbe9220 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1283,10 +1283,6 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): if '__slots__' in cls.__dict__: raise TypeError(f'{cls.__name__} already specifies __slots__') - # gh-102069: Remove existing __weakref__ descriptor. - # gh-135228: Make sure the original class can be garbage collected. - sys._clear_type_descriptors(cls) - # Create a new dict for our new class. cls_dict = dict(cls.__dict__) field_names = tuple(f.name for f in fields(cls)) @@ -1304,6 +1300,11 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # available in _MARKER. cls_dict.pop(field_name, None) + # Remove __dict__ and `__weakref__` descriptors. + # They'll be added back if applicable. + cls_dict.pop('__dict__', None) + cls_dict.pop('__weakref__', None) # gh-102069 + # And finally create the class. qualname = getattr(cls, '__qualname__', None) newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) diff --git a/Lib/inspect.py b/Lib/inspect.py index 183e67fabf966e..d7814bfeb2b885 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1698,7 +1698,8 @@ def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): class_dict = dunder_dict['__dict__'] if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and - class_dict.__objclass__ is entry): + (class_dict.__objclass__ is object or + class_dict.__objclass__ is entry)): return class_dict return _sentinel diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 8da6647c3f71fc..9dfeeccb81b34d 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -6013,5 +6013,69 @@ class A(metaclass=M): pass +class TestGenericDescriptors(unittest.TestCase): + def test___dict__(self): + class CustomClass: + pass + class SlotClass: + __slots__ = ['foo'] + class SlotSubClass(SlotClass): + pass + class IntSubclass(int): + pass + + dict_descriptor = CustomClass.__dict__['__dict__'] + self.assertEqual(dict_descriptor.__objclass__, object) + + for cls in CustomClass, SlotSubClass, IntSubclass: + with self.subTest(cls=cls): + self.assertIs(cls.__dict__['__dict__'], dict_descriptor) + instance = cls() + instance.attr = 123 + self.assertEqual( + dict_descriptor.__get__(instance, cls), + {'attr': 123}, + ) + with self.assertRaises(AttributeError): + print(dict_descriptor.__get__(True, bool)) + with self.assertRaises(AttributeError): + print(dict_descriptor.__get__(SlotClass(), SlotClass)) + + # delegation to type.__dict__ + self.assertIsInstance( + dict_descriptor.__get__(type, type), + types.MappingProxyType, + ) + + def test___weakref__(self): + class CustomClass: + pass + class SlotClass: + __slots__ = ['foo'] + class SlotSubClass(SlotClass): + pass + class IntSubclass(int): + pass + + weakref_descriptor = CustomClass.__dict__['__weakref__'] + self.assertEqual(weakref_descriptor.__objclass__, object) + + for cls in CustomClass, SlotSubClass: + with self.subTest(cls=cls): + self.assertIs(cls.__dict__['__weakref__'], weakref_descriptor) + instance = cls() + instance.attr = 123 + self.assertEqual( + weakref_descriptor.__get__(instance, cls), + None, + ) + with self.assertRaises(AttributeError): + weakref_descriptor.__get__(True, bool) + with self.assertRaises(AttributeError): + weakref_descriptor.__get__(SlotClass(), SlotClass) + with self.assertRaises(AttributeError): + weakref_descriptor.__get__(IntSubclass(), IntSubclass) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-10-22-15.gh-issue-136966.J5lrE0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-10-22-15.gh-issue-136966.J5lrE0.rst new file mode 100644 index 00000000000000..aafd9ca4db4cd3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-10-22-15.gh-issue-136966.J5lrE0.rst @@ -0,0 +1,4 @@ +The :attr:`object.__dict__` and :attr:`!__weakref__` descriptors now use a +single descriptor instance per interpreter, shared across all types that +need them. +This speeds up class creation, and helps avoid reference cycles. diff --git a/Objects/descrobject.c b/Objects/descrobject.c index d3d17e92b6d1e8..06a81a4fdbd865 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -39,41 +39,41 @@ descr_name(PyDescrObject *descr) } static PyObject * -descr_repr(PyDescrObject *descr, const char *format) +descr_repr(PyDescrObject *descr, const char *kind) { PyObject *name = NULL; if (descr->d_name != NULL && PyUnicode_Check(descr->d_name)) name = descr->d_name; - return PyUnicode_FromFormat(format, name, "?", descr->d_type->tp_name); + if (descr->d_type == &PyBaseObject_Type) { + return PyUnicode_FromFormat("<%s '%V'>", kind, name, "?"); + } + return PyUnicode_FromFormat("<%s '%V' of '%s' objects>", + kind, name, "?", descr->d_type->tp_name); } static PyObject * method_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<method '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "method"); } static PyObject * member_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<member '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "member"); } static PyObject * getset_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<attribute '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "attribute"); } static PyObject * wrapperdescr_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<slot wrapper '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "slot wrapper"); } static int diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fb33bc747d885b..9cead729b6fe7a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4039,26 +4039,15 @@ subtype_getweakref(PyObject *obj, void *context) return Py_NewRef(result); } -/* Three variants on the subtype_getsets list. */ - -static PyGetSetDef subtype_getsets_full[] = { - {"__dict__", subtype_dict, subtype_setdict, - PyDoc_STR("dictionary for instance variables")}, - {"__weakref__", subtype_getweakref, NULL, - PyDoc_STR("list of weak references to the object")}, - {0} -}; - -static PyGetSetDef subtype_getsets_dict_only[] = { - {"__dict__", subtype_dict, subtype_setdict, - PyDoc_STR("dictionary for instance variables")}, - {0} +/* getset definitions for common descriptors */ +static PyGetSetDef subtype_getset_dict = { + "__dict__", subtype_dict, subtype_setdict, + PyDoc_STR("dictionary for instance variables"), }; -static PyGetSetDef subtype_getsets_weakref_only[] = { - {"__weakref__", subtype_getweakref, NULL, - PyDoc_STR("list of weak references to the object")}, - {0} +static PyGetSetDef subtype_getset_weakref = { + "__weakref__", subtype_getweakref, NULL, + PyDoc_STR("list of weak references to the object"), }; static int @@ -4594,10 +4583,36 @@ type_new_classmethod(PyObject *dict, PyObject *attr) return 0; } +/* Add __dict__ or __weakref__ descriptor */ +static int +type_add_common_descriptor(PyInterpreterState *interp, + PyObject **cache, + PyGetSetDef *getset_def, + PyObject *dict) +{ +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->cached_objects.descriptor_mutex); +#endif + PyObject *descr = *cache; + if (!descr) { + descr = PyDescr_NewGetSet(&PyBaseObject_Type, getset_def); + *cache = descr; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->cached_objects.descriptor_mutex); +#endif + if (!descr) { + return -1; + } + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { + return -1; + } + return 0; +} /* Add descriptors for custom slots from __slots__, or for __dict__ */ static int -type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) +type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type, PyObject *dict) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; Py_ssize_t slotoffset = ctx->base->tp_basicsize; @@ -4635,6 +4650,30 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) type->tp_basicsize = slotoffset; type->tp_itemsize = ctx->base->tp_itemsize; type->tp_members = _PyHeapType_GET_MEMBERS(et); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (type->tp_dictoffset) { + if (type_add_common_descriptor( + interp, + &interp->cached_objects.dict_descriptor, + &subtype_getset_dict, + dict) < 0) + { + return -1; + } + } + if (type->tp_weaklistoffset) { + if (type_add_common_descriptor( + interp, + &interp->cached_objects.weakref_descriptor, + &subtype_getset_weakref, + dict) < 0) + { + return -1; + } + } + return 0; } @@ -4642,18 +4681,7 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) static void type_new_set_slots(const type_new_ctx *ctx, PyTypeObject *type) { - if (type->tp_weaklistoffset && type->tp_dictoffset) { - type->tp_getset = subtype_getsets_full; - } - else if (type->tp_weaklistoffset && !type->tp_dictoffset) { - type->tp_getset = subtype_getsets_weakref_only; - } - else if (!type->tp_weaklistoffset && type->tp_dictoffset) { - type->tp_getset = subtype_getsets_dict_only; - } - else { - type->tp_getset = NULL; - } + type->tp_getset = NULL; /* Special case some slots */ if (type->tp_dictoffset != 0 || ctx->nslot > 0) { @@ -4758,7 +4786,7 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) return -1; } - if (type_new_descriptors(ctx, type) < 0) { + if (type_new_descriptors(ctx, type, dict) < 0) { return -1; } @@ -6642,6 +6670,14 @@ _PyStaticType_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) } +void +_PyTypes_FiniCachedDescriptors(PyInterpreterState *interp) +{ + Py_CLEAR(interp->cached_objects.dict_descriptor); + Py_CLEAR(interp->cached_objects.weakref_descriptor); +} + + static void type_dealloc(PyObject *self) { diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 09ce77fd12608f..a47e4d11b54441 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1793,37 +1793,6 @@ sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__baserepl_impl(module); } -PyDoc_STRVAR(sys__clear_type_descriptors__doc__, -"_clear_type_descriptors($module, type, /)\n" -"--\n" -"\n" -"Private function for clearing certain descriptors from a type\'s dictionary.\n" -"\n" -"See gh-135228 for context."); - -#define SYS__CLEAR_TYPE_DESCRIPTORS_METHODDEF \ - {"_clear_type_descriptors", (PyCFunction)sys__clear_type_descriptors, METH_O, sys__clear_type_descriptors__doc__}, - -static PyObject * -sys__clear_type_descriptors_impl(PyObject *module, PyObject *type); - -static PyObject * -sys__clear_type_descriptors(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - PyObject *type; - - if (!PyObject_TypeCheck(arg, &PyType_Type)) { - _PyArg_BadArgument("_clear_type_descriptors", "argument", (&PyType_Type)->tp_name, arg); - goto exit; - } - type = arg; - return_value = sys__clear_type_descriptors_impl(module, type); - -exit: - return return_value; -} - PyDoc_STRVAR(sys__is_gil_enabled__doc__, "_is_gil_enabled($module, /)\n" "--\n" @@ -1979,4 +1948,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=9052f399f40ca32d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=449d16326e69dcf6 input=a9049054013a1b77]*/ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e22a9cc1c75050..b6b1d2845ec2f1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1906,6 +1906,7 @@ finalize_interp_clear(PyThreadState *tstate) _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); + _PyTypes_FiniCachedDescriptors(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 19912b4a4c6198..bedbdfc489872e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2644,46 +2644,6 @@ sys__baserepl_impl(PyObject *module) Py_RETURN_NONE; } -/*[clinic input] -sys._clear_type_descriptors - - type: object(subclass_of='&PyType_Type') - / - -Private function for clearing certain descriptors from a type's dictionary. - -See gh-135228 for context. -[clinic start generated code]*/ - -static PyObject * -sys__clear_type_descriptors_impl(PyObject *module, PyObject *type) -/*[clinic end generated code: output=5ad17851b762b6d9 input=dc536c97fde07251]*/ -{ - PyTypeObject *typeobj = (PyTypeObject *)type; - if (_PyType_HasFeature(typeobj, Py_TPFLAGS_IMMUTABLETYPE)) { - PyErr_SetString(PyExc_TypeError, "argument is immutable"); - return NULL; - } - PyObject *dict = _PyType_GetDict(typeobj); - PyObject *dunder_dict = NULL; - if (PyDict_Pop(dict, &_Py_ID(__dict__), &dunder_dict) < 0) { - return NULL; - } - PyObject *dunder_weakref = NULL; - if (PyDict_Pop(dict, &_Py_ID(__weakref__), &dunder_weakref) < 0) { - PyType_Modified(typeobj); - Py_XDECREF(dunder_dict); - return NULL; - } - PyType_Modified(typeobj); - // We try to hold onto a reference to these until after we call - // PyType_Modified(), in case their deallocation triggers somer user code - // that tries to do something to the type. - Py_XDECREF(dunder_dict); - Py_XDECREF(dunder_weakref); - Py_RETURN_NONE; -} - /*[clinic input] sys._is_gil_enabled -> bool @@ -2881,7 +2841,6 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF - SYS__CLEAR_TYPE_DESCRIPTORS_METHODDEF SYS__IS_GIL_ENABLED_METHODDEF SYS__DUMP_TRACELETS_METHODDEF {NULL, NULL} // sentinel diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index 6204353e9bd26a..6f0f464892845f 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -67,6 +67,7 @@ 'PyMethodDef', 'PyMethodDef[]', 'PyMemberDef[]', + 'PyGetSetDef', 'PyGetSetDef[]', 'PyNumberMethods', 'PySequenceMethods', _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
