https://github.com/python/cpython/commit/b5ee0258bf5bb60a5a5a65c64717853e06b64808
commit: b5ee0258bf5bb60a5a5a65c64717853e06b64808
branch: main
author: mpage <[email protected]>
committer: mpage <[email protected]>
date: 2025-01-14T11:56:11-08:00
summary:
gh-115999: Specialize `LOAD_ATTR` for instance and class receivers in
free-threaded builds (#128164)
Finish specialization for LOAD_ATTR in the free-threaded build by adding
support for class and instance receivers.
files:
M Include/cpython/pystats.h
M Include/internal/pycore_dict.h
M Include/internal/pycore_opcode_metadata.h
M Include/internal/pycore_uop_metadata.h
M Lib/test/test_capi/test_type.py
M Lib/test/test_descr.py
M Lib/test/test_generated_cases.py
M Lib/test/test_opcache.py
M Objects/dictobject.c
M Python/bytecodes.c
M Python/executor_cases.c.h
M Python/generated_cases.c.h
M Python/optimizer_bytecodes.c
M Python/optimizer_cases.c.h
M Python/specialize.c
M Tools/cases_generator/analyzer.py
M Tools/cases_generator/generators_common.py
M Tools/cases_generator/stack.py
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index 29ef0c0e4d4e72..ee8885cda7b60d 100644
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -31,7 +31,7 @@
#define PYSTATS_MAX_UOP_ID 512
-#define SPECIALIZATION_FAILURE_KINDS 36
+#define SPECIALIZATION_FAILURE_KINDS 37
/* Stats for determining who is calling PyEval_EvalFrame */
#define EVAL_CALL_TOTAL 0
diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h
index 71927006d1cd48..74ac8f2148174c 100644
--- a/Include/internal/pycore_dict.h
+++ b/Include/internal/pycore_dict.h
@@ -114,6 +114,16 @@ extern Py_ssize_t
_Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject
extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *);
extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys,
PyObject *key);
+
+/* Look up a string key in an all unicode dict keys, assign the keys object a
version, and
+ * store it in version.
+ *
+ * Returns DKIX_ERROR if key is not a string or if the keys object is not all
+ * strings.
+ *
+ * Returns DKIX_EMPTY if the key is not present.
+ */
+extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject*
dictkeys, PyObject *key, uint32_t *version);
extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys,
PyObject *key);
PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *,
PyObject *);
PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *,
PyObject *, _PyStackRef *);
diff --git a/Include/internal/pycore_opcode_metadata.h
b/Include/internal/pycore_opcode_metadata.h
index 5365e2a5c6b8cd..0c0a6145bdbb27 100644
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -1540,7 +1540,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int
*effect) {
return 0;
}
case LOAD_ATTR_WITH_HINT: {
- *effect = Py_MAX(0, (oparg & 1));
+ *effect = Py_MAX(1, (oparg & 1));
return 0;
}
case LOAD_BUILD_CLASS: {
diff --git a/Include/internal/pycore_uop_metadata.h
b/Include/internal/pycore_uop_metadata.h
index 721fa94da19615..5670fe26f72071 100644
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -862,7 +862,7 @@ int _PyUop_num_popped(int opcode, int oparg)
case _CHECK_ATTR_WITH_HINT:
return 0;
case _LOAD_ATTR_WITH_HINT:
- return 1;
+ return 2;
case _LOAD_ATTR_SLOT_0:
return 1;
case _LOAD_ATTR_SLOT_1:
diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py
index 54c83e09f892a0..92d056e802eeed 100644
--- a/Lib/test/test_capi/test_type.py
+++ b/Lib/test/test_capi/test_type.py
@@ -1,4 +1,4 @@
-from test.support import import_helper
+from test.support import import_helper, Py_GIL_DISABLED, refleak_helper
import unittest
_testcapi = import_helper.import_module('_testcapi')
@@ -37,6 +37,9 @@ class D(A, C): pass
# as well
type_freeze(D)
+ @unittest.skipIf(
+ Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(),
+ "Specialization failure triggers gh-127773")
def test_freeze_meta(self):
"""test PyType_Freeze() with overridden MRO"""
type_freeze = _testcapi.type_freeze
diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index 51f97bb51f7bd2..a7ebc9e8be0294 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -7,6 +7,7 @@
import random
import string
import sys
+import textwrap
import types
import unittest
import warnings
@@ -15,6 +16,7 @@
from copy import deepcopy
from contextlib import redirect_stdout
from test import support
+from test.support.script_helper import assert_python_ok
try:
import _testcapi
@@ -5222,6 +5224,7 @@ def test_type_lookup_mro_reference(self):
# Issue #14199: _PyType_Lookup() has to keep a strong reference to
# the type MRO because it may be modified during the lookup, if
# __bases__ is set during the lookup for example.
+ code = textwrap.dedent("""
class MyKey(object):
def __hash__(self):
return hash('mykey')
@@ -5237,12 +5240,29 @@ class Base2(object):
mykey = 'from Base2'
mykey2 = 'from Base2'
- with self.assertWarnsRegex(RuntimeWarning, 'X'):
- X = type('X', (Base,), {MyKey(): 5})
- # mykey is read from Base
- self.assertEqual(X.mykey, 'from Base')
- # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__
- self.assertEqual(X.mykey2, 'from Base2')
+ X = type('X', (Base,), {MyKey(): 5})
+
+ bases_before = ",".join([c.__name__ for c in X.__bases__])
+ print(f"before={bases_before}")
+
+ # mykey is initially read from Base, however, the lookup will be
perfomed
+ # again if specialization fails. The second lookup will use the new
+ # mro set by __eq__.
+ print(X.mykey)
+
+ bases_after = ",".join([c.__name__ for c in X.__bases__])
+ print(f"after={bases_after}")
+
+ # mykey2 is read from Base2 because MyKey.__eq__ has set __bases_
+ print(f"mykey2={X.mykey2}")
+ """)
+ _, out, err = assert_python_ok("-c", code)
+ err = err.decode()
+ self.assertRegex(err, "RuntimeWarning: .*X")
+ out = out.decode()
+ self.assertRegex(out, "before=Base")
+ self.assertRegex(out, "after=Base2")
+ self.assertRegex(out, "mykey2=from Base2")
class PicklingTests(unittest.TestCase):
diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py
index 4a1c99edacb71a..7a50a29bb0126c 100644
--- a/Lib/test/test_generated_cases.py
+++ b/Lib/test/test_generated_cases.py
@@ -1639,12 +1639,16 @@ def test_escaping_call_next_to_cmacro(self):
"""
self.run_cases_test(input, output)
- def test_pop_dead_inputs_all_live(self):
+ def test_pystackref_frompyobject_new_next_to_cmacro(self):
input = """
- inst(OP, (a, b --)) {
- POP_DEAD_INPUTS();
- HAM(a, b);
- INPUTS_DEAD();
+ inst(OP, (-- out1, out2)) {
+ PyObject *obj = SPAM();
+ #ifdef Py_GIL_DISABLED
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #else
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #endif
+ out2 = PyStackRef_FromPyObjectNew(obj);
}
"""
output = """
@@ -1652,22 +1656,28 @@ def test_pop_dead_inputs_all_live(self):
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- _PyStackRef a;
- _PyStackRef b;
- b = stack_pointer[-1];
- a = stack_pointer[-2];
- HAM(a, b);
- stack_pointer += -2;
+ _PyStackRef out1;
+ _PyStackRef out2;
+ PyObject *obj = SPAM();
+ #ifdef Py_GIL_DISABLED
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #else
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #endif
+ out2 = PyStackRef_FromPyObjectNew(obj);
+ stack_pointer[0] = out1;
+ stack_pointer[1] = out2;
+ stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
"""
self.run_cases_test(input, output)
- def test_pop_dead_inputs_some_live(self):
+ def test_pop_input(self):
input = """
- inst(OP, (a, b, c --)) {
- POP_DEAD_INPUTS();
+ inst(OP, (a, b --)) {
+ POP_INPUT(b);
HAM(a);
INPUTS_DEAD();
}
@@ -1678,8 +1688,10 @@ def test_pop_dead_inputs_some_live(self):
next_instr += 1;
INSTRUCTION_STATS(OP);
_PyStackRef a;
- a = stack_pointer[-3];
- stack_pointer += -2;
+ _PyStackRef b;
+ b = stack_pointer[-1];
+ a = stack_pointer[-2];
+ stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
HAM(a);
stack_pointer += -1;
@@ -1689,29 +1701,23 @@ def test_pop_dead_inputs_some_live(self):
"""
self.run_cases_test(input, output)
- def test_pop_dead_inputs_with_output(self):
+ def test_pop_input_with_empty_stack(self):
input = """
- inst(OP, (a, b -- c)) {
- POP_DEAD_INPUTS();
- c = SPAM();
+ inst(OP, (--)) {
+ POP_INPUT(foo);
}
"""
- output = """
- TARGET(OP) {
- frame->instr_ptr = next_instr;
- next_instr += 1;
- INSTRUCTION_STATS(OP);
- _PyStackRef c;
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- c = SPAM();
- stack_pointer[0] = c;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- DISPATCH();
+ with self.assertRaises(SyntaxError):
+ self.run_cases_test(input, "")
+
+ def test_pop_input_with_non_tos(self):
+ input = """
+ inst(OP, (a, b --)) {
+ POP_INPUT(a);
}
"""
- self.run_cases_test(input, output)
+ with self.assertRaises(SyntaxError):
+ self.run_cases_test(input, "")
def test_no_escaping_calls_in_branching_macros(self):
diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py
index c7cd4c2e8a3146..b80ccbf17f1ee6 100644
--- a/Lib/test/test_opcache.py
+++ b/Lib/test/test_opcache.py
@@ -564,6 +564,16 @@ def instantiate():
instantiate()
+def make_deferred_ref_count_obj():
+ """Create an object that uses deferred reference counting.
+
+ Only objects that use deferred refence counting may be stored in inline
+ caches in free-threaded builds. This constructs a new class named Foo,
+ which uses deferred reference counting.
+ """
+ return type("Foo", (object,), {})
+
+
@threading_helper.requires_working_threading()
class TestRacesDoNotCrash(TestBase):
# Careful with these. Bigger numbers have a higher chance of catching bugs,
@@ -714,11 +724,11 @@ def write(items):
opname = "FOR_ITER_LIST"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_class(self):
def get_items():
class C:
- a = object()
+ a = make_deferred_ref_count_obj()
items = []
for _ in range(self.ITEMS):
@@ -739,12 +749,45 @@ def write(items):
del item.a
except AttributeError:
pass
- item.a = object()
+ item.a = make_deferred_ref_count_obj()
opname = "LOAD_ATTR_CLASS"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
+ def test_load_attr_class_with_metaclass_check(self):
+ def get_items():
+ class Meta(type):
+ pass
+
+ class C(metaclass=Meta):
+ a = make_deferred_ref_count_obj()
+
+ items = []
+ for _ in range(self.ITEMS):
+ item = C
+ items.append(item)
+ return items
+
+ def read(items):
+ for item in items:
+ try:
+ item.a
+ except AttributeError:
+ pass
+
+ def write(items):
+ for item in items:
+ try:
+ del item.a
+ except AttributeError:
+ pass
+ item.a = make_deferred_ref_count_obj()
+
+ opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK"
+ self.assert_races_do_not_crash(opname, get_items, read, write)
+
+ @requires_specialization_ft
def test_load_attr_getattribute_overridden(self):
def get_items():
class C:
@@ -774,7 +817,7 @@ def write(items):
opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_instance_value(self):
def get_items():
class C:
@@ -798,7 +841,7 @@ def write(items):
opname = "LOAD_ATTR_INSTANCE_VALUE"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_method_lazy_dict(self):
def get_items():
class C(Exception):
@@ -828,7 +871,7 @@ def write(items):
opname = "LOAD_ATTR_METHOD_LAZY_DICT"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_method_no_dict(self):
def get_items():
class C:
@@ -859,7 +902,7 @@ def write(items):
opname = "LOAD_ATTR_METHOD_NO_DICT"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_method_with_values(self):
def get_items():
class C:
@@ -914,7 +957,7 @@ def write(items):
opname = "LOAD_ATTR_MODULE"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_property(self):
def get_items():
class C:
@@ -944,7 +987,34 @@ def write(items):
opname = "LOAD_ATTR_PROPERTY"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
+ def test_load_attr_slot(self):
+ def get_items():
+ class C:
+ __slots__ = ["a", "b"]
+
+ items = []
+ for i in range(self.ITEMS):
+ item = C()
+ item.a = i
+ item.b = i + self.ITEMS
+ items.append(item)
+ return items
+
+ def read(items):
+ for item in items:
+ item.a
+ item.b
+
+ def write(items):
+ for item in items:
+ item.a = 100
+ item.b = 200
+
+ opname = "LOAD_ATTR_SLOT"
+ self.assert_races_do_not_crash(opname, get_items, read, write)
+
+ @requires_specialization_ft
def test_load_attr_with_hint(self):
def get_items():
class C:
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 2a054c3f2ae0ff..82789d5e56f523 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -1129,6 +1129,24 @@ dictkeys_generic_lookup(PyDictObject *mp,
PyDictKeysObject* dk, PyObject *key, P
return do_lookup(mp, dk, key, hash, compare_generic);
}
+static bool
+check_keys_unicode(PyDictKeysObject *dk, PyObject *key)
+{
+ return PyUnicode_CheckExact(key) && (dk->dk_kind != DICT_KEYS_GENERAL);
+}
+
+static Py_ssize_t
+hash_unicode_key(PyObject *key)
+{
+ assert(PyUnicode_CheckExact(key));
+ Py_hash_t hash = unicode_get_hash(key);
+ if (hash == -1) {
+ hash = PyUnicode_Type.tp_hash(key);
+ assert(hash != -1);
+ }
+ return hash;
+}
+
#ifdef Py_GIL_DISABLED
static Py_ssize_t
unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key,
@@ -1167,21 +1185,28 @@ unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject
*key, Py_hash_t hash)
Py_ssize_t
_PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key)
{
- DictKeysKind kind = dk->dk_kind;
- if (!PyUnicode_CheckExact(key) || kind == DICT_KEYS_GENERAL) {
+ if (!check_keys_unicode(dk, key)) {
return DKIX_ERROR;
}
- Py_hash_t hash = unicode_get_hash(key);
- if (hash == -1) {
- hash = PyUnicode_Type.tp_hash(key);
- if (hash == -1) {
- PyErr_Clear();
- return DKIX_ERROR;
- }
- }
+ Py_hash_t hash = hash_unicode_key(key);
return unicodekeys_lookup_unicode(dk, key, hash);
}
+Py_ssize_t
+_PyDictKeys_StringLookupAndVersion(PyDictKeysObject *dk, PyObject *key,
uint32_t *version)
+{
+ if (!check_keys_unicode(dk, key)) {
+ return DKIX_ERROR;
+ }
+ Py_ssize_t ix;
+ Py_hash_t hash = hash_unicode_key(key);
+ LOCK_KEYS(dk);
+ ix = unicodekeys_lookup_unicode(dk, key, hash);
+ *version =
_PyDictKeys_GetVersionForCurrentState(_PyInterpreterState_GET(), dk);
+ UNLOCK_KEYS(dk);
+ return ix;
+}
+
/* Like _PyDictKeys_StringLookup() but only works on split keys. Note
* that in free-threaded builds this locks the keys object as required.
*/
@@ -1926,6 +1951,16 @@ build_indices_unicode(PyDictKeysObject *keys,
PyDictUnicodeEntry *ep, Py_ssize_t
}
}
+static void
+invalidate_and_clear_inline_values(PyDictValues *values)
+{
+ assert(values->embedded);
+ FT_ATOMIC_STORE_UINT8(values->valid, 0);
+ for (int i = 0; i < values->capacity; i++) {
+ FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL);
+ }
+}
+
/*
Restructure the table by allocating a new table and reinserting all
items again. When entries have been deleted, the new table may
@@ -2017,7 +2052,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
if (oldvalues->embedded) {
assert(oldvalues->embedded == 1);
assert(oldvalues->valid == 1);
- FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0);
+ invalidate_and_clear_inline_values(oldvalues);
}
else {
free_values(oldvalues, IS_DICT_SHARED(mp));
@@ -7007,7 +7042,13 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj,
PyObject *name, PyObject **attr
#ifdef Py_GIL_DISABLED
PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]);
- if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) {
+ if (value == NULL) {
+ if (FT_ATOMIC_LOAD_UINT8(values->valid)) {
+ *attr = NULL;
+ return true;
+ }
+ }
+ else if (_Py_TryIncrefCompare(&values->values[ix], value)) {
*attr = value;
return true;
}
@@ -7345,7 +7386,7 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
}
mp->ma_values = values;
- FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0);
+ invalidate_and_clear_inline_values(_PyObject_InlineValues(obj));
assert(_PyObject_InlineValuesConsistencyCheck(obj));
ASSERT_CONSISTENT(mp);
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index cec530fefffefb..a906ded365650c 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2190,18 +2190,23 @@ dummy_func(
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
+
DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid));
}
split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if
(oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
DEOPT_IF(attr_o == NULL);
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ DEOPT_IF(true);
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
DECREF_INPUTS();
}
@@ -2227,9 +2232,8 @@ dummy_func(
assert(index <
FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
- DEAD(mod_keys);
// Clear mod_keys from stack in case we need to deopt
- POP_DEAD_INPUTS();
+ POP_INPUT(mod_keys);
DEOPT_IF(attr_o == NULL);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o,
&attr);
@@ -2251,30 +2255,50 @@ dummy_func(
_LOAD_ATTR_MODULE_FROM_KEYS +
unused/5;
- op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
+ op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- EXIT_IF(dict == NULL);
- assert(PyDict_CheckExact((PyObject *)dict));
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ EXIT_IF(dict_o == NULL);
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
}
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1)))
{
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr,
null if (oparg & 1))) {
PyObject *attr_o;
+ if (!LOCK_OBJECT(dict)) {
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys));
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
- DEOPT_IF(ep->me_key != name);
+ if (ep->me_key != name) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL);
+ if (attr_o == NULL) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
+ DEAD(dict);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
@@ -2289,12 +2313,17 @@ dummy_func(
split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg &
1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
DEOPT_IF(attr_o == NULL);
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ DEOPT_IF(!increfed);
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
DECREF_INPUTS();
}
@@ -2309,7 +2338,7 @@ dummy_func(
EXIT_IF(!PyType_Check(owner_o));
assert(type_version != 0);
- EXIT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version);
+ EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject
*)owner_o)->tp_version_tag) != type_version);
}
split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg &
1))) {
@@ -2363,7 +2392,7 @@ dummy_func(
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(cls->tp_version_tag != type_version);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) !=
type_version);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
@@ -3281,13 +3310,15 @@ dummy_func(
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid));
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
PyTypeObject *owner_cls =
Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) !=
keys_version);
}
split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self
if (1))) {
@@ -3357,7 +3388,7 @@ dummy_func(
op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) +
MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL);
}
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 1aa80f398d7470..cda01bb768c269 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -2652,7 +2652,7 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner_o)->valid) {
+ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid))
{
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -2668,15 +2668,23 @@
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
@@ -2691,15 +2699,23 @@
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
@@ -2778,55 +2794,88 @@
case _CHECK_ATTR_WITH_HINT: {
_PyStackRef owner;
+ PyDictObject *dict;
owner = stack_pointer[-1];
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- if (dict == NULL) {
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ if (dict_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- assert(PyDict_CheckExact((PyObject *)dict));
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
+ stack_pointer[0].bits = (uintptr_t)dict;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_WITH_HINT: {
+ PyDictObject *dict;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
- owner = stack_pointer[-1];
+ dict = (PyDictObject *)stack_pointer[-1].bits;
+ owner = stack_pointer[-2];
uint16_t hint = (uint16_t)CURRENT_OPERAND0();
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
+ if (!LOCK_OBJECT(dict)) {
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- if (!DK_IS_UNICODE(dict->ma_keys)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
attr_o = ep->me_value;
if (attr_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
- stack_pointer[-1] = attr;
- if (oparg & 1) stack_pointer[0] = null;
- stack_pointer += (oparg & 1);
+ stack_pointer[-2] = attr;
+ if (oparg & 1) stack_pointer[-1] = null;
+ stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@@ -2839,15 +2888,23 @@
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ if (!increfed) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
@@ -2861,15 +2918,23 @@
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ if (!increfed) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
@@ -2890,7 +2955,7 @@
JUMP_TO_JUMP_TARGET();
}
assert(type_version != 0);
- if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) {
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject
*)owner_o)->tp_version_tag) != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -3924,7 +3989,8 @@
owner = stack_pointer[-1];
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner_o)->valid) {
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -3937,7 +4003,8 @@
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
PyTypeObject *owner_cls =
Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- if (owner_heap_type->ht_cached_keys->dk_version != keys_version) {
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) !=
keys_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
@@ -4022,7 +4089,7 @@
owner = stack_pointer[-1];
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) +
MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
if (dict != NULL) {
UOP_STAT_INC(uopcode, miss);
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 810beb61d0db5e..81408380d6b2b8 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -5345,7 +5345,7 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag !=
type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject
*)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
@@ -5380,7 +5380,7 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag !=
type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject
*)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_TYPE_VERSION
{
@@ -5421,7 +5421,7 @@
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) !=
type_version, LOAD_ATTR);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
@@ -5463,19 +5463,24 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+
DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid),
LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t offset = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) +
offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ DEOPT_IF(true, LOAD_ATTR);
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
@@ -5507,7 +5512,7 @@
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) +
MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
@@ -5586,14 +5591,16 @@
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls =
Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject
*)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) !=
keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
@@ -5716,14 +5723,16 @@
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls =
Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject
*)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) !=
keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
@@ -5824,12 +5833,17 @@
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o,
&attr);
+ DEOPT_IF(!increfed, LOAD_ATTR);
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
@@ -5846,6 +5860,7 @@
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect
cache size");
_PyStackRef owner;
+ PyDictObject *dict;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
@@ -5861,26 +5876,40 @@
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict == NULL, LOAD_ATTR);
- assert(PyDict_CheckExact((PyObject *)dict));
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ DEOPT_IF(dict_o == NULL, LOAD_ATTR);
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries,
LOAD_ATTR);
+ if (!LOCK_OBJECT(dict)) {
+ DEOPT_IF(true, LOAD_ATTR);
+ }
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR);
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) +
hint;
- DEOPT_IF(ep->me_key != name, LOAD_ATTR);
+ if (ep->me_key != name) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index 788adecca8af80..4d96ada5acf00f 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -582,11 +582,17 @@ dummy_func(void) {
}
}
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
+ op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) {
+ dict = sym_new_not_null(ctx);
+ (void)owner;
+ }
+
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg &
1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
+ (void)dict;
}
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 1a7cc6becfefb6..aff4493fdc4dd7 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -1250,22 +1250,33 @@
}
case _CHECK_ATTR_WITH_HINT: {
+ _Py_UopsSymbol *owner;
+ _Py_UopsSymbol *dict;
+ owner = stack_pointer[-1];
+ dict = sym_new_not_null(ctx);
+ (void)owner;
+ stack_pointer[0] = dict;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_WITH_HINT: {
+ _Py_UopsSymbol *dict;
_Py_UopsSymbol *owner;
_Py_UopsSymbol *attr;
_Py_UopsSymbol *null = NULL;
- owner = stack_pointer[-1];
+ dict = stack_pointer[-1];
+ owner = stack_pointer[-2];
uint16_t hint = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
- stack_pointer[-1] = attr;
- if (oparg & 1) stack_pointer[0] = null;
- stack_pointer += (oparg & 1);
+ (void)dict;
+ stack_pointer[-2] = attr;
+ if (oparg & 1) stack_pointer[-1] = null;
+ stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
diff --git a/Python/specialize.c b/Python/specialize.c
index 897005c4f1078d..8d9f19c8895187 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -3,6 +3,7 @@
#include "opcode.h"
#include "pycore_code.h"
+#include "pycore_critical_section.h"
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // DICT_KEYS_UNICODE
#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
@@ -537,6 +538,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t
size, PyObject *consts,
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33
#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34
#define SPEC_FAIL_ATTR_SPLIT_DICT 35
+#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36
/* Binary subscr and store subscr */
@@ -729,11 +731,8 @@ unspecialize(_Py_CODEUNIT *instr)
}
static int function_kind(PyCodeObject *code);
-#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int
opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
-static uint32_t type_get_version(PyTypeObject *t, int opcode);
-#endif
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr,
PyObject *name)
@@ -879,10 +878,11 @@ descriptor_is_class(PyObject *descriptor, PyObject *name)
(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
-#ifndef Py_GIL_DISABLED
static DescriptorClassification
-analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
+analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr,
unsigned int *tp_version) {
bool has_getattr = false;
+ bool have_ga_version = false;
+ unsigned int ga_version;
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
@@ -892,24 +892,27 @@ analyze_descriptor_load(PyTypeObject *type, PyObject
*name, PyObject **descr) {
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
- PyObject *getattribute = _PyType_LookupRef(type,
&_Py_ID(__getattribute__));
+ PyObject *getattribute = _PyType_LookupRefAndVersion(type,
+ &_Py_ID(__getattribute__), &ga_version);
+ have_ga_version = true;
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
- PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__));
+ PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__));
has_getattr = getattr != NULL;
- Py_XDECREF(getattr);
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
+ *tp_version = ga_version;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF(getattribute);
*descr = NULL;
+ *tp_version = ga_version;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
@@ -923,16 +926,18 @@ analyze_descriptor_load(PyTypeObject *type, PyObject
*name, PyObject **descr) {
}
else {
*descr = NULL;
+ *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag);
return GETSET_OVERRIDDEN;
}
- PyObject *descriptor = _PyType_LookupRef(type, name);
+ unsigned int descr_version;
+ PyObject *descriptor = _PyType_LookupRefAndVersion(type, name,
&descr_version);
*descr = descriptor;
+ *tp_version = have_ga_version ? ga_version : descr_version;
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, has_getattr);
}
-#endif //!Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr,
unsigned int *tp_version)
@@ -952,12 +957,13 @@ analyze_descriptor_store(PyTypeObject *type, PyObject
*name, PyObject **descr, u
static int
specialize_dict_access_inline(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
- DescriptorClassification kind, PyObject *name, unsigned int tp_version,
+ PyObject *name, unsigned int tp_version,
int base_op, int values_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner);
Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
@@ -965,6 +971,7 @@ specialize_dict_access_inline(
return 0;
}
assert(index >= 0);
+ assert(_PyObject_InlineValues(owner)->valid);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
@@ -980,10 +987,13 @@ specialize_dict_access_inline(
static int
specialize_dict_access_hint(
PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
- DescriptorClassification kind, PyObject *name, unsigned int tp_version,
+ PyObject *name, unsigned int tp_version,
int base_op, int hint_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
+
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict);
+
// We found an instance with a __dict__.
if (_PyDict_HasSplitTable(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
@@ -1027,7 +1037,7 @@ specialize_dict_access(
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
// managed dict, not materialized, inline values valid
- res = specialize_dict_access_inline(owner, instr, type, kind, name,
+ res = specialize_dict_access_inline(owner, instr, type, name,
tp_version, base_op,
values_op);
}
else {
@@ -1047,16 +1057,19 @@ specialize_dict_access(
int res;
Py_BEGIN_CRITICAL_SECTION(dict);
// materialized managed dict
- res = specialize_dict_access_hint(dict, instr, type, kind, name,
+ res = specialize_dict_access_hint(dict, instr, type, name,
tp_version, base_op, hint_op);
Py_END_CRITICAL_SECTION();
return res;
}
}
-#ifndef Py_GIL_DISABLED
-static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr,
PyObject* name,
- PyObject* descr, DescriptorClassification kind, bool is_method);
+static int
+specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
+ PyObject *name, PyObject *descr,
+ unsigned int tp_version,
+ DescriptorClassification kind, bool is_method,
+ uint32_t shared_keys_version);
static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr,
PyObject* name);
/* Returns true if instances of obj's class are
@@ -1065,7 +1078,7 @@ static int specialize_class_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyOb
* For other objects, we check their actual dictionary.
*/
static bool
-instance_has_key(PyObject *obj, PyObject* name)
+instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version)
{
PyTypeObject *cls = Py_TYPE(obj);
if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
@@ -1073,36 +1086,38 @@ instance_has_key(PyObject *obj, PyObject* name)
}
if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys;
- Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
+ Py_ssize_t index =
+ _PyDictKeys_StringLookupAndVersion(keys, name,
shared_keys_version);
return index >= 0;
}
PyDictObject *dict = _PyObject_GetManagedDict(obj);
if (dict == NULL || !PyDict_CheckExact(dict)) {
return false;
}
+ bool result;
+ Py_BEGIN_CRITICAL_SECTION(dict);
if (dict->ma_values) {
- return false;
+ result = false;
}
- Py_ssize_t index = _PyDict_LookupIndex(dict, name);
- if (index < 0) {
- return false;
+ else {
+ result = (_PyDict_LookupIndex(dict, name) >= 0);
}
- return true;
+ Py_END_CRITICAL_SECTION();
+ return result;
}
static int
-specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
name)
+do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr,
PyObject* name,
+ bool shadow, uint32_t shared_keys_version,
+ DescriptorClassification kind, PyObject
*descr, unsigned int tp_version)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
- bool shadow = instance_has_key(owner, name);
- PyObject *descr = NULL;
- DescriptorClassification kind = analyze_descriptor_load(type, name,
&descr);
- Py_XDECREF(descr); // turn strong ref into a borrowed ref
- assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
- if (type_get_version(type, LOAD_ATTR) == 0) {
+ if (tp_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
+ uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(LOAD_ATTR,
SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
@@ -1112,9 +1127,10 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
if (shadow) {
goto try_instance;
}
- int oparg = instr->op.arg;
if (oparg & 1) {
- if (specialize_attr_loadclassattr(owner, instr, name, descr,
kind, true)) {
+ if (specialize_attr_loadclassattr(owner, instr, name, descr,
+ tp_version, kind, true,
+ shared_keys_version)) {
return 0;
}
else {
@@ -1140,7 +1156,7 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
if (!function_check_args(fget, 1, LOAD_ATTR)) {
return -1;
}
- if (instr->op.arg & 1) {
+ if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
@@ -1149,8 +1165,14 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
- assert(type->tp_version_tag != 0);
- write_u32(lm_cache->type_version, type->tp_version_tag);
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(fget)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR,
SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return -1;
+ }
+ #endif
+ assert(tp_version != 0);
+ write_u32(lm_cache->type_version, tp_version);
/* borrowed */
write_obj(lm_cache->descr, fget);
specialize(instr, LOAD_ATTR_PROPERTY);
@@ -1176,7 +1198,7 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
assert(offset > 0);
cache->index = (uint16_t)offset;
- write_u32(cache->version, type->tp_version_tag);
+ write_u32(cache->version, tp_version);
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
@@ -1185,7 +1207,7 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
Py_ssize_t offset = offsetof(PyObject, ob_type);
assert(offset == (uint16_t)offset);
cache->index = (uint16_t)offset;
- write_u32(cache->version, type->tp_version_tag);
+ write_u32(cache->version, tp_version);
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
@@ -1200,13 +1222,18 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
return -1;
case GETATTRIBUTE_IS_PYTHON_FUNCTION:
{
+ #ifndef Py_GIL_DISABLED
+ // In free-threaded builds it's possible for tp_getattro to change
+ // after the call to analyze_descriptor. That is fine: the version
+ // guard will fail.
assert(type->tp_getattro == _Py_slot_tp_getattro);
+ #endif
assert(Py_IS_TYPE(descr, &PyFunction_Type));
_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
if (!function_check_args(descr, 2, LOAD_ATTR)) {
return -1;
}
- if (instr->op.arg & 1) {
+ if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
@@ -1219,10 +1246,16 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR,
SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return -1;
+ }
+ #endif
write_u32(lm_cache->keys_version, version);
/* borrowed */
write_obj(lm_cache->descr, descr);
- write_u32(lm_cache->type_version, type->tp_version_tag);
+ write_u32(lm_cache->type_version, tp_version);
specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
return 0;
}
@@ -1237,8 +1270,10 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
if (shadow) {
goto try_instance;
}
- if ((instr->op.arg & 1) == 0) {
- if (specialize_attr_loadclassattr(owner, instr, name, descr,
kind, false)) {
+ if ((oparg & 1) == 0) {
+ if (specialize_attr_loadclassattr(owner, instr, name, descr,
+ tp_version, kind, false,
+ shared_keys_version)) {
return 0;
}
}
@@ -1252,14 +1287,28 @@ specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject* na
}
Py_UNREACHABLE();
try_instance:
- if (specialize_dict_access(owner, instr, type, kind, name,
type->tp_version_tag,
+ if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE,
LOAD_ATTR_WITH_HINT))
{
return 0;
}
return -1;
}
-#endif // Py_GIL_DISABLED
+
+static int
+specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
name)
+{
+ // 0 is not a valid version
+ uint32_t shared_keys_version = 0;
+ bool shadow = instance_has_key(owner, name, &shared_keys_version);
+ PyObject *descr = NULL;
+ unsigned int tp_version = 0;
+ PyTypeObject *type = Py_TYPE(owner);
+ DescriptorClassification kind = analyze_descriptor_load(type, name,
&descr, &tp_version);
+ int result = do_specialize_instance_load_attr(owner, instr, name, shadow,
shared_keys_version, kind, descr, tp_version);
+ Py_XDECREF(descr);
+ return result;
+}
void
_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject
*name)
@@ -1281,20 +1330,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st,
_Py_CODEUNIT *instr, PyObject *nam
fail = specialize_module_load_attr(owner, instr, name);
}
else if (PyType_Check(owner)) {
- #ifdef Py_GIL_DISABLED
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
- fail = true;
- #else
fail = specialize_class_load_attr(owner, instr, name);
- #endif
}
else {
- #ifdef Py_GIL_DISABLED
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
- fail = true;
- #else
fail = specialize_instance_load_attr(owner, instr, name);
- #endif
}
if (fail) {
@@ -1402,8 +1441,6 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st,
_Py_CODEUNIT *instr, PyObject *na
return;
}
-#ifndef Py_GIL_DISABLED
-
#ifdef Py_STATS
static int
load_attr_fail_kind(DescriptorClassification kind)
@@ -1452,8 +1489,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT
*instr,
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN);
return -1;
}
- PyObject *metadescriptor = _PyType_Lookup(Py_TYPE(cls), name);
+ unsigned int meta_version = 0;
+ PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name,
&meta_version);
DescriptorClassification metakind = classify_descriptor(metadescriptor,
false);
+ Py_XDECREF(metadescriptor);
switch (metakind) {
case METHOD:
case NON_DESCRIPTOR:
@@ -1468,38 +1507,52 @@ specialize_class_load_attr(PyObject *owner,
_Py_CODEUNIT *instr,
}
PyObject *descr = NULL;
DescriptorClassification kind = 0;
- kind = analyze_descriptor_load(cls, name, &descr);
- Py_XDECREF(descr); // turn strong ref into a borrowed ref
- if (type_get_version(cls, LOAD_ATTR) == 0) {
+ unsigned int tp_version = 0;
+ kind = analyze_descriptor_load(cls, name, &descr, &tp_version);
+ if (tp_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+ Py_XDECREF(descr);
return -1;
}
bool metaclass_check = false;
if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) {
metaclass_check = true;
- if (type_get_version(Py_TYPE(cls), LOAD_ATTR) == 0) {
+ if (meta_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+ Py_XDECREF(descr);
return -1;
}
}
switch (kind) {
case METHOD:
case NON_DESCRIPTOR:
- write_u32(cache->type_version, cls->tp_version_tag);
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR,
SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ Py_XDECREF(descr);
+ return -1;
+ }
+ #endif
+ write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
if (metaclass_check) {
- write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag);
+ write_u32(cache->keys_version, meta_version);
specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
}
else {
specialize(instr, LOAD_ATTR_CLASS);
}
+ Py_XDECREF(descr);
return 0;
#ifdef Py_STATS
case ABSENT:
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
+ Py_XDECREF(descr);
return -1;
#endif
default:
SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
+ Py_XDECREF(descr);
return -1;
}
}
@@ -1508,29 +1561,41 @@ specialize_class_load_attr(PyObject *owner,
_Py_CODEUNIT *instr,
// can cause a significant drop in cache hits. A possible test is
// python.exe -m test_typing test_re test_dis test_zlib.
static int
-specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject
*name,
-PyObject *descr, DescriptorClassification kind, bool is_method)
+specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
+ PyObject *name, PyObject *descr,
+ unsigned int tp_version,
+ DescriptorClassification kind, bool is_method,
+ uint32_t shared_keys_version)
{
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
assert(descr != NULL);
assert((is_method && kind == METHOD) || (!is_method && kind ==
NON_DESCRIPTOR));
- if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
- PyDictKeysObject *keys = ((PyHeapTypeObject
*)owner_cls)->ht_cached_keys;
- assert(_PyDictKeys_StringLookup(keys, name) < 0);
- uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
- _PyInterpreterState_GET(), keys);
- if (keys_version == 0) {
+
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return 0;
+ }
+ #endif
+
+ unsigned long tp_flags = PyType_GetFlags(owner_cls);
+ if (tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ #ifndef Py_GIL_DISABLED
+ assert(_PyDictKeys_StringLookup(
+ ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0);
+ #endif
+ if (shared_keys_version == 0) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return 0;
}
- write_u32(cache->keys_version, keys_version);
+ write_u32(cache->keys_version, shared_keys_version);
specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES :
LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
}
else {
Py_ssize_t dictoffset;
- if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ if (tp_flags & Py_TPFLAGS_MANAGED_DICT) {
dictoffset = MANAGED_DICT_OFFSET;
}
else {
@@ -1576,13 +1641,11 @@ PyObject *descr, DescriptorClassification kind, bool
is_method)
* PyType_Modified usages in typeobject.c). The MCACHE has been
* working since Python 2.6 and it's battle-tested.
*/
- write_u32(cache->type_version, owner_cls->tp_version_tag);
+ write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
return 1;
}
-#endif // Py_GIL_DISABLED
-
static void
specialize_load_global_lock_held(
@@ -1729,7 +1792,6 @@ function_kind(PyCodeObject *code) {
return SIMPLE_FUNCTION;
}
-#ifndef Py_GIL_DISABLED
/* Returning false indicates a failure. */
static bool
function_check_args(PyObject *o, int expected_argcount, int opcode)
@@ -1763,19 +1825,6 @@ function_get_version(PyObject *o, int opcode)
return version;
}
-/* Returning 0 indicates a failure. */
-static uint32_t
-type_get_version(PyTypeObject *t, int opcode)
-{
- uint32_t version = t->tp_version_tag;
- if (version == 0) {
- SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
- return 0;
- }
- return version;
-}
-#endif // Py_GIL_DISABLED
-
void
_Py_Specialize_BinarySubscr(
_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
diff --git a/Tools/cases_generator/analyzer.py
b/Tools/cases_generator/analyzer.py
index 99896f32fd2b08..4013b503502df6 100644
--- a/Tools/cases_generator/analyzer.py
+++ b/Tools/cases_generator/analyzer.py
@@ -386,7 +386,7 @@ def find_assignment_target(node: parser.InstDef, idx: int)
-> list[lexer.Token]:
"""Find the tokens that make up the left-hand side of an assignment"""
offset = 0
for tkn in reversed(node.block.tokens[: idx]):
- if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}:
+ if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}:
return node.block.tokens[idx - offset : idx]
offset += 1
return []
diff --git a/Tools/cases_generator/generators_common.py
b/Tools/cases_generator/generators_common.py
index 69d84183f1c7e6..f54afbb880d2fa 100644
--- a/Tools/cases_generator/generators_common.py
+++ b/Tools/cases_generator/generators_common.py
@@ -126,7 +126,7 @@ def __init__(self, out: CWriter):
"PyStackRef_AsPyObjectSteal": self.stackref_steal,
"DISPATCH": self.dispatch,
"INSTRUCTION_SIZE": self.instruction_size,
- "POP_DEAD_INPUTS": self.pop_dead_inputs,
+ "POP_INPUT": self.pop_input,
}
self.out = out
@@ -423,7 +423,7 @@ def save_stack(
self.emit_save(storage)
return True
- def pop_dead_inputs(
+ def pop_input(
self,
tkn: Token,
tkn_iter: TokenIterator,
@@ -432,9 +432,18 @@ def pop_dead_inputs(
inst: Instruction | None,
) -> bool:
next(tkn_iter)
+ name_tkn = next(tkn_iter)
+ name = name_tkn.text
next(tkn_iter)
next(tkn_iter)
- storage.pop_dead_inputs(self.out)
+ if not storage.inputs:
+ raise analysis_error("stack is empty", tkn)
+ tos = storage.inputs[-1]
+ if tos.name != name:
+ raise analysis_error(f"'{name} is not top of stack", name_tkn)
+ tos.defined = False
+ storage.clear_dead_inputs()
+ storage.flush(self.out)
return True
def emit_reload(self, storage: Storage) -> None:
diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py
index 9471fe0e56f7d8..286f47d0cfb11b 100644
--- a/Tools/cases_generator/stack.py
+++ b/Tools/cases_generator/stack.py
@@ -512,10 +512,6 @@ def flush(self, out: CWriter, cast_type: str =
"uintptr_t", extract_bits: bool =
self._push_defined_outputs()
self.stack.flush(out, cast_type, extract_bits)
- def pop_dead_inputs(self, out: CWriter, cast_type: str = "uintptr_t",
extract_bits: bool = True) -> None:
- self.clear_dead_inputs()
- self.stack.flush(out, cast_type, extract_bits)
-
def save(self, out: CWriter) -> None:
assert self.spilled >= 0
if self.spilled == 0:
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
