https://github.com/python/cpython/commit/62fb15d8666f26883add93ae00cfff22ca95de2e
commit: 62fb15d8666f26883add93ae00cfff22ca95de2e
branch: main
author: Sergey Miryanov <[email protected]>
committer: encukou <[email protected]>
date: 2025-03-24T13:55:09+01:00
summary:
gh-131311: Extract _replace_array_elements from
PyCStructUnionType_update_stginfo (GH-131504)
files:
M Modules/_ctypes/stgdict.c
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 4779d362d6a262..31b83374917152 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -210,6 +210,11 @@ MakeAnonFields(PyObject *type)
return 0;
}
+
+int
+_replace_array_elements(ctypes_state *st, PyObject *layout_fields,
+ Py_ssize_t ffi_ofs, StgInfo *baseinfo, StgInfo
*stginfo);
+
/*
Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute,
and initialize StgInfo. Used for Structure and Union subclasses.
@@ -445,215 +450,9 @@ PyCStructUnionType_update_stginfo(PyObject *type,
PyObject *fields, int isStruct
#endif
if (arrays_seen && (total_size <= MAX_STRUCT_SIZE)) {
- /*
- * See bpo-22273 and gh-110190. Arrays are normally treated as
- * pointers, which is fine when an array name is being passed as
- * parameter, but not when passing structures by value that contain
- * arrays.
- * Small structures passed by value are passed in registers, and in
- * order to do this, libffi needs to know the true type of the array
- * members of structs. Treating them as pointers breaks things.
- *
- * Small structures have different sizes depending on the platform
- * where Python is running on:
- *
- * * x86-64: 16 bytes or less
- * * Arm platforms (both 32 and 64 bit): 32 bytes or less
- * * PowerPC 64 Little Endian: 64 bytes or less
- *
- * In that case, there can't be more than 16, 32 or 64 elements after
- * unrolling arrays, as we (will) disallow bitfields.
- * So we can collect the true ffi_type values in a fixed-size local
- * array on the stack and, if any arrays were seen, replace the
- * ffi_type_pointer.elements with a more accurate set, to allow
- * libffi to marshal them into registers correctly.
- * It means one more loop over the fields, but if we got here,
- * the structure is small, so there aren't too many of those.
- *
- * Although the passing in registers is specific to the above
- * platforms, the array-in-struct vs. pointer problem is general.
- * But we restrict the type transformation to small structs
- * nonetheless.
- *
- * Note that although a union may be small in terms of memory usage, it
- * could contain many overlapping declarations of arrays, e.g.
- *
- * union {
- * unsigned int_8 foo [16];
- * unsigned uint_8 bar [16];
- * unsigned int_16 baz[8];
- * unsigned uint_16 bozz[8];
- * unsigned int_32 fizz[4];
- * unsigned uint_32 buzz[4];
- * }
- *
- * which is still only 16 bytes in size. We need to convert this into
- * the following equivalent for libffi:
- *
- * union {
- * struct { int_8 e1; int_8 e2; ... int_8 e_16; } f1;
- * struct { uint_8 e1; uint_8 e2; ... uint_8 e_16; } f2;
- * struct { int_16 e1; int_16 e2; ... int_16 e_8; } f3;
- * struct { uint_16 e1; uint_16 e2; ... uint_16 e_8; } f4;
- * struct { int_32 e1; int_32 e2; ... int_32 e_4; } f5;
- * struct { uint_32 e1; uint_32 e2; ... uint_32 e_4; } f6;
- * }
- *
- * The same principle applies for a struct 32 or 64 bytes in size.
- *
- * So the struct/union needs setting up as follows: all non-array
- * elements copied across as is, and all array elements replaced with
- * an equivalent struct which has as many fields as the array has
- * elements, plus one NULL pointer.
- */
-
- Py_ssize_t num_ffi_type_pointers = 0; /* for the dummy fields */
- Py_ssize_t num_ffi_types = 0; /* for the dummy structures */
- size_t alloc_size; /* total bytes to allocate */
- void *type_block; /* to hold all the type information needed */
- ffi_type **element_types; /* of this struct/union */
- ffi_type **dummy_types; /* of the dummy struct elements */
- ffi_type *structs; /* point to struct aliases of arrays */
- Py_ssize_t element_index; /* index into element_types for this */
- Py_ssize_t dummy_index = 0; /* index into dummy field pointers */
- Py_ssize_t struct_index = 0; /* index into dummy structs */
-
- /* first pass to see how much memory to allocate */
- for (Py_ssize_t i = 0; i < len; ++i) {
- PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); //
borrowed
- assert(prop_obj);
- assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type));
- CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed
-
- StgInfo *info;
- if (PyStgInfo_FromType(st, prop->proto, &info) < 0) {
- goto error;
- }
- assert(info);
-
- if (!PyCArrayTypeObject_Check(st, prop->proto)) {
- /* Not an array. Just need an ffi_type pointer. */
- num_ffi_type_pointers++;
- }
- else {
- /* It's an array. */
- Py_ssize_t length = info->length;
-
- StgInfo *einfo;
- if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) {
- goto error;
- }
- if (einfo == NULL) {
- PyErr_Format(PyExc_TypeError,
- "second item in _fields_ tuple (index %zd) must be a C
type",
- i);
- goto error;
- }
- /*
- * We need one extra ffi_type to hold the struct, and one
- * ffi_type pointer per array element + one for a NULL to
- * mark the end.
- */
- num_ffi_types++;
- num_ffi_type_pointers += length + 1;
- }
- }
-
- /*
- * At this point, we know we need storage for some ffi_types and some
- * ffi_type pointers. We'll allocate these in one block.
- * There are three sub-blocks of information: the ffi_type pointers to
- * this structure/union's elements, the ffi_type_pointers to the
- * dummy fields standing in for array elements, and the
- * ffi_types representing the dummy structures.
- */
- alloc_size = (ffi_ofs + 1 + len + num_ffi_type_pointers) *
sizeof(ffi_type *) +
- num_ffi_types * sizeof(ffi_type);
- type_block = PyMem_Malloc(alloc_size);
-
- if (type_block == NULL) {
- PyErr_NoMemory();
+ if (_replace_array_elements(st, layout_fields, ffi_ofs, baseinfo,
stginfo) < 0) {
goto error;
}
- /*
- * the first block takes up ffi_ofs + len + 1 which is the pointers *
- * for this struct/union. The second block takes up
- * num_ffi_type_pointers, so the sum of these is ffi_ofs + len + 1 +
- * num_ffi_type_pointers as allocated above. The last bit is the
- * num_ffi_types structs.
- */
- element_types = (ffi_type **) type_block;
- dummy_types = &element_types[ffi_ofs + len + 1];
- structs = (ffi_type *) &dummy_types[num_ffi_type_pointers];
-
- if (num_ffi_types > 0) {
- memset(structs, 0, num_ffi_types * sizeof(ffi_type));
- }
- if (ffi_ofs && (baseinfo != NULL)) {
- memcpy(element_types,
- baseinfo->ffi_type_pointer.elements,
- ffi_ofs * sizeof(ffi_type *));
- }
- element_index = ffi_ofs;
-
- /* second pass to actually set the type pointers */
- for (Py_ssize_t i = 0; i < len; ++i) {
- PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); //
borrowed
- assert(prop_obj);
- assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type));
- CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed
-
- StgInfo *info;
- if (PyStgInfo_FromType(st, prop->proto, &info) < 0) {
- PyMem_Free(type_block);
- goto error;
- }
- assert(info);
-
- assert(element_index < (ffi_ofs + len)); /* will be used below */
- if (!PyCArrayTypeObject_Check(st, prop->proto)) {
- /* Not an array. Just copy over the element ffi_type. */
- element_types[element_index++] = &info->ffi_type_pointer;
- }
- else {
- Py_ssize_t length = info->length;
- StgInfo *einfo;
- if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) {
- PyMem_Free(type_block);
- goto error;
- }
- if (einfo == NULL) {
- PyMem_Free(type_block);
- PyErr_Format(PyExc_TypeError,
- "second item in _fields_ tuple (index %zd)
must be a C type",
- i);
- goto error;
- }
- element_types[element_index++] = &structs[struct_index];
- structs[struct_index].size = length *
einfo->ffi_type_pointer.size;
- structs[struct_index].alignment =
einfo->ffi_type_pointer.alignment;
- structs[struct_index].type = FFI_TYPE_STRUCT;
- structs[struct_index].elements = &dummy_types[dummy_index];
- ++struct_index;
- /* Copy over the element's type, length times. */
- while (length > 0) {
- assert(dummy_index < (num_ffi_type_pointers));
- dummy_types[dummy_index++] = &einfo->ffi_type_pointer;
- length--;
- }
- assert(dummy_index < (num_ffi_type_pointers));
- dummy_types[dummy_index++] = NULL;
- }
- }
-
- element_types[element_index] = NULL;
- /*
- * Replace the old elements with the new, taking into account
- * base class elements where necessary.
- */
- assert(stginfo->ffi_type_pointer.elements);
- PyMem_Free(stginfo->ffi_type_pointer.elements);
- stginfo->ffi_type_pointer.elements = element_types;
}
/* We did check that this flag was NOT set above, it must not
@@ -677,3 +476,227 @@ PyCStructUnionType_update_stginfo(PyObject *type,
PyObject *fields, int isStruct
Py_XDECREF(format_spec_obj);
return retval;
}
+
+/*
+ Replace array elements at stginfo->ffi_type_pointer.elements.
+ Return -1 if error occured.
+*/
+int
+_replace_array_elements(ctypes_state *st, PyObject *layout_fields,
+ Py_ssize_t ffi_ofs, StgInfo *baseinfo, StgInfo
*stginfo)
+{
+ /*
+ * See bpo-22273 and gh-110190. Arrays are normally treated as
+ * pointers, which is fine when an array name is being passed as
+ * parameter, but not when passing structures by value that contain
+ * arrays.
+ * Small structures passed by value are passed in registers, and in
+ * order to do this, libffi needs to know the true type of the array
+ * members of structs. Treating them as pointers breaks things.
+ *
+ * Small structures have different sizes depending on the platform
+ * where Python is running on:
+ *
+ * * x86-64: 16 bytes or less
+ * * Arm platforms (both 32 and 64 bit): 32 bytes or less
+ * * PowerPC 64 Little Endian: 64 bytes or less
+ *
+ * In that case, there can't be more than 16, 32 or 64 elements after
+ * unrolling arrays, as we (will) disallow bitfields.
+ * So we can collect the true ffi_type values in a fixed-size local
+ * array on the stack and, if any arrays were seen, replace the
+ * ffi_type_pointer.elements with a more accurate set, to allow
+ * libffi to marshal them into registers correctly.
+ * It means one more loop over the fields, but if we got here,
+ * the structure is small, so there aren't too many of those.
+ *
+ * Although the passing in registers is specific to the above
+ * platforms, the array-in-struct vs. pointer problem is general.
+ * But we restrict the type transformation to small structs
+ * nonetheless.
+ *
+ * Note that although a union may be small in terms of memory usage, it
+ * could contain many overlapping declarations of arrays, e.g.
+ *
+ * union {
+ * unsigned int_8 foo [16];
+ * unsigned uint_8 bar [16];
+ * unsigned int_16 baz[8];
+ * unsigned uint_16 bozz[8];
+ * unsigned int_32 fizz[4];
+ * unsigned uint_32 buzz[4];
+ * }
+ *
+ * which is still only 16 bytes in size. We need to convert this into
+ * the following equivalent for libffi:
+ *
+ * union {
+ * struct { int_8 e1; int_8 e2; ... int_8 e_16; } f1;
+ * struct { uint_8 e1; uint_8 e2; ... uint_8 e_16; } f2;
+ * struct { int_16 e1; int_16 e2; ... int_16 e_8; } f3;
+ * struct { uint_16 e1; uint_16 e2; ... uint_16 e_8; } f4;
+ * struct { int_32 e1; int_32 e2; ... int_32 e_4; } f5;
+ * struct { uint_32 e1; uint_32 e2; ... uint_32 e_4; } f6;
+ * }
+ *
+ * The same principle applies for a struct 32 or 64 bytes in size.
+ *
+ * So the struct/union needs setting up as follows: all non-array
+ * elements copied across as is, and all array elements replaced with
+ * an equivalent struct which has as many fields as the array has
+ * elements, plus one NULL pointer.
+ */
+
+ Py_ssize_t num_ffi_type_pointers = 0; /* for the dummy fields */
+ Py_ssize_t num_ffi_types = 0; /* for the dummy structures */
+ size_t alloc_size; /* total bytes to allocate */
+ void *type_block = NULL; /* to hold all the type information needed */
+ ffi_type **element_types; /* of this struct/union */
+ ffi_type **dummy_types; /* of the dummy struct elements */
+ ffi_type *structs; /* point to struct aliases of arrays */
+ Py_ssize_t element_index; /* index into element_types for this */
+ Py_ssize_t dummy_index = 0; /* index into dummy field pointers */
+ Py_ssize_t struct_index = 0; /* index into dummy structs */
+
+ Py_ssize_t len = PyTuple_GET_SIZE(layout_fields);
+
+ /* first pass to see how much memory to allocate */
+ for (Py_ssize_t i = 0; i < len; ++i) {
+ PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed
+ assert(prop_obj);
+ assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type));
+ CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed
+
+ StgInfo *info;
+ if (PyStgInfo_FromType(st, prop->proto, &info) < 0) {
+ goto error;
+ }
+ assert(info);
+
+ if (!PyCArrayTypeObject_Check(st, prop->proto)) {
+ /* Not an array. Just need an ffi_type pointer. */
+ num_ffi_type_pointers++;
+ }
+ else {
+ /* It's an array. */
+ Py_ssize_t length = info->length;
+
+ StgInfo *einfo;
+ if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) {
+ goto error;
+ }
+ if (einfo == NULL) {
+ PyErr_Format(PyExc_TypeError,
+ "second item in _fields_ tuple (index %zd) must be a C
type",
+ i);
+ goto error;
+ }
+ /*
+ * We need one extra ffi_type to hold the struct, and one
+ * ffi_type pointer per array element + one for a NULL to
+ * mark the end.
+ */
+ num_ffi_types++;
+ num_ffi_type_pointers += length + 1;
+ }
+ }
+
+ /*
+ * At this point, we know we need storage for some ffi_types and some
+ * ffi_type pointers. We'll allocate these in one block.
+ * There are three sub-blocks of information: the ffi_type pointers to
+ * this structure/union's elements, the ffi_type_pointers to the
+ * dummy fields standing in for array elements, and the
+ * ffi_types representing the dummy structures.
+ */
+ alloc_size = (ffi_ofs + 1 + len + num_ffi_type_pointers) * sizeof(ffi_type
*) +
+ num_ffi_types * sizeof(ffi_type);
+ type_block = PyMem_Malloc(alloc_size);
+
+ if (type_block == NULL) {
+ PyErr_NoMemory();
+ goto error;
+ }
+ /*
+ * the first block takes up ffi_ofs + len + 1 which is the pointers *
+ * for this struct/union. The second block takes up
+ * num_ffi_type_pointers, so the sum of these is ffi_ofs + len + 1 +
+ * num_ffi_type_pointers as allocated above. The last bit is the
+ * num_ffi_types structs.
+ */
+ element_types = (ffi_type **) type_block;
+ dummy_types = &element_types[ffi_ofs + len + 1];
+ structs = (ffi_type *) &dummy_types[num_ffi_type_pointers];
+
+ if (num_ffi_types > 0) {
+ memset(structs, 0, num_ffi_types * sizeof(ffi_type));
+ }
+ if (ffi_ofs && (baseinfo != NULL)) {
+ memcpy(element_types,
+ baseinfo->ffi_type_pointer.elements,
+ ffi_ofs * sizeof(ffi_type *));
+ }
+ element_index = ffi_ofs;
+
+ /* second pass to actually set the type pointers */
+ for (Py_ssize_t i = 0; i < len; ++i) {
+ PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed
+ assert(prop_obj);
+ assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type));
+ CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed
+
+ StgInfo *info;
+ if (PyStgInfo_FromType(st, prop->proto, &info) < 0) {
+ goto error;
+ }
+ assert(info);
+
+ assert(element_index < (ffi_ofs + len)); /* will be used below */
+ if (!PyCArrayTypeObject_Check(st, prop->proto)) {
+ /* Not an array. Just copy over the element ffi_type. */
+ element_types[element_index++] = &info->ffi_type_pointer;
+ }
+ else {
+ Py_ssize_t length = info->length;
+ StgInfo *einfo;
+ if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) {
+ goto error;
+ }
+ if (einfo == NULL) {
+ PyErr_Format(PyExc_TypeError,
+ "second item in _fields_ tuple (index %zd) must be a C
type",
+ i);
+ goto error;
+ }
+ element_types[element_index++] = &structs[struct_index];
+ structs[struct_index].size = length * einfo->ffi_type_pointer.size;
+ structs[struct_index].alignment =
einfo->ffi_type_pointer.alignment;
+ structs[struct_index].type = FFI_TYPE_STRUCT;
+ structs[struct_index].elements = &dummy_types[dummy_index];
+ ++struct_index;
+ /* Copy over the element's type, length times. */
+ while (length > 0) {
+ assert(dummy_index < (num_ffi_type_pointers));
+ dummy_types[dummy_index++] = &einfo->ffi_type_pointer;
+ length--;
+ }
+ assert(dummy_index < (num_ffi_type_pointers));
+ dummy_types[dummy_index++] = NULL;
+ }
+ }
+
+ element_types[element_index] = NULL;
+ /*
+ * Replace the old elements with the new, taking into account
+ * base class elements where necessary.
+ */
+ assert(stginfo->ffi_type_pointer.elements);
+ PyMem_Free(stginfo->ffi_type_pointer.elements);
+ stginfo->ffi_type_pointer.elements = element_types;
+
+ return 0;
+
+error:
+ PyMem_Free(type_block);
+ return -1;
+}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
