This is an automated email from the ASF dual-hosted git repository. raulcd pushed a commit to branch maint-25.0.x in repository https://gitbox.apache.org/repos/asf/arrow.git
commit d1eaaedef3c7b05668e620fee1478fc884f36296 Author: Liang-Chi Hsieh <[email protected]> AuthorDate: Thu Jul 23 00:15:05 2026 +0800 GH-50326: [Python] Convert arrays to Python objects without per-element Scalars in to_pylist (#50327) ### Rationale for this change `pa.Array.to_pylist()` converts one element at a time through `Array::GetScalar` plus a Python `Scalar` wrapper; for list types each row additionally allocates a Python `Array` wrapper for the row's values slice and a fresh generator before recursing per element. A `sample` profile shows ~20% of runtime in CPython GC (triggered by the per-row GC-tracked allocations), ~25% in `GetScalar`, and only ~7% doing the useful work of creating the output objects — making `to_pylist` several tim [...] ### What changes are included in this PR? Following review feedback, this adds a general scalar-free conversion mechanism instead of per-type `to_pylist` overrides: - `Array` gains `cdef object _getitem_py(self, int64_t i)`, returning `self[i]` as a Python object. The base implementation is `GetScalar` + `Scalar.as_py`, so any type without a specialization behaves exactly as today (dates, times, timestamps, durations, decimals, dictionary, extension, unions, views, ...). - The baseline `Array.to_pylist` becomes a single loop over `_getitem_py`. `maps_as_pydicts != None` keeps the Scalar-based path, since map→dict conversion has per-entry duplicate-key semantics. - Specializations avoid all per-element Scalar and per-row Array-wrapper allocation: - integers and floats (a `type_id` switch on `NumericArray`; date/time/timestamp subclasses fall through to the exact base), - boolean, - string/binary and large variants (`GetValue` + `PyUnicode_DecodeUTF8` / `PyBytes_FromStringAndSize`, matching `str(buf, 'utf8')` / `to_pybytes()` exactly), - list/large_list/fixed_size_list (each row's list is built from the child's `_getitem_py` over the offset range; the wrapped child is cached on the parent array), - map (association list of key/value tuples, matching `MapScalar.as_py`), - struct (one dict per row; duplicate field names fall back to the Scalar path so they raise `ValueError` like `StructScalar.as_py`). Nested types compose without any per-row wrappers. `ChunkedArray.to_pylist`, `Table.to_pylist` and `ListScalar.as_py` delegate here and speed up automatically. Follow-up candidates: string/binary views, run-end-encoded, dictionary, a fast path for date32. Benchmarks (macOS arm64, M4 Max): | benchmark | before | after | speedup | |---|---|---|---| | flat `int64` with nulls (4M) | 0.39 s | 0.028 s | 14x (~7 ns/element, on par with `ndarray.tolist`) | | flat `string` (4M) | 0.83 s | 0.06 s | 14x | | `list<string>` (2M rows) | 1.93 s | 0.46 s | 4.2x | | `list<list<int32>>` (1M rows) | 2.10 s | 0.40 s | 5.2x | | `struct<int64,string>` (1M rows) | 0.91 s | 0.07 s | 13x | | `map<string,int64>` (1M rows) | 2.77 s | 0.74 s | 3.8x | ### Are these changes tested? `test_to_pylist_bulk_paths` (added here) compares against the per-scalar conversion with exact element types for representative arrays including sliced views. Additionally verified with a randomized differential test against `[x.as_py() for x in arr]` with exact-type equality: all integer widths (incl. values beyond 2^62), floats (NaN/inf), boolean, string/binary (+large, multibyte), all list kinds, nested lists, struct (incl. empty struct, duplicate-field-name `ValueError`), map (inc [...] ### Are there any user-facing changes? No behavior changes, only performance. * GitHub Issue: #50326 This pull request and its description were written by Isaac. Lead-authored-by: Liang-Chi Hsieh <[email protected]> Co-authored-by: Isaac Signed-off-by: Antoine Pitrou <[email protected]> --- python/pyarrow/array.pxi | 157 ++++++++++++++++++++++++++++++++++- python/pyarrow/includes/libarrow.pxd | 90 +++++++++++--------- python/pyarrow/lib.pxd | 7 ++ python/pyarrow/tests/test_array.py | 58 +++++++++++++ 4 files changed, 270 insertions(+), 42 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5fc74969ab..26bb6482a4 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1864,7 +1864,24 @@ cdef class Array(_PandasConvertible): lst : list """ self._assert_cpu() - return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + cdef int64_t i, n = self.length() + if maps_as_pydicts is not None: + # Converting maps to dicts has per-entry semantics (duplicate-key + # detection); use the Scalar-based conversion for exact behavior. + # TODO(GH-50429): this falls back to the Scalar path for the whole + # array even when the type contains no maps; threading + # maps_as_pydicts through _getitem_py keeps the fast paths instead. + return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + # TODO(GH-50448): convert per range instead of per element to cut + # the per-element call overhead further. + return [self._getitem_py(i) for i in range(n)] + + cdef object _getitem_py(self, int64_t i): + # Return self[i] as a Python object, without creating a Python Scalar + # (nor, for nested types, per-row Array wrappers) where a subclass + # provides a specialization; this base implementation goes through + # Scalar.as_py and thus preserves its semantics exactly (see GH-50326). + return self.getitem(i).as_py() def tolist(self): """ @@ -2444,6 +2461,12 @@ cdef class BooleanArray(Array): """ Concrete class for Arrow arrays of boolean data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + return (<CBooleanArray*> self.ap).Value(i) + @property def false_count(self): return (<CBooleanArray*> self.ap).false_count() @@ -2458,6 +2481,34 @@ cdef class NumericArray(Array): A base class for Arrow numeric arrays. """ + cdef object _getitem_py(self, int64_t i): + cdef Type tid = self.ap.type_id() + if self.ap.IsNull(i): + return None + if tid == _Type_INT8: + return (<CInt8Array*> self.ap).Value(i) + elif tid == _Type_INT16: + return (<CInt16Array*> self.ap).Value(i) + elif tid == _Type_INT32: + return (<CInt32Array*> self.ap).Value(i) + elif tid == _Type_INT64: + return (<CInt64Array*> self.ap).Value(i) + elif tid == _Type_UINT8: + return (<CUInt8Array*> self.ap).Value(i) + elif tid == _Type_UINT16: + return (<CUInt16Array*> self.ap).Value(i) + elif tid == _Type_UINT32: + return (<CUInt32Array*> self.ap).Value(i) + elif tid == _Type_UINT64: + return (<CUInt64Array*> self.ap).Value(i) + elif tid == _Type_FLOAT: + return (<CFloatArray*> self.ap).Value(i) + elif tid == _Type_DOUBLE: + return (<CDoubleArray*> self.ap).Value(i) + # Subclasses whose as_py returns non-primitive objects (dates, times, + # timestamps, durations, half floats, ...) use the exact Scalar path. + return Array._getitem_py(self, i) + cdef class IntegerArray(NumericArray): """ @@ -2776,6 +2827,16 @@ cdef class ListArray(BaseListArray): Concrete class for Arrow arrays of a list data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = <CListArray*> self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = <Array> self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -2961,6 +3022,16 @@ cdef class LargeListArray(BaseListArray): Identical to ListArray, but 64-bit offsets. """ + cdef object _getitem_py(self, int64_t i): + cdef CLargeListArray* arr = <CLargeListArray*> self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = <Array> self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3551,6 +3622,19 @@ cdef class MapArray(ListArray): Concrete class for Arrow arrays of a map data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = <CListArray*> self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = (self.keys, self.items) + cdef Array keys = <Array> (<tuple> self._children_cache)[0] + cdef Array items = <Array> (<tuple> self._children_cache)[1] + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + # Matches MapScalar.as_py with the default maps_as_pydicts=None: + # an association list of (key, value) tuples. + return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)] + @staticmethod def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3688,6 +3772,16 @@ cdef class FixedSizeListArray(BaseListArray): Concrete class for Arrow arrays of a fixed size list data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CFixedSizeListArray* arr = <CFixedSizeListArray*> self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = <Array> self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(values, list_size=None, DataType type=None, mask=None): """ @@ -3974,6 +4068,13 @@ cdef class StringArray(Array): Concrete class for Arrow arrays of string (or utf8) data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = (<CBinaryArray*> self.ap).GetView(i) + # Matches StringScalar.as_py, which is str(buf, 'utf8'). + return cp.PyUnicode_DecodeUTF8(view.data(), <Py_ssize_t> view.size(), NULL) + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4006,6 +4107,12 @@ cdef class LargeStringArray(Array): Concrete class for Arrow arrays of large string (or utf8) data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = (<CLargeBinaryArray*> self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), <Py_ssize_t> view.size(), NULL) + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4038,11 +4145,24 @@ cdef class StringViewArray(Array): Concrete class for Arrow arrays of string (or utf8) view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = (<CBinaryViewArray*> self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), <Py_ssize_t> view.size(), NULL) + cdef class BinaryArray(Array): """ Concrete class for Arrow arrays of variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = (<CBinaryArray*> self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), <Py_ssize_t> view.size()) + @property def total_values_length(self): """ @@ -4056,6 +4176,13 @@ cdef class LargeBinaryArray(Array): """ Concrete class for Arrow arrays of large variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = (<CLargeBinaryArray*> self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), <Py_ssize_t> view.size()) + @property def total_values_length(self): """ @@ -4070,6 +4197,12 @@ cdef class BinaryViewArray(Array): Concrete class for Arrow arrays of variable-sized binary view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = (<CBinaryViewArray*> self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), <Py_ssize_t> view.size()) + cdef class DictionaryArray(Array): """ @@ -4229,6 +4362,28 @@ cdef class StructArray(Array): Concrete class for Arrow arrays of a struct data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef int64_t k, num_fields = self.type.num_fields + if self._children_cache is None: + names = [self.type.field(k).name for k in range(num_fields)] + if len(set(names)) != len(names): + # Matches StructScalar.as_py + raise ValueError( + "Converting to Python dictionary is not supported when " + "duplicate field names are present") + self._children_cache = ( + names, [self.field(k) for k in range(num_fields)]) + names = (<tuple> self._children_cache)[0] + fields = (<tuple> self._children_cache)[1] + cdef Array field_arr + result = {} + for k in range(num_fields): + field_arr = <Array> fields[k] + result[names[k]] = field_arr._getitem_py(i) + return result + def field(self, index): """ Retrieves the child array belonging to field. diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8b4786ecbf..e57c6d0d92 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -264,7 +264,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string Diff(const CArray& other) c_bool Equals(const CArray& arr) - c_bool IsNull(int i) + c_bool IsNull(int64_t i) shared_ptr[CArrayData] data() @@ -675,87 +675,87 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string* result) cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray): - c_bool Value(int i) + c_bool Value(int64_t i) int64_t false_count() int64_t true_count() cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray): - uint8_t Value(int i) + uint8_t Value(int64_t i) cdef cppclass CInt8Array" arrow::Int8Array"(CArray): - int8_t Value(int i) + int8_t Value(int64_t i) cdef cppclass CUInt16Array" arrow::UInt16Array"(CArray): - uint16_t Value(int i) + uint16_t Value(int64_t i) cdef cppclass CInt16Array" arrow::Int16Array"(CArray): - int16_t Value(int i) + int16_t Value(int64_t i) cdef cppclass CUInt32Array" arrow::UInt32Array"(CArray): - uint32_t Value(int i) + uint32_t Value(int64_t i) cdef cppclass CInt32Array" arrow::Int32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CUInt64Array" arrow::UInt64Array"(CArray): - uint64_t Value(int i) + uint64_t Value(int64_t i) cdef cppclass CInt64Array" arrow::Int64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CDate32Array" arrow::Date32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CDate64Array" arrow::Date64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CTime32Array" arrow::Time32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CTime64Array" arrow::Time64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CDurationArray" arrow::DurationArray"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CMonthDayNanoIntervalArray \ "arrow::MonthDayNanoIntervalArray"(CArray): pass cdef cppclass CHalfFloatArray" arrow::HalfFloatArray"(CArray): - uint16_t Value(int i) + uint16_t Value(int64_t i) cdef cppclass CFloatArray" arrow::FloatArray"(CArray): - float Value(int i) + float Value(int64_t i) cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray): - double Value(int i) + double Value(int64_t i) cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray): - const uint8_t* GetValue(int i) + const uint8_t* GetValue(int64_t i) cdef cppclass CDecimal32Array" arrow::Decimal32Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal64Array" arrow::Decimal64Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal128Array" arrow::Decimal128Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal256Array" arrow::Decimal256Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CListArray" arrow::ListArray"(CArray): @staticmethod @@ -776,8 +776,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: ) const int32_t* raw_value_offsets() - int32_t value_offset(int i) - int32_t value_length(int i) + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CDataType] value_type() @@ -800,8 +800,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CBuffer] null_bitmap ) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CDataType] value_type() @@ -819,8 +819,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CDataType], shared_ptr[CBuffer] null_bitmap) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CDataType] value_type() @@ -850,8 +850,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const int32_t* raw_value_offsets() const int32_t* raw_value_sizes() - int32_t value_offset(int i) - int32_t value_length(int i) + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CArray] sizes() @@ -881,8 +881,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CMemoryPool* pool ) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CArray] sizes() @@ -911,8 +911,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CArray] keys() shared_ptr[CArray] items() CMapType* map_type() - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CDataType] value_type() @@ -941,18 +941,20 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const vector[c_string]& field_names, const vector[int8_t]& type_codes) - int32_t value_offset(int i) + int32_t value_offset(int64_t i) shared_ptr[CBuffer] value_offsets() cdef cppclass CBinaryArray" arrow::BinaryArray"(CArray): - const uint8_t* GetValue(int i, int32_t* length) + const uint8_t* GetValue(int64_t i, int32_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int32_t value_offset(int64_t i) int32_t value_length(int64_t i) int32_t total_values_length() cdef cppclass CLargeBinaryArray" arrow::LargeBinaryArray"(CArray): - const uint8_t* GetValue(int i, int64_t* length) + const uint8_t* GetValue(int64_t i, int64_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int64_t value_offset(int64_t i) int64_t value_length(int64_t i) @@ -965,7 +967,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int64_t null_count, int64_t offset) - c_string GetString(int i) + c_string GetString(int64_t i) cdef cppclass CLargeStringArray" arrow::LargeStringArray" \ (CLargeBinaryArray): @@ -975,7 +977,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int64_t null_count, int64_t offset) - c_string GetString(int i) + c_string GetString(int64_t i) + + cdef cppclass CBinaryViewArray" arrow::BinaryViewArray"(CArray): + cpp_string_view GetView(int64_t i) + + cdef cppclass CStringViewArray" arrow::StringViewArray"(CBinaryViewArray): + pass cdef cppclass CStructArray" arrow::StructArray"(CArray): CStructArray(shared_ptr[CDataType]& type, int64_t length, diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 683faa7855..38f1ac69a8 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -288,8 +288,15 @@ cdef class Array(_PandasConvertible): # To allow Table to propagate metadata to pandas.Series object _name + cdef: + # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326). + # Appended after the pre-existing attributes to keep their offsets + # stable for extensions compiled against an older pyarrow. + object _children_cache + cdef void init(self, const shared_ptr[CArray]& sp_array) except * cdef getitem(self, int64_t i) + cdef object _getitem_py(self, int64_t i) cdef int64_t length(self) cdef void _assert_cpu(self) except * diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index adc3e097b5..c1e3f0128b 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -465,6 +465,64 @@ def test_array_getitem_numpy_scalars(): assert arr[np.int32(idx)].as_py() == lst[idx] +def test_to_pylist_bulk_paths(): + # GH-50326: to_pylist converts through scalar-free _getitem_py + # specializations; the result must match the per-scalar conversion + # exactly. + arrays = [ + pa.array([[1, None, 3], None, [], [4]], type=pa.list_(pa.int32())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.list_(pa.string())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.large_list(pa.large_string())), + pa.array([[1, None], None, [3, 4]], type=pa.list_(pa.int32(), 2)), + pa.array([[[1], [2, None]], None, [None, [3]]], + type=pa.list_(pa.list_(pa.int32()))), + pa.array([[("k1", 1), ("k2", None)], None, []], + type=pa.map_(pa.string(), pa.int32())), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.string()), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.large_string()), + pa.array([b"a\x00b", None, b"", b"\xff"], type=pa.binary()), + pa.array([b"a\x00b", None, b""], type=pa.large_binary()), + # View types store short values inline and long values out-of-line; + # cover both, plus NUL bytes and non-ASCII data. + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}", + "long string exceeding the inline view size"], + type=pa.string_view()), + pa.array([b"a\x00b", None, b"", b"\xff", + b"long binary value exceeding the inline view size"], + type=pa.binary_view()), + pa.array([[b"x", None, b"\x00y"], None, []], + type=pa.list_(pa.binary())), + pa.array([1, None, -(2**62), 2**62], type=pa.int64()), + pa.array([0, None, 2**63 + 7], type=pa.uint64()), + pa.array([-128, 127, None], type=pa.int8()), + pa.array([1.5, None, -0.5], type=pa.float64()), + pa.array([1.5, None], type=pa.float32()), + pa.array([True, None, False], type=pa.bool_()), + pa.array([{"a": 1, "b": "x"}, None, {"a": None, "b": None}], + type=pa.struct([("a", pa.int32()), ("b", pa.string())])), + pa.array([], type=pa.list_(pa.int32())), + pa.array([None, None], type=pa.list_(pa.string())), + ] + for arr in arrays: + for view in (arr, arr.slice(1), arr.slice(0, 2), arr.slice(2)): + assert view.to_pylist() == [x.as_py() for x in view] + + # Values inside numeric lists must stay Python ints/None, never floats + result = pa.array([[1, None, 3]], type=pa.list_(pa.int32())).to_pylist() + assert result == [[1, None, 3]] + assert [type(x) for x in result[0]] == [int, type(None), int] + + # Duplicate struct field names raise like StructScalar.as_py does + dup = pa.StructArray.from_arrays( + [pa.array([1, 2]), pa.array(["a", "b"])], names=["x", "x"]) + with pytest.raises(ValueError, match="duplicate field names"): + dup.to_pylist() + + def test_array_slice(): arr = pa.array(range(10))
