rok commented on code in PR #50327:
URL: https://github.com/apache/arrow/pull/50327#discussion_r3629728508
##########
python/pyarrow/array.pxi:
##########
@@ -1864,7 +1864,21 @@ cdef class Array(_PandasConvertible):
lst : list
"""
self._assert_cpu()
- return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
+ cdef int64_t i, n = self.length()
+ if maps_as_pydicts is not None:
+ # Converting maps to dicts has per-entry semantics (duplicate-key
+ # detection); use the Scalar-based conversion for exact behavior.
+ return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
Review Comment:
Won't this proposal also ignore user-defined
`CustomArray(pa.ExtensionArray).to_pylist` here?
How about we check for custom `to_pylist` and use this path too. See sketch
below:
```diff
diff --git i/python/pyarrow/array.pxi w/python/pyarrow/array.pxi
index 71c18fbdfe..523a603b29 100644
--- i/python/pyarrow/array.pxi
+++ w/python/pyarrow/array.pxi
@@ -1123,6 +1123,17 @@ cdef PandasOptions _convert_pandas_options(dict
options):
return result
+cdef bint _has_custom_extension_to_pylist(DataType type):
+ cdef int i
+ if type.id == _Type_EXTENSION:
+ array_class = type.__arrow_ext_class__()
+ return array_class.to_pylist is not ExtensionArray.to_pylist
+ for i in range(type.num_fields):
+ if _has_custom_extension_to_pylist(type.field(i).type):
+ return True
+ return False
+
+
cdef class Array(_PandasConvertible):
"""
The base class for all Arrow arrays.
@@ -1865,9 +1876,11 @@ cdef class Array(_PandasConvertible):
"""
self._assert_cpu()
cdef int64_t i, n = self.length()
- if maps_as_pydicts is not None:
+ if (maps_as_pydicts is not None or
+ _has_custom_extension_to_pylist(self.type)):
# Converting maps to dicts has per-entry semantics
(duplicate-key
- # detection); use the Scalar-based conversion for exact
behavior.
+ # detection), while extension arrays can customize to_pylist;
+ # use the Scalar-based conversion for exact behavior.
return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
# TODO(GH-50448): convert per range instead of per element to cut
# the per-element call overhead further.
diff --git i/python/pyarrow/tests/test_extension_type.py
w/python/pyarrow/tests/test_extension_type.py
index 35b801eca8..3f5eca5222 100644
--- i/python/pyarrow/tests/test_extension_type.py
+++ w/python/pyarrow/tests/test_extension_type.py
@@ -391,6 +391,21 @@ def test_ext_array_to_pylist():
assert arr.to_pylist() == [b"foo", b"bar", None]
+def test_nested_custom_ext_array_to_pylist():
+ class CustomArray(pa.ExtensionArray):
+ def to_pylist(self, **kwargs):
+ return ["custom"] * len(self)
+
+ class CustomType(IntegerType):
+ def __arrow_ext_class__(self):
+ return CustomArray
+
+ values = pa.ExtensionArray.from_storage(CustomType(), pa.array([1]))
+ arr = pa.ListArray.from_arrays([0, 1], values)
+
+ assert arr.to_pylist() == [["custom"]]
+
+
def test_ext_array_errors():
ty = ParamExtType(4)
storage = pa.array([b"foo", b"bar"], type=pa.binary(3))
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
