This is an automated email from the ASF dual-hosted git repository.
alenka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 639201bfa4 GH-44500: [Python][Parquet] Map Parquet logical types to
Arrow extension types by default (#46772)
639201bfa4 is described below
commit 639201bfa412db26ce45e73851432018af6c945e
Author: Dewey Dunnington <[email protected]>
AuthorDate: Tue Jun 17 03:11:24 2025 -0500
GH-44500: [Python][Parquet] Map Parquet logical types to Arrow extension
types by default (#46772)
### Rationale for this change
The Parquet C++ implementation now supports reading four logical types
(JSON, UUID, Geometry, Geography) as Arrow extension types; however, users have
to opt-in to avoid loosing the logical type on read.
### What changes are included in this PR?
This PR sets the default value of `arrow_extensions_enabled` to `True` (in
Python).
### Are these changes tested?
Yes, the behaviour of `arrow_extensions_enabled` was already tested (and
tests were updated to reflect the new default value).
### Are there any user-facing changes?
**This PR includes breaking changes to public APIs.**
Reading Parquet files that contained a JSON or UUID logical type will now
have an extension type rather than string or fixed size binary, respectively.
Python users that were relying on the previous behaviour would have to
explicitly cast to storage or use `read_table(...,
arrow_extensions_enabled=False)` after this PR:
```python
import uuid
import pyarrow as pa
json_array = pa.array(['{"k": "v"}'], pa.json_())
json_array.cast(pa.string())
#> [
#> "{"k": "v"}"
#> ]
uuid_array = pa.array([uuid.uuid4().bytes], pa.uuid())
uuid_array.cast(pa.binary(16))
#> <pyarrow.lib.FixedSizeBinaryArray object at 0x11e42b1c0>
#> [
#> 746C1022AB434A97972E1707EC3EE8F4
#> ]
```
* GitHub Issue: #44500
Authored-by: Dewey Dunnington <[email protected]>
Signed-off-by: AlenkaF <[email protected]>
---
python/pyarrow/parquet/core.py | 12 ++++++------
python/pyarrow/tests/parquet/test_data_types.py | 7 +++++--
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py
index a84fd5e8b7..e98a813395 100644
--- a/python/pyarrow/parquet/core.py
+++ b/python/pyarrow/parquet/core.py
@@ -261,7 +261,7 @@ class ParquetFile:
it will be parsed as an URI to determine the filesystem.
page_checksum_verification : bool, default False
If True, verify the checksum for each page read from the file.
- arrow_extensions_enabled : bool, default False
+ arrow_extensions_enabled : bool, default True
If True, read Parquet logical types as Arrow extension types where
possible,
(e.g., read JSON as the canonical `arrow.json` extension type or UUID
as
the canonical `arrow.uuid` extension type).
@@ -314,7 +314,7 @@ class ParquetFile:
coerce_int96_timestamp_unit=None,
decryption_properties=None, thrift_string_size_limit=None,
thrift_container_size_limit=None, filesystem=None,
- page_checksum_verification=False,
arrow_extensions_enabled=False):
+ page_checksum_verification=False,
arrow_extensions_enabled=True):
self._close_source = getattr(source, 'closed', True)
@@ -1321,7 +1321,7 @@ thrift_container_size_limit : int, default None
sufficient for most Parquet files.
page_checksum_verification : bool, default False
If True, verify the page checksum for each page read from the file.
-arrow_extensions_enabled : bool, default False
+arrow_extensions_enabled : bool, default True
If True, read Parquet logical types as Arrow extension types where
possible,
(e.g., read JSON as the canonical `arrow.json` extension type or UUID as
the canonical `arrow.uuid` extension type).
@@ -1339,7 +1339,7 @@ Examples
decryption_properties=None, thrift_string_size_limit=None,
thrift_container_size_limit=None,
page_checksum_verification=False,
- arrow_extensions_enabled=False):
+ arrow_extensions_enabled=True):
import pyarrow.dataset as ds
# map format arguments
@@ -1739,7 +1739,7 @@ thrift_container_size_limit : int, default None
sufficient for most Parquet files.
page_checksum_verification : bool, default False
If True, verify the checksum for each page read from the file.
-arrow_extensions_enabled : bool, default False
+arrow_extensions_enabled : bool, default True
If True, read Parquet logical types as Arrow extension types where
possible,
(e.g., read JSON as the canonical `arrow.json` extension type or UUID as
the canonical `arrow.uuid` extension type).
@@ -1839,7 +1839,7 @@ def read_table(source, *, columns=None, use_threads=True,
decryption_properties=None, thrift_string_size_limit=None,
thrift_container_size_limit=None,
page_checksum_verification=False,
- arrow_extensions_enabled=False):
+ arrow_extensions_enabled=True):
try:
dataset = ParquetDataset(
diff --git a/python/pyarrow/tests/parquet/test_data_types.py
b/python/pyarrow/tests/parquet/test_data_types.py
index 351221f64d..c546bc1532 100644
--- a/python/pyarrow/tests/parquet/test_data_types.py
+++ b/python/pyarrow/tests/parquet/test_data_types.py
@@ -569,6 +569,7 @@ def test_json_extension_type(storage_type):
_check_roundtrip(
table,
pa.table({"ext": pa.array(data, pa.string())}),
+ {"arrow_extensions_enabled": False},
store_schema=False)
# With arrow_extensions_enabled=True on read, we get a arrow.json back
@@ -576,7 +577,7 @@ def test_json_extension_type(storage_type):
_check_roundtrip(
table,
pa.table({"ext": pa.array(data, pa.json_(pa.string()))}),
- read_table_kwargs={"arrow_extensions_enabled": True},
+ {"arrow_extensions_enabled": True},
store_schema=False)
@@ -594,11 +595,13 @@ def test_uuid_extension_type():
_check_roundtrip(
table,
pa.table({"ext": pa.array(data, pa.binary(16))}),
+ {"arrow_extensions_enabled": False},
store_schema=False)
_check_roundtrip(
table,
table,
- {"arrow_extensions_enabled": True}, store_schema=False)
+ {"arrow_extensions_enabled": True},
+ store_schema=False)
def test_undefined_logical_type(parquet_test_datadir):
