This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 6287272f39 GH-50464: [C++][Python] Simplify arrow_to_pandas DateOffset
handling for nanoseconds/milliseconds (#50465)
6287272f39 is described below
commit 6287272f39b8ebef02ca3ec8cf51650e4b240231
Author: Raúl Cumplido <[email protected]>
AuthorDate: Thu Jul 23 10:28:47 2026 +0200
GH-50464: [C++][Python] Simplify arrow_to_pandas DateOffset handling for
nanoseconds/milliseconds (#50465)
### Rationale for this change
Pandas 1.4 fixed the following:
- https://github.com/pandas-dev/pandas/issues/43892
Which made us special case microseconds extracting those from our
`MonthDayNano` and manually building `microseconds`.
`DateOffset` can be created with the following:
```python
DateOffset(years=1, months=1,
days=1, seconds=1, microseconds=1,
minutes=1, hours=1, weeks=1, nanoseconds=1)
```
We currently only roundtrip Month, Day, Microseconds and Nano due to our
MonthDayNano data type. We special cased microseconds but we were not special
casing years, seconds, minutos, hours or weeks.
As the initial issue that made us special case has been solved upstream we
can stop roundtripping microseconds and just return nanoseconds simplifying our
codebase.
### What changes are included in this PR?
Remove the unnecessary code to retrieve microseconds from nanoseconds.
Update test as microseconds won't be returned, only nanoseconds.
### Are these changes tested?
Yes via CI
### Are there any user-facing changes?
Yes, this is a change of the UX.
* GitHub Issue: #50464
Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 25 +++++++---------------
python/pyarrow/tests/test_array.py | 12 +++++------
python/pyarrow/tests/test_pandas.py | 3 +--
3 files changed, 14 insertions(+), 26 deletions(-)
diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
index 348d352a04..3cf6da31a8 100644
--- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
+++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
@@ -1292,24 +1292,15 @@ struct ObjectWriterVisitor {
auto to_date_offset = [&](const MonthDayNanoIntervalType::MonthDayNanos&
interval,
PyObject** out) {
ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr);
- // DateOffset objects do not add nanoseconds component to pd.Timestamp.
- // as of Pandas 1.3.3
- // (https://github.com/pandas-dev/pandas/issues/43892).
- // So convert microseconds and remainder to preserve data
- // but give users more expected results.
- int64_t microseconds = interval.nanoseconds / 1000;
- int64_t nanoseconds;
- if (interval.nanoseconds >= 0) {
- nanoseconds = interval.nanoseconds % 1000;
- } else {
- nanoseconds = -((-interval.nanoseconds) % 1000);
- }
- PyDict_SetItemString(kwargs.obj(), "months",
PyLong_FromLong(interval.months));
- PyDict_SetItemString(kwargs.obj(), "days",
PyLong_FromLong(interval.days));
- PyDict_SetItemString(kwargs.obj(), "microseconds",
- PyLong_FromLongLong(microseconds));
- PyDict_SetItemString(kwargs.obj(), "nanoseconds",
PyLong_FromLongLong(nanoseconds));
+ OwnedRef months(PyLong_FromLong(interval.months));
+ OwnedRef days(PyLong_FromLong(interval.days));
+ OwnedRef nanoseconds(PyLong_FromLongLong(interval.nanoseconds));
+ RETURN_IF_PYERROR();
+ PyDict_SetItemString(kwargs.obj(), "months", months.obj());
+ PyDict_SetItemString(kwargs.obj(), "days", days.obj());
+ PyDict_SetItemString(kwargs.obj(), "nanoseconds", nanoseconds.obj());
+ RETURN_IF_PYERROR();
*out =
PyObject_Call(internal::BorrowPandasDataOffsetType(), args.obj(),
kwargs.obj());
RETURN_IF_PYERROR();
diff --git a/python/pyarrow/tests/test_array.py
b/python/pyarrow/tests/test_array.py
index c1e3f0128b..a26481c575 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -2820,11 +2820,10 @@ def test_interval_array_from_relativedelta():
assert arr.equals(expected)
assert arr.to_pandas().tolist() == [
None, DateOffset(months=13, days=8,
- microseconds=(
+ nanoseconds=(
datetime.timedelta(seconds=1, microseconds=1,
minutes=1, hours=1) //
- datetime.timedelta(microseconds=1)),
- nanoseconds=0)]
+ datetime.timedelta(microseconds=1)) * 1000)]
with pytest.raises(ValueError):
pa.array([DateOffset(years=((1 << 32) // 12), months=100)])
with pytest.raises(ValueError):
@@ -2872,12 +2871,11 @@ def test_interval_array_from_dateoffset():
assert arr.equals(expected)
expected_from_pandas = [
None, DateOffset(months=13, days=8,
- microseconds=(
+ nanoseconds=(
datetime.timedelta(seconds=1, microseconds=1,
minutes=1, hours=1) //
- datetime.timedelta(microseconds=1)),
- nanoseconds=1),
- DateOffset(months=0, days=0, microseconds=0, nanoseconds=0)]
+ datetime.timedelta(microseconds=1) * 1000) + 1),
+ DateOffset(months=0, days=0, nanoseconds=0)]
assert arr.to_pandas().tolist() == expected_from_pandas
diff --git a/python/pyarrow/tests/test_pandas.py
b/python/pyarrow/tests/test_pandas.py
index c3153e82b7..4b144448ce 100644
--- a/python/pyarrow/tests/test_pandas.py
+++ b/python/pyarrow/tests/test_pandas.py
@@ -1733,8 +1733,7 @@ class TestConvertDateTimeLikeTypes:
from pandas.tseries.offsets import DateOffset
df = pd.DataFrame({
'date_offset': [None,
- DateOffset(days=3600, months=3600, microseconds=3,
- nanoseconds=600)]
+ DateOffset(days=3600, months=3600,
nanoseconds=3600)]
})
schema = pa.schema([('date_offset', pa.month_day_nano_interval())])
_check_pandas_roundtrip(
