This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 65b7d24736c GH-49826: [Python] Return NotImplemented from Scalar/Array
arithmetic dunders for unsupported types (#49845)
65b7d24736c is described below
commit 65b7d24736ce9000f2ab6c2fa45112ee50c26b61
Author: Alexandros Anastasiou <[email protected]>
AuthorDate: Wed Sep 30 10:57:57 2026 +0200
GH-49826: [Python] Return NotImplemented from Scalar/Array arithmetic
dunders for unsupported types (#49845)
### Rationale for this change
In pyarrow 24.0.0, `Scalar` and `Array` gained arithmetic dunder methods
(#32007) that unconditionally dispatch to `pyarrow.compute.call_function`. When
the other operand is an unrecognized type, `_pack_compute_args` raises
`TypeError` instead of returning `NotImplemented`. This prevents Python from
falling back to the other operand's reflected methods (`__radd__`, `__rsub__`,
etc.), breaking downstream libraries that relied on this protocol.
### What changes are included in this PR?
- Adds a `_compute_binary_op` helper in `scalar.pxi` that wraps
`call_function` in a `try/except TypeError` and returns `NotImplemented` on
failure. This follows the same pattern already used by `Scalar.__eq__` and
`Array.__eq__`.
- Updates all binary arithmetic and bitwise dunders on both `Scalar`
(`scalar.pxi`) and `Array` (`array.pxi`) to use this helper.
- Adds parametrized tests in `test_scalars.py` and `test_array.py`
verifying that reflected operators on custom types are correctly invoked.
### Are these changes tested?
Yes. New parametrized tests
(`test_dunders_return_notimplemented_for_unknown_types`) cover all 10 binary
operators for both `Scalar` and `Array`. The bug was also manually reproduced
against pyarrow 24.0.0 to confirm the tests exercise the right code path.
### Are there any user-facing changes?
Yes. `Scalar` and `Array` arithmetic dunders now return `NotImplemented`
instead of raising `TypeError` when the other operand is not a recognized
Arrow/NumPy type. This restores the pre-24.0.0 behavior where Python would fall
back to the other operand's reflected method.
### AI-generated code disclosure
This PR was developed with assistance from an AI coding tool (Claude,
Anthropic). All changes have been reviewed, understood, and verified.
* GitHub Issue: #49826
Closes #49826
Lead-authored-by: AlenkaF <[email protected]>
Co-authored-by: Alexandros Anastasiou <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
python/pyarrow/array.pxi | 33 +++++++++++++++++++++++----------
python/pyarrow/scalar.pxi | 20 ++++++++++----------
python/pyarrow/tests/test_array.py | 36 ++++++++++++++++++++++++++++++++++++
python/pyarrow/tests/test_scalars.py | 36 ++++++++++++++++++++++++++++++++++++
4 files changed, 105 insertions(+), 20 deletions(-)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 865d2079fc0..223b77dd4fa 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -1124,6 +1124,19 @@ cdef PandasOptions _convert_pandas_options(dict options):
return result
+def _compute_binary_op(func_name, left, right):
+ """
+ Helper for arithmetic/bitwise dunder methods.
+
+ Only use for ops that can't raise ArrowTypeError as it
+ subclasses TypeError, so will get swallowed.
+ """
+ try:
+ return _pc().call_function(func_name, [left, right])
+ except TypeError:
+ return NotImplemented
+
+
cdef class Array(_PandasConvertible):
"""
The base class for all Arrow arrays.
@@ -2445,15 +2458,15 @@ cdef class Array(_PandasConvertible):
def __add__(self, object other):
self._assert_cpu()
- return _pc().call_function('add_checked', [self, other])
+ return _compute_binary_op('add_checked', self, other)
def __truediv__(self, object other):
self._assert_cpu()
- return _pc().call_function('divide_checked', [self, other])
+ return _compute_binary_op('divide_checked', self, other)
def __mul__(self, object other):
self._assert_cpu()
- return _pc().call_function('multiply_checked', [self, other])
+ return _compute_binary_op('multiply_checked', self, other)
def __neg__(self):
self._assert_cpu()
@@ -2461,31 +2474,31 @@ cdef class Array(_PandasConvertible):
def __pow__(self, object other):
self._assert_cpu()
- return _pc().call_function('power_checked', [self, other])
+ return _compute_binary_op('power_checked', self, other)
def __sub__(self, object other):
self._assert_cpu()
- return _pc().call_function('subtract_checked', [self, other])
+ return _compute_binary_op('subtract_checked', self, other)
def __and__(self, object other):
self._assert_cpu()
- return _pc().call_function('bit_wise_and', [self, other])
+ return _compute_binary_op('bit_wise_and', self, other)
def __or__(self, object other):
self._assert_cpu()
- return _pc().call_function('bit_wise_or', [self, other])
+ return _compute_binary_op('bit_wise_or', self, other)
def __xor__(self, object other):
self._assert_cpu()
- return _pc().call_function('bit_wise_xor', [self, other])
+ return _compute_binary_op('bit_wise_xor', self, other)
def __lshift__(self, object other):
self._assert_cpu()
- return _pc().call_function('shift_left_checked', [self, other])
+ return _compute_binary_op('shift_left_checked', self, other)
def __rshift__(self, object other):
self._assert_cpu()
- return _pc().call_function('shift_right_checked', [self, other])
+ return _compute_binary_op('shift_right_checked', self, other)
cdef _array_like_to_pandas(obj, options, types_mapper):
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 863ab2b66a8..f96fb40572b 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -199,37 +199,37 @@ cdef class Scalar(_Weakrefable):
return _pc().call_function('abs_checked', [self])
def __add__(self, object other):
- return _pc().call_function('add_checked', [self, other])
+ return _compute_binary_op('add_checked', self, other)
def __truediv__(self, object other):
- return _pc().call_function('divide_checked', [self, other])
+ return _compute_binary_op('divide_checked', self, other)
def __mul__(self, object other):
- return _pc().call_function('multiply_checked', [self, other])
+ return _compute_binary_op('multiply_checked', self, other)
def __neg__(self):
return _pc().call_function('negate_checked', [self])
def __pow__(self, object other):
- return _pc().call_function('power_checked', [self, other])
+ return _compute_binary_op('power_checked', self, other)
def __sub__(self, object other):
- return _pc().call_function('subtract_checked', [self, other])
+ return _compute_binary_op('subtract_checked', self, other)
def __and__(self, object other):
- return _pc().call_function('bit_wise_and', [self, other])
+ return _compute_binary_op('bit_wise_and', self, other)
def __or__(self, object other):
- return _pc().call_function('bit_wise_or', [self, other])
+ return _compute_binary_op('bit_wise_or', self, other)
def __xor__(self, object other):
- return _pc().call_function('bit_wise_xor', [self, other])
+ return _compute_binary_op('bit_wise_xor', self, other)
def __lshift__(self, object other):
- return _pc().call_function('shift_left_checked', [self, other])
+ return _compute_binary_op('shift_left_checked', self, other)
def __rshift__(self, object other):
- return _pc().call_function('shift_right_checked', [self, other])
+ return _compute_binary_op('shift_right_checked', self, other)
_NULL = NA = None
diff --git a/python/pyarrow/tests/test_array.py
b/python/pyarrow/tests/test_array.py
index eb81a4409e8..87040c0800f 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -21,6 +21,7 @@ import decimal
import hypothesis as h
import hypothesis.strategies as st
import itertools
+import operator
import pytest
import struct
import subprocess
@@ -4669,3 +4670,38 @@ def test_dictionary_uint64_index_to_pandas():
result = arr.to_pandas()
assert list(result.cat.categories) == ["a", "b"]
assert result.cat.codes.tolist() == [0, 1, -1, 0]
+
+
[email protected]("op", [
+ operator.add,
+ operator.sub,
+ operator.mul,
+ operator.truediv,
+ operator.pow,
+ operator.and_,
+ operator.or_,
+ operator.xor,
+ operator.lshift,
+ operator.rshift,
+])
+def test_arithmetic_dunders_unknown_types(op):
+ # GH-49826
+ class MyObj:
+ def __radd__(self, other):
+ return "reflected"
+
+ __rsub__ = __rmul__ = __rtruediv__ = __rpow__ = __radd__
+ __rand__ = __ror__ = __rxor__ = __rlshift__ = __rrshift__ = __radd__
+
+ assert op(pa.array([1, 2, 3]), MyObj()) == "reflected"
+
+ # If NotImplemented is returned for both sides of the operation
+ # Python will fallback to a TypeError
+ with pytest.raises(TypeError, match="unsupported operand type\\(s\\)"):
+ op(pa.array([1, 2, 3]), object())
+
+
+def test_arithmetic_dunder_raises_arrow_invalid():
+ # GH-49826
+ with pytest.raises(pa.ArrowInvalid, match="divide by zero"):
+ pa.array([1, 2, 3]) / pa.scalar(0)
diff --git a/python/pyarrow/tests/test_scalars.py
b/python/pyarrow/tests/test_scalars.py
index 08f9fcd55ce..533a0585052 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -17,6 +17,7 @@
import datetime
import decimal
+import operator
import pytest
import weakref
from collections.abc import Sequence, Mapping
@@ -1051,3 +1052,38 @@ def test_dunders_checked_overflow():
scl ** scl
with pytest.raises(pa.ArrowInvalid, match=error_match):
scl * scl
+
+
[email protected]("op", [
+ operator.add,
+ operator.sub,
+ operator.mul,
+ operator.truediv,
+ operator.pow,
+ operator.and_,
+ operator.or_,
+ operator.xor,
+ operator.lshift,
+ operator.rshift,
+])
+def test_arithmetic_dunders_unknown_types(op):
+ # GH-49826
+ class MyObj:
+ def __radd__(self, other):
+ return "reflected"
+
+ __rsub__ = __rmul__ = __rtruediv__ = __rpow__ = __radd__
+ __rand__ = __ror__ = __rxor__ = __rlshift__ = __rrshift__ = __radd__
+
+ assert op(pa.scalar(5), MyObj()) == "reflected"
+
+ # If NotImplemented is returned for both sides of the operation
+ # Python will fallback to a TypeError
+ with pytest.raises(TypeError, match="unsupported operand type\\(s\\)"):
+ op(pa.scalar(1), object())
+
+
+def test_arithmetic_dunder_raises_arrow_invalid():
+ # GH-49826
+ with pytest.raises(pa.ArrowInvalid, match="divide by zero"):
+ pa.scalar(1) / pa.scalar(0)
