This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new b60d43701b GH-48473: [CI][Python] Require numpy 2.0 (#50769)
b60d43701b is described below
commit b60d43701bae097566bda6b96cd514690ca9dc9d
Author: Ádám Lippai <[email protected]>
AuthorDate: Fri Aug 21 13:16:28 2026 -0400
GH-48473: [CI][Python] Require numpy 2.0 (#50769)
### Rationale for this change
Require Numpy 2.0 to support newer features, allow newer numpy API/ABI
### What changes are included in this PR?
- Numpy 2.0+ requirement
- Pandas 2.2.2+ requirement (first wheel compatible with numpy 2.0)
- Pyodide 0.27.1 usage (numpy 2.0 api)
### Are there any user-facing changes?
Yes, in contrast to pyarrow 25, after merging this PR pyarrow 26 can't be
used with Numpy 1.x or pandas 2.0-2.2.1
**This PR includes breaking changes to public APIs.**
This PR was created using GPT-5.6-Sol-xhigh, every line read & reviewed by
me.
* GitHub Issue: #48473
Lead-authored-by: Adam Lippai <[email protected]>
Co-authored-by: Ádám Lippai <[email protected]>
Co-authored-by: Raúl Cumplido <[email protected]>
Co-authored-by: Joris Van den Bossche <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
.github/workflows/python.yml | 10 ++---
ci/conda_env_python.txt | 2 +-
ci/docker/conda-python-emscripten.dockerfile | 2 +-
compose.yaml | 2 +-
cpp/src/arrow/util/float16_test.cc | 4 +-
dev/tasks/tasks.yml | 4 +-
docs/source/python/extending_types.rst | 4 +-
docs/source/python/install.rst | 4 +-
python/CMakeLists.txt | 15 +++-----
python/benchmarks/common.py | 2 +-
python/pyarrow/array.pxi | 33 +++++++---------
python/pyarrow/feather.py | 5 ---
python/pyarrow/lib.pyx | 4 ++
python/pyarrow/pandas-shim.pxi | 50 +++++--------------------
python/pyarrow/pandas_compat.py | 34 +++++------------
python/pyarrow/src/arrow/python/numpy_interop.h | 25 ++++---------
python/pyarrow/tests/parquet/test_pandas.py | 15 +++-----
python/pyarrow/tests/test_array.py | 22 +++--------
python/pyarrow/tests/test_convert_builtin.py | 2 +-
python/pyarrow/tests/test_dlpack.py | 18 ++-------
python/pyarrow/tests/test_extension_type.py | 19 ++++------
python/pyarrow/tests/test_feather.py | 14 -------
python/pyarrow/tests/test_io.py | 8 +---
python/pyarrow/tests/test_pandas.py | 18 ++++-----
python/pyarrow/tests/test_schema.py | 2 +-
python/pyarrow/tests/test_table.py | 12 +-----
python/pyarrow/tests/test_types.py | 4 +-
python/pyarrow/types.pxi | 5 +--
python/pyproject.toml | 2 +-
python/requirements-build.txt | 2 +-
python/requirements-wheel-test.txt | 4 +-
python/setup.cfg | 1 -
32 files changed, 108 insertions(+), 240 deletions(-)
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 9b35d6f84d..d4710e1158 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -67,7 +67,7 @@ jobs:
name:
- conda-python-docs
- conda-python-3.12-nopandas
- - conda-python-3.11-pandas-2.0.3
+ - conda-python-3.11-pandas-2.2.2-numpy-2.0.2
- conda-python-3.14-pandas-latest
- conda-python-3.13-no-numpy
include:
@@ -79,12 +79,12 @@ jobs:
image: conda-python
title: AMD64 Conda Python 3.12 Without Pandas
python: "3.12"
- - name: conda-python-3.11-pandas-2.0.3
+ - name: conda-python-3.11-pandas-2.2.2-numpy-2.0.2
image: conda-python-pandas
- title: AMD64 Conda Python 3.11 Pandas 2.0.3
+ title: AMD64 Conda Python 3.11 Pandas 2.2.2 NumPy 2.0.2
python: "3.11"
- pandas: "2.0.3"
- numpy: "1.23.2"
+ pandas: "2.2.2"
+ numpy: "2.0.2"
- name: conda-python-3.14-pandas-latest
image: conda-python-pandas
title: AMD64 Conda Python 3.14 Pandas latest
diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
index a0dca4eac1..4602446043 100644
--- a/ci/conda_env_python.txt
+++ b/ci/conda_env_python.txt
@@ -25,7 +25,7 @@ cloudpickle
fsspec
hypothesis
libcst>=1.8.6
-numpy>=1.16.6
+numpy>=2.0
pytest
pytest-faulthandler
s3fs>=2023.10.0
diff --git a/ci/docker/conda-python-emscripten.dockerfile
b/ci/docker/conda-python-emscripten.dockerfile
index 5154d8c419..e5fb9ff238 100644
--- a/ci/docker/conda-python-emscripten.dockerfile
+++ b/ci/docker/conda-python-emscripten.dockerfile
@@ -22,7 +22,7 @@ ARG python="3.12"
FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
ARG selenium_version="4.41.0"
-ARG pyodide_version="0.26.0"
+ARG pyodide_version="0.27.1"
ARG chrome_version="latest"
ARG required_python_min="(3,12)"
# fail if python version < 3.12
diff --git a/compose.yaml b/compose.yaml
index dd42cde609..6fa4e6fee5 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -981,7 +981,7 @@ services:
arch_short: ${ARCH_SHORT}
clang_tools: ${CLANG_TOOLS}
llvm: ${LLVM}
- pyodide_version: "0.26.0"
+ pyodide_version: "0.27.1"
chrome_version: "latest"
selenium_version: "4.41.0"
required_python_min: "(3,12)"
diff --git a/cpp/src/arrow/util/float16_test.cc
b/cpp/src/arrow/util/float16_test.cc
index 284bf71883..bdf168ed39 100644
--- a/cpp/src/arrow/util/float16_test.cc
+++ b/cpp/src/arrow/util/float16_test.cc
@@ -109,7 +109,7 @@ class Float16ConversionTest : public ::testing::Test {
template <>
void Float16ConversionTest<float>::TestRoundTrip() {
- // Expected values were also manually validated with numpy-1.24.3
+ // Expected values were also manually validated with NumPy 1.24.3
const RoundTripTestCase test_cases[] = {
// +/-0.0f
{F32(0x80000000u), 0b1000000000000000u, -0.0f},
@@ -148,7 +148,7 @@ void Float16ConversionTest<float>::TestRoundTrip() {
template <>
void Float16ConversionTest<double>::TestRoundTrip() {
- // Expected values were also manually validated with numpy-1.24.3
+ // Expected values were also manually validated with NumPy 1.24.3
const RoundTripTestCase test_cases[] = {
// +/-0.0
{F64(0x8000000000000000u), 0b1000000000000000u, -0.0},
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index a2230c2fba..523e9a1fbf 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -730,9 +730,9 @@ tasks:
############################## Integration tests ############################
-{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11",
"2.0.3", "1.23.2", True),
+{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11",
"2.2.2", "2.0.2", True),
("3.12",
"latest", "latest", False),
- ("3.13",
"latest", "1.26.2", False),
+ ("3.13",
"latest", "2.1.3", False),
("3.13",
"latest", "latest", False),
("3.14",
"nightly", "nightly", False),
("3.14",
"upstream_devel", "nightly", False)] %}
diff --git a/docs/source/python/extending_types.rst
b/docs/source/python/extending_types.rst
index fec04c182a..c38d27f937 100644
--- a/docs/source/python/extending_types.rst
+++ b/docs/source/python/extending_types.rst
@@ -328,9 +328,9 @@ a built-in :class:`ExtensionArray` object. Nevertheless,
one could want to subcl
type. Arrow allows to do so by adding a special method ``__arrow_ext_class__``
to the
definition of the extension type.
-For instance, let us consider the example from the `Numpy Quickstart
<https://docs.scipy.org/doc/numpy-1.13.0/user/quickstart.html>`_ of points in
3D space.
+For instance, let us consider the example from the `NumPy Quickstart
<https://numpy.org/doc/stable/user/quickstart.html>`_ of points in 3D space.
We can store these as a fixed-size list, where we wish to be able to extract
-the data as a 2-D Numpy array ``(N, 3)`` without any copy:
+the data as a 2-D NumPy array ``(N, 3)`` without any copy:
.. code-block:: python
diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst
index d076ca9643..b57c462f99 100644
--- a/docs/source/python/install.rst
+++ b/docs/source/python/install.rst
@@ -73,8 +73,8 @@ Dependencies
Optional dependencies
-* **NumPy 1.23.2** or higher.
-* **pandas 2.0.3** or higher,
+* **NumPy 2.0** or higher.
+* **pandas 2.2.2** or higher,
* **cffi**.
Additional packages PyArrow is compatible with are :ref:`fsspec
<filesystem-fsspec>`
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 408153bb43..4bea6e7d8d 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -167,16 +167,7 @@ if($ENV{PYODIDE})
set(Python3_INCLUDE_DIR $ENV{PYTHONINCLUDE})
set(Python3_LIBRARY $ENV{CPYTHONLIB})
set(Python3_EXECUTABLE)
- execute_process(COMMAND ${Python3_EXECUTABLE} -c
- "import numpy; print(numpy.__version__)"
- OUTPUT_VARIABLE PYODIDE_NUMPY_VERSION
- OUTPUT_STRIP_TRAILING_WHITESPACE)
- string(REGEX MATCH "^([0-9]+)" PYODIDE_NUMPY_MAJOR_VERSION
${PYODIDE_NUMPY_VERSION})
- if(PYODIDE_NUMPY_MAJOR_VERSION GREATER_EQUAL 2)
- set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/_core/include)
- else()
- set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/core/include)
- endif()
+ set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/_core/include)
set(ENV{_PYTHON_SYSCONFIGDATA_NAME} $ENV{SYSCONFIG_NAME})
# we set the c and cxx compiler manually to bypass pywasmcross
# which is pyodide's way of messing with C++ build parameters.
@@ -278,6 +269,10 @@ set(EXECUTABLE_OUTPUT_PATH
"${BUILD_OUTPUT_ROOT_DIRECTORY}")
# Python and Numpy libraries
find_package(Python3Alt REQUIRED)
+if(Python3_NumPy_VERSION AND Python3_NumPy_VERSION VERSION_LESS "2.0")
+ message(FATAL_ERROR "PyArrow requires NumPy 2.0 or newer, found
${Python3_NumPy_VERSION}"
+ )
+endif()
message(STATUS "Found NumPy version: ${Python3_NumPy_VERSION}")
message(STATUS "NumPy include dir: ${NUMPY_INCLUDE_DIRS}")
diff --git a/python/benchmarks/common.py b/python/benchmarks/common.py
index 8317ff3171..4e27b22295 100644
--- a/python/benchmarks/common.py
+++ b/python/benchmarks/common.py
@@ -149,7 +149,7 @@ class BuiltinsGenerator(object):
Generate a list of Python bools with *none_prob* probability of
an entry being None.
"""
- # Make sure we get Python bools, not np.bool_
+ # Make sure we get Python bools, not np.bool
data = [bool(x >= 0.5) for x in self.rnd.uniform(0.0, 1.0, n)]
assert len(data) == n
self.sprinkle_nones(data, none_prob)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index d8bbd001fd..e31feb1cb0 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -342,7 +342,7 @@ def array(object obj, type=None, mask=None, size=None,
from_pandas=None,
values = values.data
if mask is not None:
- if mask.dtype != np.bool_:
+ if mask.dtype != np.bool:
raise TypeError("Mask must be boolean dtype")
if mask.ndim != 1:
raise ValueError("Mask must be 1D array")
@@ -945,14 +945,12 @@ cdef class _PandasConvertible(_Weakrefable):
Cast integers with nulls to objects
date_as_object : bool, default True
Cast dates to objects. If False, convert to datetime64 dtype with
- the equivalent time unit (if supported). Note: in pandas version
- < 2.0, only datetime64[ns] conversion is supported.
+ the equivalent time unit (if supported).
timestamp_as_object : bool, default False
- Cast non-nanosecond timestamps (np.datetime64) to objects. This is
- useful in pandas version 1.x if you have timestamps that don't fit
- in the normal date range of nanosecond timestamps (1678 CE-2262
CE).
- Non-nanosecond timestamps are supported in pandas version 2.0.
- If False, all timestamps are converted to datetime64 dtype.
+ Cast non-nanosecond timestamps (np.datetime64) to objects. This can
+ be useful when Python datetime objects are required, such as for
+ compatibility with code expecting object dtype. If False, all
+ timestamps are converted to datetime64 dtype.
use_threads : bool, default True
Whether to parallelize the conversion using multiple threads.
deduplicate_objects : bool, default True
@@ -963,9 +961,8 @@ cdef class _PandasConvertible(_Weakrefable):
DataFrame index, if present
safe : bool, default True
For certain data types, a cast is needed in order to store the
- data in a pandas DataFrame or Series (e.g. timestamps are always
- stored as nanoseconds in pandas). This option controls whether it
- is a safe cast or not.
+ data in a pandas DataFrame or Series. This option controls whether
+ it is a safe cast or not.
split_blocks : bool, default False
If True, generate one internal "block" for each column when
creating a pandas.DataFrame from a RecordBatch or Table. While this
@@ -1002,12 +999,10 @@ cdef class _PandasConvertible(_Weakrefable):
default conversion should be used for that type. If you have
a dictionary mapping, you can pass ``dict.get`` as function.
coerce_temporal_nanoseconds : bool, default False
- Only applicable to pandas version >= 2.0.
A legacy option to coerce date32, date64, duration, and timestamp
- time units to nanoseconds when converting to pandas. This is the
- default behavior in pandas version 1.x. Set this option to True if
- you'd like to use this coercion when using pandas version >= 2.0
- for backwards compatibility (not recommended otherwise).
+ time units to nanoseconds when converting to pandas. Set this
+ option only if nanosecond coercion is required for compatibility
+ with older application behavior.
Returns
-------
@@ -5130,7 +5125,7 @@ cdef class Bool8Array(ExtensionArray):
"""
if not writable:
try:
- return self.storage.to_numpy().view(np.bool_)
+ return self.storage.to_numpy().view(np.bool)
except ArrowInvalid as e:
if zero_copy_only:
raise e
@@ -5171,7 +5166,7 @@ cdef class Bool8Array(ExtensionArray):
--------
>>> import pyarrow as pa
>>> import numpy as np
- >>> arr = np.array([True, False, True], dtype=np.bool_)
+ >>> arr = np.array([True, False, True], dtype=np.bool)
>>> pa.Bool8Array.from_numpy(arr)
<pyarrow.lib.Bool8Array object at ...>
[
@@ -5184,7 +5179,7 @@ cdef class Bool8Array(ExtensionArray):
if obj.ndim != 1:
raise ValueError(f"Cannot convert {obj.ndim}-D array to bool8
array")
- if obj.dtype not in [np.bool_, np.int8]:
+ if obj.dtype not in [np.bool, np.int8]:
raise TypeError(f"Array dtype {obj.dtype} incompatible with bool8
storage")
storage_arr = array(obj.view(np.int8), type=int8())
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 60d59b0e0b..effe30ba9b 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -150,11 +150,6 @@ def write_feather(df, dest, compression=None,
compression_level=None,
DeprecationWarning,
stacklevel=2
)
- if _pandas_api.have_pandas:
- if (_pandas_api.has_sparse and
- isinstance(df, _pandas_api.pd.SparseDataFrame)):
- df = df.to_dense()
-
if _pandas_api.is_data_frame(df):
# Feather v1 creates a new column in the resultant Table to
# store index information if index type is not RangeIndex
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 7e97177a6e..d949720ca8 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -38,6 +38,10 @@ cimport cpython as cp
# Initialize NumPy C API only if numpy was able to be imported
if np is not None:
+ if int(np.__version__.partition('.')[0]) < 2:
+ raise ImportError(
+ f"pyarrow requires NumPy 2.0 or newer, found {np.__version__}"
+ )
arrow_init_numpy()
# Initialize PyArrow C++ API
diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi
index 94802da6b0..603b2b09d4 100644
--- a/python/pyarrow/pandas-shim.pxi
+++ b/python/pyarrow/pandas-shim.pxi
@@ -35,10 +35,8 @@ cdef class _PandasAPIShim(object):
object _pd, _types_api, _compat_module
object _data_frame, _index, _series, _categorical_type
object _datetimetz_type, _extension_array, _extension_dtype
- object _array_like_types, _is_extension_array_dtype, _lock
- bint has_sparse
- bint _pd024
- bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
+ object _array_like_types, _lock
+ bint _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
def __init__(self):
self._lock = Lock()
@@ -62,23 +60,22 @@ cdef class _PandasAPIShim(object):
self._version = pd.__version__
self._loose_version = Version(pd.__version__)
- if self._loose_version < Version('2.0.3'):
+ if self._loose_version < Version('2.2.2'):
self._have_pandas = False
if raise_:
raise ImportError(
- f"pyarrow requires pandas 2.0.3 or above, pandas
{self._version} is "
+ f"pyarrow requires pandas 2.2.2 or above, pandas
{self._version} is "
"installed"
)
else:
warnings.warn(
- f"pyarrow requires pandas 2.0.3 or above, pandas
{self._version} is "
+ f"pyarrow requires pandas 2.2.2 or above, pandas
{self._version} is "
"installed. Therefore, pandas-specific integration is not "
"used.",
stacklevel=2
)
return
- self._is_ge_v21 = self._loose_version >= Version('2.1.0')
self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0')
self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0')
self._is_ge_v3_strict = self._loose_version >= Version('3.0.0')
@@ -93,12 +90,9 @@ cdef class _PandasAPIShim(object):
self._series, self._index, self._categorical_type,
self._extension_array)
self._extension_dtype = pd.api.extensions.ExtensionDtype
- self._is_extension_array_dtype = (
- pd.api.types.is_extension_array_dtype)
self._types_api = pd.api.types
self._datetimetz_type = pd.api.types.DatetimeTZDtype
self._have_pandas = True
- self.has_sparse = False
cdef inline _check_import(self, bint raise_=True):
if not self._tried_importing_pandas:
@@ -142,17 +136,11 @@ cdef class _PandasAPIShim(object):
cpdef infer_dtype(self, obj):
self._check_import()
- try:
- return self._types_api.infer_dtype(obj, skipna=False)
- except AttributeError:
- return self._pd.lib.infer_dtype(obj)
+ return self._types_api.infer_dtype(obj, skipna=False)
cpdef pandas_dtype(self, dtype):
self._check_import()
- try:
- return self._types_api.pandas_dtype(dtype)
- except AttributeError:
- return None
+ return self._types_api.pandas_dtype(dtype)
@property
def loose_version(self):
@@ -164,10 +152,6 @@ cdef class _PandasAPIShim(object):
self._check_import()
return self._version
- def is_ge_v21(self):
- self._check_import()
- return self._is_ge_v21
-
def is_ge_v23(self):
self._check_import()
return self._is_ge_v23
@@ -183,12 +167,7 @@ cdef class _PandasAPIShim(object):
def uses_string_dtype(self):
if self.is_ge_v3_strict():
return True
- try:
- if self.is_ge_v23() and self.pd.options.future.infer_string:
- return True
- except:
- pass
- return False
+ return self.is_ge_v23() and self.pd.options.future.infer_string
@property
def categorical_type(self):
@@ -223,10 +202,7 @@ cdef class _PandasAPIShim(object):
cpdef is_extension_array_dtype(self, obj):
self._check_import()
- if self._is_extension_array_dtype:
- return self._is_extension_array_dtype(obj)
- else:
- return False
+ return self._types_api.is_extension_array_dtype(obj)
cpdef is_sparse(self, obj):
if self._have_pandas_internal():
@@ -265,13 +241,5 @@ cdef class _PandasAPIShim(object):
return obj.array
return obj.values
- def get_rangeindex_attribute(self, level, name):
- # public start/stop/step attributes added in pandas 0.25.0
- self._check_import()
- if hasattr(level, name):
- return getattr(level, name)
- return getattr(level, '_' + name)
-
-
cdef _PandasAPIShim pandas_api = _PandasAPIShim()
_pandas_api = pandas_api
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index ccb89fc05d..36d336d513 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -93,7 +93,7 @@ def get_numpy_logical_type_map():
global _numpy_logical_type_map
if not _numpy_logical_type_map:
_numpy_logical_type_map.update({
- np.bool_: 'bool',
+ np.bool: 'bool',
np.int8: 'int8',
np.int16: 'int16',
np.int32: 'int32',
@@ -277,7 +277,7 @@ def construct_metadata(columns_to_convert, df,
column_names, index_levels,
else:
index_descriptors = index_column_metadata = column_indexes = []
- attributes = df.attrs if hasattr(df, "attrs") else {}
+ attributes = df.attrs
try:
json.dumps(attributes)
@@ -537,13 +537,12 @@ def _level_name(name):
def _get_range_index_descriptor(level):
- # public start/stop/step attributes added in pandas 0.25.0
return {
'kind': 'range',
'name': _level_name(level.name),
- 'start': _pandas_api.get_rangeindex_attribute(level, 'start'),
- 'stop': _pandas_api.get_rangeindex_attribute(level, 'stop'),
- 'step': _pandas_api.get_rangeindex_attribute(level, 'step')
+ 'start': level.start,
+ 'stop': level.stop,
+ 'step': level.step
}
@@ -759,17 +758,9 @@ def _reconstruct_block(item, columns=None,
extension_columns=None, return_block=
elif 'timezone' in item:
unit, _ = np.datetime_data(block_arr.dtype)
dtype = make_datetimetz(unit, item['timezone'])
- if _pandas_api.is_ge_v21():
- arr = _pandas_api.pd.array(
- block_arr.view("int64"), dtype=dtype, copy=False
- )
- else:
- arr = block_arr
- if return_block:
- block = _int.make_block(block_arr, placement=placement,
- klass=_int.DatetimeTZBlock,
- dtype=dtype)
- return block
+ arr = _pandas_api.pd.array(
+ block_arr.view("int64"), dtype=dtype, copy=False
+ )
elif 'py_array' in item:
# create ExtensionBlock
arr = item['py_array']
@@ -846,10 +837,7 @@ def table_to_dataframe(
]
axes = [columns, index]
mgr = BlockManager(blocks, axes)
- if _pandas_api.is_ge_v21():
- df = DataFrame._from_mgr(mgr, mgr.axes)
- else:
- df = DataFrame(mgr)
+ df = DataFrame._from_mgr(mgr, mgr.axes)
df.attrs = attributes
@@ -884,10 +872,6 @@ def _get_extension_dtypes(table, columns_metadata,
types_mapper, options, catego
ext_columns = {}
- # older pandas version that does not yet support extension dtypes
- if _pandas_api.extension_dtype is None:
- return ext_columns
-
# use the specified mapping of built-in arrow types to pandas dtypes
if types_mapper:
for field in table.schema:
diff --git a/python/pyarrow/src/arrow/python/numpy_interop.h
b/python/pyarrow/src/arrow/python/numpy_interop.h
index a83ae4a62b..40e02d5bec 100644
--- a/python/pyarrow/src/arrow/python/numpy_interop.h
+++ b/python/pyarrow/src/arrow/python/numpy_interop.h
@@ -19,16 +19,14 @@
#include "arrow/python/platform.h" // IWYU pragma: export
+// Require the NumPy 2.0 C API and hide deprecated APIs.
+#define NPY_TARGET_VERSION NPY_2_0_API_VERSION
+#define NPY_NO_DEPRECATED_API NPY_2_0_API_VERSION
+
#include <numpy/numpyconfig.h> // IWYU pragma: export
-// Don't use the deprecated Numpy functions
-#ifdef NPY_1_7_API_VERSION
-# define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-#else
-# define NPY_ARRAY_NOTSWAPPED NPY_NOTSWAPPED
-# define NPY_ARRAY_ALIGNED NPY_ALIGNED
-# define NPY_ARRAY_WRITEABLE NPY_WRITEABLE
-# define NPY_ARRAY_UPDATEIFCOPY NPY_UPDATEIFCOPY
+#if NPY_ABI_VERSION < 0x02000000
+# error "PyArrow requires NumPy 2.0 or newer"
#endif
// This is required to be able to access the NumPy C API properly in C++ files
@@ -67,20 +65,13 @@
# define NPY_INT32_IS_INT 0
#endif
-// Backported NumPy 2 API (can be removed if numpy 2 is required)
-#if NPY_ABI_VERSION < 0x02000000
-# define PyDataType_ELSIZE(descr) ((descr)->elsize)
-# define PyDataType_C_METADATA(descr) ((descr)->c_metadata)
-# define PyDataType_FIELDS(descr) ((descr)->fields)
-#endif
-
namespace arrow {
namespace py {
inline int import_numpy() {
#ifdef NUMPY_IMPORT_ARRAY
- import_array1(-1);
- import_umath1(-1);
+ if (PyArray_ImportNumPyAPI() < 0) return -1;
+ if (PyUFunc_ImportUFuncAPI() < 0) return -1;
#endif
return 0;
diff --git a/python/pyarrow/tests/parquet/test_pandas.py
b/python/pyarrow/tests/parquet/test_pandas.py
index d12b43f4e0..f919770ee9 100644
--- a/python/pyarrow/tests/parquet/test_pandas.py
+++ b/python/pyarrow/tests/parquet/test_pandas.py
@@ -27,7 +27,6 @@ import pytest
import pyarrow as pa
from pyarrow.fs import LocalFileSystem, SubTreeFileSystem
from pyarrow.util import guid
-from pyarrow.vendored.version import Version
try:
import pyarrow.parquet as pq
@@ -430,22 +429,20 @@ carat cut color clarity depth table price
x y z
@pytest.mark.pandas
def test_backwards_compatible_column_metadata_handling(datadir):
- if Version("2.2.0") <= Version(pd.__version__):
- # TODO: regression in pandas
- # https://github.com/pandas-dev/pandas/issues/56775
- pytest.skip("Regression in pandas 2.2.0")
+ dates = pd.date_range(
+ "2017-01-01", periods=3, tz='Europe/Brussels'
+ ).as_unit("ns")
expected = pd.DataFrame(
{'a': [1, 2, 3], 'b': [.1, .2, .3],
- 'c': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
+ 'c': dates})
expected.index = pd.MultiIndex.from_arrays(
- [['a', 'b', 'c'],
- pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')],
+ [['a', 'b', 'c'], dates],
names=['index', None])
path = datadir / 'v0.7.1.column-metadata-handling.parquet'
table = _read_table(path)
result = table.to_pandas()
- tm.assert_frame_equal(result, expected)
+ tm.assert_frame_equal(result, expected, check_freq=False)
table = _read_table(
path, columns=['a'])
diff --git a/python/pyarrow/tests/test_array.py
b/python/pyarrow/tests/test_array.py
index bc4e521dca..a1e3616c9c 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -34,7 +34,6 @@ except ImportError:
import pyarrow as pa
import pyarrow.tests.strategies as past
-from pyarrow.vendored.version import Version
import pyarrow.compute as pc
@@ -3800,21 +3799,12 @@ def test_numpy_array_protocol():
result = np.asarray(arr)
np.testing.assert_array_equal(result, expected)
- if Version(np.__version__) < Version("2.0.0.dev0"):
- # copy keyword is not strict and not passed down to __array__
- result = np.array(arr, copy=False)
- np.testing.assert_array_equal(result, expected)
+ with pytest.raises(ValueError, match="Unable to avoid a copy"):
+ np.array(arr, copy=False)
- result = np.array(arr, dtype="float64", copy=False)
- np.testing.assert_array_equal(result, expected)
- else:
- # starting with numpy 2.0, the copy=False keyword is assumed to be
strict
- with pytest.raises(ValueError, match="Unable to avoid a copy"):
- np.array(arr, copy=False)
-
- arr = pa.array([1, 2, 3])
- with pytest.raises(ValueError):
- np.array(arr, dtype="float64", copy=False)
+ arr = pa.array([1, 2, 3])
+ with pytest.raises(ValueError):
+ np.array(arr, dtype="float64", copy=False)
# copy=True -> not yet passed by numpy, so we have to call this directly
to test
arr = pa.array([1, 2, 3])
@@ -4436,7 +4426,7 @@ def test_non_cpu_array():
ctx = cuda.Context(0)
data = np.arange(4, dtype=np.int32)
- validity = np.array([True, False, True, False], dtype=np.bool_)
+ validity = np.array([True, False, True, False], dtype=np.bool)
cuda_data_buf = ctx.buffer_from_data(data)
cuda_validity_buf = ctx.buffer_from_data(validity)
arr = pa.Array.from_buffers(pa.int32(), 4, [None, cuda_data_buf])
diff --git a/python/pyarrow/tests/test_convert_builtin.py
b/python/pyarrow/tests/test_convert_builtin.py
index bb2813f3b5..3ac7be88ea 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -240,7 +240,7 @@ def test_sequence_boolean(seq):
@pytest.mark.numpy
@parametrize_with_sequence_types
def test_sequence_numpy_boolean(seq):
- expected = [np.bool_(True), None, np.bool_(False), None]
+ expected = [np.bool(True), None, np.bool(False), None]
arr = pa.array(seq(expected))
assert arr.type == pa.bool_()
assert arr.to_pylist() == [True, None, False, None]
diff --git a/python/pyarrow/tests/test_dlpack.py
b/python/pyarrow/tests/test_dlpack.py
index 09a510122f..7f5f98d866 100644
--- a/python/pyarrow/tests/test_dlpack.py
+++ b/python/pyarrow/tests/test_dlpack.py
@@ -92,11 +92,6 @@ def check_bytes_allocated(f):
]
)
def test_dlpack(value_type, np_type_str):
- if Version(np.__version__) < Version("1.24.0"):
- pytest.skip("No dlpack support in numpy versions older than 1.22.0, "
- "strict keyword in assert_array_equal added in numpy
version "
- "1.24.0")
-
expected = np.array([1, 2, 3], dtype=np.dtype(np_type_str))
arr = pa.array(expected, type=value_type)
check_dlpack_export(arr, expected)
@@ -130,11 +125,6 @@ def test_dlpack(value_type, np_type_str):
np.int8, np.int16, np.int32, np.int64,
np.float16, np.float32, np.float64,])
def test_tensor_dlpack(np_type):
- if Version(np.__version__) < Version("1.24.0"):
- pytest.skip("No dlpack support in numpy versions older than 1.22.0, "
- "strict keyword in assert_array_equal added in numpy
version "
- "1.24.0")
-
arr = np.array([1, 2, 3, 4, 5, 6, 1, 1])
expected = np.array(arr, dtype=np_type).reshape((2, 2, 2), order='C')
t = pa.Tensor.from_numpy(expected)
@@ -227,8 +217,9 @@ def test_dlpack_versioned_roundtrip(obj):
@check_bytes_allocated
def test_dlpack_copy_is_writeable():
- if Version(np.__version__) < Version("2.1.0"):
- pytest.skip("Read-only DLPack flag requires numpy 2.1.0 or later")
+ # NumPy did not set the writeable flag on DLPack imports before 2.2.5.
+ if Version(np.__version__) < Version("2.2.5"):
+ pytest.skip("Writable DLPack imports require numpy 2.2.5 or later")
arr = pa.array([1, 2, 3], type=pa.int32())
@@ -244,9 +235,6 @@ def test_dlpack_copy_is_writeable():
def test_dlpack_not_supported():
- if Version(np.__version__) < Version("1.22.0"):
- pytest.skip("No dlpack support in numpy versions older than 1.22.0.")
-
arr = pa.array([1, None, 3])
with pytest.raises(TypeError, match="Can only use DLPack "
"on arrays with no nulls."):
diff --git a/python/pyarrow/tests/test_extension_type.py
b/python/pyarrow/tests/test_extension_type.py
index f98bd4dfe5..bdd898767b 100644
--- a/python/pyarrow/tests/test_extension_type.py
+++ b/python/pyarrow/tests/test_extension_type.py
@@ -31,7 +31,6 @@ except ImportError:
np = None
import pyarrow as pa
-from pyarrow.vendored.version import Version
@contextlib.contextmanager
@@ -1965,13 +1964,9 @@ def
test_extension_to_pandas_storage_type(registered_period_type):
assert result["ext"].dtype == pandas_dtype
import pandas as pd
- # Skip tests for 2.0.x, See: GH-35821
- if (
- Version(pd.__version__) >= Version("2.1.0")
- ):
- # Check the usage of types_mapper
- result = table.to_pandas(types_mapper=pd.ArrowDtype)
- assert isinstance(result["ext"].dtype, pd.ArrowDtype)
+ # Check the usage of types_mapper
+ result = table.to_pandas(types_mapper=pd.ArrowDtype)
+ assert isinstance(result["ext"].dtype, pd.ArrowDtype)
def test_tensor_type_is_picklable(pickle_module):
@@ -2127,7 +2122,7 @@ def test_bool8_to_numpy_conversion():
)
# zero-copy possible with non-null array
- np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool_)
+ np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool)
arr_no_nulls = pa.ExtensionArray.from_storage(
pa.bool8(),
pa.array([-1, 0, 1, 2], pa.int8()),
@@ -2149,7 +2144,7 @@ def test_bool8_to_numpy_conversion():
@pytest.mark.numpy
def test_bool8_from_numpy_conversion():
- np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool_)
+ np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool)
canonical_bool8_arr_no_nulls = pa.ExtensionArray.from_storage(
pa.bool8(),
pa.array([1, 0, 1, 1], pa.int8()),
@@ -2167,14 +2162,14 @@ def test_bool8_from_numpy_conversion():
match="Cannot convert 2-D array to bool8 array",
):
pa.Bool8Array.from_numpy(
- np.array([[True, False], [False, True]], dtype=np.bool_),
+ np.array([[True, False], [False, True]], dtype=np.bool),
)
with pytest.raises(
ValueError,
match="Cannot convert 0-D array to bool8 array",
):
- pa.Bool8Array.from_numpy(np.bool_())
+ pa.Bool8Array.from_numpy(np.bool())
# must use compatible storage type
with pytest.raises(
diff --git a/python/pyarrow/tests/test_feather.py
b/python/pyarrow/tests/test_feather.py
index 8c9e7eb437..d9cee7a31d 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -588,20 +588,6 @@ def test_filelike_objects(version):
assert_frame_equal(result, df)
[email protected]
[email protected]("ignore:Sparse:FutureWarning")
[email protected]("ignore:DataFrame.to_sparse:FutureWarning")
-def test_sparse_dataframe(version):
- if not pa.pandas_compat._pandas_api.has_sparse:
- pytest.skip("version of pandas does not support SparseDataFrame")
- # GH #221
- data = {'A': [0, 1, 2],
- 'B': [1, 0, 1]}
- df = pd.DataFrame(data).to_sparse(fill_value=1)
- expected = df.to_dense()
- _check_pandas_roundtrip(df, expected, version=version)
-
-
@pytest.mark.pandas
def test_duplicate_columns_pandas():
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 3d4ba997b3..8494a0ee66 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -917,16 +917,12 @@ def test_compression_level(compression):
with pytest.raises(ValueError):
codec.decompress(compressed_bytes)
- # The ability to set a seed this way is not present on older versions of
- # numpy (currently in our python 3.6 CI build). Some inputs might just
- # happen to compress the same between the two levels so using seeded
- # random numbers is necessary to help get more reliable results
+ # Some inputs might just happen to compress the same between the two
levels,
+ # so using seeded random numbers makes the results more reliable.
#
# The goal of this part is to ensure the compression_level is being
# passed down to the C++ layer, not to verify the compression algs
# themselves
- if not hasattr(np.random, 'default_rng'):
- pytest.skip('Requires newer version of numpy')
rng = np.random.default_rng(seed=42)
values = rng.integers(0, 100, 1000)
arr = pa.array(values)
diff --git a/python/pyarrow/tests/test_pandas.py
b/python/pyarrow/tests/test_pandas.py
index 0f35edd516..dd20a0aa97 100644
--- a/python/pyarrow/tests/test_pandas.py
+++ b/python/pyarrow/tests/test_pandas.py
@@ -254,13 +254,13 @@ class TestConvertMetadata:
result = table.to_pandas()
tm.assert_frame_equal(result, df)
assert isinstance(result.index, pd.RangeIndex)
- assert _pandas_api.get_rangeindex_attribute(result.index, 'step') == 2
+ assert result.index.step == 2
assert result.index.name == index_name
result2 = table_no_index_name.to_pandas()
tm.assert_frame_equal(result2, df2)
assert isinstance(result2.index, pd.RangeIndex)
- assert _pandas_api.get_rangeindex_attribute(result2.index, 'step') == 1
+ assert result2.index.step == 1
assert result2.index.name is None
def test_range_index_force_serialization(self):
@@ -1686,10 +1686,6 @@ class TestConvertDateTimeLikeTypes:
expected = pd.Series([None, date(1991, 1, 1), None])
assert pa.Array.from_pandas(expected).equals(result)
- @pytest.mark.skipif(
- np is not None and Version('1.16.0') <= Version(
- np.__version__) < Version('1.16.1'),
- reason='Until numpy/numpy#12745 is resolved')
def test_fixed_offset_timezone(self):
df = pd.DataFrame({
'a': [
@@ -2783,7 +2779,7 @@ class TestConvertStructTypes:
def test_from_numpy(self):
dt = np.dtype([('x', np.int32),
- (('y_title', 'y'), np.bool_)])
+ (('y_title', 'y'), np.bool)])
ty = pa.struct([pa.field('x', pa.int32()),
pa.field('y', pa.bool_())])
@@ -2797,7 +2793,7 @@ class TestConvertStructTypes:
{'x': 43, 'y': False}]
# With mask
- arr = pa.array(data, mask=np.bool_([False, True]), type=ty)
+ arr = pa.array(data, mask=np.bool([False, True]), type=ty)
assert arr.to_pylist() == [{'x': 42, 'y': True}, None]
# Trivial struct type
@@ -2815,7 +2811,7 @@ class TestConvertStructTypes:
def test_from_numpy_nested(self):
# Note: an object field inside a struct
dt = np.dtype([('x', np.dtype([('xx', np.int8),
- ('yy', np.bool_)])),
+ ('yy', np.bool)])),
('y', np.int16),
('z', np.object_)])
# Note: itemsize is not necessarily a multiple of sizeof(object)
@@ -2899,7 +2895,7 @@ class TestConvertStructTypes:
ty = pa.struct([pa.field('x', pa.int32()),
pa.field('y', pa.bool_())])
dt = np.dtype([('x', np.int32),
- ('z', np.bool_)])
+ ('z', np.bool)])
data = np.array([], dtype=dt)
with pytest.raises(ValueError,
@@ -3859,7 +3855,7 @@ def test_array_uses_memory_pool():
# ARROW-6570
N = 10000
arr = pa.array(np.arange(N, dtype=np.int64),
- mask=np.random.randint(0, 2, size=N).astype(np.bool_))
+ mask=np.random.randint(0, 2, size=N).astype(np.bool))
# In the case the gc is caught loading
gc.collect()
diff --git a/python/pyarrow/tests/test_schema.py
b/python/pyarrow/tests/test_schema.py
index db44ad3211..27b6419f9a 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -51,7 +51,7 @@ def test_type_to_pandas_dtype():
M8 = np.dtype('datetime64[ms]')
cases = [
(pa.null(), np.object_),
- (pa.bool_(), np.bool_),
+ (pa.bool_(), np.bool),
(pa.int8(), np.int8),
(pa.int16(), np.int16),
(pa.int32(), np.int32),
diff --git a/python/pyarrow/tests/test_table.py
b/python/pyarrow/tests/test_table.py
index cb010f4387..bf6e5773dd 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -28,7 +28,6 @@ import pytest
import pyarrow as pa
import pyarrow.compute as pc
from pyarrow.interchange import from_dataframe
-from pyarrow.vendored.version import Version
def test_chunked_array_basics():
@@ -3556,16 +3555,9 @@ def test_numpy_asarray(constructor):
@pytest.mark.parametrize("constructor", [pa.table, pa.record_batch])
def test_numpy_array_protocol(constructor):
table = constructor([[1, 2, 3], [4.0, 5.0, 6.0]], names=["a", "b"])
- expected = np.array([[1, 4], [2, 5], [3, 6]], dtype="float64")
- if Version(np.__version__) < Version("2.0.0.dev0"):
- # copy keyword is not strict and not passed down to __array__
- result = np.array(table, copy=False)
- np.testing.assert_array_equal(result, expected)
- else:
- # starting with numpy 2.0, the copy=False keyword is assumed to be
strict
- with pytest.raises(ValueError, match="Unable to avoid a copy"):
- np.array(table, copy=False)
+ with pytest.raises(ValueError, match="Unable to avoid a copy"):
+ np.array(table, copy=False)
@pytest.mark.acero
diff --git a/python/pyarrow/tests/test_types.py
b/python/pyarrow/tests/test_types.py
index 9b5c5efe1c..e0d74775e0 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -1350,8 +1350,8 @@ def test_is_boolean_value():
assert pa.types.is_boolean_value(True)
assert pa.types.is_boolean_value(False)
if np is not None:
- assert pa.types.is_boolean_value(np.bool_(True))
- assert pa.types.is_boolean_value(np.bool_(False))
+ assert pa.types.is_boolean_value(np.bool(True))
+ assert pa.types.is_boolean_value(np.bool(False))
@h.settings(suppress_health_check=(h.HealthCheck.too_slow,))
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 8eea34224a..f9530a3436 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -31,8 +31,7 @@ import sys
import warnings
from cython import sizeof
-# These are imprecise because the type (in pandas 0.x) depends on the presence
-# of nulls
+# These are imprecise because the type depends on the presence of nulls
cdef dict _pandas_type_map = {}
@@ -41,7 +40,7 @@ def _get_pandas_type_map():
if not _pandas_type_map:
_pandas_type_map.update({
_Type_NA: np.object_, # NaNs
- _Type_BOOL: np.bool_,
+ _Type_BOOL: np.bool,
_Type_INT8: np.int8,
_Type_INT16: np.int16,
_Type_INT32: np.int32,
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 7b7ba8ba73..68c1a807dd 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -22,7 +22,7 @@ requires = [
# Needed for build-time stub docstring extraction
"libcst>=1.9.0; python_version >= '3.15'",
"libcst>=1.8.6; python_version < '3.15'",
- "numpy>=1.25",
+ "numpy>=2.0",
"setuptools_scm[toml]>=8",
]
build-backend = "scikit_build_core.build"
diff --git a/python/requirements-build.txt b/python/requirements-build.txt
index 65acde0c80..353c042fab 100644
--- a/python/requirements-build.txt
+++ b/python/requirements-build.txt
@@ -2,6 +2,6 @@ build
cython>=3.1
libcst>=1.9.0; python_version >= "3.15"
libcst>=1.8.6; python_version < "3.15"
-numpy>=1.25
+numpy>=2.0
scikit-build-core>=1.0
setuptools_scm>=8
diff --git a/python/requirements-wheel-test.txt
b/python/requirements-wheel-test.txt
index 00c0fcc468..ce75cbc3c0 100644
--- a/python/requirements-wheel-test.txt
+++ b/python/requirements-wheel-test.txt
@@ -12,9 +12,7 @@ tzdata; sys_platform == 'win32'
# version. However, there is no need to make this strictly the oldest version,
# so it can be broadened to have a single version specification across
platforms.
# (`~=x.y.z` specifies a compatible release as `>=x.y.z, == x.y.*`)
-numpy~=1.23.2; python_version < "3.11"
-numpy~=1.23.2; python_version == "3.11"
-numpy~=1.26.0; python_version == "3.12"
+numpy~=2.0.0; python_version <= "3.12"
numpy~=2.1.0; python_version == "3.13"
numpy~=2.3.3; python_version == "3.14"
numpy~=2.5.2; python_version == "3.15" # TODO: 2.5.2 is for 3.15-rc1, after
release, update to correct version
diff --git a/python/setup.cfg b/python/setup.cfg
index 58e401185a..28a05b2180 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -24,7 +24,6 @@ addopts = --ignore=scripts
filterwarnings =
# https://github.com/apache/arrow/issues/49227
ignore:pyarrow.gandiva is deprecated:FutureWarning
- error:The SparseDataFrame:FutureWarning
# https://github.com/apache/arrow/issues/38239
ignore:Setting custom ClientSession:DeprecationWarning
# https://github.com/apache/arrow/issues/49255
