This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new b78ca94af fix(python/adbc_driver_manager): reimport PyArrow if
installed mid-session (#4641)
b78ca94af is described below
commit b78ca94af4bc31be0433bcbf464573884faa1f74
Author: Bryce Mecum <[email protected]>
AuthorDate: Sun Aug 2 15:45:05 2026 -0700
fix(python/adbc_driver_manager): reimport PyArrow if installed mid-session
(#4641)
Currently, with the dbapi module in the Python driver manager, if I am
using a REPL or notebook session started without PyArrow installed, I
have to restart my session to pick PyArrow up so I can use
PyArrow-enabled methods. This is because the module attempts to import
PyArrow at startup and sets a persistent flag that never changes after
import.
It would be nicer if we could rescue the user so they can install
PyArrow in their environment while a REPL or notebook is running, make
the call again, and have it work.
Steps to reproduce:
1. Create an environment without PyArrow:
```sh
python3 -m venv .venv
source .venv/bin/activate
pip install adbc_driver_manager
```
2. See we get the expected error when we try to use a PyArrow-enabled
routine:
```python
.venv $ python
Python 3.14.0 (main, Oct 14 2025, 21:10:22) [Clang 20.1.4 ] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> from adbc_driver_manager import dbapi
>>> con = dbapi.connect("sqlite://:memory:")
>>> cur = con.cursor()
>>> cur.execute("SELECT 1").fetch_arrow_table()
Traceback (most recent call last):
File "<python-input-3>", line 1, in <module>
cur.execute("SELECT 1").fetch_arrow_table()
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File
"/Users/bryce/src/apache/arrow-adbc/test_env/.venv/lib/python3.14/site-packages/adbc_driver_manager/dbapi.py",
line 1355, in fetch_arrow_table
return self._results.fetch_arrow_table()
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File
"/Users/bryce/src/apache/arrow-adbc/test_env/.venv/lib/python3.14/site-packages/adbc_driver_manager/dbapi.py",
line 1532, in fetch_arrow_table
return _blocking_call(self.reader.read_all, (), {}, self._stmt.cancel)
^^^^^^^^^^^
File
"/Users/bryce/src/apache/arrow-adbc/test_env/.venv/lib/python3.14/site-packages/adbc_driver_manager/dbapi.py",
line 1466, in reader
_requires_pyarrow()
~~~~~~~~~~~~~~~~~^^
File
"/Users/bryce/src/apache/arrow-adbc/test_env/.venv/lib/python3.14/site-packages/adbc_driver_manager/dbapi.py",
line 1588, in _requires_pyarrow
raise ProgrammingError(
...<2 lines>...
)
adbc_driver_manager.ProgrammingError: This API requires PyArrow to be
installed
```
3. Try to fix it in another shell
```sh
source .venv/bin/activate
pip install pyarrow
```
4. Run (2) again and get the same error
The Python driver manager's dbapi module guards PyArrow-enabled routines
with `_requires_pyarrow()`. This PR tweaks that helper and lets it
attempt to re-import PyArrow but only if PyArrow wasn't enabled at
import time.
This PR was generated using an LLM.
---
.../adbc_driver_manager/dbapi.py | 23 +++++++++----
python/adbc_driver_manager/tests/test_dbapi.py | 38 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 6 deletions(-)
diff --git a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py
b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py
index d3cb15750..254ab34f5 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py
+++ b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py
@@ -69,6 +69,8 @@ if typing.TYPE_CHECKING:
import pyarrow
from typing_extensions import CapsuleType, Self
+ from ._reader import AdbcRecordBatchReader
+
# ----------------------------------------------------------
# Globals
@@ -1445,7 +1447,7 @@ class _RowIterator(_Closeable):
self._stmt = stmt
self._handle: Optional[_lib.ArrowArrayStreamHandle] = handle
self._backend = dbapi_backend
- self._reader: Optional["_reader.AdbcRecordBatchReader"] = None
+ self._reader: Optional["AdbcRecordBatchReader"] = None
self._current_batch = None
self._next_row = 0
self._finished = False
@@ -1461,7 +1463,7 @@ class _RowIterator(_Closeable):
handle.release()
@property
- def reader(self) -> "_reader.AdbcRecordBatchReader":
+ def reader(self) -> "AdbcRecordBatchReader":
if self._reader is None:
_requires_pyarrow()
if self._handle is None:
@@ -1584,8 +1586,17 @@ def _is_arrow_data(data) -> bool:
def _requires_pyarrow() -> None:
+ global _has_pyarrow, _reader, pyarrow
if not _has_pyarrow:
- raise ProgrammingError(
- "This API requires PyArrow to be installed",
- status_code=_lib.AdbcStatusCode.INVALID_STATE,
- )
+ try:
+ import pyarrow
+ import pyarrow.dataset
+ except ImportError:
+ raise ProgrammingError(
+ "This API requires PyArrow to be installed",
+ status_code=_lib.AdbcStatusCode.INVALID_STATE,
+ )
+ from . import _reader as _reader_mod
+
+ _reader = _reader_mod
+ _has_pyarrow = True
diff --git a/python/adbc_driver_manager/tests/test_dbapi.py
b/python/adbc_driver_manager/tests/test_dbapi.py
index 8653d94e8..d266dbd12 100644
--- a/python/adbc_driver_manager/tests/test_dbapi.py
+++ b/python/adbc_driver_manager/tests/test_dbapi.py
@@ -722,6 +722,44 @@ def test_close_connection_suppresses_cursor_close_error()
-> None:
assert real_cursor._closed
+def test_requires_pyarrow_late_install(monkeypatch) -> None:
+ """Test that _requires_pyarrow recovers if PyArrow becomes available
+ after import."""
+ # Simulate the state where PyArrow wasn't available at module load time
+ monkeypatch.setattr(dbapi, "_has_pyarrow", False)
+ monkeypatch.delattr(dbapi, "_reader", raising=False)
+ monkeypatch.delattr(dbapi, "pyarrow", raising=False)
+
+ # Should not raise because PyArrow is actually importable now
+ dbapi._requires_pyarrow()
+
+ # The flag should be updated so subsequent calls don't re-import
+ assert dbapi._has_pyarrow is True
+ # The _reader Cython extension should now be available at module level
+ assert hasattr(dbapi, "_reader")
+ assert hasattr(dbapi._reader, "AdbcRecordBatchReader")
+ # pyarrow must be in module globals for callers that use it directly
+ assert hasattr(dbapi, "pyarrow")
+
+
+def test_requires_pyarrow_truly_missing(monkeypatch) -> None:
+ """Test that _requires_pyarrow still raises when PyArrow is genuinely
missing."""
+ import builtins
+
+ real_import = builtins.__import__
+
+ def mock_import(name, *args, **kwargs):
+ if name.startswith("pyarrow"):
+ raise ImportError("mocked")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(dbapi, "_has_pyarrow", False)
+ monkeypatch.setattr(builtins, "__import__", mock_import)
+
+ with pytest.raises(dbapi.ProgrammingError, match="requires PyArrow"):
+ dbapi._requires_pyarrow()
+
+
@pytest.mark.sqlite
def test_connect(tmp_path: pathlib.Path, monkeypatch) -> None:
with dbapi.connect(driver="adbc_driver_sqlite") as conn:
