This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 2be1827bf REST: Add pagination support for list_tables (#3348)
2be1827bf is described below
commit 2be1827bf9a9a8e82f0cb4e8cb6a2ae535bafd7f
Author: Yuya Ebihara <[email protected]>
AuthorDate: Sun May 24 08:38:08 2026 +0900
REST: Add pagination support for list_tables (#3348)
<!--
Thanks for opening a pull request!
-->
<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->
# Rationale for this change
Follows REST catalog spec:
- /v1/{prefix}/tables:
https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L525-L562
- ListTablesResponse:
https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L4210-L4219
- PageToken:
https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L2233-L2256
## Are these changes tested?
Yes, includes unit tests for paginated cases.
## Are there any user-facing changes?
- Before: returned incomplete table list when server paginated (only
first page)
- After: returns complete table list (fetches all pages)
<!-- In the case of user-facing changes, please add the changelog label.
-->
---------
Co-authored-by: Kevin Liu <[email protected]>
---
pyiceberg/catalog/rest/__init__.py | 30 +++++++++++---
tests/catalog/test_rest.py | 84 ++++++++++++++++++++++++++++++++++++++
2 files changed, 108 insertions(+), 6 deletions(-)
diff --git a/pyiceberg/catalog/rest/__init__.py
b/pyiceberg/catalog/rest/__init__.py
index 81996b6bd..4db3b7920 100644
--- a/pyiceberg/catalog/rest/__init__.py
+++ b/pyiceberg/catalog/rest/__init__.py
@@ -383,6 +383,7 @@ class ListViewResponseEntry(IcebergBaseModel):
class ListTablesResponse(IcebergBaseModel):
identifiers: list[ListTableResponseEntry] = Field()
+ next_page_token: str | None = Field(default=None, alias="next-page-token")
class ListViewsResponse(IcebergBaseModel):
@@ -1039,12 +1040,29 @@ class RestCatalog(Catalog):
self._check_endpoint(Capability.V1_LIST_TABLES)
namespace_tuple = self._check_valid_namespace_identifier(namespace)
namespace_concat = self._encode_namespace_path(namespace_tuple)
- response = self._session.get(self.url(Endpoints.list_tables,
namespace=namespace_concat))
- try:
- response.raise_for_status()
- except HTTPError as exc:
- _handle_non_200_response(exc, {404: NoSuchNamespaceError})
- return [(*table.namespace, table.name) for table in
ListTablesResponse.model_validate_json(response.text).identifiers]
+ url = self.url(Endpoints.list_tables, namespace=namespace_concat)
+
+ tables: list[Identifier] = []
+ page_token: str | None = None
+
+ while True:
+ params: dict[str, str] = {}
+ if page_token:
+ params["pageToken"] = page_token
+ response = self._session.get(url, params=params)
+ try:
+ response.raise_for_status()
+ except HTTPError as exc:
+ _handle_non_200_response(exc, {404: NoSuchNamespaceError})
+
+ parsed = ListTablesResponse.model_validate_json(response.text)
+ tables.extend([(*table.namespace, table.name) for table in
parsed.identifiers])
+
+ if not parsed.next_page_token:
+ break
+ page_token = parsed.next_page_token
+
+ return tables
@retry(**_RETRY_ARGS)
@override
diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py
index 51f47744a..857cfb28a 100644
--- a/tests/catalog/test_rest.py
+++ b/tests/catalog/test_rest.py
@@ -480,6 +480,90 @@ def test_list_tables_200(rest_mock: Mocker) -> None:
assert RestCatalog("rest", uri=TEST_URI,
token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")]
+def test_list_tables_paginated_200(rest_mock: Mocker) -> None:
+ namespace = "examples"
+ # First page with next-page-token
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/tables",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "table1"},
+ {"namespace": ["examples"], "name": "table2"},
+ ],
+ "next-page-token": "page2token",
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+ # Second page with next-page-token
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/tables?pageToken=page2token",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "table3"},
+ ],
+ "next-page-token": "page3token",
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+ # Third page without next-page-token (last page)
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/tables?pageToken=page3token",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "table4"},
+ ],
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+
+ result = RestCatalog("rest", uri=TEST_URI,
token=TEST_TOKEN).list_tables(namespace)
+ assert result == [
+ ("examples", "table1"),
+ ("examples", "table2"),
+ ("examples", "table3"),
+ ("examples", "table4"),
+ ]
+
+
+def test_list_tables_paginated_200_none_next_page_token(rest_mock: Mocker) ->
None:
+ namespace = "examples"
+ # First page with next-page-token
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/tables",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "table1"},
+ {"namespace": ["examples"], "name": "table2"},
+ ],
+ "next-page-token": "page2token",
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+ # The last page with NONE next-page-token
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/tables?pageToken=page2token",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "table3"},
+ ],
+ "next-page-token": None,
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+
+ result = RestCatalog("rest", uri=TEST_URI,
token=TEST_TOKEN).list_tables(namespace)
+ assert result == [
+ ("examples", "table1"),
+ ("examples", "table2"),
+ ("examples", "table3"),
+ ]
+
+
def test_list_tables_200_sigv4(rest_mock: Mocker) -> None:
namespace = "examples"
rest_mock.get(
