This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 8055a60af REST: Add support for page-size in list_namespaces,
list_tables, and list_views (#3377)
8055a60af is described below
commit 8055a60af5d32ec4d2e1d02cd020a78273785558
Author: Yuya Ebihara <[email protected]>
AuthorDate: Sun May 24 12:04:50 2026 +0900
REST: Add support for page-size in list_namespaces, list_tables, and
list_views (#3377)
<!--
Thanks for opening a pull request!
-->
<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->
# Rationale for this change
Follows REST catalog spec:
* `/v1/{prefix}/views`:
https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L1525-L1538
* `page-size`:
https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L2029-L2038
## Are these changes tested?
Yes.
## Are there any user-facing changes?
Add support for `page-size` in list_views in REST catalog.
<!-- In the case of user-facing changes, please add the changelog label.
-->
---
pyiceberg/catalog/rest/__init__.py | 29 +++++++++++--
tests/catalog/test_rest.py | 85 ++++++++++++++++++++++++++++++++++++++
2 files changed, 111 insertions(+), 3 deletions(-)
diff --git a/pyiceberg/catalog/rest/__init__.py
b/pyiceberg/catalog/rest/__init__.py
index 4db3b7920..7a09e376f 100644
--- a/pyiceberg/catalog/rest/__init__.py
+++ b/pyiceberg/catalog/rest/__init__.py
@@ -267,6 +267,8 @@ SCAN_PLANNING_MODE_DEFAULT = ScanPlanningMode.CLIENT.value
VIEW_ENDPOINTS_SUPPORTED = "view-endpoints-supported"
VIEW_ENDPOINTS_SUPPORTED_DEFAULT = False
+PAGE_SIZE = "rest-page-size"
+
NAMESPACE_SEPARATOR_PROPERTY = "namespace-separator"
DEFAULT_NAMESPACE_SEPARATOR = b"\x1f".decode(UTF8)
@@ -1042,11 +1044,17 @@ class RestCatalog(Catalog):
namespace_concat = self._encode_namespace_path(namespace_tuple)
url = self.url(Endpoints.list_tables, namespace=namespace_concat)
+ params: dict[str, str] = {}
+ page_size = property_as_int(self.properties, PAGE_SIZE, None)
+ if page_size is not None:
+ if page_size <= 0:
+ raise ValueError(f"{PAGE_SIZE} must be a positive integer")
+ params["pageSize"] = str(page_size)
+
tables: list[Identifier] = []
page_token: str | None = None
while True:
- params: dict[str, str] = {}
if page_token:
params["pageToken"] = page_token
response = self._session.get(url, params=params)
@@ -1150,11 +1158,20 @@ class RestCatalog(Catalog):
namespace_concat = self._encode_namespace_path(namespace_tuple)
url = self.url(Endpoints.list_views, namespace=namespace_concat)
+ params: dict[str, str] = {}
+ page_size = property_as_int(self.properties, PAGE_SIZE, None)
+ if page_size is not None:
+ if page_size <= 0:
+ raise ValueError(f"{PAGE_SIZE} must be a positive integer")
+ params["pageSize"] = str(page_size)
+
views: list[Identifier] = []
page_token: str | None = None
while True:
- params = {"pageToken": page_token} if page_token else None
+ if page_token:
+ params["pageToken"] = page_token
+
response = self._session.get(url, params=params)
try:
response.raise_for_status()
@@ -1263,11 +1280,17 @@ class RestCatalog(Catalog):
self._check_endpoint(Capability.V1_LIST_NAMESPACES)
namespace_tuple = self.identifier_to_tuple(namespace)
+ params: dict[str, str] = {}
+ page_size = property_as_int(self.properties, PAGE_SIZE, None)
+ if page_size is not None:
+ if page_size <= 0:
+ raise ValueError(f"{PAGE_SIZE} must be a positive integer")
+ params["pageSize"] = str(page_size)
+
namespaces: list[Identifier] = []
page_token: str | None = None
while True:
- params: dict[str, str] = {}
if namespace_tuple:
params["parent"] = self._encode_namespace_path(namespace_tuple)
if page_token:
diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py
index 857cfb28a..1eb9f26a5 100644
--- a/tests/catalog/test_rest.py
+++ b/tests/catalog/test_rest.py
@@ -35,6 +35,7 @@ from pyiceberg.catalog.rest import (
DEFAULT_ENDPOINTS,
EMPTY_BODY_SHA256,
OAUTH2_SERVER_URI,
+ PAGE_SIZE,
SIGV4_MAX_RETRIES,
SIGV4_MAX_RETRIES_DEFAULT,
SNAPSHOT_LOADING_MODE,
@@ -564,6 +565,29 @@ def
test_list_tables_paginated_200_none_next_page_token(rest_mock: Mocker) -> No
]
+def test_list_tables_page_size(rest_mock: Mocker) -> None:
+ namespace = "examples"
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/tables",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "table1"},
+ {"namespace": ["examples"], "name": "table2"},
+ ],
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+
+ result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE:
"100"}).list_tables(namespace)
+ assert rest_mock.last_request.url ==
f"{TEST_URI}v1/namespaces/examples/tables?pageSize=100"
+
+ assert result == [
+ ("examples", "table1"),
+ ("examples", "table2"),
+ ]
+
+
def test_list_tables_200_sigv4(rest_mock: Mocker) -> None:
namespace = "examples"
rest_mock.get(
@@ -810,6 +834,48 @@ def
test_list_views_paginated_200_none_next_page_token(rest_mock: Mocker) -> Non
]
+def test_list_views_page_size(rest_mock: Mocker) -> None:
+ namespace = "examples"
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/views",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "view1"},
+ {"namespace": ["examples"], "name": "view2"},
+ ],
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+
+ result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE:
"100"}).list_views(namespace)
+ assert rest_mock.last_request.url ==
f"{TEST_URI}v1/namespaces/examples/views?pageSize=100"
+
+ assert result == [
+ ("examples", "view1"),
+ ("examples", "view2"),
+ ]
+
+
+def test_list_views_invalid_page_size(rest_mock: Mocker) -> None:
+ namespace = "examples"
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces/{namespace}/views",
+ json={
+ "identifiers": [
+ {"namespace": ["examples"], "name": "view1"},
+ {"namespace": ["examples"], "name": "view2"},
+ ],
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+
+ with pytest.raises(ValueError) as e:
+ RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE:
"0"}).list_views(namespace)
+ assert str(e.value) == "rest-page-size must be a positive integer"
+
+
def test_list_views_200_sigv4(rest_mock: Mocker) -> None:
namespace = "examples"
rest_mock.get(
@@ -1006,6 +1072,25 @@ def
test_list_namespaces_paginated_200_none_next_page_token(rest_mock: Mocker) -
]
+def test_list_namespaces_page_size(rest_mock: Mocker) -> None:
+ rest_mock.get(
+ f"{TEST_URI}v1/namespaces",
+ json={
+ "namespaces": [["ns1"], ["ns2"]],
+ },
+ status_code=200,
+ request_headers=TEST_HEADERS,
+ )
+
+ result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE:
"100"}).list_namespaces()
+ assert rest_mock.last_request.url ==
f"{TEST_URI}v1/namespaces?pageSize=100"
+
+ assert result == [
+ ("ns1",),
+ ("ns2",),
+ ]
+
+
def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None:
rest_mock.get(
f"{TEST_URI}v1/namespaces?parent=some_namespace",
