This is an automated email from the ASF dual-hosted git repository. github-actions[bot] pushed a commit to branch cherry-pick-b98b4be9-to-branch-1.3 in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 5c5b95b0243a63dbf471ca2b4621cec3ef9e1979 Author: Akshay Thorat <[email protected]> AuthorDate: Wed Jun 10 18:45:12 2026 -0700 [#10669] feat(iceberg-rest): Add pagination support for list endpoints (#10671) ### What changes were proposed in this pull request? Add `pageToken` and `pageSize` query parameter support to the Iceberg REST server's `listNamespaces`, `listTables`, and `listViews` endpoints. Pagination is handled at the REST layer in `IcebergPaginationHelper`, a package-private utility that paginates the full response after authorization filtering. Pagination is implemented as cursor/keyset-based in-memory pagination. Results are sorted deterministically by name, and `nextPageToken` is set to the name of the last item on the current page. On the next request, items strictly after the cursor (by string comparison) are returned. When no pagination parameters are provided, existing behavior (return all results) is preserved. **Known limitations** (documented in `IcebergPaginationHelper` Javadoc): - The full list is materialized from the underlying catalog on every page request (the Iceberg `Catalog` interface does not support native pagination). - Items created after the cursor but before the current page's last item may be missed — this is consistent with standard keyset pagination behavior. ### Why are the changes needed? The Iceberg REST spec defines `pageToken` and `pageSize` query parameters on list endpoints, but Gravitino's Iceberg REST server did not accept or handle them. Clients with large catalogs cannot paginate results — everything was returned in a single response. Fix: #10669 ### Does this PR introduce _any_ user-facing change? Yes. The following REST endpoints now accept optional `pageToken` and `pageSize` query parameters: - `GET /v1/{prefix}/namespaces` - `GET /v1/{prefix}/namespaces/{namespace}/tables` - `GET /v1/{prefix}/namespaces/{namespace}/views` ### How was this patch tested? - Added comprehensive unit tests in `TestIcebergPaginationHelper` covering cursor-based pagination logic (first page, middle page, last page, beyond-end cursor, deterministic sort, empty list, single item, page size exceeding list size) - Added `testListNamespaceWithPagination` in `TestIcebergNamespaceOperations` - Added `testListTablesWithPagination` in `TestIcebergTableOperations` - Added `testListViewsWithPagination` in `TestIcebergViewOperations` - All existing unit tests pass (`./gradlew :iceberg:iceberg-rest-server:test -PskipITs`) --- .../service/rest/IcebergNamespaceOperations.java | 8 +- .../service/rest/IcebergPaginationHelper.java | 155 +++++++++++++++ .../service/rest/IcebergTableOperations.java | 9 +- .../service/rest/IcebergViewOperations.java | 11 +- .../rest/TestIcebergNamespaceOperations.java | 47 +++++ .../service/rest/TestIcebergPaginationHelper.java | 216 +++++++++++++++++++++ .../service/rest/TestIcebergTableOperations.java | 43 ++++ .../service/rest/TestIcebergViewOperations.java | 44 +++++ 8 files changed, 530 insertions(+), 3 deletions(-) diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java index ca5807e2f4..9d33d5ddfd 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; @@ -101,7 +102,9 @@ public class IcebergNamespaceOperations { accessMetadataType = MetadataObject.Type.CATALOG) public Response listNamespaces( @DefaultValue("") @Encoded() @QueryParam("parent") String parent, - @AuthorizationMetadata(type = Entity.EntityType.CATALOG) @PathParam("prefix") String prefix) { + @AuthorizationMetadata(type = Entity.EntityType.CATALOG) @PathParam("prefix") String prefix, + @QueryParam("pageToken") String pageToken, + @QueryParam("pageSize") Integer pageSize) { String catalogName = IcebergRESTUtils.getCatalogName(prefix); Namespace parentNamespace = parent.isEmpty() @@ -124,6 +127,9 @@ public class IcebergNamespaceOperations { response = filterListNamespacesResponse(response, authContext.metalakeName(), catalogName); } + response = + IcebergPaginationHelper.paginateNamespaces( + response, Optional.ofNullable(pageToken), Optional.ofNullable(pageSize)); return IcebergRESTUtils.ok(response); }); } catch (Exception e) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergPaginationHelper.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergPaginationHelper.java new file mode 100644 index 0000000000..625a342562 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergPaginationHelper.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.service.rest; + +import com.google.common.base.Preconditions; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; + +/** + * Utility for applying cursor-based in-memory pagination to Iceberg list responses. + * + * <p>Items are sorted deterministically by name and the page token is the name of the last item on + * the current page. On the next request, items strictly after the cursor (by string comparison) are + * returned. This is more resilient than offset-based pagination when the underlying catalog is + * modified between page requests. + * + * <p><b>Known limitations:</b> + * + * <ul> + * <li>This is in-memory pagination: the full item list is materialized from the underlying + * catalog on every page request, so server-side memory usage is unchanged. Paging through N + * items is O(N²) total work. True server-push-down pagination would require catalog + * implementations to add native support. + * <li>Items created after the cursor but alphabetically before the last item on the current page + * will be missed on subsequent pages. This is the same behavior as most keyset pagination + * implementations and is generally acceptable. + * <li>When {@code pageToken} is provided without {@code pageSize}, the response returns all items + * after the cursor with no {@code nextPageToken}. + * </ul> + */ +class IcebergPaginationHelper { + + private IcebergPaginationHelper() {} + + /** + * Paginate a {@link ListNamespacesResponse}. + * + * @param response the full (unpaginated) response + * @param pageToken cursor-based page token (name of last item on previous page), or empty for the + * first page + * @param pageSize maximum items per page, or empty for no limit + * @return a new response containing only the requested page + */ + static ListNamespacesResponse paginateNamespaces( + ListNamespacesResponse response, Optional<String> pageToken, Optional<Integer> pageSize) { + PaginatedPage<Namespace> page = + paginate(response.namespaces(), pageToken, pageSize, Namespace::toString); + ListNamespacesResponse.Builder builder = ListNamespacesResponse.builder().addAll(page.items); + page.nextPageToken.ifPresent(builder::nextPageToken); + return builder.build(); + } + + /** + * Paginate a {@link ListTablesResponse}. Works for both table and view list responses. + * + * @param response the full (unpaginated) response + * @param pageToken cursor-based page token (name of last item on previous page), or empty for the + * first page + * @param pageSize maximum items per page, or empty for no limit + * @return a new response containing only the requested page + */ + static ListTablesResponse paginateTables( + ListTablesResponse response, Optional<String> pageToken, Optional<Integer> pageSize) { + PaginatedPage<TableIdentifier> page = + paginate(response.identifiers(), pageToken, pageSize, TableIdentifier::toString); + ListTablesResponse.Builder builder = ListTablesResponse.builder().addAll(page.items); + page.nextPageToken.ifPresent(builder::nextPageToken); + return builder.build(); + } + + /** + * Core pagination logic shared by all list endpoints. + * + * <p>Items are sorted by {@code keyExtractor} to produce a stable ordering, then sliced using the + * cursor token. The next-page token is the key of the last item on the returned page. + * + * @param items the complete list of items to paginate + * @param pageToken cursor string (key of last item on previous page), or empty for the first page + * @param pageSize maximum items per page, or empty for no limit + * @param keyExtractor function to extract a comparable cursor key from each item + * @return a {@link PaginatedPage} containing the requested page of items and optional next token + */ + static <T> PaginatedPage<T> paginate( + List<T> items, + Optional<String> pageToken, + Optional<Integer> pageSize, + Function<T, String> keyExtractor) { + String token = pageToken.orElse(""); + if (!pageSize.isPresent() && token.isEmpty()) { + return new PaginatedPage<>(items, Optional.empty()); + } + + pageSize.ifPresent( + size -> Preconditions.checkArgument(size > 0, "pageSize must be positive, got: %s", size)); + + List<T> sorted = + items.stream().sorted(Comparator.comparing(keyExtractor)).collect(Collectors.toList()); + + int startIdx = 0; + if (!token.isEmpty()) { + startIdx = sorted.size(); + for (int i = 0; i < sorted.size(); i++) { + if (keyExtractor.apply(sorted.get(i)).compareTo(token) > 0) { + startIdx = i; + break; + } + } + } + + int limit = pageSize.orElse(sorted.size()); + int end = Math.min(startIdx + limit, sorted.size()); + List<T> page = sorted.subList(startIdx, end); + + Optional<String> nextToken = Optional.empty(); + if (end < sorted.size() && !page.isEmpty()) { + nextToken = Optional.of(keyExtractor.apply(page.get(page.size() - 1))); + } + + return new PaginatedPage<>(page, nextToken); + } + + /** Holds a page of items and an optional token for the next page. */ + static class PaginatedPage<T> { + final List<T> items; + final Optional<String> nextPageToken; + + PaginatedPage(List<T> items, Optional<String> nextPageToken) { + this.items = items; + this.nextPageToken = nextPageToken; + } + } +} diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java index a3eec15dd0..029722cfd8 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java @@ -123,7 +123,9 @@ public class IcebergTableOperations { public Response listTable( @AuthorizationMetadata(type = Entity.EntityType.CATALOG) @PathParam("prefix") String prefix, @AuthorizationMetadata(type = EntityType.SCHEMA) @Encoded() @PathParam("namespace") - String namespace) { + String namespace, + @QueryParam("pageToken") String pageToken, + @QueryParam("pageSize") Integer pageSize) { String catalogName = IcebergRESTUtils.getCatalogName(prefix); Namespace icebergNS = RESTUtil.decodeNamespace(namespace, IcebergRESTUtils.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); @@ -143,6 +145,11 @@ public class IcebergTableOperations { filterListTablesResponse( listTablesResponse, authContext.metalakeName(), catalogName); } + listTablesResponse = + IcebergPaginationHelper.paginateTables( + listTablesResponse, + Optional.ofNullable(pageToken), + Optional.ofNullable(pageSize)); return IcebergRESTUtils.ok(listTablesResponse); }); } catch (Exception e) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java index ff01399e0a..b33a53b2e6 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; @@ -36,6 +37,7 @@ import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -97,7 +99,9 @@ public class IcebergViewOperations { public Response listView( @AuthorizationMetadata(type = Entity.EntityType.CATALOG) @PathParam("prefix") String prefix, @AuthorizationMetadata(type = EntityType.SCHEMA) @Encoded() @PathParam("namespace") - String namespace) { + String namespace, + @QueryParam("pageToken") String pageToken, + @QueryParam("pageSize") Integer pageSize) { String catalogName = IcebergRESTUtils.getCatalogName(prefix); Namespace icebergNS = RESTUtil.decodeNamespace(namespace, IcebergRESTUtils.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); @@ -117,6 +121,11 @@ public class IcebergViewOperations { filterListViewsResponse( listTablesResponse, authContext.metalakeName(), catalogName); } + listTablesResponse = + IcebergPaginationHelper.paginateTables( + listTablesResponse, + Optional.ofNullable(pageToken), + Optional.ofNullable(pageSize)); return IcebergRESTUtils.ok(listTablesResponse); }); } catch (Exception e) { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java index c7c8f95bfb..dab90563fc 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java @@ -18,8 +18,12 @@ */ package org.apache.gravitino.iceberg.service.rest; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.client.Entity; import javax.ws.rs.core.Application; @@ -58,6 +62,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.iceberg.rest.requests.ImmutableRegisterTableRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StringType; @@ -307,6 +312,48 @@ public class TestIcebergNamespaceOperations extends IcebergNamespaceTestBase { verifyListNamespaceFail(Optional.of(Namespace.of("list_foo3", "a", "x")), 404); } + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListNamespaceWithPagination(String prefix) { + setUrlPathWithPrefix(prefix); + dropAllExistingNamespace(); + + doCreateNamespace(Namespace.of("page_ns1")); + doCreateNamespace(Namespace.of("page_ns2")); + doCreateNamespace(Namespace.of("page_ns3")); + + // First page: pageSize=2, no pageToken + Response firstPageResponse = + getNamespaceClientBuilder( + Optional.empty(), Optional.empty(), Optional.of(ImmutableMap.of("pageSize", "2"))) + .get(); + Assertions.assertEquals(Status.OK.getStatusCode(), firstPageResponse.getStatus()); + ListNamespacesResponse firstPage = firstPageResponse.readEntity(ListNamespacesResponse.class); + Assertions.assertEquals(2, firstPage.namespaces().size()); + Assertions.assertNotNull(firstPage.nextPageToken()); + + // Second page using the nextPageToken + Response secondPageResponse = + getNamespaceClientBuilder( + Optional.empty(), + Optional.empty(), + Optional.of( + ImmutableMap.of("pageToken", firstPage.nextPageToken(), "pageSize", "2"))) + .get(); + Assertions.assertEquals(Status.OK.getStatusCode(), secondPageResponse.getStatus()); + ListNamespacesResponse secondPage = secondPageResponse.readEntity(ListNamespacesResponse.class); + Assertions.assertEquals(1, secondPage.namespaces().size()); + Assertions.assertNull(secondPage.nextPageToken()); + + // Verify combined results + Set<String> paginatedNames = + java.util.stream.Stream.concat( + firstPage.namespaces().stream(), secondPage.namespaces().stream()) + .map(Namespace::toString) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of("page_ns1", "page_ns2", "page_ns3"), paginatedNames); + } + @Test @SuppressWarnings("deprecation") void testIcebergListNamespacesEventDeprecatedConstructorReturnsNegativeCount() { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergPaginationHelper.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergPaginationHelper.java new file mode 100644 index 0000000000..63804c9752 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergPaginationHelper.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.service.rest; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TestIcebergPaginationHelper { + + @Test + void testPaginateNamespacesNoPagination() { + ListNamespacesResponse response = + ListNamespacesResponse.builder().add(Namespace.of("ns1")).add(Namespace.of("ns2")).build(); + ListNamespacesResponse result = + IcebergPaginationHelper.paginateNamespaces(response, Optional.empty(), Optional.empty()); + Assertions.assertEquals(2, result.namespaces().size()); + Assertions.assertNull(result.nextPageToken()); + } + + @Test + void testPaginateNamespacesFirstPage() { + ListNamespacesResponse response = + ListNamespacesResponse.builder() + .add(Namespace.of("ns1")) + .add(Namespace.of("ns2")) + .add(Namespace.of("ns3")) + .build(); + ListNamespacesResponse result = + IcebergPaginationHelper.paginateNamespaces(response, Optional.empty(), Optional.of(2)); + Assertions.assertEquals(2, result.namespaces().size()); + Assertions.assertNotNull(result.nextPageToken()); + } + + @Test + void testPaginateNamespacesSecondPage() { + ListNamespacesResponse response = + ListNamespacesResponse.builder() + .add(Namespace.of("ns1")) + .add(Namespace.of("ns2")) + .add(Namespace.of("ns3")) + .build(); + // First page + ListNamespacesResponse firstPage = + IcebergPaginationHelper.paginateNamespaces(response, Optional.empty(), Optional.of(2)); + // Second page using cursor from first page + ListNamespacesResponse secondPage = + IcebergPaginationHelper.paginateNamespaces( + response, Optional.ofNullable(firstPage.nextPageToken()), Optional.of(2)); + Assertions.assertEquals(1, secondPage.namespaces().size()); + Assertions.assertNull(secondPage.nextPageToken()); + } + + @Test + void testPaginateNamespacesSortsDeterministically() { + // Items added in reverse order should still paginate in sorted order + ListNamespacesResponse response = + ListNamespacesResponse.builder() + .add(Namespace.of("ns3")) + .add(Namespace.of("ns1")) + .add(Namespace.of("ns2")) + .build(); + ListNamespacesResponse result = + IcebergPaginationHelper.paginateNamespaces(response, Optional.empty(), Optional.of(2)); + List<String> names = + result.namespaces().stream().map(Namespace::toString).collect(Collectors.toList()); + Assertions.assertEquals(Arrays.asList("ns1", "ns2"), names); + } + + @Test + void testPaginateNamespacesCursorBeyondAllItems() { + ListNamespacesResponse response = + ListNamespacesResponse.builder().add(Namespace.of("ns1")).add(Namespace.of("ns2")).build(); + // Cursor after all items alphabetically + ListNamespacesResponse result = + IcebergPaginationHelper.paginateNamespaces(response, Optional.of("zzzz"), Optional.of(10)); + Assertions.assertEquals(0, result.namespaces().size()); + Assertions.assertNull(result.nextPageToken()); + } + + @Test + void testPaginateNamespacesPageTokenWithoutPageSize() { + ListNamespacesResponse response = + ListNamespacesResponse.builder() + .add(Namespace.of("ns1")) + .add(Namespace.of("ns2")) + .add(Namespace.of("ns3")) + .build(); + // pageToken without pageSize returns all items after cursor + ListNamespacesResponse result = + IcebergPaginationHelper.paginateNamespaces(response, Optional.of("ns1"), Optional.empty()); + List<String> names = + result.namespaces().stream().map(Namespace::toString).collect(Collectors.toList()); + Assertions.assertEquals(Arrays.asList("ns2", "ns3"), names); + Assertions.assertNull(result.nextPageToken()); + } + + @Test + void testPaginateNamespacesZeroPageSizeThrows() { + ListNamespacesResponse response = + ListNamespacesResponse.builder().add(Namespace.of("ns1")).build(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + IcebergPaginationHelper.paginateNamespaces(response, Optional.empty(), Optional.of(0))); + } + + @Test + void testPaginateNamespacesNegativePageSizeThrows() { + ListNamespacesResponse response = + ListNamespacesResponse.builder().add(Namespace.of("ns1")).build(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + IcebergPaginationHelper.paginateNamespaces( + response, Optional.empty(), Optional.of(-1))); + } + + @Test + void testPaginateNamespacesEmptyList() { + ListNamespacesResponse response = ListNamespacesResponse.builder().build(); + ListNamespacesResponse result = + IcebergPaginationHelper.paginateNamespaces(response, Optional.empty(), Optional.of(10)); + Assertions.assertEquals(0, result.namespaces().size()); + Assertions.assertNull(result.nextPageToken()); + } + + @Test + void testPaginateTablesFirstPage() { + Namespace ns = Namespace.of("db"); + ListTablesResponse response = + ListTablesResponse.builder() + .add(TableIdentifier.of(ns, "t1")) + .add(TableIdentifier.of(ns, "t2")) + .add(TableIdentifier.of(ns, "t3")) + .build(); + ListTablesResponse result = + IcebergPaginationHelper.paginateTables(response, Optional.empty(), Optional.of(2)); + Assertions.assertEquals(2, result.identifiers().size()); + Assertions.assertNotNull(result.nextPageToken()); + } + + @Test + void testPaginateTablesFullWalk() { + Namespace ns = Namespace.of("db"); + ListTablesResponse response = + ListTablesResponse.builder() + .add(TableIdentifier.of(ns, "t1")) + .add(TableIdentifier.of(ns, "t2")) + .add(TableIdentifier.of(ns, "t3")) + .add(TableIdentifier.of(ns, "t4")) + .add(TableIdentifier.of(ns, "t5")) + .build(); + + // Walk all pages with pageSize=2 + ListTablesResponse page1 = + IcebergPaginationHelper.paginateTables(response, Optional.empty(), Optional.of(2)); + Assertions.assertEquals(2, page1.identifiers().size()); + Assertions.assertNotNull(page1.nextPageToken()); + + ListTablesResponse page2 = + IcebergPaginationHelper.paginateTables( + response, Optional.ofNullable(page1.nextPageToken()), Optional.of(2)); + Assertions.assertEquals(2, page2.identifiers().size()); + Assertions.assertNotNull(page2.nextPageToken()); + + ListTablesResponse page3 = + IcebergPaginationHelper.paginateTables( + response, Optional.ofNullable(page2.nextPageToken()), Optional.of(2)); + Assertions.assertEquals(1, page3.identifiers().size()); + Assertions.assertNull(page3.nextPageToken()); + + // Verify all 5 items were returned + int totalItems = + page1.identifiers().size() + page2.identifiers().size() + page3.identifiers().size(); + Assertions.assertEquals(5, totalItems); + } + + @Test + void testPaginateTablesExactPageSize() { + Namespace ns = Namespace.of("db"); + ListTablesResponse response = + ListTablesResponse.builder() + .add(TableIdentifier.of(ns, "t1")) + .add(TableIdentifier.of(ns, "t2")) + .build(); + // pageSize equals total items + ListTablesResponse result = + IcebergPaginationHelper.paginateTables(response, Optional.empty(), Optional.of(2)); + Assertions.assertEquals(2, result.identifiers().size()); + Assertions.assertNull(result.nextPageToken()); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java index 13accb3c73..14d2782b4f 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java @@ -310,6 +310,49 @@ public class TestIcebergTableOperations extends IcebergNamespaceTestBase { Assertions.assertEquals(2, ((IcebergListTableEvent) listTablePostEvent).resultCount()); } + @ParameterizedTest + @MethodSource( + "org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testPrefixesAndNamespaces") + void testListTablesWithPagination(String prefix, Namespace namespace) { + setUrlPathWithPrefix(prefix); + verifyCreateNamespaceSucc(namespace); + verifyCreateTableSucc(namespace, "page_t1"); + verifyCreateTableSucc(namespace, "page_t2"); + verifyCreateTableSucc(namespace, "page_t3"); + + dummyEventListener.clearEvent(); + + // First page: pageSize=2 + String tablePath = + IcebergRestTestUtil.NAMESPACE_PATH + "/" + RESTUtil.encodeNamespace(namespace) + "/tables"; + Response firstPageResponse = + getIcebergClientBuilder(tablePath, Optional.of(ImmutableMap.of("pageSize", "2"))).get(); + Assertions.assertEquals(Status.OK.getStatusCode(), firstPageResponse.getStatus()); + ListTablesResponse firstPage = firstPageResponse.readEntity(ListTablesResponse.class); + Assertions.assertEquals(2, firstPage.identifiers().size()); + Assertions.assertNotNull(firstPage.nextPageToken()); + + // Second page using nextPageToken + Response secondPageResponse = + getIcebergClientBuilder( + tablePath, + Optional.of( + ImmutableMap.of("pageToken", firstPage.nextPageToken(), "pageSize", "2"))) + .get(); + Assertions.assertEquals(Status.OK.getStatusCode(), secondPageResponse.getStatus()); + ListTablesResponse secondPage = secondPageResponse.readEntity(ListTablesResponse.class); + Assertions.assertEquals(1, secondPage.identifiers().size()); + Assertions.assertNull(secondPage.nextPageToken()); + + // Verify combined results + Set<String> paginatedNames = + java.util.stream.Stream.concat( + firstPage.identifiers().stream(), secondPage.identifiers().stream()) + .map(id -> id.name()) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of("page_t1", "page_t2", "page_t3"), paginatedNames); + } + @ParameterizedTest @MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces") void testTableExits(Namespace namespace) { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java index 2c47a44739..4cb65b64ce 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java @@ -19,6 +19,7 @@ package org.apache.gravitino.iceberg.service.rest; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.Optional; @@ -134,6 +135,49 @@ public class TestIcebergViewOperations extends IcebergNamespaceTestBase { Assertions.assertEquals(2, ((IcebergListViewEvent) listViewPostEvent).resultCount()); } + @ParameterizedTest + @MethodSource( + "org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testPrefixesAndNamespaces") + void testListViewsWithPagination(String prefix, Namespace namespace) { + setUrlPathWithPrefix(prefix); + verifyCreateNamespaceSucc(namespace); + verifyCreateViewSucc(namespace, "page_v1"); + verifyCreateViewSucc(namespace, "page_v2"); + verifyCreateViewSucc(namespace, "page_v3"); + + dummyEventListener.clearEvent(); + + // First page: pageSize=2 + String viewPath = + IcebergRestTestUtil.NAMESPACE_PATH + "/" + RESTUtil.encodeNamespace(namespace) + "/views"; + Response firstPageResponse = + getIcebergClientBuilder(viewPath, Optional.of(ImmutableMap.of("pageSize", "2"))).get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), firstPageResponse.getStatus()); + ListTablesResponse firstPage = firstPageResponse.readEntity(ListTablesResponse.class); + Assertions.assertEquals(2, firstPage.identifiers().size()); + Assertions.assertNotNull(firstPage.nextPageToken()); + + // Second page using nextPageToken + Response secondPageResponse = + getIcebergClientBuilder( + viewPath, + Optional.of( + ImmutableMap.of("pageToken", firstPage.nextPageToken(), "pageSize", "2"))) + .get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), secondPageResponse.getStatus()); + ListTablesResponse secondPage = secondPageResponse.readEntity(ListTablesResponse.class); + Assertions.assertEquals(1, secondPage.identifiers().size()); + Assertions.assertNull(secondPage.nextPageToken()); + + // Verify combined results + Set<String> paginatedNames = + java.util.stream.Stream.concat( + firstPage.identifiers().stream(), secondPage.identifiers().stream()) + .map(id -> id.name()) + .collect(Collectors.toSet()); + Assertions.assertEquals(ImmutableSet.of("page_v1", "page_v2", "page_v3"), paginatedNames); + } + @ParameterizedTest @MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces") void testCreateView(Namespace namespace) {
