From dc44cfcc24ced7301f5f290b731d61d8463bc3de Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Sat, 9 May 2026 10:03:22 +0900 Subject: [PATCH 1/2] REST: Add pagination support for list_tables --- pyiceberg/catalog/rest/__init__.py | 28 +++++++--- tests/catalog/test_rest.py | 84 ++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 7fa81312d1..6cb8705378 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -377,6 +377,7 @@ class ListViewResponseEntry(IcebergBaseModel): class ListTablesResponse(IcebergBaseModel): identifiers: list[ListTableResponseEntry] = Field() + next_page_token: str | None = Field(default=None, alias="next-page-token") class ListViewsResponse(IcebergBaseModel): @@ -1034,12 +1035,27 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: self._check_endpoint(Capability.V1_LIST_TABLES) namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace_concat = self._encode_namespace_path(namespace_tuple) - response = self._session.get(self.url(Endpoints.list_tables, namespace=namespace_concat)) - try: - response.raise_for_status() - except HTTPError as exc: - _handle_non_200_response(exc, {404: NoSuchNamespaceError}) - return [(*table.namespace, table.name) for table in ListTablesResponse.model_validate_json(response.text).identifiers] + url = self.url(Endpoints.list_tables, namespace=namespace_concat) + + tables: list[Identifier] = [] + page_token: str | None = None + + while True: + params = {"pageToken": page_token} if page_token else None + response = self._session.get(url, params=params) + try: + response.raise_for_status() + except HTTPError as exc: + _handle_non_200_response(exc, {404: NoSuchNamespaceError}) + + parsed = ListTablesResponse.model_validate_json(response.text) + tables.extend([(*table.namespace, table.name) for table in parsed.identifiers]) + + if not parsed.next_page_token: + break + page_token = parsed.next_page_token + + return tables @retry(**_RETRY_ARGS) @override diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 2adfe9f06e..643e9482f4 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -479,6 +479,90 @@ def test_list_tables_200(rest_mock: Mocker) -> None: assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] +def test_list_tables_paginated_200(rest_mock: Mocker) -> None: + namespace = "examples" + # First page with next-page-token + rest_mock.get( + f"{TEST_URI}v1/namespaces/{namespace}/tables", + json={ + "identifiers": [ + {"namespace": ["examples"], "name": "table1"}, + {"namespace": ["examples"], "name": "table2"}, + ], + "next-page-token": "page2token", + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + # Second page with next-page-token + rest_mock.get( + f"{TEST_URI}v1/namespaces/{namespace}/tables?pageToken=page2token", + json={ + "identifiers": [ + {"namespace": ["examples"], "name": "table3"}, + ], + "next-page-token": "page3token", + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + # Third page without next-page-token (last page) + rest_mock.get( + f"{TEST_URI}v1/namespaces/{namespace}/tables?pageToken=page3token", + json={ + "identifiers": [ + {"namespace": ["examples"], "name": "table4"}, + ], + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + + result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) + assert result == [ + ("examples", "table1"), + ("examples", "table2"), + ("examples", "table3"), + ("examples", "table4"), + ] + + +def test_list_tables_paginated_200_none_next_page_token(rest_mock: Mocker) -> None: + namespace = "examples" + # First page with next-page-token + rest_mock.get( + f"{TEST_URI}v1/namespaces/{namespace}/tables", + json={ + "identifiers": [ + {"namespace": ["examples"], "name": "table1"}, + {"namespace": ["examples"], "name": "table2"}, + ], + "next-page-token": "page2token", + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + # The last page with NONE next-page-token + rest_mock.get( + f"{TEST_URI}v1/namespaces/{namespace}/tables?pageToken=page2token", + json={ + "identifiers": [ + {"namespace": ["examples"], "name": "table3"}, + ], + "next-page-token": None, + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + + result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) + assert result == [ + ("examples", "table1"), + ("examples", "table2"), + ("examples", "table3"), + ] + + def test_list_tables_200_sigv4(rest_mock: Mocker) -> None: namespace = "examples" rest_mock.get( From f7d2323ea0996d1a8b3b9cc7f6570589ecf9de22 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 23 May 2026 16:26:08 -0700 Subject: [PATCH 2/2] Update pyiceberg/catalog/rest/__init__.py --- pyiceberg/catalog/rest/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 6cb8705378..af49ecef27 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -1041,7 +1041,9 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: page_token: str | None = None while True: - params = {"pageToken": page_token} if page_token else None + params: dict[str, str] = {} + if page_token: + params["pageToken"] = page_token response = self._session.get(url, params=params) try: response.raise_for_status()