This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 704b8cb8b8 [python] Support getting a table by id in the REST catalog
(#10697)
704b8cb8b8 is described below
commit 704b8cb8b8960430d3bf9e9e6bed5e321fce4984
Author: Jiajia Li <[email protected]>
AuthorDate: Mon Oct 26 12:14:51 2026 +0800
[python] Support getting a table by id in the REST catalog (#10697)
---
docs/docs/pypaimon/catalogs.mdx | 3 ++
paimon-python/pypaimon/api/api_response.py | 8 +++++
paimon-python/pypaimon/api/resource_paths.py | 6 ++++
paimon-python/pypaimon/api/rest_api.py | 10 ++++++
paimon-python/pypaimon/catalog/catalog.py | 6 ++++
.../pypaimon/catalog/catalog_exception.py | 16 ++++++++++
.../pypaimon/catalog/rest/rest_catalog.py | 17 ++++++++++
.../pypaimon/tests/native_plan_rest_test.py | 6 ++--
paimon-python/pypaimon/tests/rest/api_test.py | 27 ++++++++++++++++
.../pypaimon/tests/rest/resource_paths_test.py | 20 +++++++-----
.../pypaimon/tests/rest/rest_catalog_test.py | 36 ++++++++++++++++++++++
paimon-python/pypaimon/tests/rest/rest_server.py | 15 +++++++++
12 files changed, 160 insertions(+), 10 deletions(-)
diff --git a/docs/docs/pypaimon/catalogs.mdx b/docs/docs/pypaimon/catalogs.mdx
index 47f491a06f..17f8f354b4 100644
--- a/docs/docs/pypaimon/catalogs.mdx
+++ b/docs/docs/pypaimon/catalogs.mdx
@@ -264,6 +264,9 @@ catalog.create_table(
# Get Table
table = catalog.get_table('database_name.table_name')
+
+# Get Table by id (REST catalog only)
+table = catalog.get_table_by_id('table_id')
```
## Alter Table
diff --git a/paimon-python/pypaimon/api/api_response.py
b/paimon-python/pypaimon/api/api_response.py
index 837e2ba7ff..8854556714 100644
--- a/paimon-python/pypaimon/api/api_response.py
+++ b/paimon-python/pypaimon/api/api_response.py
@@ -247,6 +247,7 @@ class GetTableResponse(AuditRESTResponse):
# Field constants for JSON serialization
FIELD_ID = "id"
+ FIELD_DATABASE = "database"
FIELD_NAME = "name"
FIELD_PATH = "path"
FIELD_IS_EXTERNAL = "isExternal"
@@ -254,6 +255,8 @@ class GetTableResponse(AuditRESTResponse):
FIELD_SCHEMA = "schema"
id: Optional[str] = json_field(FIELD_ID, default=None)
+ # Omitted when None: native readers compare it with the table's database
+ database: Optional[str] = optional_json_field(FIELD_DATABASE, "non_null")
name: Optional[str] = json_field(FIELD_NAME, default=None)
path: Optional[str] = json_field(FIELD_PATH, default=None)
is_external: Optional[bool] = json_field(FIELD_IS_EXTERNAL, default=None)
@@ -273,9 +276,11 @@ class GetTableResponse(AuditRESTResponse):
created_by: Optional[str] = None,
updated_at: Optional[int] = None,
updated_by: Optional[str] = None,
+ database: Optional[str] = None,
):
super().__init__(owner, created_at, created_by, updated_at, updated_by)
self.id = id
+ self.database = database
self.name = name
self.path = path
self.is_external = is_external
@@ -285,6 +290,9 @@ class GetTableResponse(AuditRESTResponse):
def get_id(self) -> str:
return self.id
+ def get_database(self) -> Optional[str]:
+ return self.database
+
def get_name(self) -> str:
return self.name
diff --git a/paimon-python/pypaimon/api/resource_paths.py
b/paimon-python/pypaimon/api/resource_paths.py
index ac5e5dca95..b47f74cc99 100644
--- a/paimon-python/pypaimon/api/resource_paths.py
+++ b/paimon-python/pypaimon/api/resource_paths.py
@@ -27,6 +27,7 @@ class ResourcePaths:
V1 = "v1"
DATABASES = "databases"
TABLES = "tables"
+ ID = "id"
TABLE_DETAILS = "table-details"
PARTITIONS = "partitions"
FUNCTIONS = "functions"
@@ -70,6 +71,11 @@ class ResourcePaths:
return ("{}/{}/{}/{}/{}".format(self.base_path, self.DATABASES,
RESTUtil.encode_string(database_name),
self.TABLES, RESTUtil.encode_string(table_name)))
+ def table_by_id(self, table_id: str) -> str:
+ # Keep '~' raw, as in table_details.
+ table_id = urllib.parse.quote_plus(table_id, safe="*~",
errors="replace")
+ return "{}/{}/{}/{}".format(self.base_path, self.TABLES, self.ID,
table_id)
+
def table_details(self, database_name: str) -> str:
database_name = urllib.parse.quote_plus(database_name, safe="*~",
errors="replace")
return "{}/{}/{}/{}".format(
diff --git a/paimon-python/pypaimon/api/rest_api.py
b/paimon-python/pypaimon/api/rest_api.py
index 8564795490..bb22875295 100755
--- a/paimon-python/pypaimon/api/rest_api.py
+++ b/paimon-python/pypaimon/api/rest_api.py
@@ -313,6 +313,16 @@ class RESTApi:
self.rest_auth_function,
)
+ def get_table_by_id(self, table_id: str) -> GetTableResponse:
+ if not table_id or not table_id.strip():
+ raise ValueError("Table id cannot be empty")
+
+ return self.client.get(
+ self.resource_paths.table_by_id(table_id),
+ GetTableResponse,
+ self.rest_auth_function,
+ )
+
def drop_table(self, identifier: Identifier) -> GetTableResponse:
database_name, table_name = self.__validate_identifier(identifier)
diff --git a/paimon-python/pypaimon/catalog/catalog.py
b/paimon-python/pypaimon/catalog/catalog.py
index 1e161abfe8..68874831ed 100644
--- a/paimon-python/pypaimon/catalog/catalog.py
+++ b/paimon-python/pypaimon/catalog/catalog.py
@@ -97,6 +97,12 @@ class Catalog(ABC):
def get_table(self, identifier: Union[str, Identifier]) -> Table:
"""Get paimon table identified by the given Identifier."""
+ def get_table_by_id(self, table_id: str) -> Table:
+ """Get paimon table identified by the given table id."""
+ raise NotImplementedError(
+ "get_table_by_id is not supported by this catalog."
+ )
+
@abstractmethod
def create_table(self, identifier: Union[str, Identifier], schema: Schema,
ignore_if_exists: bool) -> None:
"""Create table with schema."""
diff --git a/paimon-python/pypaimon/catalog/catalog_exception.py
b/paimon-python/pypaimon/catalog/catalog_exception.py
index ffcecff21f..f3f73bea3a 100644
--- a/paimon-python/pypaimon/catalog/catalog_exception.py
+++ b/paimon-python/pypaimon/catalog/catalog_exception.py
@@ -71,6 +71,22 @@ class TableNoPermissionException(CatalogException):
super().__init__(f"No permission to access table
{identifier.get_full_name()}")
+class TableIdNotExistException(CatalogException):
+ """Table id not exist exception"""
+
+ def __init__(self, table_id: str):
+ self.table_id = table_id
+ super().__init__(f"Table id {table_id} does not exist")
+
+
+class TableIdNoPermissionException(CatalogException):
+ """Table id no permission exception"""
+
+ def __init__(self, table_id: str):
+ self.table_id = table_id
+ super().__init__(f"No permission to access table id {table_id}")
+
+
class ViewNotExistException(CatalogException):
"""View not exist exception"""
diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py
b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
index a7069d5345..0766140e00 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
@@ -32,6 +32,7 @@ from pypaimon.catalog.catalog_exception import (
TableNotExistException, DatabaseAlreadyExistException,
TableAlreadyExistException, DatabaseNotExistException,
TableNoPermissionException, DatabaseNoPermissionException,
+ TableIdNotExistException, TableIdNoPermissionException,
FunctionNotExistException, FunctionAlreadyExistException,
DefinitionAlreadyExistException, DefinitionNotExistException,
TagNotExistException, TagAlreadyExistException,
@@ -250,6 +251,22 @@ class RESTCatalog(Catalog):
return self._load_system_table(identifier)
return self._load_data_table(identifier)
+ def get_table_by_id(self, table_id: str) -> Table:
+ try:
+ response = self.rest_api.get_table_by_id(table_id)
+ except NoSuchResourceException as e:
+ raise TableIdNotExistException(table_id) from e
+ except ForbiddenException as e:
+ raise TableIdNoPermissionException(table_id) from e
+ database = response.get_database()
+ identifier = Identifier.create(database, response.get_name())
+ return self.load_table(
+ identifier,
+ lambda path: self.file_io_for_data(path, identifier),
+ self.file_io_from_options,
+ lambda _: self.to_table_metadata(database, response),
+ )
+
def _load_data_table(self, identifier: Identifier) -> Union[FormatTable,
IcebergTable, ObjectTable, FileStoreTable]:
return self.load_table(
identifier,
diff --git a/paimon-python/pypaimon/tests/native_plan_rest_test.py
b/paimon-python/pypaimon/tests/native_plan_rest_test.py
index d8e98ae957..8bf8f3646b 100644
--- a/paimon-python/pypaimon/tests/native_plan_rest_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_rest_test.py
@@ -208,8 +208,8 @@ def test_native_rest_reuses_python_data_token(
assert load.call_count == 1
[email protected]('branch', [None, 'dev'])
-def test_rest_dotted_database_and_table_keep_identity(rest_catalog, branch):
[email protected]('branch,by_id', [(None, False), ('dev', False),
(None, True)])
+def test_rest_dotted_database_and_table_keep_identity(rest_catalog, branch,
by_id):
from pypaimon.common.identifier import Identifier
from pypaimon.tests.native_plan_resolved_schema_test import
_assert_parity, _write
catalog, server = rest_catalog
@@ -219,6 +219,8 @@ def
test_rest_dotted_database_and_table_keep_identity(rest_catalog, branch):
pa.schema([('id', pa.int64()), ('value', pa.string())])), False)
table = catalog.get_table(identifier)
_write(table, [{'id': 1, 'value': 'old'}])
+ if by_id:
+ table = catalog.get_table_by_id(table.catalog_environment.uuid)
if branch:
catalog.create_tag(identifier, 'first', 1)
catalog.create_branch(identifier, branch, tag_name='first')
diff --git a/paimon-python/pypaimon/tests/rest/api_test.py
b/paimon-python/pypaimon/tests/rest/api_test.py
index a09a3b9c89..21c4ab8568 100644
--- a/paimon-python/pypaimon/tests/rest/api_test.py
+++ b/paimon-python/pypaimon/tests/rest/api_test.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import json
import logging
import unittest
import uuid
@@ -156,6 +157,22 @@ class ApiTest(unittest.TestCase):
self.assertFalse(vector_type.element.nullable)
self.assertEqual(vector_type.length, 7)
+ def test_get_table_response_database(self):
+ from pypaimon.api.api_response import GetTableResponse
+ from pypaimon.schema.schema import Schema
+
+ # JSON as the Java server sends it
+ parsed = JSON.from_json(
+ '{"id": "id", "database": "db", "name": "t", "path": "path",
"isExternal": false, '
+ '"schemaId": 0, "schema": {"fields": [], "partitionKeys": [],
"primaryKeys": [], '
+ '"options": {}, "comment": null}}', GetTableResponse)
+ self.assertEqual(parsed.get_database(), 'db')
+ self.assertEqual(parsed.get_name(), 't')
+ self.assertEqual(json.loads(JSON.to_json(parsed))['database'], 'db')
+
+ response = GetTableResponse('id', 't', 'path', False, 0, Schema())
+ self.assertNotIn('database', json.loads(JSON.to_json(response)))
+
def test_api(self):
"""Example usage of RESTCatalogServer"""
# Setup logging
@@ -207,6 +224,10 @@ class ApiTest(unittest.TestCase):
self.assertEqual(rest_api.get_database('default'),
test_databases.get('default'))
table = rest_api.get_table(Identifier.from_string('default.user'))
self.assertEqual(table.id, str(test_tables['default.user'].uuid))
+ self.assertEqual(table.get_database(), 'default')
+ table_by_id = rest_api.get_table_by_id(table.id)
+ self.assertEqual(table_by_id.get_database(), 'default')
+ self.assertEqual(table_by_id.get_name(), 'user')
finally:
# Shutdown server
@@ -293,6 +314,12 @@ class ApiTest(unittest.TestCase):
rest_api.get_database(" ")
self.assertIn("Database name cannot be empty", str(context.exception))
+ # Test get_table_by_id with empty id
+ for table_id in ("", " "):
+ with self.assertRaises(ValueError) as context:
+ rest_api.get_table_by_id(table_id)
+ self.assertIn("Table id cannot be empty", str(context.exception))
+
# Test drop_database with empty name
with self.assertRaises(ValueError) as context:
rest_api.drop_database("")
diff --git a/paimon-python/pypaimon/tests/rest/resource_paths_test.py
b/paimon-python/pypaimon/tests/rest/resource_paths_test.py
index eb3b9ed660..eccd72f68f 100644
--- a/paimon-python/pypaimon/tests/rest/resource_paths_test.py
+++ b/paimon-python/pypaimon/tests/rest/resource_paths_test.py
@@ -16,6 +16,7 @@
# under the License.
+import itertools
import threading
import unittest
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -48,8 +49,9 @@ class ResourcePathsTest(unittest.TestCase):
paths.table("sales db", "orders/all"))
self.assertEqual(
"/v1/catalog%2Fid/databases/sales+db/table-details",
paths.table_details("sales db"))
+ self.assertEqual("/v1/catalog%2Fid/tables/id/a%2Fb+c",
paths.table_by_id("a/b c"))
- def test_table_details_signature_matches_the_path_sent(self):
+ def test_signature_matches_the_path_sent(self):
received = []
class Handler(BaseHTTPRequestHandler):
@@ -74,22 +76,24 @@ class ResourcePathsTest(unittest.TestCase):
token = DLFToken("test-ak", "test-sk", "test-token", None)
paths = ResourcePaths("catalog~id")
+ builders = ((paths.table_details,
"/v1/catalog~id/databases/{}/table-details"),
+ (paths.table_by_id, "/v1/catalog~id/tables/id/{}"))
for algorithm in ("default", "openapi-v4"):
provider = DLFAuthProvider(uri, "cn-hangzhou", algorithm,
token=token)
- for database, encoded in (("sales~db", "sales~db"),
- ("sales~ db/a+b*", "sales~+db%2Fa%2Bb*"),
- ("sales%7Edb", "sales%257Edb")):
- with self.subTest(algorithm=algorithm, database=database):
+ for (build, template), (name, encoded) in itertools.product(
+ builders, (("sales~db", "sales~db"),
+ ("sales~ db/a+b*", "sales~+db%2Fa%2Bb*"),
+ ("sales%7Edb", "sales%257Edb"))):
+ with self.subTest(algorithm=algorithm, path=template,
name=name):
signed = []
def sign(parameter):
signed.append(parameter.path)
return provider.merge_auth_header({}, parameter)
- client.get(paths.table_details(database), None, sign)
+ client.get(build(name), None, sign)
path, headers = received[-1]
- self.assertEqual(
-
"/v1/catalog~id/databases/{}/table-details".format(encoded), path)
+ self.assertEqual(template.format(encoded), path)
expected = provider.signer.authorization(
RESTAuthParameter("GET", path, "", {}), token,
provider.extract_host(uri), headers)
diff --git a/paimon-python/pypaimon/tests/rest/rest_catalog_test.py
b/paimon-python/pypaimon/tests/rest/rest_catalog_test.py
index 295153cf38..a691c38ab8 100644
--- a/paimon-python/pypaimon/tests/rest/rest_catalog_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_catalog_test.py
@@ -335,6 +335,42 @@ class RESTCatalogTest(RESTBaseTest):
self.assertIn("no_such_table_for_snapshot", str(context.exception))
self.assertIn("does not exist", str(context.exception))
+ def test_get_table_by_id(self):
+ from pypaimon.catalog.catalog_exception import
(TableIdNoPermissionException,
+
TableIdNotExistException)
+ from pypaimon.execution.native.table import
_resolved_rest_table_response
+ # a dotted database name must come back unchanged
+ identifier = Identifier.create("test.table_db", "get_table_by_id")
+ self.rest_catalog.create_database(identifier.get_database_name(),
False)
+ pa_schema = pa.schema([('col1', pa.int32())])
+ self.rest_catalog.create_table(identifier,
Schema.from_pyarrow_schema(pa_schema), False)
+ table = self.rest_catalog.get_table(identifier)
+ write_builder = table.new_batch_write_builder()
+ table_write = write_builder.new_write()
+ table_commit = write_builder.new_commit()
+ table_write.write_arrow(pa.Table.from_pydict({'col1': [1]},
schema=pa_schema))
+ table_commit.commit(table_write.prepare_commit())
+ table_write.close()
+ table_commit.close()
+
+ table_id = table.catalog_environment.uuid
+ table_by_id = self.rest_catalog.get_table_by_id(table_id)
+ self.assertEqual(table_by_id.catalog_environment.uuid, table_id)
+ self.assertEqual(table_by_id.identifier.get_database_name(),
"test.table_db")
+ self.assertEqual(table_by_id.identifier.get_table_name(),
"get_table_by_id")
+ self.assertEqual(table_by_id.table_path, table.table_path)
+ # native readers can reuse the response
+ self.assertIsNotNone(_resolved_rest_table_response(table_by_id))
+ read_builder = table_by_id.new_read_builder()
+ result =
read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
+ self.assertEqual(result.column('col1').to_pylist(), [1])
+
+ with self.assertRaises(TableIdNotExistException):
+ self.rest_catalog.get_table_by_id("missing_table_id")
+ self.server.add_no_permission_table(identifier)
+ with self.assertRaises(TableIdNoPermissionException):
+ self.rest_catalog.get_table_by_id(table_id)
+
def test_catalog_load_snapshot_after_rollback(self):
"""Test load_snapshot returns correct snapshot after rollback."""
table_name = "default.table_load_snapshot_after_rollback"
diff --git a/paimon-python/pypaimon/tests/rest/rest_server.py
b/paimon-python/pypaimon/tests/rest/rest_server.py
index a5a31397f5..f38c648ed8 100755
--- a/paimon-python/pypaimon/tests/rest/rest_server.py
+++ b/paimon-python/pypaimon/tests/rest/rest_server.py
@@ -1104,6 +1104,9 @@ class RESTCatalogServer:
self.resource_paths.revoke_permission()):
return self._permissions_handle(method, resource_path, data,
parameters)
+ if method == "GET" and
resource_path.startswith(self.resource_paths.table_by_id("")):
+ return self._table_by_id_handle(resource_path)
+
database = resource_path.split("/")[4]
# Database-specific endpoints
if
resource_path.startswith(self.resource_paths.database(database)):
@@ -2054,6 +2057,17 @@ class RESTCatalogServer:
return self._mock_response(ErrorResponse(None, None, "Method Not
Allowed", 405), 405)
+ def _table_by_id_handle(self, resource_path: str) -> Tuple[str, int]:
+ table_id = RESTUtil.decode_string(resource_path.rsplit("/", 1)[-1])
+ for full_name, table_metadata in self.table_metadata_store.items():
+ if str(table_metadata.uuid) == table_id:
+ identifier = self._table_identifier(full_name)
+ if full_name in self.no_permission_tables:
+ raise TableNoPermissionException(identifier)
+ return self._table_handle("GET", "", identifier)
+ return self._mock_response(
+ ErrorResponse(ErrorResponse.RESOURCE_TYPE_TABLE, table_id, "Table
not exist.", 404), 404)
+
def _table_query_auth_handle(self, method, data, identifier):
if method != "POST":
return self._mock_response(ErrorResponse(None, None, "Method Not
Allowed", 405), 405)
@@ -2718,6 +2732,7 @@ class RESTCatalogServer:
schema: Schema) -> GetTableResponse:
return GetTableResponse(
id=str(table_metadata.uuid),
+ database=identifier.get_database_name(),
name=identifier.get_object_name(),
path=path,
is_external=table_metadata.is_external,
