This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 8ee17fc3bd [python] Change Schema to TableSchema in Class
GetTableResponse. (#5990)
8ee17fc3bd is described below
commit 8ee17fc3bdb78ebd98e41fa6887cb27bc6057ea2
Author: HeavenZH <[email protected]>
AuthorDate: Thu Jul 31 10:29:22 2025 +0800
[python] Change Schema to TableSchema in Class GetTableResponse. (#5990)
---
paimon-python/pypaimon/api/__init__.py | 3 +-
paimon-python/pypaimon/api/api_response.py | 24 +++---
paimon-python/pypaimon/api/api_resquest.py | 2 +-
paimon-python/pypaimon/common/rest_json.py | 2 +-
paimon-python/pypaimon/rest/rest_catalog.py | 3 +-
paimon-python/pypaimon/schema/table_schema.py | 26 +++++++
paimon-python/pypaimon/tests/api_test.py | 59 ---------------
paimon-python/pypaimon/tests/rest_catalog_test.py | 90 +++++++++++++++++++++++
paimon-python/pypaimon/tests/rest_server.py | 11 +--
9 files changed, 139 insertions(+), 81 deletions(-)
diff --git a/paimon-python/pypaimon/api/__init__.py
b/paimon-python/pypaimon/api/__init__.py
index 2b35cd12df..8e05df6c01 100644
--- a/paimon-python/pypaimon/api/__init__.py
+++ b/paimon-python/pypaimon/api/__init__.py
@@ -27,7 +27,7 @@ from pypaimon.api.api_response import (
GetDatabaseResponse,
ConfigResponse,
PagedResponse,
- GetTableTokenResponse, Schema,
+ GetTableTokenResponse,
)
from pypaimon.api.api_resquest import CreateDatabaseRequest,
AlterDatabaseRequest, RenameTableRequest, \
CreateTableRequest
@@ -35,6 +35,7 @@ from pypaimon.api.config import CatalogOptions
from pypaimon.api.client import HttpClient
from pypaimon.api.identifier import Identifier
from pypaimon.api.typedef import T
+from pypaimon.schema.schema import Schema
class RESTException(Exception):
diff --git a/paimon-python/pypaimon/api/api_response.py
b/paimon-python/pypaimon/api/api_response.py
index b50754b3cf..9709c4f05c 100644
--- a/paimon-python/pypaimon/api/api_response.py
+++ b/paimon-python/pypaimon/api/api_response.py
@@ -22,7 +22,7 @@ from dataclasses import dataclass
from pypaimon.common.rest_json import json_field
from .typedef import T
-from .. import Schema
+from ..schema.table_schema import TableSchema
@dataclass
@@ -151,15 +151,15 @@ class GetTableResponse(AuditRESTResponse):
FIELD_NAME = "name"
FIELD_PATH = "path"
FIELD_IS_EXTERNAL = "isExternal"
- FIELD_SCHEMA_ID = "schemaId"
- FIELD_SCHEMA = "schema"
+ FIELD_TABLE_SCHEMA_ID = "tableSchemaId"
+ FIELD_TABLE_SCHEMA = "tableSchema"
id: Optional[str] = json_field(FIELD_ID, default=None)
name: Optional[str] = json_field(FIELD_NAME, default=None)
path: Optional[str] = json_field(FIELD_PATH, default=None)
is_external: Optional[bool] = json_field(FIELD_IS_EXTERNAL, default=None)
- schema_id: Optional[int] = json_field(FIELD_SCHEMA_ID, default=None)
- schema: Optional[Schema] = json_field(FIELD_SCHEMA, default=None)
+ table_schema_id: Optional[int] = json_field(FIELD_TABLE_SCHEMA_ID,
default=None)
+ table_schema: Optional[TableSchema] = json_field(FIELD_TABLE_SCHEMA,
default=None)
def __init__(
self,
@@ -167,8 +167,8 @@ class GetTableResponse(AuditRESTResponse):
name: str,
path: str,
is_external: bool,
- schema_id: int,
- schema: Schema,
+ table_schema_id: int,
+ table_schema: TableSchema,
owner: Optional[str] = None,
created_at: Optional[int] = None,
created_by: Optional[str] = None,
@@ -180,8 +180,8 @@ class GetTableResponse(AuditRESTResponse):
self.name = name
self.path = path
self.is_external = is_external
- self.schema_id = schema_id
- self.schema = schema
+ self.table_schema_id = table_schema_id
+ self.table_schema = table_schema
def get_id(self) -> str:
return self.id
@@ -196,10 +196,10 @@ class GetTableResponse(AuditRESTResponse):
return self.is_external
def get_schema_id(self) -> int:
- return self.schema_id
+ return self.table_schema_id
- def get_schema(self) -> Schema:
- return self.schema
+ def get_schema(self) -> TableSchema:
+ return self.table_schema
@dataclass
diff --git a/paimon-python/pypaimon/api/api_resquest.py
b/paimon-python/pypaimon/api/api_resquest.py
index 1ddc3ebe5a..dfc517ae89 100644
--- a/paimon-python/pypaimon/api/api_resquest.py
+++ b/paimon-python/pypaimon/api/api_resquest.py
@@ -20,9 +20,9 @@ from abc import ABC
from dataclasses import dataclass
from typing import Dict, List
-from .api_response import Schema
from .identifier import Identifier
from pypaimon.common.rest_json import json_field
+from ..schema.schema import Schema
class RESTRequest(ABC):
diff --git a/paimon-python/pypaimon/common/rest_json.py
b/paimon-python/pypaimon/common/rest_json.py
index b944bb625a..b6db836fc1 100644
--- a/paimon-python/pypaimon/common/rest_json.py
+++ b/paimon-python/pypaimon/common/rest_json.py
@@ -87,7 +87,7 @@ class JSON:
for json_name, value in data.items():
if json_name in field_mapping:
field_name = field_mapping[json_name]
- if field_name in type_mapping:
+ if json_name in type_mapping:
kwargs[field_name] = JSON.__from_dict(value,
type_mapping[json_name])
else:
kwargs[field_name] = value
diff --git a/paimon-python/pypaimon/rest/rest_catalog.py
b/paimon-python/pypaimon/rest/rest_catalog.py
index 2b66ea9981..32b5c5c7ad 100644
--- a/paimon-python/pypaimon/rest/rest_catalog.py
+++ b/paimon-python/pypaimon/rest/rest_catalog.py
@@ -25,7 +25,6 @@ from pypaimon.api.core_options import CoreOptions
from pypaimon.api.identifier import Identifier
from pypaimon.api.options import Options
-from pypaimon.schema.table_schema import TableSchema
from pypaimon.catalog.catalog_context import CatalogContext
from pypaimon.catalog.catalog_utils import CatalogUtils
@@ -102,7 +101,7 @@ class RESTCatalog(Catalog):
return self.to_table_metadata(identifier.get_database_name(), response)
def to_table_metadata(self, db: str, response: GetTableResponse) ->
TableMetadata:
- schema = TableSchema.from_schema(response.get_schema_id(),
response.get_schema())
+ schema = response.get_schema()
options: Dict[str, str] = dict(schema.options)
options[CoreOptions.PATH] = response.get_path()
response.put_audit_options_to(options)
diff --git a/paimon-python/pypaimon/schema/table_schema.py
b/paimon-python/pypaimon/schema/table_schema.py
index 7d3f6aea64..635cb3464d 100644
--- a/paimon-python/pypaimon/schema/table_schema.py
+++ b/paimon-python/pypaimon/schema/table_schema.py
@@ -15,25 +15,51 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import json
import time
+from dataclasses import dataclass
from pathlib import Path
from typing import List, Dict, Optional
import pyarrow
from pypaimon import Schema
+from pypaimon.common.rest_json import json_field
from pypaimon.schema import data_types
from pypaimon.api.core_options import CoreOptions
from pypaimon.common.file_io import FileIO
from pypaimon.schema.data_types import DataField
+@dataclass
class TableSchema:
PAIMON_07_VERSION = 1
PAIMON_08_VERSION = 2
CURRENT_VERSION = 3
+ FIELD_VERSION = "version"
+ FIELD_ID = "id"
+ FIELD_FIELDS = "fields"
+ FIELD_HEIGHEST_FIELD_ID = "highestFieldId"
+ FIELD_PARTITION_KEYS = "partitionKeys"
+ FIELD_PRIMARY_KEYS = "primaryKeys"
+ FIELD_OPTIONS = "options"
+ FIELD_COMMENT = "comment"
+ FIELD_TIME_MILLIS = "timeMillis"
+
+ version: int = json_field(FIELD_VERSION, default=CURRENT_VERSION)
+ id: int = json_field(FIELD_ID, default=0)
+ fields: List[DataField] = json_field(FIELD_FIELDS, default_factory=list)
+ highest_field_id: int = json_field("highestFieldId", default=0)
+ partition_keys: List[str] = json_field(
+ FIELD_PARTITION_KEYS, default_factory=list)
+ primary_keys: List[str] = json_field(
+ FIELD_PRIMARY_KEYS, default_factory=list)
+ options: Dict[str, str] = json_field(FIELD_OPTIONS, default_factory=dict)
+ comment: Optional[str] = json_field(FIELD_COMMENT, default=None)
+ time_millis: int = json_field("timeMillis", default_factory=lambda:
int(time.time() * 1000))
+
def __init__(self, version: int, id: int, fields: List[DataField],
highest_field_id: int,
partition_keys: List[str], primary_keys: List[str], options:
Dict[str, str],
comment: Optional[str] = None, time_millis: Optional[int] =
None):
diff --git a/paimon-python/pypaimon/tests/api_test.py
b/paimon-python/pypaimon/tests/api_test.py
index 9a604dad60..b84bc405b5 100644
--- a/paimon-python/pypaimon/tests/api_test.py
+++ b/paimon-python/pypaimon/tests/api_test.py
@@ -25,15 +25,12 @@ from ..api.api_response import (ConfigResponse)
from ..api import RESTApi
from ..api.auth import BearTokenAuthProvider
from ..api.identifier import Identifier
-from ..api.options import Options
from pypaimon.common.rest_json import JSON
from pypaimon.schema.table_schema import TableSchema
from ..api.token_loader import DLFTokenLoaderFactory, DLFToken
from pypaimon.schema.data_types import AtomicInteger, DataTypeParser,
AtomicType, ArrayType, MapType, RowType, DataField
-from ..catalog.catalog_context import CatalogContext
from ..catalog.table_metadata import TableMetadata
-from ..rest.rest_catalog import RESTCatalog
class ApiTestCase(unittest.TestCase):
@@ -182,62 +179,6 @@ class ApiTestCase(unittest.TestCase):
server.shutdown()
print("Server stopped")
- def test_rest_catalog(self):
- """Example usage of RESTCatalogServer"""
- # Setup logging
- logging.basicConfig(level=logging.INFO)
-
- # Create config
- config = ConfigResponse(defaults={"prefix": "mock-test"})
- token = str(uuid.uuid4())
- # Create server
- server = RESTCatalogServer(
- data_path="/tmp/test_warehouse",
- auth_provider=BearTokenAuthProvider(token),
- config=config,
- warehouse="test_warehouse"
- )
- try:
- # Start server
- server.start()
- print(f"Server started at: {server.get_url()}")
- test_databases = {
- "default": server.mock_database("default", {"env": "test"}),
- "test_db1": server.mock_database("test_db1", {"env": "test"}),
- "test_db2": server.mock_database("test_db2", {"env": "test"}),
- "prod_db": server.mock_database("prod_db", {"env": "prod"})
- }
- data_fields = [
- DataField(0, "name", AtomicType('INT'), 'desc name'),
- DataField(1, "arr11", ArrayType(True, AtomicType('INT')),
'desc arr11'),
- DataField(2, "map11", MapType(False, AtomicType('INT'),
- MapType(False,
AtomicType('INT'), AtomicType('INT'))),
- 'desc arr11'),
- ]
- schema = TableSchema(TableSchema.CURRENT_VERSION,
len(data_fields), data_fields, len(data_fields),
- [], [], {}, "")
- test_tables = {
- "default.user": TableMetadata(uuid=str(uuid.uuid4()),
is_external=True, schema=schema),
- }
- server.table_metadata_store.update(test_tables)
- server.database_store.update(test_databases)
- options = {
- 'uri': f"http://localhost:{server.port}";,
- 'warehouse': 'test_warehouse',
- 'dlf.region': 'cn-hangzhou',
- "token.provider": "bear",
- 'token': token
- }
- rest_catalog =
RESTCatalog(CatalogContext.create_from_options(Options(options)))
- self.assertSetEqual(set(rest_catalog.list_databases()),
{*test_databases})
- self.assertEqual(rest_catalog.get_database('default').name,
test_databases.get('default').name)
- table =
rest_catalog.get_table(Identifier.from_string('default.user'))
- self.assertEqual(table.identifier.get_full_name(), 'default.user')
- finally:
- # Shutdown server
- server.shutdown()
- print("Server stopped")
-
def test_ecs_loader_token(self):
token = DLFToken(
access_key_id='AccessKeyId',
diff --git a/paimon-python/pypaimon/tests/rest_catalog_test.py
b/paimon-python/pypaimon/tests/rest_catalog_test.py
new file mode 100644
index 0000000000..be5989a171
--- /dev/null
+++ b/paimon-python/pypaimon/tests/rest_catalog_test.py
@@ -0,0 +1,90 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import logging
+import unittest
+import uuid
+
+from pypaimon.api import ConfigResponse, Identifier
+from pypaimon.api.auth import BearTokenAuthProvider
+
+from pypaimon.api.options import Options
+from pypaimon.catalog.catalog_context import CatalogContext
+from pypaimon.catalog.table_metadata import TableMetadata
+from pypaimon.rest.rest_catalog import RESTCatalog
+from pypaimon.schema.data_types import DataField, ArrayType, AtomicType,
MapType
+from pypaimon.schema.table_schema import TableSchema
+from pypaimon.tests.rest_server import RESTCatalogServer
+
+
+class RESTCatalogTestCase(unittest.TestCase):
+
+ def test_rest_catalog(self):
+ """Example usage of RESTCatalogServer"""
+ # Setup logging
+ logging.basicConfig(level=logging.INFO)
+
+ # Create config
+ config = ConfigResponse(defaults={"prefix": "mock-test"})
+ token = str(uuid.uuid4())
+ # Create server
+ server = RESTCatalogServer(
+ data_path="/tmp/test_warehouse",
+ auth_provider=BearTokenAuthProvider(token),
+ config=config,
+ warehouse="test_warehouse"
+ )
+ try:
+ # Start server
+ server.start()
+ print(f"Server started at: {server.get_url()}")
+ test_databases = {
+ "default": server.mock_database("default", {"env": "test"}),
+ "test_db1": server.mock_database("test_db1", {"env": "test"}),
+ "test_db2": server.mock_database("test_db2", {"env": "test"}),
+ "prod_db": server.mock_database("prod_db", {"env": "prod"})
+ }
+ data_fields = [
+ DataField(0, "name", AtomicType('INT'), 'desc name'),
+ DataField(1, "arr11", ArrayType(True, AtomicType('INT')),
'desc arr11'),
+ DataField(2, "map11", MapType(False, AtomicType('INT'),
+ MapType(False,
AtomicType('INT'), AtomicType('INT'))),
+ 'desc arr11'),
+ ]
+ schema = TableSchema(TableSchema.CURRENT_VERSION,
len(data_fields), data_fields, len(data_fields),
+ [], [], {}, "")
+ test_tables = {
+ "default.user": TableMetadata(uuid=str(uuid.uuid4()),
is_external=True, schema=schema),
+ }
+ server.table_metadata_store.update(test_tables)
+ server.database_store.update(test_databases)
+ options = {
+ 'uri': f"http://localhost:{server.port}";,
+ 'warehouse': 'test_warehouse',
+ 'dlf.region': 'cn-hangzhou',
+ "token.provider": "bear",
+ 'token': token
+ }
+ rest_catalog =
RESTCatalog(CatalogContext.create_from_options(Options(options)))
+ self.assertSetEqual(set(rest_catalog.list_databases()),
{*test_databases})
+ self.assertEqual(rest_catalog.get_database('default').name,
test_databases.get('default').name)
+ table =
rest_catalog.get_table(Identifier.from_string('default.user'))
+ self.assertEqual(table.identifier.get_full_name(), 'default.user')
+ finally:
+ # Shutdown server
+ server.shutdown()
+ print("Server stopped")
diff --git a/paimon-python/pypaimon/tests/rest_server.py
b/paimon-python/pypaimon/tests/rest_server.py
index 44be0e9e42..2070acd626 100644
--- a/paimon-python/pypaimon/tests/rest_server.py
+++ b/paimon-python/pypaimon/tests/rest_server.py
@@ -30,13 +30,14 @@ from urllib.parse import urlparse
import pypaimon.api as api
from ..api import RenameTableRequest, CreateTableRequest,
CreateDatabaseRequest, Identifier
from ..api.api_response import (ConfigResponse, ListDatabasesResponse,
GetDatabaseResponse,
- Schema, GetTableResponse, ListTablesResponse,
+ GetTableResponse, ListTablesResponse,
RESTResponse, PagedList)
from pypaimon.common.rest_json import JSON
from pypaimon.schema.table_schema import TableSchema
from ..catalog.catalog_exception import DatabaseNoPermissionException,
TableNotExistException, \
DatabaseNotExistException, TableNoPermissionException
from ..catalog.table_metadata import TableMetadata
+from ..schema.schema import Schema
@dataclass
@@ -394,7 +395,7 @@ class RESTCatalogServer:
raise TableNotExistException(identifier)
table_metadata =
self.table_metadata_store[identifier.get_full_name()]
table_path =
f'file://{self.data_path}/{self.warehouse}/{identifier.database_name}/{identifier.object_name}'
- schema = table_metadata.schema.to_schema()
+ schema = table_metadata.schema
response = self.mock_table(identifier, table_metadata, table_path,
schema)
return self._mock_response(response, 200)
#
@@ -593,14 +594,14 @@ class RESTCatalogServer:
)
def mock_table(self, identifier: Identifier, table_metadata:
TableMetadata, path: str,
- schema: Schema) -> GetTableResponse:
+ schema: TableSchema) -> GetTableResponse:
return GetTableResponse(
id=str(table_metadata.uuid),
name=identifier.get_object_name(),
path=path,
is_external=table_metadata.is_external,
- schema_id=table_metadata.schema.id,
- schema=schema,
+ table_schema_id=table_metadata.schema.id,
+ table_schema=schema,
owner="owner",
created_at=1,
created_by="created",
