This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fd605b8a11 [python] Support get_table method of python RestCatalog
(#5962)
fd605b8a11 is described below
commit fd605b8a118759ecc37cd9addc477d3b0054e1a5
Author: HeavenZH <[email protected]>
AuthorDate: Mon Jul 28 13:45:17 2025 +0800
[python] Support get_table method of python RestCatalog (#5962)
---
paimon-python/pypaimon/api/__init__.py | 2 +-
paimon-python/pypaimon/api/api_response.py | 76 +++++++-------------
paimon-python/pypaimon/api/api_resquest.py | 2 +-
paimon-python/pypaimon/api/core_options.py | 44 ++++++++++++
.../pypaimon/api/{typedef.py => identifier.py} | 30 +++-----
paimon-python/pypaimon/api/schema.py | 52 ++++++++++++++
paimon-python/pypaimon/api/table_schema.py | 83 ++++++++++++++++++++++
paimon-python/pypaimon/api/typedef.py | 46 +-----------
paimon-python/pypaimon/catalog/catalog_loader.py | 24 +++++++
paimon-python/pypaimon/catalog/catalog_utils.py | 55 ++++++++++++++
paimon-python/pypaimon/catalog/table_metadata.py | 40 +++++++++++
paimon-python/pypaimon/common/__init__.py | 17 +++++
paimon-python/pypaimon/common/file_io.py | 25 +++++++
paimon-python/pypaimon/pvfs/__init__.py | 3 +-
paimon-python/pypaimon/rest/rest_catalog.py | 47 +++++++++++-
paimon-python/pypaimon/rest/rest_token_file_io.py | 34 +++++++++
paimon-python/pypaimon/schema/__init__.py | 17 +++++
paimon-python/pypaimon/schema/schema_manager.py | 29 ++++++++
paimon-python/pypaimon/table/__init__.py | 17 +++++
.../pypaimon/table/catalog_environment.py | 36 ++++++++++
paimon-python/pypaimon/table/file_store_table.py | 42 +++++++++++
.../pypaimon/table/file_store_table_factory.py | 35 +++++++++
paimon-python/pypaimon/table/table.py | 23 ++++++
paimon-python/pypaimon/tests/api_test.py | 11 ++-
paimon-python/pypaimon/tests/pvfs_test.py | 3 +-
paimon-python/pypaimon/tests/rest_server.py | 9 +--
26 files changed, 676 insertions(+), 126 deletions(-)
diff --git a/paimon-python/pypaimon/api/__init__.py
b/paimon-python/pypaimon/api/__init__.py
index f235c2815e..3c8d1fb2ef 100644
--- a/paimon-python/pypaimon/api/__init__.py
+++ b/paimon-python/pypaimon/api/__init__.py
@@ -31,9 +31,9 @@ from pypaimon.api.api_response import (
)
from pypaimon.api.api_resquest import CreateDatabaseRequest,
AlterDatabaseRequest, RenameTableRequest, \
CreateTableRequest
-from pypaimon.api.typedef import Identifier
from pypaimon.api.config import RESTCatalogOptions
from pypaimon.api.client import HttpClient
+from pypaimon.api.identifier import Identifier
from pypaimon.api.typedef import T
diff --git a/paimon-python/pypaimon/api/api_response.py
b/paimon-python/pypaimon/api/api_response.py
index 7e0b6a564a..b8d559d611 100644
--- a/paimon-python/pypaimon/api/api_response.py
+++ b/paimon-python/pypaimon/api/api_response.py
@@ -21,8 +21,8 @@ from typing import Dict, Optional, Generic, List
from dataclasses import dataclass
from .rest_json import json_field
+from .schema import Schema
from .typedef import T
-from .data_types import DataField
@dataclass
@@ -84,6 +84,14 @@ class AuditRESTResponse(RESTResponse):
def get_updated_by(self) -> Optional[str]:
return self.updated_by
+ def put_audit_options_to(self, options: dict[str, str]) -> None:
+ """Puts audit-related options into the provided dictionary."""
+ options[self.FIELD_OWNER] = self.get_owner()
+ options[self.FIELD_CREATED_BY] = str(self.get_created_by())
+ options[self.FIELD_CREATED_AT] = str(self.get_created_at())
+ options[self.FIELD_UPDATED_BY] = str(self.get_updated_by())
+ options[self.FIELD_UPDATED_AT] = str(self.get_updated_at())
+
class PagedResponse(RESTResponse, Generic[T]):
FIELD_NEXT_PAGE_TOKEN = "nextPageToken"
@@ -126,54 +134,6 @@ class ListTablesResponse(PagedResponse[str]):
return self.next_page_token
-@dataclass
-class Schema:
- FIELD_FIELDS = "fields"
- FIELD_PARTITION_KEYS = "partitionKeys"
- FIELD_PRIMARY_KEYS = "primaryKeys"
- FIELD_OPTIONS = "options"
- FIELD_COMMENT = "comment"
-
- fields: List[DataField] = json_field(FIELD_FIELDS, default_factory=list)
- partition_keys: List[str] = json_field(
- FIELD_PARTITION_KEYS, default_factory=list)
- primary_keys: List[str] = json_field(
- FIELD_PRIMARY_KEYS, default_factory=list)
- options: Dict[str, str] = json_field(FIELD_OPTIONS, default_factory=dict)
- comment: Optional[str] = json_field(FIELD_COMMENT, default=None)
-
-
-@dataclass
-class TableSchema:
- """Table schema with ID"""
-
- id: int
- fields: List[DataField]
- highest_field_id: int
- partition_keys: List[str]
- primary_keys: List[str]
- options: Dict[str, str]
- comment: Optional[str]
-
- def to_schema(self) -> Schema:
- return Schema(
- fields=self.fields,
- partition_keys=self.partition_keys,
- primary_keys=self.primary_keys,
- options=self.options,
- comment=self.comment,
- )
-
-
-@dataclass
-class TableMetadata:
- """Table metadata"""
-
- schema: TableSchema
- is_external: bool
- uuid: str
-
-
@dataclass
class RESTToken:
"""REST authentication token"""
@@ -223,6 +183,24 @@ class GetTableResponse(AuditRESTResponse):
self.schema_id = schema_id
self.schema = schema
+ def get_id(self) -> str:
+ return self.id
+
+ def get_name(self) -> str:
+ return self.name
+
+ def get_path(self) -> str:
+ return self.path
+
+ def get_is_external(self) -> bool:
+ return self.is_external
+
+ def get_schema_id(self) -> int:
+ return self.schema_id
+
+ def get_schema(self) -> Schema:
+ return self.schema
+
@dataclass
class GetDatabaseResponse(AuditRESTResponse):
diff --git a/paimon-python/pypaimon/api/api_resquest.py
b/paimon-python/pypaimon/api/api_resquest.py
index 7bc4f6b2a1..3ec7c12edc 100644
--- a/paimon-python/pypaimon/api/api_resquest.py
+++ b/paimon-python/pypaimon/api/api_resquest.py
@@ -21,7 +21,7 @@ from dataclasses import dataclass
from typing import Dict, List
from .api_response import Schema
-from .typedef import Identifier
+from .identifier import Identifier
from .rest_json import json_field
diff --git a/paimon-python/pypaimon/api/core_options.py
b/paimon-python/pypaimon/api/core_options.py
new file mode 100644
index 0000000000..2745d1c281
--- /dev/null
+++ b/paimon-python/pypaimon/api/core_options.py
@@ -0,0 +1,44 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from enum import Enum
+
+
+class CoreOptions(str, Enum):
+ """Core options for paimon."""
+
+ def __str__(self):
+ return self.value
+
+ # Basic options
+ AUTO_CREATE = "auto-create"
+ PATH = "path"
+ TYPE = "type"
+ BRANCH = "branch"
+ BUCKET = "bucket"
+ BUCKET_KEY = "bucket-key"
+ WAREHOUSE = "warehouse"
+ # File format options
+ FILE_FORMAT = "file.format"
+ FILE_FORMAT_ORC = "orc"
+ FILE_FORMAT_AVRO = "avro"
+ FILE_FORMAT_PARQUET = "parquet"
+ FILE_COMPRESSION = "file.compression"
+ FILE_COMPRESSION_PER_LEVEL = "file.compression.per.level"
+ FILE_FORMAT_PER_LEVEL = "file.format.per.level"
+ FILE_BLOCK_SIZE = "file.block-size"
diff --git a/paimon-python/pypaimon/api/typedef.py
b/paimon-python/pypaimon/api/identifier.py
similarity index 74%
copy from paimon-python/pypaimon/api/typedef.py
copy to paimon-python/pypaimon/api/identifier.py
index 0501d5e1e1..0d280bbf21 100644
--- a/paimon-python/pypaimon/api/typedef.py
+++ b/paimon-python/pypaimon/api/identifier.py
@@ -1,3 +1,4 @@
+################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -6,26 +7,25 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
from dataclasses import dataclass
-from typing import Optional, TypeVar, Dict
+from typing import Optional
from pypaimon.api.rest_json import json_field
-T = TypeVar("T")
+SYSTEM_TABLE_SPLITTER = '$'
+SYSTEM_BRANCH_PREFIX = 'branch-'
@dataclass
class Identifier:
- """Table/View/Function identifier"""
database_name: str = json_field("database", default=None)
object_name: str = json_field("object", default=None)
@@ -64,11 +64,3 @@ class Identifier:
def is_system_table(self) -> bool:
return self.object_name.startswith('$')
-
-
-@dataclass
-class RESTAuthParameter:
- method: str
- path: str
- data: str
- parameters: Dict[str, str]
diff --git a/paimon-python/pypaimon/api/schema.py
b/paimon-python/pypaimon/api/schema.py
new file mode 100644
index 0000000000..4bc06806c8
--- /dev/null
+++ b/paimon-python/pypaimon/api/schema.py
@@ -0,0 +1,52 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+from dataclasses import dataclass
+from typing import Optional, List, Dict
+
+import pyarrow as pa
+from pypaimon.api.data_types import DataField
+from pypaimon.api.rest_json import json_field
+
+
+@dataclass
+class Schema:
+ FIELD_FIELDS = "fields"
+ FIELD_PARTITION_KEYS = "partitionKeys"
+ FIELD_PRIMARY_KEYS = "primaryKeys"
+ FIELD_OPTIONS = "options"
+ FIELD_COMMENT = "comment"
+
+ pa_schema: Optional[pa.Schema] = None
+ fields: List[DataField] = json_field(FIELD_FIELDS, default_factory=list)
+ partition_keys: List[str] = json_field(
+ FIELD_PARTITION_KEYS, default_factory=list)
+ primary_keys: List[str] = json_field(
+ FIELD_PRIMARY_KEYS, default_factory=list)
+ options: Dict[str, str] = json_field(FIELD_OPTIONS, default_factory=dict)
+ comment: Optional[str] = json_field(FIELD_COMMENT, default=None)
+
+ @staticmethod
+ def from_dict(data: dict):
+ fields = [DataField.from_dict(field) for field in data["fields"]]
+ return Schema(
+ fields=fields,
+ partition_keys=data["partitionKeys"],
+ primary_keys=data["primaryKeys"],
+ options=data["options"],
+ comment=data.get("comment"),
+ )
diff --git a/paimon-python/pypaimon/api/table_schema.py
b/paimon-python/pypaimon/api/table_schema.py
new file mode 100644
index 0000000000..ac0c2b7a11
--- /dev/null
+++ b/paimon-python/pypaimon/api/table_schema.py
@@ -0,0 +1,83 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import time
+from typing import List, Dict, Optional
+
+from pypaimon.api.schema import Schema
+from pypaimon.api.data_types import DataField
+
+
+class TableSchema:
+
+ def __init__(self, id: int, fields: List[DataField], highest_field_id: int,
+ partition_keys: List[str], primary_keys: List[str], options:
Dict[str, str],
+ comment: Optional[str] = None, time_millis: Optional[int] =
None):
+ self.id = id
+ self.fields = fields
+ self.highest_field_id = highest_field_id
+ self.partition_keys = partition_keys or []
+ self.primary_keys = primary_keys or []
+ self.options = options or {}
+ self.comment = comment
+ self.time_millis = time_millis if time_millis is not None else
int(time.time() * 1000)
+
+ def to_schema(self) -> Schema:
+ # pa_schema = schema_util.convert_data_fields_to_pa_schema(self.fields)
+ return Schema(
+ fields=self.fields,
+ partition_keys=self.partition_keys,
+ primary_keys=self.primary_keys,
+ options=self.options,
+ comment=self.comment
+ )
+
+ @staticmethod
+ def create(schema_id: int, schema: Schema) -> "TableSchema":
+ fields: List[DataField] = schema.fields
+
+ partition_keys: List[str] = schema.partition_keys
+
+ primary_keys: List[str] = schema.primary_keys
+
+ options: Dict[str, str] = schema.options
+
+ highest_field_id: int = None
+
+ return TableSchema(
+ schema_id,
+ fields,
+ highest_field_id,
+ partition_keys,
+ primary_keys,
+ options,
+ schema.comment,
+ int(time.time())
+ )
+
+ def copy(self, new_options: Optional[Dict[str, str]] = None) ->
"TableSchema":
+ return TableSchema(
+ id=self.id,
+ fields=self.fields,
+ highest_field_id=self.highest_field_id,
+ partition_keys=self.partition_keys,
+ primary_keys=self.primary_keys,
+ options=new_options,
+ comment=self.comment,
+ time_millis=self.time_millis
+ )
diff --git a/paimon-python/pypaimon/api/typedef.py
b/paimon-python/pypaimon/api/typedef.py
index 0501d5e1e1..f70ef9adf4 100644
--- a/paimon-python/pypaimon/api/typedef.py
+++ b/paimon-python/pypaimon/api/typedef.py
@@ -16,56 +16,12 @@
# under the License.
from dataclasses import dataclass
-from typing import Optional, TypeVar, Dict
+from typing import TypeVar, Dict
-from pypaimon.api.rest_json import json_field
T = TypeVar("T")
-@dataclass
-class Identifier:
- """Table/View/Function identifier"""
-
- database_name: str = json_field("database", default=None)
- object_name: str = json_field("object", default=None)
- branch_name: Optional[str] = json_field("branch", default=None)
-
- @classmethod
- def create(cls, database_name: str, object_name: str) -> "Identifier":
- return cls(database_name, object_name)
-
- @classmethod
- def from_string(cls, full_name: str) -> "Identifier":
- parts = full_name.split(".")
- if len(parts) == 2:
- return cls(parts[0], parts[1])
- elif len(parts) == 3:
- return cls(parts[0], parts[1], parts[2])
- else:
- raise ValueError(f"Invalid identifier format: {full_name}")
-
- def get_full_name(self) -> str:
- if self.branch_name:
- return
f"{self.database_name}.{self.object_name}.{self.branch_name}"
- return f"{self.database_name}.{self.object_name}"
-
- def get_database_name(self) -> str:
- return self.database_name
-
- def get_table_name(self) -> str:
- return self.object_name
-
- def get_object_name(self) -> str:
- return self.object_name
-
- def get_branch_name(self) -> Optional[str]:
- return self.branch_name
-
- def is_system_table(self) -> bool:
- return self.object_name.startswith('$')
-
-
@dataclass
class RESTAuthParameter:
method: str
diff --git a/paimon-python/pypaimon/catalog/catalog_loader.py
b/paimon-python/pypaimon/catalog/catalog_loader.py
new file mode 100644
index 0000000000..09b62492ff
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/catalog_loader.py
@@ -0,0 +1,24 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from abc import ABC, abstractmethod
+
+
+class CatalogLoader(ABC):
+ @abstractmethod
+ def load(self):
+ pass
diff --git a/paimon-python/pypaimon/catalog/catalog_utils.py
b/paimon-python/pypaimon/catalog/catalog_utils.py
new file mode 100644
index 0000000000..d51de0754d
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/catalog_utils.py
@@ -0,0 +1,55 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from pathlib import Path
+from typing import Callable, Any
+
+from pypaimon.api.core_options import CoreOptions
+from pypaimon.api.identifier import Identifier
+
+from pypaimon.catalog.catalog import Catalog
+from pypaimon.catalog.table_metadata import TableMetadata
+from pypaimon.table.catalog_environment import CatalogEnvironment
+from pypaimon.table.file_store_table import FileStoreTable
+from pypaimon.table.file_store_table_factory import FileStoreTableFactory
+
+
+class CatalogUtils:
+ @staticmethod
+ def load_table(
+ identifier: Identifier,
+ internal_file_io: Callable[[Path], Any],
+ external_file_io: Callable[[Path], Any],
+ metadata_loader: Callable[[Identifier], TableMetadata],
+ ) -> FileStoreTable:
+ metadata = metadata_loader(identifier)
+ schema = metadata.schema
+ data_file_io = external_file_io if metadata.is_external else
internal_file_io
+ catalog_env = CatalogEnvironment(
+ identifier=identifier,
+ uuid=metadata.uuid,
+ catalog_loader=None,
+ supports_version_management=False
+ )
+
+ path = Path(schema.options.get(CoreOptions.PATH))
+ table = FileStoreTableFactory.create(data_file_io(path), path, schema,
catalog_env)
+ return table
+
+ @staticmethod
+ def is_system_database(database_name: str) -> bool:
+ return Catalog.SYSTEM_DATABASE_NAME.equals(database_name)
diff --git a/paimon-python/pypaimon/catalog/table_metadata.py
b/paimon-python/pypaimon/catalog/table_metadata.py
new file mode 100644
index 0000000000..db79ac522a
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/table_metadata.py
@@ -0,0 +1,40 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from typing import Optional
+
+from pypaimon.api.table_schema import TableSchema
+
+
+class TableMetadata:
+
+ def __init__(self, schema: TableSchema, is_external: bool, uuid:
Optional[str] = None):
+ self._schema = schema
+ self._is_external = is_external
+ self._uuid = uuid
+
+ @property
+ def schema(self) -> TableSchema:
+ return self._schema
+
+ @property
+ def is_external(self) -> bool:
+ return self._is_external
+
+ @property
+ def uuid(self) -> Optional[str]:
+ return self._uuid
diff --git a/paimon-python/pypaimon/common/__init__.py
b/paimon-python/pypaimon/common/__init__.py
new file mode 100644
index 0000000000..53ed4d36c2
--- /dev/null
+++ b/paimon-python/pypaimon/common/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/paimon-python/pypaimon/common/file_io.py
b/paimon-python/pypaimon/common/file_io.py
new file mode 100644
index 0000000000..438b73ce1d
--- /dev/null
+++ b/paimon-python/pypaimon/common/file_io.py
@@ -0,0 +1,25 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class FileIO(ABC):
+ @abstractmethod
+ def exists(self, path: Path) -> bool:
+ raise NotImplementedError("Method 'exists' must be implemented by
subclasses.")
diff --git a/paimon-python/pypaimon/pvfs/__init__.py
b/paimon-python/pypaimon/pvfs/__init__.py
index 36ba2f2909..25dd7df7ea 100644
--- a/paimon-python/pypaimon/pvfs/__init__.py
+++ b/paimon-python/pypaimon/pvfs/__init__.py
@@ -31,9 +31,8 @@ import fsspec
from fsspec import AbstractFileSystem
from fsspec.implementations.local import LocalFileSystem
-from pypaimon.api import RESTApi, GetTableTokenResponse, Schema,
GetTableResponse
+from pypaimon.api import RESTApi, GetTableTokenResponse, Schema,
GetTableResponse, Identifier
from pypaimon.api.client import NoSuchResourceException, AlreadyExistsException
-from pypaimon.api.typedef import Identifier
from pypaimon.api.config import RESTCatalogOptions, OssOptions, PVFSOptions
PROTOCOL_NAME = "pvfs"
diff --git a/paimon-python/pypaimon/rest/rest_catalog.py
b/paimon-python/pypaimon/rest/rest_catalog.py
index 8fca50501b..2f76aa5a1d 100644
--- a/paimon-python/pypaimon/rest/rest_catalog.py
+++ b/paimon-python/pypaimon/rest/rest_catalog.py
@@ -15,16 +15,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+from pathlib import Path
from typing import List, Dict, Optional
from pypaimon.api import RESTApi, RESTCatalogOptions
-from pypaimon.api.api_response import PagedList
+from pypaimon.api.api_response import PagedList, GetTableResponse
+from pypaimon.api.core_options import CoreOptions
+from pypaimon.api.identifier import Identifier
from pypaimon.api.options import Options
+from pypaimon.api.schema import Schema
+from pypaimon.api.table_schema import TableSchema
+
from pypaimon.catalog.catalog import Catalog
from pypaimon.catalog.catalog_context import CatalogContext
+from pypaimon.catalog.catalog_utils import CatalogUtils
from pypaimon.catalog.database import Database
from pypaimon.catalog.property_change import PropertyChange
+from pypaimon.catalog.table_metadata import TableMetadata
+from pypaimon.rest.rest_token_file_io import RESTTokenFileIO
+from pypaimon.table.file_store_table import FileStoreTable
class RESTCatalog(Catalog):
@@ -48,6 +58,7 @@ class RESTCatalog(Catalog):
response = self.api.get_database(name)
options = response.options
options[Catalog.DB_LOCATION_PROP] = response.location
+ response.put_audit_options_to(options)
if response is not None:
return Database(name, options)
@@ -74,3 +85,37 @@ class RESTCatalog(Catalog):
page_token,
table_name_pattern
)
+
+ def get_table(self, identifier: Identifier) -> FileStoreTable:
+ return CatalogUtils.load_table(
+ identifier,
+ lambda path: self.file_io_for_data(path, identifier),
+ self.file_io_from_options,
+ self.load_table_metadata,
+ )
+
+ def load_table_metadata(self, identifier: Identifier) -> TableMetadata:
+ response = self.api.get_table(identifier)
+ return self.to_table_metadata(identifier.get_database_name(), response)
+
+ def to_table_metadata(self, db: str, response: GetTableResponse) ->
TableMetadata:
+ schema = TableSchema.create(response.get_schema_id(),
Schema.from_dict(response.get_schema()))
+ options: Dict[str, str] = dict(schema.options)
+ options[CoreOptions.PATH] = response.get_path()
+ response.put_audit_options_to(options)
+
+ identifier = Identifier.create(db, response.get_name())
+ if identifier.get_branch_name() is not None:
+ options[CoreOptions.BRANCH] = identifier.get_branch_name()
+
+ return TableMetadata(
+ schema=schema.copy(options),
+ is_external=response.get_is_external(),
+ uuid=response.get_id()
+ )
+
+ def file_io_from_options(self, path: Path):
+ return None
+
+ def file_io_for_data(self, path: Path, identifier: Identifier):
+ return RESTTokenFileIO(identifier, path, None, None) if
self.data_token_enabled else None
diff --git a/paimon-python/pypaimon/rest/rest_token_file_io.py
b/paimon-python/pypaimon/rest/rest_token_file_io.py
new file mode 100644
index 0000000000..5b179b95f0
--- /dev/null
+++ b/paimon-python/pypaimon/rest/rest_token_file_io.py
@@ -0,0 +1,34 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from pathlib import Path
+from typing import Optional
+
+from pypaimon.api.identifier import Identifier
+from pypaimon.common.file_io import FileIO
+
+
+class RESTTokenFileIO(FileIO):
+
+ def __init__(self, identifier: Identifier, path: Path, warehouse:
Optional[str] = None,
+ catalog_options: Optional[dict] = None):
+ super().__init__(warehouse, catalog_options)
+ self.identifier = identifier
+ self.path = path
+
+ def exists(self, path: Path) -> bool:
+ pass
diff --git a/paimon-python/pypaimon/schema/__init__.py
b/paimon-python/pypaimon/schema/__init__.py
new file mode 100644
index 0000000000..53ed4d36c2
--- /dev/null
+++ b/paimon-python/pypaimon/schema/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/paimon-python/pypaimon/schema/schema_manager.py
b/paimon-python/pypaimon/schema/schema_manager.py
new file mode 100644
index 0000000000..c87240062b
--- /dev/null
+++ b/paimon-python/pypaimon/schema/schema_manager.py
@@ -0,0 +1,29 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from pathlib import Path
+
+from pypaimon.common.file_io import FileIO
+
+
+class SchemaManager:
+
+ def __init__(self, file_io: FileIO, table_path: Path):
+ self.schema_prefix = "schema-"
+ self.file_io = file_io
+ self.table_path = table_path
+ self.schema_path = table_path / "schema"
diff --git a/paimon-python/pypaimon/table/__init__.py
b/paimon-python/pypaimon/table/__init__.py
new file mode 100644
index 0000000000..53ed4d36c2
--- /dev/null
+++ b/paimon-python/pypaimon/table/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/paimon-python/pypaimon/table/catalog_environment.py
b/paimon-python/pypaimon/table/catalog_environment.py
new file mode 100644
index 0000000000..3150e2d78e
--- /dev/null
+++ b/paimon-python/pypaimon/table/catalog_environment.py
@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from typing import Optional
+
+from pypaimon.api.identifier import Identifier
+from pypaimon.catalog.catalog_loader import CatalogLoader
+
+
+class CatalogEnvironment:
+
+ def __init__(
+ self,
+ identifier: Optional[Identifier] = None,
+ uuid: Optional[str] = None,
+ catalog_loader: Optional[CatalogLoader] = None,
+ supports_version_management: bool = False
+ ):
+ self.identifier = identifier
+ self.uuid = uuid
+ self.catalog_loader = catalog_loader
+ self.supports_version_management = supports_version_management
diff --git a/paimon-python/pypaimon/table/file_store_table.py
b/paimon-python/pypaimon/table/file_store_table.py
new file mode 100644
index 0000000000..de5f8a870f
--- /dev/null
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -0,0 +1,42 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from pathlib import Path
+
+from pypaimon.api.identifier import Identifier
+from pypaimon.api.table_schema import TableSchema
+from pypaimon.common.file_io import FileIO
+from pypaimon.schema.schema_manager import SchemaManager
+from pypaimon.table.table import Table
+
+
+class FileStoreTable(Table):
+ def __init__(self, file_io: FileIO, identifier: Identifier, table_path:
Path,
+ table_schema: TableSchema):
+ self.file_io = file_io
+ self.identifier = identifier
+ self.table_path = table_path
+
+ self.fields = table_schema.fields
+ self.primary_keys = table_schema.primary_keys
+ self.partition_keys = table_schema.partition_keys
+
+ self.options = table_schema.options
+ self.table_schema = table_schema
+ self.schema_manager = SchemaManager(file_io, table_path)
+ self.is_primary_key_table = bool(self.primary_keys)
diff --git a/paimon-python/pypaimon/table/file_store_table_factory.py
b/paimon-python/pypaimon/table/file_store_table_factory.py
new file mode 100644
index 0000000000..5569f60d8e
--- /dev/null
+++ b/paimon-python/pypaimon/table/file_store_table_factory.py
@@ -0,0 +1,35 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from pathlib import Path
+
+from pypaimon.api.table_schema import TableSchema
+from pypaimon.common.file_io import FileIO
+from pypaimon.table.catalog_environment import CatalogEnvironment
+from pypaimon.table.file_store_table import FileStoreTable
+
+
+class FileStoreTableFactory:
+ @staticmethod
+ def create(
+ file_io: FileIO,
+ table_path: Path,
+ table_schema: TableSchema,
+ catalog_environment: CatalogEnvironment
+ ) -> FileStoreTable:
+ """Create FileStoreTable with dynamic options and catalog
environment"""
+ return FileStoreTable(file_io, catalog_environment.identifier,
table_path, table_schema)
diff --git a/paimon-python/pypaimon/table/table.py
b/paimon-python/pypaimon/table/table.py
new file mode 100644
index 0000000000..6f15b049ae
--- /dev/null
+++ b/paimon-python/pypaimon/table/table.py
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+
+from abc import ABC
+
+
+class Table(ABC):
+ """A table provides basic abstraction for table read and write."""
diff --git a/paimon-python/pypaimon/tests/api_test.py
b/paimon-python/pypaimon/tests/api_test.py
index 63bd994ab7..d32c5c201c 100644
--- a/paimon-python/pypaimon/tests/api_test.py
+++ b/paimon-python/pypaimon/tests/api_test.py
@@ -21,15 +21,18 @@ import unittest
import pypaimon.api as api
from .rest_server import RESTCatalogServer
-from ..api.api_response import (ConfigResponse, TableMetadata, TableSchema,
DataField)
+from ..api.api_response import (ConfigResponse)
from ..api import RESTApi
from ..api.auth import BearTokenAuthProvider
+from ..api.identifier import Identifier
from ..api.options import Options
from ..api.rest_json import JSON
+from ..api.table_schema import TableSchema
from ..api.token_loader import DLFTokenLoaderFactory, DLFToken
-from ..api.typedef import Identifier
-from ..api.data_types import AtomicInteger, DataTypeParser, AtomicType,
ArrayType, MapType, RowType
+
+from ..api.data_types import AtomicInteger, DataTypeParser, AtomicType,
ArrayType, MapType, RowType, DataField
from ..catalog.catalog_context import CatalogContext
+from ..catalog.table_metadata import TableMetadata
from ..rest.rest_catalog import RESTCatalog
@@ -226,6 +229,8 @@ class ApiTestCase(unittest.TestCase):
rest_catalog =
RESTCatalog(CatalogContext.create_from_options(Options(options)))
self.assertSetEqual(set(rest_catalog.list_databases()),
{*test_databases})
self.assertEqual(rest_catalog.get_database('default').name,
test_databases.get('default').name)
+ table =
rest_catalog.get_table(Identifier.from_string('default.user'))
+ self.assertEqual(table.identifier.get_full_name(), 'default.user')
finally:
# Shutdown server
server.shutdown()
diff --git a/paimon-python/pypaimon/tests/pvfs_test.py
b/paimon-python/pypaimon/tests/pvfs_test.py
index c3248dcea7..3b084d0019 100644
--- a/paimon-python/pypaimon/tests/pvfs_test.py
+++ b/paimon-python/pypaimon/tests/pvfs_test.py
@@ -23,9 +23,10 @@ import pandas
from pathlib import Path
from pypaimon.api import ConfigResponse
-from pypaimon.api.api_response import TableSchema, TableMetadata
from pypaimon.api.auth import BearTokenAuthProvider
from pypaimon.api.data_types import DataField, AtomicType
+from pypaimon.api.table_schema import TableSchema
+from pypaimon.catalog.table_metadata import TableMetadata
from pypaimon.pvfs import PaimonVirtualFileSystem
from pypaimon.tests.api_test import RESTCatalogServer
diff --git a/paimon-python/pypaimon/tests/rest_server.py
b/paimon-python/pypaimon/tests/rest_server.py
index 5439402c6a..0bcf5d1dd3 100644
--- a/paimon-python/pypaimon/tests/rest_server.py
+++ b/paimon-python/pypaimon/tests/rest_server.py
@@ -28,12 +28,13 @@ from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse
import pypaimon.api as api
-from ..api import RenameTableRequest, CreateTableRequest, CreateDatabaseRequest
+from ..api import RenameTableRequest, CreateTableRequest,
CreateDatabaseRequest, Identifier
from ..api.api_response import (ConfigResponse, ListDatabasesResponse,
GetDatabaseResponse,
- TableMetadata, Schema, GetTableResponse,
ListTablesResponse,
- TableSchema, RESTResponse, PagedList)
+ Schema, GetTableResponse, ListTablesResponse,
+ RESTResponse, PagedList)
from ..api.rest_json import JSON
-from ..api.typedef import Identifier
+from ..api.table_schema import TableSchema
+from ..catalog.table_metadata import TableMetadata
@dataclass
