This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0dfdcfda2c [python] Support database query function of python rest
catalog (#5961)
0dfdcfda2c is described below
commit 0dfdcfda2cd8bdc61bc21c2e4722d5e83256caac
Author: HeavenZH <[email protected]>
AuthorDate: Fri Jul 25 18:41:57 2025 +0800
[python] Support database query function of python rest catalog (#5961)
---
paimon-python/pypaimon/api/config.py | 1 +
paimon-python/pypaimon/api/options.py | 35 +++++++++++
paimon-python/pypaimon/catalog/__init__.py | 17 +++++
paimon-python/pypaimon/catalog/catalog.py | 37 +++++++++++
paimon-python/pypaimon/catalog/catalog_context.py | 34 ++++++++++
paimon-python/pypaimon/catalog/database.py | 35 +++++++++++
paimon-python/pypaimon/catalog/property_change.py | 53 ++++++++++++++++
paimon-python/pypaimon/rest/__init__.py | 17 +++++
paimon-python/pypaimon/rest/rest_catalog.py | 76 +++++++++++++++++++++++
paimon-python/pypaimon/tests/api_test.py | 56 +++++++++++++++++
10 files changed, 361 insertions(+)
diff --git a/paimon-python/pypaimon/api/config.py
b/paimon-python/pypaimon/api/config.py
index dcdbb95c50..e9a6561ab2 100644
--- a/paimon-python/pypaimon/api/config.py
+++ b/paimon-python/pypaimon/api/config.py
@@ -27,6 +27,7 @@ class RESTCatalogOptions:
WAREHOUSE = "warehouse"
TOKEN_PROVIDER = "token.provider"
TOKEN = "token"
+ DATA_TOKEN_ENABLED = "data.token.enabled"
DLF_REGION = "dlf.region"
DLF_ACCESS_KEY_ID = "dlf.access-key-id"
DLF_ACCESS_KEY_SECRET = "dlf.access-key-secret"
diff --git a/paimon-python/pypaimon/api/options.py
b/paimon-python/pypaimon/api/options.py
new file mode 100644
index 0000000000..21fa8f0815
--- /dev/null
+++ b/paimon-python/pypaimon/api/options.py
@@ -0,0 +1,35 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+class Options:
+ def __init__(self, data):
+ self.data = data
+
+ @classmethod
+ def from_none(cls):
+ return cls({})
+
+ def to_map(self) -> dict:
+ return self.data
+
+ def get(self, key: str, default=None):
+ return self.data.get(key, default)
+
+ def set(self, key: str, value):
+ self.data[key] = value
diff --git a/paimon-python/pypaimon/catalog/__init__.py
b/paimon-python/pypaimon/catalog/__init__.py
new file mode 100644
index 0000000000..53ed4d36c2
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/paimon-python/pypaimon/catalog/catalog.py
b/paimon-python/pypaimon/catalog/catalog.py
new file mode 100644
index 0000000000..c133b362c5
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/catalog.py
@@ -0,0 +1,37 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class Catalog(ABC):
+ """
+ This interface is responsible for reading and writing
+ metadata such as database/table from a paimon catalog.
+ """
+ SYSTEM_DATABASE_NAME = "sys"
+ DB_LOCATION_PROP = "location"
+
+ @abstractmethod
+ def get_database(self, name: str) -> 'Database':
+ """Get paimon database identified by the given name."""
+
+ @abstractmethod
+ def create_database(self, name: str, properties: Optional[dict] = None):
+ """Create a database with properties."""
diff --git a/paimon-python/pypaimon/catalog/catalog_context.py
b/paimon-python/pypaimon/catalog/catalog_context.py
new file mode 100644
index 0000000000..881f09734a
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/catalog_context.py
@@ -0,0 +1,34 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from pypaimon.api.options import Options
+
+
+class CatalogContext:
+ def __init__(self, options: Options, hadoop_conf, prefer_loader,
fallback_io_loader):
+ self.options = options
+ self.hadoop_conf = hadoop_conf
+ self.prefer_io_loader = prefer_loader
+ self.fallback_io_loader = fallback_io_loader
+
+ @staticmethod
+ def create(options: Options, hadoop_conf, prefer_loader,
fallback_io_loader):
+ return CatalogContext(options, hadoop_conf, prefer_loader,
fallback_io_loader)
+
+ @staticmethod
+ def create_from_options(options: Options):
+ return CatalogContext(options, None, None, None)
diff --git a/paimon-python/pypaimon/catalog/database.py
b/paimon-python/pypaimon/catalog/database.py
new file mode 100644
index 0000000000..35da9ed2e7
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/database.py
@@ -0,0 +1,35 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+
+from typing import Optional
+
+
+class Database:
+ """Structure of a Database."""
+
+ def __init__(self, name: str, options: dict, comment: Optional[str] =
None):
+ self.name = name
+ self.options = options
+ self.comment = comment
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, Database):
+ return False
+ return (self.name == other.name and
+ self.options == other.options and
+ self.comment == other.comment)
diff --git a/paimon-python/pypaimon/catalog/property_change.py
b/paimon-python/pypaimon/catalog/property_change.py
new file mode 100644
index 0000000000..e21b0c2dbe
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/property_change.py
@@ -0,0 +1,53 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from abc import ABC
+from typing import List, Tuple, Dict, Set
+
+
+class PropertyChange(ABC):
+ @staticmethod
+ def set_property(property: str, value: str) -> "PropertyChange":
+ return SetProperty(property, value)
+
+ @staticmethod
+ def remove_property(property: str) -> "PropertyChange":
+ return RemoveProperty(property)
+
+ @staticmethod
+ def get_set_properties_to_remove_keys(changes: List["PropertyChange"]) ->
Tuple[Dict[str, str], Set[str]]:
+ set_properties: Dict[str, str] = {}
+ remove_keys: Set[str] = set()
+
+ for change in changes:
+ if isinstance(change, SetProperty):
+ set_properties[change.property] = change.value
+ elif isinstance(change, RemoveProperty):
+ remove_keys.add(change.property)
+
+ return set_properties, remove_keys
+
+
+class SetProperty(PropertyChange):
+ def __init__(self, property: str, value: str):
+ self.property = property
+ self.value = value
+
+
+class RemoveProperty(PropertyChange):
+ def __init__(self, property: str):
+ self.property = property
diff --git a/paimon-python/pypaimon/rest/__init__.py
b/paimon-python/pypaimon/rest/__init__.py
new file mode 100644
index 0000000000..53ed4d36c2
--- /dev/null
+++ b/paimon-python/pypaimon/rest/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/paimon-python/pypaimon/rest/rest_catalog.py
b/paimon-python/pypaimon/rest/rest_catalog.py
new file mode 100644
index 0000000000..8fca50501b
--- /dev/null
+++ b/paimon-python/pypaimon/rest/rest_catalog.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from typing import List, Dict, Optional
+
+from pypaimon.api import RESTApi, RESTCatalogOptions
+from pypaimon.api.api_response import PagedList
+from pypaimon.api.options import Options
+
+from pypaimon.catalog.catalog import Catalog
+from pypaimon.catalog.catalog_context import CatalogContext
+from pypaimon.catalog.database import Database
+from pypaimon.catalog.property_change import PropertyChange
+
+
+class RESTCatalog(Catalog):
+ def __init__(self, context: CatalogContext, config_required:
Optional[bool] = True):
+ self.api = RESTApi(context.options.to_map(), config_required)
+ self.context = CatalogContext.create(Options(self.api.options),
context.hadoop_conf, context.prefer_io_loader,
+ context.fallback_io_loader)
+ self.data_token_enabled =
self.api.options.get(RESTCatalogOptions.DATA_TOKEN_ENABLED)
+
+ def list_databases(self) -> List[str]:
+ return self.api.list_databases()
+
+ def list_databases_paged(self, max_results: Optional[int] = None,
page_token: Optional[str] = None,
+ database_name_pattern: Optional[str] = None) ->
PagedList[str]:
+ return self.api.list_databases_paged(max_results, page_token,
database_name_pattern)
+
+ def create_database(self, name: str, properties: Dict[str, str] = None):
+ self.api.create_database(name, properties)
+
+ def get_database(self, name: str) -> Database:
+ response = self.api.get_database(name)
+ options = response.options
+ options[Catalog.DB_LOCATION_PROP] = response.location
+ if response is not None:
+ return Database(name, options)
+
+ def drop_database(self, name: str):
+ self.api.drop_database(name)
+
+ def alter_database(self, name: str, changes: List[PropertyChange]):
+ set_properties, remove_keys =
PropertyChange.get_set_properties_to_remove_keys(changes)
+ self.api.alter_database(name, list(remove_keys), set_properties)
+
+ def list_tables(self, database_name: str) -> List[str]:
+ return self.api.list_tables(database_name)
+
+ def list_tables_paged(
+ self,
+ database_name: str,
+ max_results: Optional[int] = None,
+ page_token: Optional[str] = None,
+ table_name_pattern: Optional[str] = None
+ ) -> PagedList[str]:
+ return self.api.list_tables_paged(
+ database_name,
+ max_results,
+ page_token,
+ table_name_pattern
+ )
diff --git a/paimon-python/pypaimon/tests/api_test.py
b/paimon-python/pypaimon/tests/api_test.py
index 9559296652..63bd994ab7 100644
--- a/paimon-python/pypaimon/tests/api_test.py
+++ b/paimon-python/pypaimon/tests/api_test.py
@@ -24,10 +24,13 @@ from .rest_server import RESTCatalogServer
from ..api.api_response import (ConfigResponse, TableMetadata, TableSchema,
DataField)
from ..api import RESTApi
from ..api.auth import BearTokenAuthProvider
+from ..api.options import Options
from ..api.rest_json import JSON
from ..api.token_loader import DLFTokenLoaderFactory, DLFToken
from ..api.typedef import Identifier
from ..api.data_types import AtomicInteger, DataTypeParser, AtomicType,
ArrayType, MapType, RowType
+from ..catalog.catalog_context import CatalogContext
+from ..rest.rest_catalog import RESTCatalog
class ApiTestCase(unittest.TestCase):
@@ -175,6 +178,59 @@ class ApiTestCase(unittest.TestCase):
server.shutdown()
print("Server stopped")
+ def test_rest_catalog(self):
+ """Example usage of RESTCatalogServer"""
+ # Setup logging
+ logging.basicConfig(level=logging.INFO)
+
+ # Create config
+ config = ConfigResponse(defaults={"prefix": "mock-test"})
+ token = str(uuid.uuid4())
+ # Create server
+ server = RESTCatalogServer(
+ data_path="/tmp/test_warehouse",
+ auth_provider=BearTokenAuthProvider(token),
+ config=config,
+ warehouse="test_warehouse"
+ )
+ try:
+ # Start server
+ server.start()
+ print(f"Server started at: {server.get_url()}")
+ test_databases = {
+ "default": server.mock_database("default", {"env": "test"}),
+ "test_db1": server.mock_database("test_db1", {"env": "test"}),
+ "test_db2": server.mock_database("test_db2", {"env": "test"}),
+ "prod_db": server.mock_database("prod_db", {"env": "prod"})
+ }
+ data_fields = [
+ DataField(0, "name", AtomicType('INT'), 'desc name'),
+ DataField(1, "arr11", ArrayType(True, AtomicType('INT')),
'desc arr11'),
+ DataField(2, "map11", MapType(False, AtomicType('INT'),
+ MapType(False,
AtomicType('INT'), AtomicType('INT'))),
+ 'desc arr11'),
+ ]
+ schema = TableSchema(len(data_fields), data_fields,
len(data_fields), [], [], {}, "")
+ test_tables = {
+ "default.user": TableMetadata(uuid=str(uuid.uuid4()),
is_external=True, schema=schema),
+ }
+ server.table_metadata_store.update(test_tables)
+ server.database_store.update(test_databases)
+ options = {
+ 'uri': f"http://localhost:{server.port}";,
+ 'warehouse': 'test_warehouse',
+ 'dlf.region': 'cn-hangzhou',
+ "token.provider": "bear",
+ 'token': token
+ }
+ rest_catalog =
RESTCatalog(CatalogContext.create_from_options(Options(options)))
+ self.assertSetEqual(set(rest_catalog.list_databases()),
{*test_databases})
+ self.assertEqual(rest_catalog.get_database('default').name,
test_databases.get('default').name)
+ finally:
+ # Shutdown server
+ server.shutdown()
+ print("Server stopped")
+
def test_ecs_loader_token(self):
token = DLFToken(
access_key_id='AccessKeyId',
