This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d49e4f1c49 [python] Unify the REST User-Agent format (#10309)
d49e4f1c49 is described below
commit d49e4f1c49b2a36209eff3b56de72f1157866f09
Author: Dapeng Sun(孙大鹏) <[email protected]>
AuthorDate: Fri Oct 2 09:41:44 2026 +0800
[python] Unify the REST User-Agent format (#10309)
---
paimon-python/pypaimon/api/client.py | 10 +-
paimon-python/pypaimon/api/rest_api.py | 4 +-
paimon-python/pypaimon/api/token_loader.py | 2 +
paimon-python/pypaimon/build_info.py | 8 ++
paimon-python/pypaimon/common/options/config.py | 12 +-
paimon-python/pypaimon/common/user_agent.py | 64 +++++++++
paimon-python/pypaimon/filesystem/pvfs.py | 3 +-
.../pypaimon/tests/rest_user_agent_test.py | 152 +++++++++++++++++++++
8 files changed, 250 insertions(+), 5 deletions(-)
diff --git a/paimon-python/pypaimon/api/client.py
b/paimon-python/pypaimon/api/client.py
index 9e73c41723..4e0214d179 100644
--- a/paimon-python/pypaimon/api/client.py
+++ b/paimon-python/pypaimon/api/client.py
@@ -37,6 +37,7 @@ from pypaimon.api.rest_exception import
(AlreadyExistsException,
ServiceFailureException,
ServiceUnavailableException)
from pypaimon.api.typedef import RESTAuthParameter
+from pypaimon.common import user_agent
from pypaimon.common.json_util import JSON
T = TypeVar('T', bound='RESTResponse')
@@ -286,7 +287,7 @@ class HttpClient(RESTClient):
_READ_TIMEOUT_SECONDS = 180
_MAX_RETRIES = 5
- def __init__(self, uri: str):
+ def __init__(self, uri: str, user_agent_value: Optional[str] = None):
self.logger = logging.getLogger(self.__class__.__name__)
self.uri = _normalize_uri(uri)
self.error_handler = DefaultErrorHandler.get_instance()
@@ -301,10 +302,15 @@ class HttpClient(RESTClient):
# and pass it explicitly on every call (see ``_execute_request``).
self._timeout = (self._CONNECT_TIMEOUT_SECONDS,
self._READ_TIMEOUT_SECONDS)
+ # A per-request User-Agent, e.g. from ``header.User-Agent``, still
takes precedence.
self.session.headers.update({
- 'Accept': 'application/json'
+ 'Accept': 'application/json',
+ 'User-Agent': user_agent_value or user_agent.rest_user_agent(),
})
+ def set_user_agent(self, value: str) -> None:
+ self.session.headers['User-Agent'] = value
+
def set_error_handler(self, error_handler: ErrorHandler) -> None:
self.error_handler = error_handler
diff --git a/paimon-python/pypaimon/api/rest_api.py
b/paimon-python/pypaimon/api/rest_api.py
index aff1e8793b..f9ecb27faa 100755
--- a/paimon-python/pypaimon/api/rest_api.py
+++ b/paimon-python/pypaimon/api/rest_api.py
@@ -50,6 +50,7 @@ from pypaimon.api.client import HttpClient
from pypaimon.api.resource_paths import ResourcePaths
from pypaimon.api.rest_util import RESTUtil
from pypaimon.api.typedef import T
+from pypaimon.common import user_agent
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions
from pypaimon.common.identifier import Identifier
@@ -85,7 +86,7 @@ class RESTApi:
raise ValueError("URI cannot be empty")
self.logger = logging.getLogger(self.__class__.__name__)
- self.client = HttpClient(uri)
+ self.client = HttpClient(uri, user_agent.rest_user_agent(options))
auth_provider = AuthProviderFactory.create_auth_provider(options)
base_headers = RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX)
@@ -105,6 +106,7 @@ class RESTApi:
RESTAuthFunction(base_headers, auth_provider),
)
options = config_response.merge(options)
+ self.client.set_user_agent(user_agent.rest_user_agent(options))
base_headers.update(
RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX)
)
diff --git a/paimon-python/pypaimon/api/token_loader.py
b/paimon-python/pypaimon/api/token_loader.py
index 15e8035b0d..8f2affd220 100644
--- a/paimon-python/pypaimon/api/token_loader.py
+++ b/paimon-python/pypaimon/api/token_loader.py
@@ -27,6 +27,7 @@ from requests.adapters import HTTPAdapter
from requests.exceptions import RequestException
from pypaimon.api.client import ExponentialRetry
+from pypaimon.common import user_agent
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions
from pypaimon.common.json_util import JSON, json_field, json_ignore_field
@@ -136,6 +137,7 @@ class HTTPClient:
self.connect_timeout = connect_timeout
self.read_timeout = read_timeout
self.session = requests.Session()
+ self.session.headers['User-Agent'] = user_agent.rest_user_agent()
# Add retry adapter
retry_interceptor = ExponentialRetry(max_retries=3)
diff --git a/paimon-python/pypaimon/build_info.py
b/paimon-python/pypaimon/build_info.py
index e034f356fd..22649caf77 100644
--- a/paimon-python/pypaimon/build_info.py
+++ b/paimon-python/pypaimon/build_info.py
@@ -84,3 +84,11 @@ _FULL_VERSION = _load_full_version()
def full_version():
"""Return ``<pypaimon-version>-<commit-id>`` for snapshot provenance."""
return _FULL_VERSION
+
+
+def version():
+ """Return the pypaimon version embedded at build time, or None when
unknown."""
+ prefix = "python-"
+ if not _FULL_VERSION.startswith(prefix):
+ return None
+ return _FULL_VERSION[len(prefix):].rsplit("-", 1)[0] or None
diff --git a/paimon-python/pypaimon/common/options/config.py
b/paimon-python/pypaimon/common/options/config.py
index 1792dbd1df..d6d627bd3e 100644
--- a/paimon-python/pypaimon/common/options/config.py
+++ b/paimon-python/pypaimon/common/options/config.py
@@ -131,7 +131,17 @@ class CatalogOptions:
"If not set, will be automatically selected based on endpoint host.")
PREFIX =
ConfigOptions.key("prefix").string_type().no_default_value().with_description("Prefix")
HTTP_USER_AGENT_HEADER = ConfigOptions.key(
-
"header.HTTP_USER_AGENT").string_type().no_default_value().with_description("HTTP
User Agent header")
+ "header.User-Agent").string_type().no_default_value().with_description(
+ "The User-Agent of REST catalog requests, replacing Paimon's unified
one")
+ USER_AGENT_MODULE = ConfigOptions.key(
+ "user-agent.module").string_type().no_default_value().with_description(
+ "The module of Paimon's unified User-Agent, 'pypaimon/<version>' by
default")
+ USER_AGENT_FEATURES = ConfigOptions.key(
+
"user-agent.features").string_type().no_default_value().with_description(
+ "Space-separated features of Paimon's unified User-Agent")
+ USER_AGENT_EXTENDED = ConfigOptions.key(
+
"user-agent.extended").string_type().no_default_value().with_description(
+ "Free text appended to Paimon's unified User-Agent")
SYNC_ALL_PROPERTIES =
ConfigOptions.key("sync-all-properties").boolean_type().default_value(True).with_description(
"Sync all table properties to the catalog metastore")
RESOLVING_FILE_IO_ENABLED = (
diff --git a/paimon-python/pypaimon/common/user_agent.py
b/paimon-python/pypaimon/common/user_agent.py
new file mode 100644
index 0000000000..6bfac5677f
--- /dev/null
+++ b/paimon-python/pypaimon/common/user_agent.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Paimon's unified User-Agent ``module(transport;features) extended``."""
+
+import functools
+from typing import Iterable, Optional
+
+import requests
+
+from pypaimon import build_info
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import CatalogOptions
+
+NAME = "pypaimon"
+
+
[email protected]_cache(maxsize=None)
+def identity() -> str:
+ """``pypaimon/<version>`` with the version embedded at build time, or bare
``pypaimon``."""
+ version = build_info.version()
+ return "{}/{}".format(NAME, version) if version else NAME
+
+
+def format_user_agent(module: str, transport: str, features: Iterable[str] =
(),
+ extended: Optional[str] = None) -> str:
+ """Render ``module(transport;feature...) extended``, skipping empty
parts."""
+ user_agent = "{}({})".format(module, ";".join([transport] + [f for f in
features if f]))
+ return "{} {}".format(user_agent, extended) if extended else user_agent
+
+
+def rest_user_agent(options: Optional[Options] = None) -> str:
+ """The User-Agent of REST requests, built from the catalog's
``user-agent.*`` options."""
+ data = options.to_map() if options is not None else {}
+ module = _value(data, CatalogOptions.USER_AGENT_MODULE.key()) or identity()
+ features = (_value(data, CatalogOptions.USER_AGENT_FEATURES.key()) or
"").split()
+ return format_user_agent(module, "python-requests/" +
requests.__version__, features,
+ _value(data,
CatalogOptions.USER_AGENT_EXTENDED.key()))
+
+
+def with_feature(options: Options, feature: str) -> None:
+ """Put ``feature`` first in the ``user-agent.features`` option."""
+ key = CatalogOptions.USER_AGENT_FEATURES.key()
+ features = [f for f in (_value(options.to_map(), key) or "").split() if f
!= feature]
+ options.set(CatalogOptions.USER_AGENT_FEATURES, " ".join([feature] +
features))
+
+
+def _value(data, key: str) -> Optional[str]:
+ value = data.get(key)
+ return str(value).strip() if value is not None and str(value).strip() else
None
diff --git a/paimon-python/pypaimon/filesystem/pvfs.py
b/paimon-python/pypaimon/filesystem/pvfs.py
index a56128ca40..13030f08b8 100644
--- a/paimon-python/pypaimon/filesystem/pvfs.py
+++ b/paimon-python/pypaimon/filesystem/pvfs.py
@@ -34,6 +34,7 @@ from readerwriterlock import rwlock
from pypaimon.api.api_response import GetTableResponse, GetTableTokenResponse
from pypaimon.api.client import AlreadyExistsException, NoSuchResourceException
from pypaimon.api.rest_api import RESTApi
+from pypaimon.common import user_agent
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions, OssOptions,
PVFSOptions
from pypaimon.common.identifier import Identifier
@@ -149,7 +150,7 @@ class PaimonVirtualFileSystem(fsspec.AbstractFileSystem):
def __init__(self, options: Union[Options, Dict[str, str]] = None,
**kwargs):
if isinstance(options, dict):
options = Options(options)
- options.set(CatalogOptions.HTTP_USER_AGENT_HEADER, 'PythonPVFS')
+ user_agent.with_feature(options, 'PythonPVFS')
self.options = options
self.warehouse = options.get(CatalogOptions.WAREHOUSE)
cache_expired_time = (
diff --git a/paimon-python/pypaimon/tests/rest_user_agent_test.py
b/paimon-python/pypaimon/tests/rest_user_agent_test.py
new file mode 100644
index 0000000000..f2d254fb96
--- /dev/null
+++ b/paimon-python/pypaimon/tests/rest_user_agent_test.py
@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from unittest import mock
+
+import requests
+
+from pypaimon import PaimonVirtualFileSystem, build_info
+from pypaimon.api.rest_api import RESTApi
+from pypaimon.api.token_loader import HTTPClient
+from pypaimon.common import user_agent
+from pypaimon.common.options import Options
+
+IDENTITY = "pypaimon/2.2.dev0"
+REST_USER_AGENT = "{}(python-requests/{})".format(IDENTITY,
requests.__version__)
+
+
+class UserAgentIdentityTest(unittest.TestCase):
+
+ def setUp(self):
+ user_agent.identity.cache_clear()
+ self.addCleanup(user_agent.identity.cache_clear)
+
+ def test_identity_uses_build_version(self):
+ with mock.patch.object(build_info, "version", return_value="2.2.dev"):
+ self.assertEqual("pypaimon/2.2.dev", user_agent.identity())
+
+ def test_identity_without_build_version(self):
+ with mock.patch.object(build_info, "version", return_value=None):
+ self.assertEqual("pypaimon", user_agent.identity())
+
+ def test_build_version_from_full_version(self):
+ for full_version, expected in (("python-2.2.dev-abc123", "2.2.dev"),
+ ("python-2.2.0-UNKNOWN", "2.2.0"),
+ ("UNKNOWN", None)):
+ with mock.patch.object(build_info, "_FULL_VERSION", full_version):
+ self.assertEqual(expected, build_info.version())
+
+
+class UserAgentFormatTest(unittest.TestCase):
+
+ def test_format(self):
+ self.assertEqual("m/1(t/2)", user_agent.format_user_agent("m/1",
"t/2"))
+ self.assertEqual("m/1(t/2;a;b) ext k/v",
+ user_agent.format_user_agent("m/1", "t/2", ["a", "",
"b"], "ext k/v"))
+
+ def test_rest_user_agent(self):
+ with mock.patch.object(user_agent, "identity", return_value=IDENTITY):
+ self.assertEqual(REST_USER_AGENT, user_agent.rest_user_agent())
+ self.assertEqual(REST_USER_AGENT[:-1] + ";Flink) vvr",
user_agent.rest_user_agent(
+ Options({"user-agent.features": " Flink ",
"user-agent.extended": "vvr"})))
+
self.assertEqual("MyApp/1.0(python-requests/{})".format(requests.__version__),
+
user_agent.rest_user_agent(Options({"user-agent.module": "MyApp/1.0"})))
+
+ def test_with_feature_goes_first_once(self):
+ options = Options({"user-agent.features": "Flink PythonPVFS"})
+ user_agent.with_feature(options, "PythonPVFS")
+ user_agent.with_feature(options, "PythonPVFS")
+ self.assertEqual("PythonPVFS Flink",
options.to_map()["user-agent.features"])
+
+
+class RestUserAgentTest(unittest.TestCase):
+
+ def setUp(self):
+ self.requests = []
+ recorded = self.requests
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ recorded.append(dict(self.headers))
+ body = ({"defaults": {}} if "/config" in self.path
+ else {"databases": [], "nextPageToken": None})
+ data = json.dumps(body).encode("utf-8")
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def log_message(self, *args):
+ pass
+
+ self.server = HTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
+ self.addCleanup(self.server.server_close)
+ self.addCleanup(self.server.shutdown)
+ self.uri = "http://127.0.0.1:{}".format(self.server.server_port)
+
+ patcher = mock.patch.object(user_agent, "identity",
return_value=IDENTITY)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
+ def _options(self, **extra):
+ options = {"uri": self.uri, "warehouse": "wh", "token.provider":
"bear", "token": "t"}
+ options.update(extra)
+ return options
+
+ def _user_agents(self):
+ return [headers.get("User-Agent") for headers in self.requests]
+
+ def test_default_user_agent(self):
+ RESTApi(Options(self._options())).list_databases()
+
+ self.assertEqual([REST_USER_AGENT, REST_USER_AGENT],
self._user_agents())
+
+ def test_common_options_user_agent(self):
+ options = self._options(**{"user-agent.features": "Flink",
"user-agent.extended": "vvr"})
+ RESTApi(Options(options)).list_databases()
+
+ expected = REST_USER_AGENT[:-1] + ";Flink) vvr"
+ self.assertEqual([expected, expected], self._user_agents())
+
+ def test_user_set_user_agent_wins(self):
+ options = self._options(**{"header.User-Agent": "starrocks/user",
"user-agent.extended": "vvr"})
+ RESTApi(Options(options)).list_databases()
+
+ self.assertEqual(["starrocks/user", "starrocks/user"],
self._user_agents())
+
+ def test_pvfs_user_agent(self):
+ PaimonVirtualFileSystem(self._options(**{"user-agent.features":
"Flink"})).ls("pvfs://wh/")
+
+ pvfs_user_agent = REST_USER_AGENT[:-1] + ";PythonPVFS;Flink)"
+ self.assertEqual([pvfs_user_agent, pvfs_user_agent],
self._user_agents())
+ for headers in self.requests:
+ self.assertNotIn("http_user_agent", {key.lower() for key in
headers})
+
+ def test_token_loader_user_agent(self):
+ HTTPClient().get(self.uri + "/v1/config")
+
+ self.assertEqual([REST_USER_AGENT], self._user_agents())
+
+
+if __name__ == "__main__":
+ unittest.main()
