This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new e6f59c6184 [#14146] feat(client-python): Support Semantic Model metric
discovery (#14147)
e6f59c6184 is described below
commit e6f59c618451dcd9762c7422bc94d30c1237b91a
Author: mchades <[email protected]>
AuthorDate: Thu Oct 29 13:45:48 2026 +0800
[#14146] feat(client-python): Support Semantic Model metric discovery
(#14147)
### What changes were proposed in this pull request?
Add `list_metrics` and `list_metric_infos` to the Python
`SemanticModelCatalog` API, with a `SemanticModelMetric` result type and
REST response decoding. Each call uses one schema-level request and
preserves complete metrics, including Decimal precision.
### Why are the changes needed?
Let Python clients discover metrics without loading each Semantic Model,
following #14038 under #14036.
Builds on #14143, now merged into `main`.
Fixes #14146.
### Does this PR introduce _any_ user-facing change?
Adds two discovery methods with catalog-relative schema/model
namespaces. Existing `SemanticModelCatalog` implementations retain
compatible unsupported-operation defaults.
### How was this patch tested?
138 Semantic Model unit tests and 14 real-server integration tests
passed on the rebased branch with Python 3.12 and H2, covering discovery
lifecycle, authorization, and Ossie regressions.
From `clients/client-python`:
```sh
python -m unittest discover -s tests/unittests -t . -p '*semantic*.py'
python -m unittest -v
tests.integration.test_semantic_model_metric_discovery
tests.integration.test_semantic_model_authorization
tests.integration.test_semantic_model_ossie
```
Integration tests used `GRAVITINO_HOME` and
`START_EXTERNAL_GRAVITINO=true` against the freshly built server.
`./gradlew compileDistribution spotlessApply -PskipWeb=true -x test`
passed. Black and Pylint passed on all nine changed Python files, and
`git diff --check` passed.
---
.../api/semantic/semantic_model_catalog.py | 40 +++
.../api/semantic/semantic_model_metric.py | 56 ++++
.../client/semantic_model_catalog_operations.py | 57 ++++
.../dto/responses/metric_info_list_response.py | 49 +++
.../dto/semantic/semantic_model_metric_dto.py | 51 +++
.../test_semantic_model_authorization.py | 75 +++++
.../test_semantic_model_metric_discovery.py | 195 +++++++++++
.../api/semantic/test_semantic_model_catalog.py | 11 +
.../test_semantic_model_metric_discovery.py | 367 +++++++++++++++++++++
9 files changed, 901 insertions(+)
diff --git
a/clients/client-python/gravitino/api/semantic/semantic_model_catalog.py
b/clients/client-python/gravitino/api/semantic/semantic_model_catalog.py
index 815781a699..9c0c809493 100644
--- a/clients/client-python/gravitino/api/semantic/semantic_model_catalog.py
+++ b/clients/client-python/gravitino/api/semantic/semantic_model_catalog.py
@@ -21,6 +21,7 @@ from typing import Optional
from gravitino.api.semantic.semantic_model import SemanticModel
from gravitino.api.semantic.semantic_model_change import SemanticModelChange
from gravitino.api.semantic.semantic_model_definition import
SemanticModelDefinition
+from gravitino.api.semantic.semantic_model_metric import SemanticModelMetric
from gravitino.exceptions.base import (
NoSuchSemanticModelException,
UnsupportedOperationException,
@@ -55,6 +56,45 @@ class SemanticModelCatalog(ABC):
NoSuchSchemaException: If the schema does not exist.
"""
+ def list_metrics(self, namespace: Namespace) -> list[NameIdentifier]:
+ """List visible metric identifiers across the Semantic Models in a
schema.
+
+ Identifiers are relative to this catalog. Each identifier's namespace
+ contains the schema and Semantic Model names, and its name is the exact
+ metric name.
+ Results are ordered by model identifier and then case-sensitive metric
name.
+
+ Args:
+ namespace (Namespace): A single-level schema namespace.
+
+ Returns:
+ list[NameIdentifier]: The visible metric identifiers, or an empty
list.
+
+ Raises:
+ NoSuchSchemaException: If the schema does not exist.
+ UnsupportedOperationException: If metric discovery is not
supported.
+ """
+ raise UnsupportedOperationException("Metric listing is not supported")
+
+ def list_metric_infos(self, namespace: Namespace) ->
list[SemanticModelMetric]:
+ """List complete visible metrics across the Semantic Models in a
schema.
+
+ Each result contains a namespace of schema and model names and the
complete
+ metric definition. Visibility and ordering match :meth:`list_metrics`.
+ The request fails as a whole if metric definitions cannot be decoded.
+
+ Args:
+ namespace (Namespace): A single-level schema namespace.
+
+ Returns:
+ list[SemanticModelMetric]: The visible metrics, or an empty list.
+
+ Raises:
+ NoSuchSchemaException: If the schema does not exist.
+ UnsupportedOperationException: If metric discovery is not
supported.
+ """
+ raise UnsupportedOperationException("Metric info listing is not
supported")
+
@abstractmethod
def load_semantic_model(self, identifier: NameIdentifier) -> SemanticModel:
"""Load Semantic Model metadata by `NameIdentifier` from the catalog.
diff --git
a/clients/client-python/gravitino/api/semantic/semantic_model_metric.py
b/clients/client-python/gravitino/api/semantic/semantic_model_metric.py
new file mode 100644
index 0000000000..4d50a72b9c
--- /dev/null
+++ b/clients/client-python/gravitino/api/semantic/semantic_model_metric.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from gravitino.api.semantic.metric import Metric
+from gravitino.namespace import Namespace
+from gravitino.utils.precondition import Precondition
+
+
+class SemanticModelMetric:
+ """A discovered metric with its containing Semantic Model's namespace.
+
+ Catalog-scoped discovery returns a namespace containing the schema and
model
+ names. The metric's name remains part of its complete :class:`Metric`
value.
+ """
+
+ def __init__(self, namespace: Namespace, metric: Metric):
+ Precondition.check_argument(namespace is not None, "namespace must not
be null")
+ Precondition.check_argument(metric is not None, "metric must not be
null")
+ self._namespace = Namespace.of(*namespace.levels())
+ self._metric = metric
+
+ def namespace(self) -> Namespace:
+ """Return a copy of the namespace ending with the containing model's
name."""
+ return Namespace.of(*self._namespace.levels())
+
+ def metric(self) -> Metric:
+ """Return the complete metric definition."""
+ return self._metric
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, SemanticModelMetric):
+ return False
+ return self._namespace == other._namespace and self._metric ==
other._metric
+
+ def __hash__(self) -> int:
+ return hash((self._namespace, self._metric))
+
+ def __repr__(self) -> str:
+ return (
+ f"SemanticModelMetric(namespace={self._namespace.levels()!r}, "
+ f"metric={self._metric!r})"
+ )
diff --git
a/clients/client-python/gravitino/client/semantic_model_catalog_operations.py
b/clients/client-python/gravitino/client/semantic_model_catalog_operations.py
index 6e90451fef..f0e41ce709 100644
---
a/clients/client-python/gravitino/client/semantic_model_catalog_operations.py
+++
b/clients/client-python/gravitino/client/semantic_model_catalog_operations.py
@@ -28,6 +28,7 @@ from gravitino.api.semantic.semantic_model_change import (
UpdateComment,
)
from gravitino.api.semantic.semantic_model_definition import
SemanticModelDefinition
+from gravitino.api.semantic.semantic_model_metric import SemanticModelMetric
from gravitino.client.generic_semantic_model import GenericSemanticModel
from gravitino.dto.requests.semantic_model_create_request import (
SemanticModelCreateRequest,
@@ -41,6 +42,7 @@ from gravitino.dto.requests.semantic_model_updates_request
import (
)
from gravitino.dto.responses.drop_response import DropResponse
from gravitino.dto.responses.entity_list_response import EntityListResponse
+from gravitino.dto.responses.metric_info_list_response import
MetricInfoListResponse
from gravitino.dto.responses.semantic_model_response import
SemanticModelResponse
from gravitino.dto.semantic.semantic_model_definition_dto import (
SemanticModelDefinitionDTO,
@@ -86,6 +88,42 @@ class SemanticModelCatalogOperations(SemanticModelCatalog):
for ident in entity_list_resp.identifiers()
]
+ def list_metrics(self, namespace: Namespace) -> list[NameIdentifier]:
+ self._check_semantic_model_namespace(namespace)
+ resp = self._rest_client.get(
+ self._format_metric_request_path(
+ self._get_semantic_model_full_namespace(namespace)
+ ),
+ params={"details": "false"},
+ error_handler=SEMANTIC_MODEL_ERROR_HANDLER,
+ )
+ response = EntityListResponse.from_json(resp.body, infer_missing=True)
+ response.validate()
+ return [
+ NameIdentifier.builder(
+ self._relative_metric_namespace(ident.namespace()),
ident.name()
+ )
+ for ident in response.identifiers()
+ ]
+
+ def list_metric_infos(self, namespace: Namespace) ->
list[SemanticModelMetric]:
+ self._check_semantic_model_namespace(namespace)
+ resp = self._rest_client.get(
+ self._format_metric_request_path(
+ self._get_semantic_model_full_namespace(namespace)
+ ),
+ params={"details": "true"},
+ error_handler=SEMANTIC_MODEL_ERROR_HANDLER,
+ )
+ response = MetricInfoListResponse.from_json(resp.body,
infer_missing=True)
+ response.validate()
+ return [
+ SemanticModelMetric(
+ self._relative_metric_namespace(info.namespace()),
info.metric()
+ )
+ for info in response.metrics()
+ ]
+
def load_semantic_model(self, identifier: NameIdentifier) -> SemanticModel:
self._check_semantic_model_name_identifier(identifier)
full_namespace =
self._get_semantic_model_full_namespace(identifier.namespace())
@@ -291,6 +329,25 @@ class SemanticModelCatalogOperations(SemanticModelCatalog):
"/semantic-models"
)
+ @staticmethod
+ def _format_metric_request_path(namespace: Namespace) -> str:
+ return (
+ f"api/metalakes/{encode_string(namespace.level(0))}"
+ f"/catalogs/{encode_string(namespace.level(1))}"
+ f"/schemas/{encode_string(namespace.level(2))}"
+ "/semantic-model-metrics"
+ )
+
+ @staticmethod
+ def _relative_metric_namespace(namespace: Namespace) -> Namespace:
+ level_count = None if namespace is None else namespace.length()
+ if level_count != 4:
+ raise IllegalArgumentException(
+ "Expected metric namespace with 4 levels
(metalake/catalog/schema/model), "
+ f"got {level_count}"
+ )
+ return Namespace.of(namespace.level(2), namespace.level(3))
+
def _get_semantic_model_full_namespace(
self, semantic_model_namespace: Namespace
) -> Namespace:
diff --git
a/clients/client-python/gravitino/dto/responses/metric_info_list_response.py
b/clients/client-python/gravitino/dto/responses/metric_info_list_response.py
new file mode 100644
index 0000000000..5e5d3215ee
--- /dev/null
+++ b/clients/client-python/gravitino/dto/responses/metric_info_list_response.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass, field
+
+from dataclasses_json import config
+
+from gravitino.dto.responses.base_response import BaseResponse
+from gravitino.dto.semantic.json_serdes.semantic_json_mixin import
SemanticJsonMixin
+from gravitino.dto.semantic.semantic_model_metric_dto import
SemanticModelMetricDTO
+from gravitino.utils.precondition import Precondition
+
+
+@dataclass
+class MetricInfoListResponse(SemanticJsonMixin, BaseResponse):
+ """Response containing complete metrics discovered in a schema."""
+
+ _metrics: list[SemanticModelMetricDTO] = field(
+ metadata=config(field_name="metrics")
+ )
+
+ def metrics(self) -> list[SemanticModelMetricDTO]:
+ """Return the metric discovery results."""
+ return self._metrics
+
+ def validate(self):
+ """Validate the response code and metric result list."""
+ super().validate()
+ Precondition.check_argument(
+ self._metrics is not None, "metrics must not be null"
+ )
+ Precondition.check_argument(
+ all(metric is not None for metric in self._metrics),
+ "metrics must not contain null",
+ )
diff --git
a/clients/client-python/gravitino/dto/semantic/semantic_model_metric_dto.py
b/clients/client-python/gravitino/dto/semantic/semantic_model_metric_dto.py
new file mode 100644
index 0000000000..3667d5eaa1
--- /dev/null
+++ b/clients/client-python/gravitino/dto/semantic/semantic_model_metric_dto.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+from dataclasses_json import config
+
+from gravitino.api.semantic.metric import Metric
+from gravitino.dto.semantic.json_serdes.semantic_json_mixin import
SemanticJsonMixin
+from gravitino.dto.semantic.metric_dto import MetricDTO
+from gravitino.namespace import Namespace
+from gravitino.utils.precondition import Precondition
+
+
+@dataclass
+class SemanticModelMetricDTO(SemanticJsonMixin):
+ """A metric's containing namespace and complete definition from
discovery."""
+
+ _namespace: Optional[list[str]] = field(
+ default=None, metadata=config(field_name="namespace")
+ )
+ _metric: Optional[MetricDTO] = field(
+ default=None, metadata=config(field_name="metric")
+ )
+
+ def namespace(self) -> Namespace:
+ """Return the namespace ending with the containing model's name."""
+ Precondition.check_argument(
+ self._namespace is not None, "namespace must not be null"
+ )
+ return Namespace.of(*self._namespace)
+
+ def metric(self) -> Metric:
+ """Decode the complete metric definition."""
+ Precondition.check_argument(self._metric is not None, "metric must not
be null")
+ return self._metric.to_metric()
diff --git
a/clients/client-python/tests/integration/test_semantic_model_authorization.py
b/clients/client-python/tests/integration/test_semantic_model_authorization.py
index e12c7ec13c..258e5f9db6 100644
---
a/clients/client-python/tests/integration/test_semantic_model_authorization.py
+++
b/clients/client-python/tests/integration/test_semantic_model_authorization.py
@@ -22,13 +22,18 @@ import uuid
from unittest.mock import patch
from gravitino import Catalog, GravitinoClient, NameIdentifier
+from gravitino.api.authorization.owner import Owner
from gravitino.api.authorization.privileges import Privileges
from gravitino.api.authorization.securable_objects import SecurableObjects
from gravitino.api.rel.column import Column
from gravitino.api.rel.types.types import Types
from gravitino.api.semantic.dataset import Dataset
+from gravitino.api.semantic.dialect_expression import DialectExpression
+from gravitino.api.semantic.expression import Expression
+from gravitino.api.semantic.metric import Metric
from gravitino.api.semantic.semantic_model_change import SemanticModelChange
from gravitino.api.semantic.semantic_model_definition import
SemanticModelDefinition
+from gravitino.api.semantic.semantic_model_metric import SemanticModelMetric
from gravitino.auth.simple_auth_provider import SimpleAuthProvider
from gravitino.client.metadata_object_role_operations import (
MetadataObjectRoleOperations,
@@ -155,6 +160,76 @@ class
TestSemanticModelAuthorization(AuthorizationIntegrationTestEnv):
[],
self._reader_models.list_semantic_models(Namespace.of("semantic"))
)
+ def test_metric_discovery_follows_model_visibility_and_parent_access(self):
+ metrics = [
+ Metric(name, Expression([DialectExpression("ANSI_SQL",
"COUNT(*)")]))
+ for name in ("Revenue", "revenue")
+ ]
+ definition = SemanticModelDefinition(
+ self._definition.datasets(), metrics=metrics
+ )
+ models = self._gravitino_client.load_catalog(
+ "iceberg"
+ ).as_semantic_model_catalog()
+ for model_name in ("sales", "hidden"):
+ models.alter_semantic_model(
+ NameIdentifier.of("semantic", model_name),
+ SemanticModelChange.replace_definition(definition),
+ )
+ namespace = Namespace.of("semantic")
+ expected = [
+ SemanticModelMetric(Namespace.of("semantic", "sales"), metric)
+ for metric in metrics
+ ]
+
+ def assert_discovery(infos):
+ self.assertEqual(infos,
self._reader_models.list_metric_infos(namespace))
+ self.assertEqual(
+ [
+ NameIdentifier.builder(info.namespace(),
info.metric().name())
+ for info in infos
+ ],
+ self._reader_models.list_metrics(namespace),
+ )
+
+ assert_discovery([])
+ model = SecurableObjects.of_semantic_model(self._schema, "sales", [])
+ for name in ("USE_SEMANTIC_MODEL", "MODIFY_SEMANTIC_MODEL"):
+ with self.subTest(privilege=name):
+ privilege = Privileges.allow(name)
+ self._gravitino_client.grant_privileges_to_role(
+ "reader", model, [privilege]
+ )
+ assert_discovery(expected)
+ self._gravitino_client.revoke_privileges_from_role(
+ "reader", model, [privilege]
+ )
+ assert_discovery([])
+
+ self._gravitino_client.set_owner(model, "semantic_reader",
Owner.Type.USER)
+ assert_discovery(expected)
+ for obj, privilege in (
+ (self._schema, Privileges.allow("USE_SCHEMA")),
+ (
+ SecurableObjects.of_catalog("iceberg", []),
+ Privileges.allow("USE_CATALOG"),
+ ),
+ ):
+ with self.subTest(privilege=privilege.name()):
+ self._gravitino_client.revoke_privileges_from_role(
+ "reader", obj, [privilege]
+ )
+ for method in (
+ self._reader_models.list_metrics,
+ self._reader_models.list_metric_infos,
+ ):
+ with self.assertRaises(ForbiddenException):
+ method(namespace)
+ self._gravitino_client.grant_privileges_to_role(
+ "reader", obj, [privilege]
+ )
+ assert_discovery(expected)
+
def test_create_requires_privilege_and_parent_access(self):
ident = NameIdentifier.of("semantic", "created")
with self.assertRaises(ForbiddenException):
diff --git
a/clients/client-python/tests/integration/test_semantic_model_metric_discovery.py
b/clients/client-python/tests/integration/test_semantic_model_metric_discovery.py
new file mode 100644
index 0000000000..5a41eac8b0
--- /dev/null
+++
b/clients/client-python/tests/integration/test_semantic_model_metric_discovery.py
@@ -0,0 +1,195 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from decimal import Decimal
+from uuid import uuid4
+
+from gravitino import Catalog, GravitinoAdminClient, NameIdentifier
+from gravitino.api.rel.column import Column
+from gravitino.api.rel.types.types import Types
+from gravitino.api.semantic.ai_context import AIContext
+from gravitino.api.semantic.ai_context_object import AIContextObject
+from gravitino.api.semantic.custom_extension import CustomExtension
+from gravitino.api.semantic.data_type import DataType
+from gravitino.api.semantic.dataset import Dataset
+from gravitino.api.semantic.dialect_expression import DialectExpression
+from gravitino.api.semantic.expression import Expression
+from gravitino.api.semantic.metric import Metric
+from gravitino.api.semantic.semantic_model_change import SemanticModelChange
+from gravitino.api.semantic.semantic_model_definition import
SemanticModelDefinition
+from gravitino.api.semantic.semantic_model_metric import SemanticModelMetric
+from gravitino.exceptions.base import NoSuchSchemaException
+from gravitino.namespace import Namespace
+from tests.integration.integration_test_env import IntegrationTestEnv
+
+
+class TestSemanticModelMetricDiscovery(IntegrationTestEnv):
+ """Exercise metric discovery through the public SDK and a real server."""
+
+ def setUp(self):
+ super().setUp()
+ admin = GravitinoAdminClient(uri="http://localhost:8090";)
+ self.addCleanup(admin.close)
+ metalake = "metrics_it_" + uuid4().hex
+ client = self.create_metalake_client(admin, metalake)
+ self.addCleanup(client.close)
+ self.addCleanup(admin.drop_metalake, metalake, force=True)
+ self.catalog = client.create_catalog(
+ "metrics_catalog", Catalog.Type.RELATIONAL, "lakehouse-generic",
"", {}
+ )
+ for schema in ("source", "semantic", "other"):
+ self.catalog.as_schemas().create_schema(schema, "", {})
+ self.catalog.as_table_catalog().create_table(
+ NameIdentifier.of("source", "orders"),
+ [Column.of("amount", Types.LongType.get())],
+ properties={
+ "format": "delta",
+ "external": "true",
+ "location": f"file:///tmp/{metalake}/orders",
+ },
+ )
+ self.models = self.catalog.as_semantic_model_catalog()
+ self.namespace = Namespace.of("semantic")
+
+ def
test_discovery_preserves_complete_metrics_order_and_schema_isolation(self):
+ self.assertEqual([], self.models.list_metrics(self.namespace))
+ self.assertEqual([], self.models.list_metric_infos(self.namespace))
+ metrics = [
+ self._metric("revenue"),
+ self._metric("Revenue"),
+ self._metric("收入"),
+ ]
+ self._create("b", [metrics[1]])
+ self._create("a", metrics)
+ self._create("empty", [])
+ self._create("null_metrics", None)
+ self.models.create_semantic_model(
+ NameIdentifier.of("other", "a"), "", self._definition(metrics), {}
+ )
+
+ expected = [
+ SemanticModelMetric(Namespace.of("semantic", "a"), metrics[1]),
+ SemanticModelMetric(Namespace.of("semantic", "a"), metrics[0]),
+ SemanticModelMetric(Namespace.of("semantic", "a"), metrics[2]),
+ SemanticModelMetric(Namespace.of("semantic", "b"), metrics[1]),
+ ]
+ self._assert_discovery(expected)
+ self.assertEqual(3,
len(self.models.list_metrics(Namespace.of("other"))))
+ actual = self.models.list_metric_infos(self.namespace)[0].metric()
+ self.assertIsInstance(
+ actual.ai_context().object().additional_properties()["threshold"],
Decimal
+ )
+
+ def test_discovery_tracks_definition_replacement_rename_and_drop(self):
+ identifier = NameIdentifier.of("semantic", "sales")
+ original = self._metric("Revenue")
+ self._create("sales", [original, self._metric("removed")])
+ changed = Metric(
+ "Revenue",
+ Expression([DialectExpression("ANSI_SQL", "COUNT(*)")]),
+ ai_context=AIContext.of("Updated"),
+ )
+ self.models.alter_semantic_model(
+ identifier,
+
SemanticModelChange.replace_definition(self._definition([changed])),
+ )
+ self._assert_discovery(
+ [SemanticModelMetric(Namespace.of("semantic", "sales"), changed)]
+ )
+
+ self.models.alter_semantic_model(
+ identifier, SemanticModelChange.rename("renamed")
+ )
+ identifier = NameIdentifier.of("semantic", "renamed")
+ self._assert_discovery(
+ [SemanticModelMetric(Namespace.of("semantic", "renamed"), changed)]
+ )
+ self.models.alter_semantic_model(
+ identifier,
SemanticModelChange.replace_definition(self._definition([]))
+ )
+ self._assert_discovery([])
+ self.models.alter_semantic_model(
+ identifier,
+
SemanticModelChange.replace_definition(self._definition([original])),
+ )
+ self._assert_discovery(
+ [SemanticModelMetric(Namespace.of("semantic", "renamed"),
original)]
+ )
+ self.assertTrue(self.models.drop_semantic_model(identifier))
+ self._assert_discovery([])
+
+ def test_missing_and_dropped_schema(self):
+ for method in (self.models.list_metrics,
self.models.list_metric_infos):
+ with self.subTest(method=method),
self.assertRaises(NoSuchSchemaException):
+ method(Namespace.of("missing"))
+ self._create("sales", [self._metric("Revenue")])
+ self.assertTrue(self.catalog.as_schemas().drop_schema("semantic",
True))
+ for method in (self.models.list_metrics,
self.models.list_metric_infos):
+ with self.subTest(method=method),
self.assertRaises(NoSuchSchemaException):
+ method(self.namespace)
+ self.catalog.as_schemas().create_schema("semantic", "", {})
+ self._assert_discovery([])
+
+ def _create(self, name, metrics):
+ self.models.create_semantic_model(
+ NameIdentifier.of("semantic", name), "",
self._definition(metrics), {}
+ )
+
+ def _assert_discovery(self, expected):
+ self.assertEqual(expected,
self.models.list_metric_infos(self.namespace))
+ self.assertEqual(
+ [
+ NameIdentifier.builder(info.namespace(), info.metric().name())
+ for info in expected
+ ],
+ self.models.list_metrics(self.namespace),
+ )
+
+ @staticmethod
+ def _definition(metrics):
+ return SemanticModelDefinition(
+ [
+ Dataset(
+ "orders", NameIdentifier.of("metrics_catalog", "source",
"orders")
+ )
+ ],
+ metrics=metrics,
+ )
+
+ @staticmethod
+ def _metric(name):
+ return Metric(
+ name,
+ Expression([DialectExpression("ANSI_SQL", "SUM(orders.amount)")]),
+ "Revenue - café 雪",
+ DataType.DECIMAL,
+ AIContext.of(
+ AIContextObject(
+ instructions="Use certified metrics",
+ synonyms=["Sales"],
+ examples=["By year"],
+ additional_properties={
+ "threshold":
Decimal("0.12345678901234567890123456789"),
+ "nested": {
+ "large": 1234567890123456789012345678901234567890,
+ "values": [True, None, "extra"],
+ },
+ },
+ )
+ ),
+ [CustomExtension("vendor", '{"opaque": "雪"}')],
+ )
diff --git
a/clients/client-python/tests/unittests/api/semantic/test_semantic_model_catalog.py
b/clients/client-python/tests/unittests/api/semantic/test_semantic_model_catalog.py
index ccf6008e53..75dff8ad96 100644
---
a/clients/client-python/tests/unittests/api/semantic/test_semantic_model_catalog.py
+++
b/clients/client-python/tests/unittests/api/semantic/test_semantic_model_catalog.py
@@ -92,6 +92,17 @@ class _InMemorySemanticModelCatalog(SemanticModelCatalog):
class TestSemanticModelCatalog(unittest.TestCase):
+ def test_metric_discovery_defaults_preserve_existing_implementations(self):
+ catalog = _InMemorySemanticModelCatalog()
+ for method, message in (
+ (catalog.list_metrics, "Metric listing is not supported"),
+ (catalog.list_metric_infos, "Metric info listing is not
supported"),
+ ):
+ with self.subTest(method=method), self.assertRaisesRegex(
+ UnsupportedOperationException, message
+ ):
+ method(Namespace.of("semantic"))
+
def test_semantic_model_exists_uses_load(self):
catalog = _InMemorySemanticModelCatalog()
identifier = NameIdentifier.of("semantic", "sales_model")
diff --git
a/clients/client-python/tests/unittests/test_semantic_model_metric_discovery.py
b/clients/client-python/tests/unittests/test_semantic_model_metric_discovery.py
new file mode 100644
index 0000000000..49531b6fa1
--- /dev/null
+++
b/clients/client-python/tests/unittests/test_semantic_model_metric_discovery.py
@@ -0,0 +1,367 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+import json
+import unittest
+from decimal import Decimal
+from unittest.mock import patch
+from urllib.error import HTTPError
+
+import simplejson
+
+from gravitino.api.semantic.ai_context import AIContext
+from gravitino.api.semantic.ai_context_object import AIContextObject
+from gravitino.api.semantic.custom_extension import CustomExtension
+from gravitino.api.semantic.data_type import DataType
+from gravitino.api.semantic.dialect_expression import DialectExpression
+from gravitino.api.semantic.expression import Expression
+from gravitino.api.semantic.metric import Metric
+from gravitino.api.semantic.semantic_model_metric import SemanticModelMetric
+from gravitino.client.relational_catalog import RelationalCatalog
+from gravitino.dto.audit_dto import AuditDTO
+from gravitino.dto.responses.metric_info_list_response import
MetricInfoListResponse
+from gravitino.exceptions.base import (
+ ForbiddenException,
+ IllegalArgumentException,
+ IllegalNamespaceException,
+ InternalError,
+ NoSuchCatalogException,
+ NoSuchMetalakeException,
+ NoSuchSchemaException,
+ UnauthorizedException,
+ UnsupportedOperationException,
+)
+from gravitino.name_identifier import NameIdentifier
+from gravitino.namespace import Namespace
+from gravitino.utils.http_client import HTTPClient
+
+
+class TestSemanticModelMetricDiscovery(unittest.TestCase):
+ def setUp(self):
+ self.models = RelationalCatalog(
+ catalog_namespace=Namespace.of("metalake /雪"),
+ name="catalog +%",
+ catalog_type=RelationalCatalog.Type.RELATIONAL,
+ provider="lakehouse-generic",
+ audit=AuditDTO("test"),
+ rest_client=HTTPClient("http://localhost:8090";),
+ ).as_semantic_model_catalog()
+ self.namespace = Namespace.of("schema #?")
+ self.full_namespace = ["metalake /雪", "catalog +%", "schema #?",
"a.model"]
+ self.path = (
+ "http://localhost:8090/api/metalakes/metalake%20%2F%E9%9B%AA";
+
"/catalogs/catalog%20%2B%25/schemas/schema%20%23%3F/semantic-model-metrics"
+ )
+ opener_patch = patch("gravitino.utils.http_client.build_opener")
+ self.opener = opener_patch.start().return_value
+ self.addCleanup(opener_patch.stop)
+ self.opener.open.return_value.getcode.return_value = 200
+
+ def
test_list_metrics_uses_one_get_and_preserves_namespaces_and_order(self):
+ identifiers = [
+ {"namespace": self.full_namespace, "name": name}
+ for name in ("Revenue", "revenue", "收入 /é+%")
+ ] + [{"namespace": self.full_namespace[:-1] + ["b"], "name":
"Revenue"}]
+ self._respond({"code": 0, "identifiers": identifiers})
+
+ actual = self.models.list_metrics(self.namespace)
+
+ self._assert_request("false")
+ self.assertEqual(
+ [
+ NameIdentifier.of(*item["namespace"][2:], item["name"])
+ for item in identifiers
+ ],
+ actual,
+ )
+
+ def test_list_metric_infos_preserves_complete_metrics_and_precision(self):
+ metric = self._metric_json()
+ text_metric = {**metric, "name": "revenue", "aiContext": "Use
certified values"}
+ minimal_metric = {"name": "Revenue", "expression":
metric["expression"]}
+ self._respond(
+ {
+ "code": 0,
+ "metrics": [
+ {"namespace": self.full_namespace, "metric": metric},
+ {"namespace": self.full_namespace, "metric": text_metric},
+ {
+ "namespace": self.full_namespace[:-1] + ["b"],
+ "metric": minimal_metric,
+ },
+ ],
+ }
+ )
+
+ actual = self.models.list_metric_infos(self.namespace)
+
+ self._assert_request("true")
+ expression = Expression(
+ [
+ DialectExpression("ANSI_SQL", "SUM(orders.amount)"),
+ DialectExpression("SPARK_SQL", "sum(orders.amount)"),
+ ]
+ )
+ expected = Metric(
+ "Revenue",
+ expression,
+ "Revenue - café 雪",
+ DataType.DECIMAL,
+ AIContext.of(
+ AIContextObject(
+ instructions="Use certified values",
+ synonyms=["Sales"],
+ examples=["Revenue by year"],
+ additional_properties={
+ "nested": {
+ "precise":
Decimal("0.12345678901234567890123456789"),
+ "large": 1234567890123456789012345678901234567890,
+ "values": [True, None, "extra"],
+ }
+ },
+ )
+ ),
+ [CustomExtension("vendor", '{"opaque": "雪"}')],
+ )
+ expected_info = SemanticModelMetric(
+ Namespace.of("schema #?", "a.model"), expected
+ )
+ self.assertEqual(expected_info, actual[0])
+ self.assertEqual(hash(expected_info), hash(actual[0]))
+ self.assertEqual(
+ ["Revenue", "revenue", "Revenue"], [info.metric().name() for info
in actual]
+ )
+ self.assertEqual("Use certified values",
actual[1].metric().ai_context().text())
+ self.assertEqual(Namespace.of("schema #?", "b"), actual[2].namespace())
+ self.assertEqual(Metric("Revenue", expression), actual[2].metric())
+ self.assertIsInstance(
+ actual[0]
+ .metric()
+ .ai_context()
+ .object()
+ .additional_properties()["nested"]["precise"],
+ Decimal,
+ )
+
+ def test_empty_responses(self):
+ for method, field, details in (
+ (self.models.list_metrics, "identifiers", "false"),
+ (self.models.list_metric_infos, "metrics", "true"),
+ ):
+ with self.subTest(details=details):
+ self.opener.open.reset_mock()
+ self._respond({"code": 0, field: []})
+ self.assertEqual([], method(self.namespace))
+ self._assert_request(details)
+
+ def test_invalid_namespaces_do_not_send_requests(self):
+ for method in (self.models.list_metrics,
self.models.list_metric_infos):
+ for namespace in (
+ None,
+ Namespace.empty(),
+ Namespace.of("catalog", "schema"),
+ ):
+ with self.subTest(
+ method=method, namespace=namespace
+ ), self.assertRaises(IllegalNamespaceException):
+ method(namespace)
+ self.opener.open.assert_not_called()
+
+ def test_invalid_response_envelopes(self):
+ for method, field in (
+ (self.models.list_metrics, "identifiers"),
+ (self.models.list_metric_infos, "metrics"),
+ ):
+ for body in (
+ {"code": 0},
+ {"code": 0, field: None},
+ {"code": -1, field: []},
+ ):
+ with self.subTest(method=method, body=body), self.assertRaises(
+ IllegalArgumentException
+ ):
+ self._respond(body)
+ method(self.namespace)
+
+ def test_invalid_full_namespaces_are_rejected(self):
+ for namespace in (
+ [],
+ ["schema", "model"],
+ ["lake", "catalog", "schema"],
+ self.full_namespace + ["extra"],
+ ):
+ with self.subTest(namespace=namespace):
+ expected_message = (
+ "Expected metric namespace with 4 levels
(metalake/catalog/schema/model), "
+ f"got {len(namespace)}"
+ )
+ self._respond(
+ {
+ "code": 0,
+ "identifiers": [{"namespace": namespace, "name":
"Revenue"}],
+ }
+ )
+ with self.assertRaises(IllegalArgumentException) as error:
+ self.models.list_metrics(self.namespace)
+ self.assertEqual(expected_message, str(error.exception))
+ self._respond(
+ {
+ "code": 0,
+ "metrics": [
+ {"namespace": namespace, "metric":
self._metric_json()}
+ ],
+ }
+ )
+ with self.assertRaises(IllegalArgumentException) as error:
+ self.models.list_metric_infos(self.namespace)
+ self.assertEqual(expected_message, str(error.exception))
+
+ def test_missing_identifier_namespace_is_rejected(self):
+ self._respond({"code": 0, "identifiers": [{"name": "Revenue"}]})
+ with self.assertRaises(IllegalArgumentException) as error:
+ self.models.list_metrics(self.namespace)
+ self.assertEqual(
+ "Expected metric namespace with 4 levels
(metalake/catalog/schema/model), got None",
+ str(error.exception),
+ )
+
+ def test_invalid_metric_fails_entire_detailed_request(self):
+ valid = {"namespace": self.full_namespace, "metric":
self._metric_json()}
+ for invalid in (
+ None,
+ {},
+ {"metric": self._metric_json()},
+ {"namespace": self.full_namespace},
+ {"namespace": self.full_namespace, "metric": {"name": "broken"}},
+ {
+ "namespace": self.full_namespace,
+ "metric": {**self._metric_json(), "name": ""},
+ },
+ {
+ "namespace": self.full_namespace,
+ "metric": {**self._metric_json(), "expression": {"dialects":
[]}},
+ },
+ ):
+ with self.subTest(invalid=invalid), self.assertRaises(
+ IllegalArgumentException
+ ):
+ self._respond({"code": 0, "metrics": [valid, invalid]})
+ self.models.list_metric_infos(self.namespace)
+
+ def test_http_error_envelopes_map_to_sdk_exceptions(self):
+ cases = (
+ (404, 1003, NoSuchMetalakeException),
+ (404, 1003, NoSuchCatalogException),
+ (404, 1003, NoSuchSchemaException),
+ (401, 1011, UnauthorizedException),
+ (403, 1008, ForbiddenException),
+ (500, 1002, InternalError),
+ (501, 1006, UnsupportedOperationException),
+ )
+ for method in (self.models.list_metrics,
self.models.list_metric_infos):
+ for status, code, exception in cases:
+ with self.subTest(method=method, exception=exception):
+ body = json.dumps(
+ {
+ "code": code,
+ "type": exception.__name__,
+ "message": "server detail",
+ }
+ ).encode()
+ self.opener.open.side_effect = HTTPError(
+ self.path, status, "error", {}, io.BytesIO(body)
+ )
+ with self.assertRaisesRegex(exception, "server detail"):
+ method(self.namespace)
+
+ def test_metric_info_response_json_round_trip(self):
+ body = {
+ "code": 0,
+ "metrics": [
+ {"namespace": self.full_namespace, "metric":
self._metric_json()}
+ ],
+ }
+ response = MetricInfoListResponse.from_json(simplejson.dumps(body))
+ response.validate()
+ self.assertEqual(body, simplejson.loads(response.to_json(),
use_decimal=True))
+
+ def test_result_value_semantics_and_namespace_copy(self):
+ namespace = Namespace.of("schema", "model")
+ metric = Metric(
+ "Revenue", Expression([DialectExpression("ANSI_SQL", "COUNT(*)")])
+ )
+ result = SemanticModelMetric(namespace, metric)
+ equal = SemanticModelMetric(Namespace.of("schema", "model"), metric)
+ original_hash = hash(result)
+ namespace.levels().clear()
+ result.namespace().levels().clear()
+ self.assertEqual(equal, result)
+ self.assertEqual(original_hash, hash(result))
+ self.assertEqual(hash(equal), hash(result))
+ self.assertNotEqual(result, metric)
+ self.assertNotEqual(
+ result, SemanticModelMetric(Namespace.of("schema", "other"),
metric)
+ )
+ self.assertIn("Revenue", repr(result))
+ self.assertIn("model", repr(result))
+ for namespace, value in (
+ (None, metric),
+ (Namespace.of("schema", "model"), None),
+ ):
+ with self.assertRaises(IllegalArgumentException):
+ SemanticModelMetric(namespace, value)
+
+ def _respond(self, body):
+ self.opener.open.return_value.read.return_value =
simplejson.dumps(body).encode(
+ "utf-8"
+ )
+
+ def _assert_request(self, details):
+ self.opener.open.assert_called_once()
+ request = self.opener.open.call_args.args[0]
+ self.assertEqual(self.path + "?details=" + details, request.full_url)
+ self.assertEqual("GET", request.get_method())
+ self.assertIsNone(request.data)
+ self.assertEqual(
+ "application/vnd.gravitino.v1+json", request.get_header("Accept")
+ )
+
+ @staticmethod
+ def _metric_json():
+ return {
+ "name": "Revenue",
+ "expression": {
+ "dialects": [
+ {"dialect": "ANSI_SQL", "expression":
"SUM(orders.amount)"},
+ {"dialect": "SPARK_SQL", "expression":
"sum(orders.amount)"},
+ ]
+ },
+ "description": "Revenue - café 雪",
+ "datatype": "Decimal",
+ "aiContext": {
+ "instructions": "Use certified values",
+ "synonyms": ["Sales"],
+ "examples": ["Revenue by year"],
+ "nested": {
+ "precise": Decimal("0.12345678901234567890123456789"),
+ "large": 1234567890123456789012345678901234567890,
+ "values": [True, None, "extra"],
+ },
+ },
+ "customExtensions": [{"vendorName": "vendor", "data": '{"opaque":
"雪"}'}],
+ }
