This is an automated email from the ASF dual-hosted git repository. jerryshao pushed a commit to branch branch-1.3 in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 50aa8e36510d01fdacd5d2a2d2a8259f569eb628 Author: Zhiguo Wu <[email protected]> AuthorDate: Mon Jul 20 21:22:50 2026 +0800 [#12089] improvement(client-python): Implement client-side view models (#12090) ### What changes were proposed in this pull request? - Add `GenericView` as the client-side implementation of `View`. - Add DTOs for views and SQL representations. - Add polymorphic JSON serialization for view representations. - Extend `DTOConverters` with representation conversions. - Add unit tests for the new models, serialization, and conversions. ### Why are the changes needed? The Python client needs concrete view models and serialization support before view catalog operations can consume and return view metadata. Fix: #12089 ### Does this PR introduce _any_ user-facing change? No. This PR adds the supporting client-side models. View catalog operations will be added separately. ### How was this patch tested? Added unit tests covering `GenericView`, view and representation DTOs, representation serialization, and DTO conversions. Python client formatting, lint, and unit tests pass. (cherry picked from commit 648fdd7c1a869f798d05e471420de6cf56e74957) --- .../client-python/gravitino/client/generic_view.py | 63 +++++ .../dto/rel/json_serdes/representation_serdes.py | 49 ++++ .../gravitino/dto/rel/representation_dto.py | 30 +++ .../gravitino/dto/rel/sql_representation_dto.py | 58 ++++ .../client-python/gravitino/dto/rel/view_dto.py | 105 ++++++++ .../gravitino/dto/util/dto_converters.py | 54 +++- .../dto/rel/test_representation_serdes.py | 54 ++++ .../dto/rel/test_sql_representation_dto.py | 55 ++++ .../tests/unittests/dto/rel/test_view_dto.py | 293 +++++++++++++++++++++ .../unittests/dto/util/test_dto_converters.py | 52 ++++ .../tests/unittests/test_generic_view.py | 61 +++++ 11 files changed, 868 insertions(+), 6 deletions(-) diff --git a/clients/client-python/gravitino/client/generic_view.py b/clients/client-python/gravitino/client/generic_view.py new file mode 100644 index 0000000000..ce4e2a765c --- /dev/null +++ b/clients/client-python/gravitino/client/generic_view.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Optional + +from gravitino.api.audit import Audit +from gravitino.api.rel.column import Column +from gravitino.api.rel.representation import Representation +from gravitino.api.rel.view import View +from gravitino.dto.rel.view_dto import ViewDTO + + +class GenericView(View): + """A generic implementation of the View interface.""" + + def __init__( + self, + view_dto: ViewDTO, + ): + """Create a GenericView from a ViewDTO. + + Args: + view_dto: The view DTO. + """ + self._view_dto = view_dto + + def name(self) -> str: + return self._view_dto.name() + + def comment(self) -> Optional[str]: + return self._view_dto.comment() + + def columns(self) -> list[Column]: + return self._view_dto.columns() + + def representations(self) -> list[Representation]: + return self._view_dto.representations() + + def default_catalog(self) -> Optional[str]: + return self._view_dto.default_catalog() + + def default_schema(self) -> Optional[str]: + return self._view_dto.default_schema() + + def properties(self) -> dict[str, str]: + return self._view_dto.properties() + + def audit_info(self) -> Audit: + return self._view_dto.audit_info() diff --git a/clients/client-python/gravitino/dto/rel/json_serdes/representation_serdes.py b/clients/client-python/gravitino/dto/rel/json_serdes/representation_serdes.py new file mode 100644 index 0000000000..1414633df2 --- /dev/null +++ b/clients/client-python/gravitino/dto/rel/json_serdes/representation_serdes.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Optional + +from gravitino.api.rel.representation import Representation +from gravitino.dto.rel.representation_dto import RepresentationDTO +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO +from gravitino.exceptions.base import IllegalArgumentException +from gravitino.utils.precondition import Precondition + + +class RepresentationSerdes: + """Serdes for view representation DTOs.""" + + @staticmethod + def serialize(value: Optional[RepresentationDTO]) -> Optional[dict]: + """Encode a representation DTO to a dictionary.""" + if value is None: + return None + return value.to_dict() + + @staticmethod + def deserialize(value: Optional[dict]) -> Optional[RepresentationDTO]: + """Decode a representation DTO from a dictionary.""" + if value is None: + return None + Precondition.check_argument( + isinstance(value, dict) and len(value) > 0, + f"Cannot parse representation from invalid JSON: {value}", + ) + if value.get("type") == Representation.TYPE_SQL: + return SQLRepresentationDTO.from_dict(value) + raise IllegalArgumentException( + f"Unsupported representation type: {value.get('type')}" + ) diff --git a/clients/client-python/gravitino/dto/rel/representation_dto.py b/clients/client-python/gravitino/dto/rel/representation_dto.py new file mode 100644 index 0000000000..dab099399a --- /dev/null +++ b/clients/client-python/gravitino/dto/rel/representation_dto.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import abstractmethod + +from dataclasses_json import DataClassJsonMixin + +from gravitino.api.rel.representation import Representation + + +class RepresentationDTO(Representation, DataClassJsonMixin): + """DTO base class for view representations.""" + + @abstractmethod + def validate(self) -> None: + """Validates the representation DTO.""" diff --git a/clients/client-python/gravitino/dto/rel/sql_representation_dto.py b/clients/client-python/gravitino/dto/rel/sql_representation_dto.py new file mode 100644 index 0000000000..45f7974b33 --- /dev/null +++ b/clients/client-python/gravitino/dto/rel/sql_representation_dto.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field + +from dataclasses_json import config + +from gravitino.api.rel.representation import Representation +from gravitino.dto.rel.representation_dto import RepresentationDTO +from gravitino.utils.precondition import Precondition + + +@dataclass +class SQLRepresentationDTO(RepresentationDTO): + """DTO for SQL view representations.""" + + _dialect: str = field(metadata=config(field_name="dialect")) + _sql: str = field(metadata=config(field_name="sql")) + _type: str = field( + default=Representation.TYPE_SQL, metadata=config(field_name="type") + ) + + def type(self) -> str: + """Returns the representation type.""" + return self._type + + def dialect(self) -> str: + """Returns the SQL dialect.""" + return self._dialect + + def sql(self) -> str: + """Returns the SQL text.""" + return self._sql + + def validate(self) -> None: + Precondition.check_string_not_empty( + self._dialect, '"dialect" field is required and cannot be empty' + ) + Precondition.check_string_not_empty( + self._sql, '"sql" field is required and cannot be empty' + ) + + def __hash__(self) -> int: + return hash((self._type, self._dialect, self._sql)) diff --git a/clients/client-python/gravitino/dto/rel/view_dto.py b/clients/client-python/gravitino/dto/rel/view_dto.py new file mode 100644 index 0000000000..3606217cab --- /dev/null +++ b/clients/client-python/gravitino/dto/rel/view_dto.py @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field +from typing import Optional + +from dataclasses_json import DataClassJsonMixin, config + +from gravitino.api.rel.column import Column +from gravitino.api.rel.representation import Representation +from gravitino.api.rel.view import View +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.rel.column_dto import ColumnDTO +from gravitino.dto.rel.json_serdes.representation_serdes import RepresentationSerdes +from gravitino.dto.rel.representation_dto import RepresentationDTO +from gravitino.dto.util.dto_converters import DTOConverters +from gravitino.utils.precondition import Precondition + + +@dataclass +class ViewDTO(View, DataClassJsonMixin): # pylint: disable=too-many-instance-attributes + """Represents a View DTO.""" + + _name: str = field(metadata=config(field_name="name")) + _columns: Optional[list[ColumnDTO]] = field( + default=None, metadata=config(field_name="columns") + ) + _representations: Optional[list[RepresentationDTO]] = field( + default=None, + metadata=config( + field_name="representations", + encoder=lambda items: [ + RepresentationSerdes.serialize(item) for item in items + ], + decoder=lambda values: [ + RepresentationSerdes.deserialize(value) for value in values + ], + ), + ) + _audit: Optional[AuditDTO] = field( + default=None, metadata=config(field_name="audit") + ) + _comment: Optional[str] = field(default=None, metadata=config(field_name="comment")) + _default_catalog: Optional[str] = field( + default=None, metadata=config(field_name="defaultCatalog") + ) + _default_schema: Optional[str] = field( + default=None, metadata=config(field_name="defaultSchema") + ) + _properties: Optional[dict[str, str]] = field( + default=None, metadata=config(field_name="properties") + ) + + def __post_init__(self): + if self._columns is None: + self._columns = [] + Precondition.check_string_not_empty(self._name, "name cannot be null or empty") + Precondition.check_argument(self._audit is not None, "audit cannot be null") + Precondition.check_argument( + self._representations is not None and len(self._representations) > 0, + "representations cannot be null or empty", + ) + for representation in self._representations: + Precondition.check_argument( + representation is not None, "representation must not be null" + ) + representation.validate() + + def name(self) -> str: + return self._name + + def comment(self) -> Optional[str]: + return self._comment + + def columns(self) -> list[Column]: + return self._columns + + def representations(self) -> list[Representation]: + return DTOConverters.from_dtos(self._representations) + + def default_catalog(self) -> Optional[str]: + return self._default_catalog + + def default_schema(self) -> Optional[str]: + return self._default_schema + + def properties(self) -> dict[str, str]: + return self._properties or {} + + def audit_info(self) -> AuditDTO: + return self._audit diff --git a/clients/client-python/gravitino/dto/util/dto_converters.py b/clients/client-python/gravitino/dto/util/dto_converters.py index 1aa128e179..9aedba7f40 100644 --- a/clients/client-python/gravitino/dto/util/dto_converters.py +++ b/clients/client-python/gravitino/dto/util/dto_converters.py @@ -41,6 +41,8 @@ from gravitino.api.rel.partitions.identity_partition import IdentityPartition from gravitino.api.rel.partitions.list_partition import ListPartition from gravitino.api.rel.partitions.partition import Partition from gravitino.api.rel.partitions.range_partition import RangePartition +from gravitino.api.rel.representation import Representation +from gravitino.api.rel.sql_representation import SQLRepresentation from gravitino.api.rel.table import Table from gravitino.api.rel.table_change import ( AddColumn, @@ -87,7 +89,9 @@ from gravitino.dto.rel.partitions.identity_partition_dto import IdentityPartitio from gravitino.dto.rel.partitions.list_partition_dto import ListPartitionDTO from gravitino.dto.rel.partitions.partition_dto import PartitionDTO from gravitino.dto.rel.partitions.range_partition_dto import RangePartitionDTO +from gravitino.dto.rel.representation_dto import RepresentationDTO from gravitino.dto.rel.sort_order_dto import SortOrderDTO +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO from gravitino.dto.rel.table_dto import TableDTO from gravitino.dto.requests.table_update_request import ( TableUpdateRequest, @@ -227,6 +231,19 @@ class DTOConverters: DTOConverters.from_function_arg(dto.default_value()), ) + @from_dto.register + @staticmethod + def _(dto: SQLRepresentationDTO) -> SQLRepresentation: + """Converts a SQLRepresentationDTO to a SQLRepresentation. + + Args: + dto (SQLRepresentationDTO): The SQLRepresentation DTO to be converted. + + Returns: + SQLRepresentation: The SQLRepresentation. + """ + return SQLRepresentation(_dialect=dto.dialect(), _sql=dto.sql()) + @from_dto.register @staticmethod def _(dto: Partitioning) -> Transform: # pylint: disable=too-many-return-statements @@ -375,22 +392,25 @@ class DTOConverters: @staticmethod def from_dtos(dtos: list[SortOrderDTO]) -> list[SortOrder]: ... # pragma: no cover + @overload + @staticmethod + def from_dtos( + dtos: list[RepresentationDTO], + ) -> list[Representation]: ... # pragma: no cover + @overload @staticmethod def from_dtos(dtos: list[Partitioning]) -> list[Transform]: ... # pragma: no cover @staticmethod def from_dtos(dtos): - """Converts list of `ColumnDTO`, `IndexDTO`, `SortOrderDTO`, or `Partitioning` - to the corresponding list of `Column`s, `Index`es, `SortOrder`s, or `Transform`s. + """Converts a list of DTOs to the corresponding API objects. Args: - dtos (list[ColumnDTO] | list[IndexDTO] | list[SortOrderDTO] | list[Partitioning]): - The DTOs to be converted. + dtos: The DTOs to be converted. Returns: - list[Column] | list[Index] | list[SortOrder] | list[Transform]: - The list of Columns, Indexes, SortOrders, or Transforms depends on the input DTOs. + The converted objects. """ if not dtos: return [] @@ -557,6 +577,16 @@ class DTOConverters: field_names=obj.field_names(), ) + @to_dto.register + @staticmethod + def _(obj: RepresentationDTO) -> RepresentationDTO: + return obj + + @to_dto.register + @staticmethod + def _(obj: SQLRepresentation) -> SQLRepresentationDTO: + return SQLRepresentationDTO(_dialect=obj.dialect(), _sql=obj.sql()) + @to_dto.register @staticmethod def _(obj: Partitioning) -> Partitioning: @@ -646,6 +676,18 @@ class DTOConverters: @staticmethod def to_dtos(dtos: list[SortOrderDTO]) -> list[SortOrderDTO]: ... # pragma: no cover + @overload + @staticmethod + def to_dtos( + dtos: list[Representation], + ) -> list[RepresentationDTO]: ... # pragma: no cover + + @overload + @staticmethod + def to_dtos( + dtos: list[RepresentationDTO], + ) -> list[RepresentationDTO]: ... # pragma: no cover + @overload @staticmethod def to_dtos(dtos: list[Transform]) -> list[Partitioning]: ... # pragma: no cover diff --git a/clients/client-python/tests/unittests/dto/rel/test_representation_serdes.py b/clients/client-python/tests/unittests/dto/rel/test_representation_serdes.py new file mode 100644 index 0000000000..e75fc0829c --- /dev/null +++ b/clients/client-python/tests/unittests/dto/rel/test_representation_serdes.py @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from gravitino.api.rel.dialects import Dialects +from gravitino.api.rel.representation import Representation +from gravitino.dto.rel.json_serdes.representation_serdes import RepresentationSerdes +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO +from gravitino.exceptions.base import IllegalArgumentException + + +class TestRepresentationSerdes(unittest.TestCase): + def test_representation_serdes(self): + representation = SQLRepresentationDTO( + _dialect=Dialects.TRINO, _sql="SELECT id FROM table" + ) + serialized = RepresentationSerdes.serialize(representation) + + self.assertEqual(Representation.TYPE_SQL, serialized["type"]) + self.assertEqual(Dialects.TRINO, serialized["dialect"]) + self.assertEqual("SELECT id FROM table", serialized["sql"]) + self.assertEqual(representation, RepresentationSerdes.deserialize(serialized)) + + def test_representation_serdes_none(self): + self.assertIsNone(RepresentationSerdes.serialize(None)) + self.assertIsNone(RepresentationSerdes.deserialize(None)) + + def test_representation_serdes_invalid_json(self): + for invalid_json in ("invalid", {}): + with self.subTest(invalid_json=invalid_json): + with self.assertRaisesRegex( + IllegalArgumentException, + "Cannot parse representation from invalid JSON", + ): + RepresentationSerdes.deserialize(invalid_json) + + def test_representation_serdes_unknown_type(self): + with self.assertRaises(IllegalArgumentException): + RepresentationSerdes.deserialize({"type": "unknown"}) diff --git a/clients/client-python/tests/unittests/dto/rel/test_sql_representation_dto.py b/clients/client-python/tests/unittests/dto/rel/test_sql_representation_dto.py new file mode 100644 index 0000000000..7cdcb85d09 --- /dev/null +++ b/clients/client-python/tests/unittests/dto/rel/test_sql_representation_dto.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from gravitino.api.rel.dialects import Dialects +from gravitino.api.rel.representation import Representation +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO + + +class TestSQLRepresentationDTO(unittest.TestCase): + def test_sql_representation_dto(self): + representation = SQLRepresentationDTO( + _dialect=Dialects.TRINO, _sql="SELECT id FROM table" + ) + + self.assertEqual(Representation.TYPE_SQL, representation.type()) + self.assertEqual(Dialects.TRINO, representation.dialect()) + self.assertEqual("SELECT id FROM table", representation.sql()) + self.assertEqual( + representation, + SQLRepresentationDTO(_dialect=Dialects.TRINO, _sql="SELECT id FROM table"), + ) + self.assertEqual( + hash(representation), + hash( + SQLRepresentationDTO( + _dialect=Dialects.TRINO, _sql="SELECT id FROM table" + ) + ), + ) + self.assertNotEqual( + representation, + SQLRepresentationDTO(_dialect=Dialects.SPARK, _sql="SELECT id FROM table"), + ) + + def test_sql_representation_dto_validate(self): + with self.assertRaises(ValueError): + SQLRepresentationDTO(_dialect="", _sql="SELECT 1").validate() + with self.assertRaises(ValueError): + SQLRepresentationDTO(_dialect=Dialects.TRINO, _sql="").validate() diff --git a/clients/client-python/tests/unittests/dto/rel/test_view_dto.py b/clients/client-python/tests/unittests/dto/rel/test_view_dto.py new file mode 100644 index 0000000000..c568646594 --- /dev/null +++ b/clients/client-python/tests/unittests/dto/rel/test_view_dto.py @@ -0,0 +1,293 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from gravitino.api.rel.dialects import Dialects +from gravitino.api.rel.sql_representation import SQLRepresentation +from gravitino.api.rel.types.types import Types +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.rel.column_dto import ColumnDTO +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO +from gravitino.dto.rel.view_dto import ViewDTO + + +class TestViewDTO(unittest.TestCase): + @staticmethod + def _column(name: str = "id") -> ColumnDTO: + return ColumnDTO( + _name=name, + _data_type=Types.IntegerType.get(), + _comment="column comment", + _nullable=False, + ) + + @staticmethod + def _representation( + dialect: str = Dialects.TRINO, sql: str = "SELECT id FROM table" + ) -> SQLRepresentationDTO: + return SQLRepresentationDTO(_dialect=dialect, _sql=sql) + + def _view_dto( + self, + name: str = "test_view", + columns: list[ColumnDTO] = None, + representations: list[SQLRepresentationDTO] = None, + comment: str = "view comment", + properties: dict[str, str] = None, + ) -> ViewDTO: + return ViewDTO( + _name=name, + _columns=columns if columns is not None else [self._column()], + _representations=( + representations + if representations is not None + else [self._representation()] + ), + _comment=comment, + _default_catalog="catalog", + _default_schema="schema", + _properties=properties if properties is not None else {"k1": "v1"}, + _audit=AuditDTO( + "creator", "2022-01-01T00:00:00Z", "modifier", "2022-01-02T00:00:00Z" + ), + ) + + def test_view_dto_serialization(self): + view = self._view_dto() + + deserialized = ViewDTO.from_json(view.to_json()) + + self.assertEqual(view, deserialized) + self.assertEqual("test_view", deserialized.name()) + self.assertEqual("view comment", deserialized.comment()) + self.assertEqual("catalog", deserialized.default_catalog()) + self.assertEqual("schema", deserialized.default_schema()) + self.assertEqual({"k1": "v1"}, deserialized.properties()) + self.assertEqual("creator", deserialized.audit_info().creator()) + self.assertEqual(1, len(deserialized.columns())) + self.assertEqual(1, len(deserialized.representations())) + self.assertIsInstance(deserialized.representations()[0], SQLRepresentation) + self.assertEqual( + "SELECT id FROM table", + deserialized.representations()[0].sql(), + ) + + def test_view_dto_deserialize_from_raw_json(self): + raw_json = """ + { + "name": "raw_view", + "comment": "raw comment", + "columns": [ + { + "name": "id", + "type": "integer", + "comment": "id column", + "nullable": false, + "autoIncrement": false + } + ], + "representations": [ + { + "type": "sql", + "dialect": "spark", + "sql": "SELECT id FROM table" + } + ], + "defaultCatalog": "catalog", + "defaultSchema": "schema", + "properties": { + "k1": "v1" + }, + "audit": { + "creator": "creator" + } + } + """ + + view = ViewDTO.from_json(raw_json) + + self.assertEqual("raw_view", view.name()) + self.assertEqual("raw comment", view.comment()) + self.assertEqual("catalog", view.default_catalog()) + self.assertEqual("schema", view.default_schema()) + self.assertEqual({"k1": "v1"}, view.properties()) + self.assertEqual("creator", view.audit_info().creator()) + self.assertEqual(1, len(view.columns())) + self.assertEqual("id", view.columns()[0].name()) + self.assertEqual(1, len(view.representations())) + self.assertIsInstance(view.representations()[0], SQLRepresentation) + self.assertEqual("spark", view.representations()[0].dialect()) + self.assertEqual("SELECT id FROM table", view.representations()[0].sql()) + + def test_view_dto_defaults(self): + view = ViewDTO( + _name="test_view", + _representations=[self._representation()], + _audit=AuditDTO("creator"), + ) + + self.assertEqual([], view.columns()) + self.assertEqual([], view.to_dict()["columns"]) + self.assertEqual({}, view.properties()) + self.assertIsNone(view.comment()) + self.assertIsNone(view.default_catalog()) + self.assertIsNone(view.default_schema()) + + def test_view_dto_deserialize_invalid_name(self): + raw_json = """ + { + "name": "", + "representations": [ + { + "type": "sql", + "dialect": "spark", + "sql": "SELECT 1" + } + ], + "audit": { + "creator": "creator" + } + } + """ + + with self.assertRaisesRegex(ValueError, "name cannot be null or empty"): + ViewDTO.from_json(raw_json) + + def test_view_dto_deserialize_missing_audit(self): + raw_json = """ + { + "name": "test_view", + "representations": [ + { + "type": "sql", + "dialect": "spark", + "sql": "SELECT 1" + } + ] + } + """ + + with self.assertRaisesRegex(ValueError, "audit cannot be null"): + ViewDTO.from_json(raw_json) + + def test_view_dto_deserialize_missing_representations(self): + raw_json = """ + { + "name": "test_view", + "audit": { + "creator": "creator" + } + } + """ + + with self.assertRaisesRegex( + ValueError, "representations cannot be null or empty" + ): + ViewDTO.from_json(raw_json) + + def test_view_dto_deserialize_invalid_representations(self): + raw_json_strings = [ + """ + { + "name": "test_view", + "representations": [], + "audit": { + "creator": "creator" + } + } + """, + """ + { + "name": "test_view", + "representations": [null], + "audit": { + "creator": "creator" + } + } + """, + """ + { + "name": "test_view", + "representations": [ + { + "type": "sql", + "dialect": "", + "sql": "SELECT 1" + } + ], + "audit": { + "creator": "creator" + } + } + """, + """ + { + "name": "test_view", + "representations": [ + { + "type": "sql", + "dialect": "spark", + "sql": "" + } + ], + "audit": { + "creator": "creator" + } + } + """, + ] + + for raw_json in raw_json_strings: + with self.assertRaises(ValueError): + ViewDTO.from_json(raw_json) + + def test_view_dto_validate_constructor_arguments(self): + with self.assertRaises(ValueError): + ViewDTO( + _name="", + _representations=[self._representation()], + _audit=AuditDTO("creator"), + ) + with self.assertRaises(ValueError): + ViewDTO(_name="test_view", _representations=[], _audit=AuditDTO("creator")) + with self.assertRaises(ValueError): + ViewDTO( + _name="test_view", _representations=[None], _audit=AuditDTO("creator") + ) + with self.assertRaises(ValueError): + ViewDTO( + _name="test_view", + _representations=[self._representation()], + _audit=None, + ) + + def test_view_dto_sql_for(self): + view = self._view_dto( + representations=[ + self._representation(Dialects.TRINO, "SELECT 1"), + self._representation(Dialects.SPARK, "SELECT 2"), + ] + ) + deserialized = ViewDTO.from_json(view.to_json()) + + self.assertEqual("SELECT 1", deserialized.sql_for("trino").sql()) + self.assertEqual("SELECT 1", deserialized.sql_for("TRINO").sql()) + self.assertEqual("SELECT 1", deserialized.sql_for("Trino").sql()) + self.assertEqual("SELECT 2", deserialized.sql_for(Dialects.SPARK).sql()) + self.assertIsNone(deserialized.sql_for("hive")) + self.assertIsNone(deserialized.sql_for(None)) diff --git a/clients/client-python/tests/unittests/dto/util/test_dto_converters.py b/clients/client-python/tests/unittests/dto/util/test_dto_converters.py index 51ebf0bdd9..f9199d28d9 100644 --- a/clients/client-python/tests/unittests/dto/util/test_dto_converters.py +++ b/clients/client-python/tests/unittests/dto/util/test_dto_converters.py @@ -23,6 +23,7 @@ from typing import Dict, cast from unittest.mock import MagicMock, patch from gravitino.api.rel.column import Column +from gravitino.api.rel.dialects import Dialects from gravitino.api.rel.expressions.distributions.distributions import Distributions from gravitino.api.rel.expressions.distributions.strategy import Strategy from gravitino.api.rel.expressions.expression import Expression @@ -41,6 +42,7 @@ from gravitino.api.rel.indexes.indexes import Indexes from gravitino.api.rel.partitions.partition import Partition from gravitino.api.rel.partitions.partitions import Partitions from gravitino.api.rel.table import Table +from gravitino.api.rel.sql_representation import SQLRepresentation from gravitino.api.rel.types.types import Types from gravitino.dto.rel.column_dto import ColumnDTO from gravitino.dto.rel.distribution_dto import DistributionDTO @@ -71,6 +73,7 @@ from gravitino.dto.rel.partitions.identity_partition_dto import IdentityPartitio from gravitino.dto.rel.partitions.list_partition_dto import ListPartitionDTO from gravitino.dto.rel.partitions.range_partition_dto import RangePartitionDTO from gravitino.dto.rel.sort_order_dto import SortOrderDTO +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO from gravitino.dto.rel.table_dto import TableDTO from gravitino.dto.util.dto_converters import DTOConverters from gravitino.exceptions.base import IllegalArgumentException @@ -386,6 +389,16 @@ class TestDTOConverters(unittest.TestCase): converted = DTOConverters.from_dto(sort_order_dto) self.assertTrue(converted == expected) + def test_from_dto_sql_representation(self): + representation_dto = SQLRepresentationDTO(Dialects.TRINO, "SELECT 1") + + converted = DTOConverters.from_dto(representation_dto) + + self.assertEqual( + SQLRepresentation(Dialects.TRINO, "SELECT 1"), + converted, + ) + def test_from_dto_column(self): column_name = "test_column" column_data_type = Types.IntegerType.get() @@ -537,6 +550,20 @@ class TestDTOConverters(unittest.TestCase): converted = DTOConverters.from_dtos(sort_order_dtos) self.assertListEqual(converted, expected) + def test_from_dtos_representations(self): + representation_dtos = [ + SQLRepresentationDTO(Dialects.TRINO, "SELECT 1"), + SQLRepresentationDTO(Dialects.SPARK, "SELECT 2"), + ] + expected = [ + SQLRepresentation(Dialects.TRINO, "SELECT 1"), + SQLRepresentation(Dialects.SPARK, "SELECT 2"), + ] + + converted = DTOConverters.from_dtos(representation_dtos) + + self.assertListEqual(converted, expected) + def test_from_dtos_partitioning(self): field_name = ["score"] field_names = [field_name] @@ -668,6 +695,16 @@ class TestDTOConverters(unittest.TestCase): with self.assertRaisesRegex(IllegalArgumentException, "Unsupported type"): DTOConverters.to_dto("test") + def test_to_dto_sql_representation(self): + representation = SQLRepresentation(Dialects.TRINO, "SELECT 1") + + dto = DTOConverters.to_dto(representation) + + self.assertIsInstance(dto, SQLRepresentationDTO) + self.assertEqual(Dialects.TRINO, dto.dialect()) + self.assertEqual("SELECT 1", dto.sql()) + self.assertIs(dto, DTOConverters.to_dto(dto)) + def test_to_dto_range_partition(self): converted = DTOConverters.to_dto( Partitions.range( @@ -858,6 +895,21 @@ class TestDTOConverters(unittest.TestCase): self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + def test_to_dtos_representations(self): + representations = [ + SQLRepresentation(Dialects.TRINO, "SELECT 1"), + SQLRepresentation(Dialects.SPARK, "SELECT 2"), + ] + expected_dtos = [ + SQLRepresentationDTO(Dialects.TRINO, "SELECT 1"), + SQLRepresentationDTO(Dialects.SPARK, "SELECT 2"), + ] + + converted_dtos = DTOConverters.to_dtos(representations) + + self.assertListEqual(converted_dtos, expected_dtos) + self.assertListEqual(DTOConverters.to_dtos(converted_dtos), converted_dtos) + def test_to_dtos_single_field_transforms(self): converted_dtos = DTOConverters.to_dtos(self.single_field_transforms.values()) for key, converted in zip(self.single_field_transforms.keys(), converted_dtos): diff --git a/clients/client-python/tests/unittests/test_generic_view.py b/clients/client-python/tests/unittests/test_generic_view.py new file mode 100644 index 0000000000..a07c15be05 --- /dev/null +++ b/clients/client-python/tests/unittests/test_generic_view.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from gravitino.api.rel.dialects import Dialects +from gravitino.client.generic_view import GenericView +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.rel.sql_representation_dto import SQLRepresentationDTO +from gravitino.dto.rel.view_dto import ViewDTO + + +class TestGenericView(unittest.TestCase): + def _generic_view( + self, + name: str = "demo_view", + comment: str = "comment", + sql: str = "SELECT 1", + default_catalog: str = "demo_catalog", + default_schema: str = "demo_schema", + ) -> GenericView: + return GenericView( + ViewDTO( + _name=name, + _representations=[ + SQLRepresentationDTO(_dialect=Dialects.TRINO, _sql=sql) + ], + _comment=comment, + _default_catalog=default_catalog, + _default_schema=default_schema, + _properties={"key": "value"}, + _audit=AuditDTO("creator"), + ), + ) + + def test_generic_view(self) -> None: + generic_view = self._generic_view() + + self.assertEqual("demo_view", generic_view.name()) + self.assertEqual("comment", generic_view.comment()) + self.assertEqual("demo_catalog", generic_view.default_catalog()) + self.assertEqual("demo_schema", generic_view.default_schema()) + self.assertEqual({"key": "value"}, generic_view.properties()) + self.assertEqual(0, len(generic_view.columns())) + self.assertEqual(1, len(generic_view.representations())) + self.assertEqual("SELECT 1", generic_view.sql_for(Dialects.TRINO).sql()) + self.assertEqual("creator", generic_view.audit_info().creator())
