This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 6dbd235225b50ec6f2721768ab44d40a7358af1a Author: XiaoHongbo <[email protected]> AuthorDate: Sun Sep 20 10:07:45 2026 +0800 [python] Fix MAP and MULTISET type JSON serialization (#9682) --- paimon-python/pypaimon/schema/data_types.py | 5 +- paimon-python/pypaimon/tests/data_types_test.py | 59 ++++++++++++++++++++++++ paimon-python/pypaimon/tests/native_plan_test.py | 34 ++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py index f01ee089ac..5cf0b3eef6 100755 --- a/paimon-python/pypaimon/schema/data_types.py +++ b/paimon-python/pypaimon/schema/data_types.py @@ -220,8 +220,7 @@ class MultisetType(DataType): def to_dict(self) -> Dict[str, Any]: return { - "type": "MULTISET{}{}".format('<' + str(self.element) + '>' if self.element else '', - " NOT NULL" if not self.nullable else ""), + "type": "MULTISET" + (" NOT NULL" if not self.nullable else ""), "element": self.element.to_dict() if self.element else None, "nullable": self.nullable, } @@ -263,7 +262,7 @@ class MapType(DataType): def to_dict(self) -> Dict[str, Any]: return { - "type": "MAP<{}, {}>".format(self.key, self.value), + "type": "MAP" + (" NOT NULL" if not self.nullable else ""), "key": self.key.to_dict() if self.key else None, "value": self.value.to_dict() if self.value else None, "nullable": self.nullable, diff --git a/paimon-python/pypaimon/tests/data_types_test.py b/paimon-python/pypaimon/tests/data_types_test.py index 94242ce780..e96c612fb8 100755 --- a/paimon-python/pypaimon/tests/data_types_test.py +++ b/paimon-python/pypaimon/tests/data_types_test.py @@ -88,10 +88,69 @@ class DataTypesTest(unittest.TestCase): paimon_type, ) + @parameterized.expand([ + (nullable, element_nullable) + for nullable in (True, False) + for element_nullable in (True, False) + ]) + def test_multiset_json_uses_canonical_type(self, nullable, element_nullable): + element = "INT" + ("" if element_nullable else " NOT NULL") + data_type = MultisetType(nullable, AtomicType("INT", element_nullable)) + self.assertEqual(data_type.to_dict(), { + "type": "MULTISET" + ("" if nullable else " NOT NULL"), + "element": element, "nullable": nullable, + }) + self.assertEqual(MultisetType.from_dict(data_type.to_dict()), data_type) + self.assertEqual(str(data_type), "MULTISET<{}>{}".format( + element, "" if nullable else " NOT NULL")) + + @parameterized.expand([(True,), (False,)]) + def test_legacy_multiset_json(self, nullable): + legacy = { + "type": "MULTISET<INT>" + ("" if nullable else " NOT NULL"), + "element": "INT", "nullable": nullable, + } + expected = MultisetType(nullable, AtomicType("INT")) + self.assertEqual(MultisetType.from_dict(legacy), expected) + legacy.pop("nullable") + self.assertEqual(MultisetType.from_dict(legacy), expected) + def test_map_type(self): self.assertEqual(str(MapType(True, AtomicType("STRING"), AtomicType("TIMESTAMP(6)"))), "MAP<STRING, TIMESTAMP(6)>") + @parameterized.expand([ + (nullable, key_nullable, value_nullable) + for nullable in (True, False) + for key_nullable in (True, False) + for value_nullable in (True, False) + ]) + def test_map_json_uses_canonical_type(self, nullable, key_nullable, value_nullable): + key = "STRING" + ("" if key_nullable else " NOT NULL") + value = "INT" + ("" if value_nullable else " NOT NULL") + data_type = MapType(nullable, AtomicType("STRING", key_nullable), + AtomicType("INT", value_nullable)) + self.assertEqual(data_type.to_dict(), { + "type": "MAP" + ("" if nullable else " NOT NULL"), + "key": key, "value": value, "nullable": nullable, + }) + self.assertEqual(str(data_type), "MAP<{}, {}>{}".format( + key, value, "" if nullable else " NOT NULL")) + + @parameterized.expand([ + ("MAP<STRING NOT NULL, INT NOT NULL>", {}, True), + ("MAP<STRING NOT NULL, INT NOT NULL>", {"nullable": True}, True), + ("MAP<STRING NOT NULL, INT NOT NULL>", {"nullable": False}, False), + ("MAP<STRING NOT NULL, INT NOT NULL> NOT NULL", {}, False), + ("MAP<STRING NOT NULL, INT NOT NULL> NOT NULL", {"nullable": None}, False), + ("MAP<STRING NOT NULL, INT NOT NULL> NOT NULL", {"nullable": True}, True), + ]) + def test_legacy_map_json_nullability(self, type_name, attributes, nullable): + legacy = dict({"type": type_name, "key": "STRING NOT NULL", "value": "INT NOT NULL"}, + **attributes) + self.assertEqual(MapType.from_dict(legacy), MapType( + nullable, AtomicType("STRING", False), AtomicType("INT", False))) + def test_map_nullability_dict_roundtrip(self): for map_nullable in (True, False): for value_nullable in (True, False): diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index b62177d8bd..8ea40f6b2b 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json import sys import unittest from types import ModuleType, SimpleNamespace @@ -34,6 +35,7 @@ from pypaimon.read.native_plan import ( _catalog_options, _predicate_to_native, _read_options, + _resolved_schema_json, _restore_python_partition_paths, native_family_search_modes_available, native_plan, @@ -41,6 +43,8 @@ from pypaimon.read.native_plan import ( ) from pypaimon.read.plan import Plan from pypaimon.read.table_scan import TableScan +from pypaimon.schema.data_types import AtomicType, DataField, MapType, MultisetType, RowType +from pypaimon.schema.table_schema import TableSchema from pypaimon.table.bucket_mode import BucketMode from pypaimon.utils.range import Range @@ -107,6 +111,36 @@ class NativePlanTest(unittest.TestCase): version_patcher.start() self.addCleanup(version_patcher.stop) + def test_resolved_schema_json_uses_canonical_nested_collection_types(self): + schema = TableSchema(id=7, highest_field_id=3, time_millis=0, fields=[ + DataField(0, 'attributes', MapType(False, AtomicType('STRING', False), RowType(True, [ + DataField(1, 'counts', MapType(True, AtomicType('STRING', False), AtomicType('INT', False))), + DataField(2, 'tags', MultisetType(False, AtomicType('STRING', False))), + DataField(3, 'groups', MultisetType(True, MapType( + False, AtomicType('STRING', False), AtomicType('INT')))) + ]))) + ]) + table = SimpleNamespace(table_schema=schema, options=CoreOptions(Options({}))) + self.assertEqual(json.loads(_resolved_schema_json(table)), { + 'version': 3, 'id': 7, 'highestFieldId': 3, 'timeMillis': 0, + 'partitionKeys': [], 'primaryKeys': [], 'comment': None, + 'options': {'source.split.target-size': '134217728', + 'source.split.open-file-cost': '4194304', + 'deletion-vectors.merge-on-read': 'false'}, + 'fields': [{'id': 0, 'name': 'attributes', 'type': { + 'type': 'MAP NOT NULL', 'nullable': False, 'key': 'STRING NOT NULL', + 'value': {'type': 'ROW', 'nullable': True, 'fields': [ + {'id': 1, 'name': 'counts', 'type': { + 'type': 'MAP', 'nullable': True, + 'key': 'STRING NOT NULL', 'value': 'INT NOT NULL'}}, + {'id': 2, 'name': 'tags', 'type': { + 'type': 'MULTISET NOT NULL', 'nullable': False, 'element': 'STRING NOT NULL'}}, + {'id': 3, 'name': 'groups', 'type': { + 'type': 'MULTISET', 'nullable': True, 'element': { + 'type': 'MAP NOT NULL', 'nullable': False, + 'key': 'STRING NOT NULL', 'value': 'INT'}}}]}}}], + }) + def test_resolved_schema_keeps_custom_io_and_rest_on_catalog_path(self): from pypaimon.catalog.catalog_environment import CatalogEnvironment from pypaimon.catalog.jdbc_catalog_loader import JdbcCatalogLoader
