This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ba091e190e [python] Fix MAP and MULTISET type JSON serialization
(#9682)
ba091e190e is described below
commit ba091e190eb6a491738d470a84c699cc5d79d643
Author: XiaoHongbo <[email protected]>
AuthorDate: Sun Sep 20 10:07:45 2026 +0800
[python] Fix MAP and MULTISET type JSON serialization (#9682)
---
paimon-python/pypaimon/schema/data_types.py | 5 +-
paimon-python/pypaimon/tests/data_types_test.py | 59 ++++++++++++++++++++++++
paimon-python/pypaimon/tests/native_plan_test.py | 34 ++++++++++++++
3 files changed, 95 insertions(+), 3 deletions(-)
diff --git a/paimon-python/pypaimon/schema/data_types.py
b/paimon-python/pypaimon/schema/data_types.py
index f01ee089ac..5cf0b3eef6 100755
--- a/paimon-python/pypaimon/schema/data_types.py
+++ b/paimon-python/pypaimon/schema/data_types.py
@@ -220,8 +220,7 @@ class MultisetType(DataType):
def to_dict(self) -> Dict[str, Any]:
return {
- "type": "MULTISET{}{}".format('<' + str(self.element) + '>' if
self.element else '',
- " NOT NULL" if not self.nullable
else ""),
+ "type": "MULTISET" + (" NOT NULL" if not self.nullable else ""),
"element": self.element.to_dict() if self.element else None,
"nullable": self.nullable,
}
@@ -263,7 +262,7 @@ class MapType(DataType):
def to_dict(self) -> Dict[str, Any]:
return {
- "type": "MAP<{}, {}>".format(self.key, self.value),
+ "type": "MAP" + (" NOT NULL" if not self.nullable else ""),
"key": self.key.to_dict() if self.key else None,
"value": self.value.to_dict() if self.value else None,
"nullable": self.nullable,
diff --git a/paimon-python/pypaimon/tests/data_types_test.py
b/paimon-python/pypaimon/tests/data_types_test.py
index 94242ce780..e96c612fb8 100755
--- a/paimon-python/pypaimon/tests/data_types_test.py
+++ b/paimon-python/pypaimon/tests/data_types_test.py
@@ -88,10 +88,69 @@ class DataTypesTest(unittest.TestCase):
paimon_type,
)
+ @parameterized.expand([
+ (nullable, element_nullable)
+ for nullable in (True, False)
+ for element_nullable in (True, False)
+ ])
+ def test_multiset_json_uses_canonical_type(self, nullable,
element_nullable):
+ element = "INT" + ("" if element_nullable else " NOT NULL")
+ data_type = MultisetType(nullable, AtomicType("INT", element_nullable))
+ self.assertEqual(data_type.to_dict(), {
+ "type": "MULTISET" + ("" if nullable else " NOT NULL"),
+ "element": element, "nullable": nullable,
+ })
+ self.assertEqual(MultisetType.from_dict(data_type.to_dict()),
data_type)
+ self.assertEqual(str(data_type), "MULTISET<{}>{}".format(
+ element, "" if nullable else " NOT NULL"))
+
+ @parameterized.expand([(True,), (False,)])
+ def test_legacy_multiset_json(self, nullable):
+ legacy = {
+ "type": "MULTISET<INT>" + ("" if nullable else " NOT NULL"),
+ "element": "INT", "nullable": nullable,
+ }
+ expected = MultisetType(nullable, AtomicType("INT"))
+ self.assertEqual(MultisetType.from_dict(legacy), expected)
+ legacy.pop("nullable")
+ self.assertEqual(MultisetType.from_dict(legacy), expected)
+
def test_map_type(self):
self.assertEqual(str(MapType(True, AtomicType("STRING"),
AtomicType("TIMESTAMP(6)"))),
"MAP<STRING, TIMESTAMP(6)>")
+ @parameterized.expand([
+ (nullable, key_nullable, value_nullable)
+ for nullable in (True, False)
+ for key_nullable in (True, False)
+ for value_nullable in (True, False)
+ ])
+ def test_map_json_uses_canonical_type(self, nullable, key_nullable,
value_nullable):
+ key = "STRING" + ("" if key_nullable else " NOT NULL")
+ value = "INT" + ("" if value_nullable else " NOT NULL")
+ data_type = MapType(nullable, AtomicType("STRING", key_nullable),
+ AtomicType("INT", value_nullable))
+ self.assertEqual(data_type.to_dict(), {
+ "type": "MAP" + ("" if nullable else " NOT NULL"),
+ "key": key, "value": value, "nullable": nullable,
+ })
+ self.assertEqual(str(data_type), "MAP<{}, {}>{}".format(
+ key, value, "" if nullable else " NOT NULL"))
+
+ @parameterized.expand([
+ ("MAP<STRING NOT NULL, INT NOT NULL>", {}, True),
+ ("MAP<STRING NOT NULL, INT NOT NULL>", {"nullable": True}, True),
+ ("MAP<STRING NOT NULL, INT NOT NULL>", {"nullable": False}, False),
+ ("MAP<STRING NOT NULL, INT NOT NULL> NOT NULL", {}, False),
+ ("MAP<STRING NOT NULL, INT NOT NULL> NOT NULL", {"nullable": None},
False),
+ ("MAP<STRING NOT NULL, INT NOT NULL> NOT NULL", {"nullable": True},
True),
+ ])
+ def test_legacy_map_json_nullability(self, type_name, attributes,
nullable):
+ legacy = dict({"type": type_name, "key": "STRING NOT NULL", "value":
"INT NOT NULL"},
+ **attributes)
+ self.assertEqual(MapType.from_dict(legacy), MapType(
+ nullable, AtomicType("STRING", False), AtomicType("INT", False)))
+
def test_map_nullability_dict_roundtrip(self):
for map_nullable in (True, False):
for value_nullable in (True, False):
diff --git a/paimon-python/pypaimon/tests/native_plan_test.py
b/paimon-python/pypaimon/tests/native_plan_test.py
index b62177d8bd..8ea40f6b2b 100644
--- a/paimon-python/pypaimon/tests/native_plan_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_test.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import json
import sys
import unittest
from types import ModuleType, SimpleNamespace
@@ -34,6 +35,7 @@ from pypaimon.read.native_plan import (
_catalog_options,
_predicate_to_native,
_read_options,
+ _resolved_schema_json,
_restore_python_partition_paths,
native_family_search_modes_available,
native_plan,
@@ -41,6 +43,8 @@ from pypaimon.read.native_plan import (
)
from pypaimon.read.plan import Plan
from pypaimon.read.table_scan import TableScan
+from pypaimon.schema.data_types import AtomicType, DataField, MapType,
MultisetType, RowType
+from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.bucket_mode import BucketMode
from pypaimon.utils.range import Range
@@ -107,6 +111,36 @@ class NativePlanTest(unittest.TestCase):
version_patcher.start()
self.addCleanup(version_patcher.stop)
+ def test_resolved_schema_json_uses_canonical_nested_collection_types(self):
+ schema = TableSchema(id=7, highest_field_id=3, time_millis=0, fields=[
+ DataField(0, 'attributes', MapType(False, AtomicType('STRING',
False), RowType(True, [
+ DataField(1, 'counts', MapType(True, AtomicType('STRING',
False), AtomicType('INT', False))),
+ DataField(2, 'tags', MultisetType(False, AtomicType('STRING',
False))),
+ DataField(3, 'groups', MultisetType(True, MapType(
+ False, AtomicType('STRING', False), AtomicType('INT'))))
+ ])))
+ ])
+ table = SimpleNamespace(table_schema=schema,
options=CoreOptions(Options({})))
+ self.assertEqual(json.loads(_resolved_schema_json(table)), {
+ 'version': 3, 'id': 7, 'highestFieldId': 3, 'timeMillis': 0,
+ 'partitionKeys': [], 'primaryKeys': [], 'comment': None,
+ 'options': {'source.split.target-size': '134217728',
+ 'source.split.open-file-cost': '4194304',
+ 'deletion-vectors.merge-on-read': 'false'},
+ 'fields': [{'id': 0, 'name': 'attributes', 'type': {
+ 'type': 'MAP NOT NULL', 'nullable': False, 'key': 'STRING NOT
NULL',
+ 'value': {'type': 'ROW', 'nullable': True, 'fields': [
+ {'id': 1, 'name': 'counts', 'type': {
+ 'type': 'MAP', 'nullable': True,
+ 'key': 'STRING NOT NULL', 'value': 'INT NOT NULL'}},
+ {'id': 2, 'name': 'tags', 'type': {
+ 'type': 'MULTISET NOT NULL', 'nullable': False,
'element': 'STRING NOT NULL'}},
+ {'id': 3, 'name': 'groups', 'type': {
+ 'type': 'MULTISET', 'nullable': True, 'element': {
+ 'type': 'MAP NOT NULL', 'nullable': False,
+ 'key': 'STRING NOT NULL', 'value': 'INT'}}}]}}}],
+ })
+
def test_resolved_schema_keeps_custom_io_and_rest_on_catalog_path(self):
from pypaimon.catalog.catalog_environment import CatalogEnvironment
from pypaimon.catalog.jdbc_catalog_loader import JdbcCatalogLoader
