This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 97b7846a28 [python] Resolve value stats columns using file schema
(#9906)
97b7846a28 is described below
commit 97b7846a2842aa26138d37ce76a935c4b39bf2b2
Author: XiaoHongbo <[email protected]>
AuthorDate: Thu Sep 17 11:30:10 2026 +0800
[python] Resolve value stats columns using file schema (#9906)
---
.../pypaimon/manifest/manifest_file_manager.py | 3 ++-
paimon-python/pypaimon/tests/reader_base_test.py | 23 ++++++++++++++++++++++
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py
b/paimon-python/pypaimon/manifest/manifest_file_manager.py
index 7b438da88f..044d8985f9 100644
--- a/paimon-python/pypaimon/manifest/manifest_file_manager.py
+++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py
@@ -317,7 +317,8 @@ class ManifestFileManager:
elif not file_dict['_VALUE_STATS_COLS']:
fields = []
else:
- fields = [self.table.field_dict[col] for col in
file_dict['_VALUE_STATS_COLS']]
+ data_field_dict = {f.name: f for f in schema_fields}
+ fields = [data_field_dict[col] for col in
file_dict['_VALUE_STATS_COLS']]
return fields
def write(self, file_name, entries: List[ManifestEntry]):
diff --git a/paimon-python/pypaimon/tests/reader_base_test.py
b/paimon-python/pypaimon/tests/reader_base_test.py
index 27c2413bb5..6faf40851d 100644
--- a/paimon-python/pypaimon/tests/reader_base_test.py
+++ b/paimon-python/pypaimon/tests/reader_base_test.py
@@ -37,6 +37,7 @@ from pypaimon.manifest.schema.manifest_entry import
ManifestEntry
from pypaimon.manifest.schema.simple_stats import SimpleStats
from pypaimon.schema.data_types import (ArrayType, AtomicType, DataField,
MapType, PyarrowFieldParser)
+from pypaimon.schema.schema_change import SchemaChange
from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.row.generic_row import GenericRow, GenericRowDeserializer
from pypaimon.write.file_store_commit import FileStoreCommit
@@ -585,6 +586,28 @@ class ReaderBasicTest(unittest.TestCase):
table_with_stats =
catalog.get_table("test_db.test_value_stats_cols_schema_match")
self._test_append_only_schema_match_case(table_with_stats, pa_schema)
+ def test_value_stats_cols_use_file_schema_after_schema_evolution(self):
+ table_name = 'default.test_value_stats_schema_evolution'
+ schema = Schema.from_pyarrow_schema(pa.schema([
+ ('id', pa.int64()), ('name', pa.string())]))
+ self.catalog.create_table(table_name, schema, False)
+ table = self.catalog.get_table(table_name)
+ self._test_value_stats_cols_case(
+ ManifestFileManager(table), table, ['id', 'name'], 2,
'schema_evolution')
+
+ for change in (SchemaChange.rename_column('name', 'renamed'),
+ SchemaChange.drop_column('renamed'),
+ SchemaChange.add_column('name', AtomicType('INT'))):
+ self.catalog.alter_table(table_name, [change], False)
+ table = self.catalog.get_table(table_name)
+ entries = ManifestFileManager(table).read(
+ 'manifest-test-schema_evolution', drop_stats=False)
+ stats = entries[0].file.value_stats
+ self.assertEqual([field.name for field in stats.min_values.fields],
+ ['id', 'name'])
+ self.assertEqual(stats.min_values.get_field(1), 'apple')
+ self.assertEqual(stats.max_values.get_field(1), 'zebra')
+
def test_primary_key_value_stats_excludes_system_fields(self):
catalog = CatalogFactory.create({
"warehouse": self.warehouse
