This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f7694c7c89 [python] Materialize unresolved BLOB placeholders as NULL
(#10590)
f7694c7c89 is described below
commit f7694c7c89f2f62b6eb442df50a1c3aac865797a
Author: Wenchao Wu <[email protected]>
AuthorDate: Fri Oct 16 13:00:43 2026 +0800
[python] Materialize unresolved BLOB placeholders as NULL (#10590)
---
.../pypaimon/read/reader/format_blob_reader.py | 25 ++++++++----------
paimon-python/pypaimon/tests/blob_test.py | 30 ++++++++++++++--------
paimon-python/pypaimon/tests/video_format_test.py | 3 +++
3 files changed, 33 insertions(+), 25 deletions(-)
diff --git a/paimon-python/pypaimon/read/reader/format_blob_reader.py
b/paimon-python/pypaimon/read/reader/format_blob_reader.py
index c95626d7fe..35f04dcb2b 100644
--- a/paimon-python/pypaimon/read/reader/format_blob_reader.py
+++ b/paimon-python/pypaimon/read/reader/format_blob_reader.py
@@ -202,12 +202,11 @@ class FormatBlobReader(RecordBatchReader):
)
)
else:
- if blob is None:
+ if blob is None or blob is Blob.PLACE_HOLDER:
+ # A placeholder, including a video frame, has no
payload
+ # in this file. BlobFallbackBatchReader resolves
an older
+ # version; this reader materializes the cell as
NULL.
pydict_data[field_name].append(None)
- elif blob is Blob.PLACE_HOLDER:
- raise RuntimeError(
- "Blob placeholder is not supported by
FormatBlobReader yet."
- )
elif self._is_video or self._blob_as_descriptor:
pydict_data[field_name].append(blob.to_descriptor().serialize())
elif self._blob_parallelism > 1:
@@ -269,12 +268,10 @@ class FormatBlobReader(RecordBatchReader):
row_index,
blobs_to_resolve,
):
- if blob_array is None:
+ if blob_array is None or blob_array is Blob.ARRAY_PLACE_HOLDER:
+ # ARRAY_PLACE_HOLDER means this file has no payload for the cell.
+ # BlobFallbackBatchReader resolves an older version; here it is
NULL.
return None
- if blob_array is Blob.ARRAY_PLACE_HOLDER:
- raise RuntimeError(
- "Blob placeholder is not supported by FormatBlobReader yet."
- )
result = []
for element_index, blob in enumerate(blob_array):
if blob is None:
@@ -297,12 +294,10 @@ class FormatBlobReader(RecordBatchReader):
row_index,
blobs_to_resolve,
):
- if blob_map is None:
+ if blob_map is None or blob_map is Blob.MAP_PLACE_HOLDER:
+ # MAP_PLACE_HOLDER means this file has no payload for the cell.
+ # BlobFallbackBatchReader resolves an older version; here it is
NULL.
return None
- if blob_map is Blob.MAP_PLACE_HOLDER:
- raise RuntimeError(
- "Blob placeholder is not supported by FormatBlobReader yet."
- )
if None in blob_map:
raise ValueError(
"MAP<X, BLOB> with null keys cannot be converted to a PyArrow
Map."
diff --git a/paimon-python/pypaimon/tests/blob_test.py
b/paimon-python/pypaimon/tests/blob_test.py
index beba5f2d34..0043fd9833 100644
--- a/paimon-python/pypaimon/tests/blob_test.py
+++ b/paimon-python/pypaimon/tests/blob_test.py
@@ -3508,8 +3508,12 @@ class BlobEndToEndTest(unittest.TestCase):
(values[0].to_descriptor().offset,
values[0].to_descriptor().length))
self.assertIsNone(values[1])
self.assertIs(Blob.PLACE_HOLDER, values[2])
- with self.assertRaisesRegex(RuntimeError, "Blob placeholder is not
supported"):
- descriptor_reader.read_arrow_batch()
+ descriptor_values =
descriptor_reader.read_arrow_batch().column(0).to_pylist()
+ last_descriptor = BlobDescriptor.deserialize(descriptor_values[0])
+ self.assertEqual(path, last_descriptor.uri)
+ self.assertEqual(len(b"last"), last_descriptor.length)
+ self.assertIsNone(descriptor_values[1])
+ self.assertIsNone(descriptor_values[2])
with self.assertRaises(IndexError):
FormatBlobReader(
file_io, path, [field.name], [field], None, True,
@@ -4445,8 +4449,9 @@ class BlobEndToEndTest(unittest.TestCase):
self.assertIsNone(next(iterator).values[0])
self.assertEqual(next(iterator).values[0].to_data(), b"world")
- with self.assertRaisesRegex(RuntimeError, "Blob placeholder is not
supported"):
- reader.read_arrow_batch()
+ self.assertEqual(
+ [b"hello", None, None, b"world"],
+ reader.read_arrow_batch().column(0).to_pylist())
reader.close()
def test_placeholder_blob_read_as_descriptor(self):
@@ -4473,8 +4478,11 @@ class BlobEndToEndTest(unittest.TestCase):
blob_as_descriptor=True
)
- with self.assertRaisesRegex(RuntimeError, "Blob placeholder is not
supported"):
- reader.read_arrow_batch()
+ values = reader.read_arrow_batch().column(0).to_pylist()
+ self.assertIsNone(values[0])
+ descriptor = BlobDescriptor.deserialize(values[1])
+ self.assertEqual(blob_file_path, descriptor.uri)
+ self.assertEqual(len(b"world"), descriptor.length)
reader.close()
def test_array_blob_write_read(self):
@@ -4785,8 +4793,9 @@ class BlobEndToEndTest(unittest.TestCase):
)
self.assertEqual(next(iterator).values[0][0].to_data(), b"hello")
self.assertIs(next(iterator).values[0], Blob.ARRAY_PLACE_HOLDER)
- with self.assertRaisesRegex(RuntimeError, "Blob placeholder is not
supported"):
- reader.read_arrow_batch()
+ self.assertEqual(
+ [[b"hello"], None],
+ reader.read_arrow_batch().column(0).to_pylist())
reader.close()
def test_reject_malformed_array_blob_payloads(self):
@@ -5386,8 +5395,9 @@ class BlobEndToEndTest(unittest.TestCase):
blob_as_descriptor=False,
row_indices=[1],
)
- with self.assertRaisesRegex(RuntimeError, "Blob placeholder is not
supported"):
- placeholder_reader.read_arrow_batch()
+ self.assertEqual(
+ [None],
+ placeholder_reader.read_arrow_batch().column(0).to_pylist())
placeholder_reader.close()
def test_reject_duplicate_map_blob_key_on_write(self):
diff --git a/paimon-python/pypaimon/tests/video_format_test.py
b/paimon-python/pypaimon/tests/video_format_test.py
index a973cb5b88..2d81f474e0 100644
--- a/paimon-python/pypaimon/tests/video_format_test.py
+++ b/paimon-python/pypaimon/tests/video_format_test.py
@@ -153,6 +153,9 @@ class VideoFormatTest(unittest.TestCase):
self.assertEqual(frames[0].payload_descriptor,
frames[3].payload_descriptor)
self.assertNotEqual(frames[0].payload_descriptor,
frames[2].payload_descriptor)
self.assertIsNone(values[4])
+ # The file index still exposes the placeholder sentinel. Materializing
+ # that cell without an older version yields NULL.
+ self.assertIsNone(self._read(target, row_indices=[5])[0])
def test_cross_language_video_v1_fixture(self):
fixture = self._fixture_bytes(self.VIDEO_FIXTURE)
