This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new eb4958c184 [python] Avoid unused keyframe allocations during video
reads (#10803)
eb4958c184 is described below
commit eb4958c1843f89de84ca1327368f7603d595abf9
Author: XiaoHongbo <[email protected]>
AuthorDate: Tue Oct 27 09:30:21 2026 +0800
[python] Avoid unused keyframe allocations during video reads (#10803)
---
.../pypaimon/table/row/video_keyframe_index.py | 18 ++++++++++++---
.../pypaimon/tests/video_keyframe_index_test.py | 26 ++++++++++++++++++++++
2 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/paimon-python/pypaimon/table/row/video_keyframe_index.py
b/paimon-python/pypaimon/table/row/video_keyframe_index.py
index 2c38322706..acabf9ce4a 100644
--- a/paimon-python/pypaimon/table/row/video_keyframe_index.py
+++ b/paimon-python/pypaimon/table/row/video_keyframe_index.py
@@ -90,6 +90,18 @@ class VideoKeyframeIndex:
previous_ordinal, previous_pts, previous_position = (
ordinal, pts, position)
+ @property
+ def keyframes(self):
+ entries = self._keyframes
+ if isinstance(entries, bytes):
+ entries = tuple(self.ENTRY.iter_unpack(entries))
+ self._keyframes = entries
+ return entries
+
+ @keyframes.setter
+ def keyframes(self, entries):
+ self._keyframes = entries
+
def serialize(self):
entries = b''.join(self.ENTRY.pack(*entry) for entry in self.keyframes)
return self.HEADER.pack(
@@ -280,11 +292,11 @@ class VideoKeyframeIndex:
"video payload and strictly increasing."
)
- # Both sections have already been validated. Preserve Python tuples
- # without repeating per-entry normalization and validation in __init__.
+ # Validation is eager, but decoders that do not consume keyframes need
+ # not allocate Python tuples and integers for every entry.
result = cls.__new__(cls)
result.metadata_ranges = tuple(metadata_ranges)
- result.keyframes = tuple(cls.ENTRY.iter_unpack(raw))
+ result._keyframes = raw
return result
@classmethod
diff --git a/paimon-python/pypaimon/tests/video_keyframe_index_test.py
b/paimon-python/pypaimon/tests/video_keyframe_index_test.py
index b4738d136d..4a5c6a3206 100644
--- a/paimon-python/pypaimon/tests/video_keyframe_index_test.py
+++ b/paimon-python/pypaimon/tests/video_keyframe_index_test.py
@@ -16,6 +16,7 @@
# under the License.
import io
+import pickle
from fractions import Fraction
import struct
import sys
@@ -30,6 +31,31 @@ from pypaimon.table.row.video_keyframe_index import
VideoKeyframeIndex
class VideoKeyframeIndexTest(unittest.TestCase):
+ def test_deserialize_materializes_keyframes_only_when_used(self):
+ index = VideoKeyframeIndex([(0, 10)], [(0, -100, 20), (12, 120, 200)])
+ data = index.serialize()
+ entry = VideoKeyframeIndex.ENTRY
+ with mock.patch.object(VideoKeyframeIndex, 'ENTRY', mock.Mock(
+ size=entry.size,
iter_unpack=mock.Mock(wraps=entry.iter_unpack))):
+ restored = VideoKeyframeIndex.deserialize(data, 1000)
+ self.assertEqual(index.metadata_ranges, restored.metadata_ranges)
+ restored = pickle.loads(pickle.dumps(restored))
+ VideoKeyframeIndex.ENTRY.iter_unpack.assert_not_called()
+
+ self.assertEqual(index.keyframes, restored.keyframes)
+ self.assertIs(restored.keyframes, restored.keyframes)
+ VideoKeyframeIndex.ENTRY.iter_unpack.assert_called_once()
+ self.assertEqual(data, restored.serialize())
+
+ def test_replace_unmaterialized_keyframes(self):
+ index = VideoKeyframeIndex([(0, 10)], [(0, 0, 20), (12, 120, 200)])
+ restored = VideoKeyframeIndex.deserialize(index.serialize(), 1000)
+ restored.keyframes = ((0, 100, 30),)
+ self.assertEqual(
+ restored.keyframes,
+ VideoKeyframeIndex.deserialize(restored.serialize()).keyframes,
+ )
+
def test_iso_bmff_metadata_ranges_exclude_padding_bodies(self):
def box(box_type, body):
return struct.pack(">I4s", len(body) + 8, box_type) + body
