Stephen0421 commented on code in PR #10570:
URL: https://github.com/apache/paimon/pull/10570#discussion_r4283306137
##########
paimon-python/pypaimon/write/file_store_commit.py:
##########
@@ -86,6 +86,80 @@ def _reject_compact_increment(messages: List[CommitMessage]):
'Committing a compact increment requires a separate COMPACT
snapshot.')
+def _delete_abort_paths(table, paths):
+ for path_to_delete in paths:
+ try:
+ table.file_io.delete_quietly(str(path_to_delete))
+ except Exception as error:
+ logger.warning(
+ "Failed to clean up file %s during abort: %s",
+ path_to_delete,
+ error,
+ )
+
+
+def _protected_managed_packs(table, paths):
+ """Keep a pack when a blobref outside this abort still names it.
+
+ One pack can be listed on every data file that referenced it. Deleting it
+ with the first aborted file would make the other files unreadable.
+ """
+ from pypaimon.blob.managed_blob_reference_file import
ManagedBlobReferenceFile
+
+ pack_suffix = ManagedBlobReferenceFile.MANAGED_BLOB_SUFFIX
+ ref_suffix = ManagedBlobReferenceFile.REFERENCE_FILE_SUFFIX
+ deleting = {str(path) for path in paths}
+ packs_by_dir = {}
+ for path in deleting:
+ if not path.endswith(pack_suffix):
+ continue
+ parent, _, name = path.rpartition("/")
+ if parent and name:
+ packs_by_dir.setdefault(parent, {})[name] = path
+ if not packs_by_dir:
+ return set()
+
+ list_status = getattr(table.file_io, "list_status", None)
+ if list_status is None:
+ return set()
+
+ protected = set()
+ for parent, packs in packs_by_dir.items():
+ try:
+ statuses = list(list_status(parent))
+ except TypeError:
+ continue
+ except Exception as error:
Review Comment:
Fixed. `commit.abort` now keeps every candidate pack in a directory when
reference discovery does not finish: `list_status` is missing, listing raises,
or a listed entry has no usable name. A pack is deleted only after the listing
completes and no surviving `.blobref` names it. Covered by
`test_commit_abort_keeps_packs_when_listing_fails`: three files share a pack,
`list_status` raises `OSError`, aborting the third file deletes the data file
and `.blobref` and leaves the pack.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
