Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package b4 for openSUSE:Factory checked in at 2025-10-15 12:46:03 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/b4 (Old) and /work/SRC/openSUSE:Factory/.b4.new.18484 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "b4" Wed Oct 15 12:46:03 2025 rev:45 rq:1311438 version:0.14.3 Changes: -------- --- /work/SRC/openSUSE:Factory/b4/b4.changes 2025-10-02 19:20:57.800374287 +0200 +++ /work/SRC/openSUSE:Factory/.b4.new.18484/b4.changes 2025-10-15 12:47:42.690746817 +0200 @@ -1,0 +2,10 @@ +Tue Oct 14 06:11:17 UTC 2025 - Jiri Slaby <[email protected]> + +- add dig support + * 0001-ez-do-a-patch-id-match-when-pulling-in-trailer-updat.patch + * 0001-dig-initial-b4-dig-implementation.patch + * 0002-dig-fix-wrong-msgid-output-for-matches.patch + * 0003-dig-actually-handle-commitish-strings.patch + * 0004-dig-first-round-of-refinement-to-dig.patch + +------------------------------------------------------------------- New: ---- 0001-dig-initial-b4-dig-implementation.patch 0001-ez-do-a-patch-id-match-when-pulling-in-trailer-updat.patch 0002-dig-fix-wrong-msgid-output-for-matches.patch 0003-dig-actually-handle-commitish-strings.patch 0004-dig-first-round-of-refinement-to-dig.patch ----------(New B)---------- New: * 0001-ez-do-a-patch-id-match-when-pulling-in-trailer-updat.patch * 0001-dig-initial-b4-dig-implementation.patch * 0002-dig-fix-wrong-msgid-output-for-matches.patch New:- add dig support * 0001-ez-do-a-patch-id-match-when-pulling-in-trailer-updat.patch * 0001-dig-initial-b4-dig-implementation.patch New: * 0001-dig-initial-b4-dig-implementation.patch * 0002-dig-fix-wrong-msgid-output-for-matches.patch * 0003-dig-actually-handle-commitish-strings.patch New: * 0002-dig-fix-wrong-msgid-output-for-matches.patch * 0003-dig-actually-handle-commitish-strings.patch * 0004-dig-first-round-of-refinement-to-dig.patch New: * 0003-dig-actually-handle-commitish-strings.patch * 0004-dig-first-round-of-refinement-to-dig.patch ----------(New E)---------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ b4.spec ++++++ --- /var/tmp/diff_new_pack.befGRs/_old 2025-10-15 12:47:43.290771911 +0200 +++ /var/tmp/diff_new_pack.befGRs/_new 2025-10-15 12:47:43.290771911 +0200 @@ -31,6 +31,11 @@ Group: Development/Tools/Other URL: https://git.kernel.org/pub/scm/utils/b4/b4.git Source0: https://github.com/mricon/b4/archive/refs/tags/v%{version}.tar.gz#/%{name}-%{version}.tar.gz +Patch0: 0001-ez-do-a-patch-id-match-when-pulling-in-trailer-updat.patch +Patch1: 0001-dig-initial-b4-dig-implementation.patch +Patch2: 0002-dig-fix-wrong-msgid-output-for-matches.patch +Patch3: 0003-dig-actually-handle-commitish-strings.patch +Patch4: 0004-dig-first-round-of-refinement-to-dig.patch BuildRequires: %{python_module base >= 3.9} BuildRequires: %{python_module dkimpy >= 1.0.5} BuildRequires: %{python_module patatt >= 0.6} ++++++ 0001-dig-initial-b4-dig-implementation.patch ++++++ From: Konstantin Ryabitsev <[email protected]> Date: Fri, 10 Oct 2025 16:30:19 -0400 Subject: dig: initial b4 dig implementation References: dig-support Git-commit: 16329336c1c8faba853b11238a16249306742505 Patch-mainline: yes For now, this only does `b4 dig -c [commitish]` and will do its best to figure out which lore message the commit may have come from. We never know for sure, so we always list multiple series when the patch exists in multiple ones and let the end-user figure out which one they want. Still requires testing and docs. Signed-off-by: Konstantin Ryabitsev <[email protected]> Signed-off-by: Jiri Slaby <[email protected]> --- src/b4/__init__.py | 39 ++++++++-- src/b4/command.py | 11 +++ src/b4/dig.py | 188 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 src/b4/dig.py diff --git a/src/b4/__init__.py b/src/b4/__init__.py index c9ed54c3ca1c..b6eab255103f 100644 --- a/src/b4/__init__.py +++ b/src/b4/__init__.py @@ -284,13 +284,25 @@ class LoreMailbox: self.trailer_map[patchid] = list() self.trailer_map[patchid] += fmsgs + + def get_latest_revision(self) -> Optional[int]: + if not len(self.series): + return None + # Use the latest revision + revs = list(self.series.keys()) + # sort by date of each series.submission_date + revs.sort(key=lambda r: self.series[r].submission_date or 0) + return revs[-1] + + def get_series(self, revision: Optional[int] = None, sloppytrailers: bool = False, reroll: bool = True, codereview_trailers: bool = True) -> Optional['LoreSeries']: if revision is None: if not len(self.series): return None - # Use the highest revision - revision = max(self.series.keys()) + revision = self.get_latest_revision() + if revision is None: + return None elif revision not in self.series.keys(): return None @@ -499,6 +511,7 @@ class LoreSeries: change_id: Optional[str] = None prereq_patch_ids: Optional[List[str]] = None prereq_base_commit: Optional[str] = None + _submission_date: Optional[datetime.datetime] = None def __init__(self, revision: int, expected: int) -> None: self.revision = revision @@ -783,8 +796,21 @@ class LoreSeries: return msgs + @property + def submission_date(self) -> Optional[datetime.datetime]: + # Find the date of the first patch we have + if self._submission_date is not None: + return self._submission_date + for lmsg in self.patches: + if lmsg is None: + continue + self._submission_date = lmsg.date + break + return self._submission_date + def populate_indexes(self): self.indexes = list() + seenfiles = set() for lmsg in self.patches[1:]: if lmsg is None or not lmsg.blob_indexes: @@ -827,12 +853,9 @@ class LoreSeries: def find_base(self, gitdir: str, branches: Optional[list] = None, maxdays: int = 30) -> Tuple[str, len, len]: # Find the date of the first patch we have - pdate = datetime.datetime.now() - for lmsg in self.patches: - if lmsg is None: - continue - pdate = lmsg.date - break + pdate = self.submission_date + if not pdate: + pdate = datetime.datetime.now() # Find the latest commit on that date guntil = pdate.strftime('%Y-%m-%d') diff --git a/src/b4/command.py b/src/b4/command.py index 5d90b4f52b4f..c90705ce9c5e 100644 --- a/src/b4/command.py +++ b/src/b4/command.py @@ -118,6 +118,11 @@ def cmd_diff(cmdargs): b4.diff.main(cmdargs) +def cmd_dig(cmdargs: argparse.Namespace) -> None: + import b4.dig + b4.dig.main(cmdargs) + + class ConfigOption(argparse.Action): """Action class for storing key=value arguments in a dict.""" def __call__(self, parser, namespace, keyval, option_string=None): @@ -383,6 +388,12 @@ def setup_parser() -> argparse.ArgumentParser: help='Submit the token received via verification email') sp_send.set_defaults(func=cmd_send) + # b4 dig + sp_dig = subparsers.add_parser('dig', help='Dig into the details of a specific commit') + sp_dig.add_argument('-c', '--commit', dest='commit_id', metavar='COMMITISH', + help='Commit-ish object to dig into') + sp_dig.set_defaults(func=cmd_dig) + return parser diff --git a/src/b4/dig.py b/src/b4/dig.py new file mode 100644 index 000000000000..8b49b90c8e05 --- /dev/null +++ b/src/b4/dig.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2025 by the Linux Foundation +# +__author__ = 'Konstantin Ryabitsev <[email protected]>' + +import os +import sys +import b4 +import argparse +import email.parser + +from email.message import EmailMessage +from typing import List, Set, Optional + +logger = b4.logger + +# Supported diff algorithms we will try to match +try_diff_algos: List[str] = [ + 'myers', + 'histogram', + 'patience', + 'minimal', +] + + +def dig_commit(cmdargs: argparse.Namespace) -> None: + config = b4.get_main_config() + cfg_llval = config.get('linkmask', '') + if isinstance(cfg_llval, str) and '%s' in cfg_llval: + linkmask = cfg_llval + else: + linkmask = f'{b4.LOREADDR}/%s/' + # Are we inside a git repo? + topdir = b4.git_get_toplevel() + if not topdir: + logger.error("Not inside a git repository.") + sys.exit(1) + + # Can we resolve this commit to an object? + commit = b4.git_revparse_obj(cmdargs.commit_id, topdir) + if not commit: + logger.error('Cannot find a commit matching %s', cmdargs.commit_id) + sys.exit(1) + + logger.info('Digging into commit %s', commit) + # Make sure it has exactly one parent (not a merge) + ecode, out = b4.git_run_command( + topdir, ['show', '--no-patch', '--format=%p', commit], + ) + if ecode > 0: + logger.error('Could not get commit info for %s', commit) + sys.exit(1) + if out.strip().count(' ') != 0: + logger.error('Merge commit detected, please specify a single-parent commit.') + sys.exit(1) + + # Find commit's author and subject from git + ecode, out = b4.git_run_command( + topdir, ['show', '--no-patch', '--format=%ae %s', commit], + ) + if ecode > 0: + logger.error('Could not get commit info for %s', commit) + sys.exit(1) + fromeml, csubj = out.strip().split(maxsplit=1) + logger.debug('fromeml=%s, csubj=%s', fromeml, csubj) + logger.info('Attempting to match by exact patch-id...') + showargs = [ + '--format=email', + '--binary', + '--encoding=utf-8', + '--find-renames', + ] + # Keep a record so we don't try git-patch-id on identical patches + bpatches: Set[bytes] = set() + lmbx: Optional[b4.LoreMailbox] = None + for algo in try_diff_algos: + logger.debug('Trying with diff-algorithm=%s', algo) + algoarg = f'--diff-algorithm={algo}' + logger.debug('showargs=%s', showargs + [algoarg]) + ecode, bpatch = b4.git_run_command( + topdir, ['show'] + showargs + [algoarg] + [commit], + decode=False, + ) + if ecode > 0: + logger.error('Could not get a patch out of %s', commit) + sys.exit(1) + if bpatch in bpatches: + logger.debug('Already saw this patch, skipping diff-algorithm=%s', algo) + continue + bpatches.add(bpatch) + gitargs = ['patch-id', '--stable'] + ecode, out = b4.git_run_command(topdir, gitargs, stdin=bpatch) + if ecode > 0 or not len(out.strip()): + logger.error('Could not compute patch-id for commit %s', commit) + sys.exit(1) + patch_id = out.split(maxsplit=1)[0] + logger.debug('Patch-id for commit %s is %s', commit, patch_id) + logger.info('Trying to find matching series by patch-id %s', patch_id) + lmbx = b4.get_series_by_patch_id(patch_id) + if lmbx: + logger.info('Found matching series by patch-id') + break + + if not lmbx: + logger.info('Attempting to match by author and subject...') + q = '(s:"%s" AND f:"%s")' % (csubj.replace('"', ''), fromeml) + msgs = b4.get_pi_search_results(q) + if msgs: + logger.info('Found %s matching messages', len(msgs)) + lmbx = b4.LoreMailbox() + for msg in msgs: + lmbx.add_message(msg) + else: + logger.error('Could not find anything matching commit %s', commit) + # Look at the commit message and find any Link: trailers + ecode, out = b4.git_run_command( + topdir, ['show', '--no-patch', '--format=%B', commit], + ) + if ecode > 0: + logger.error('Could not get commit message for %s', commit) + sys.exit(1) + trailers, _ = b4.LoreMessage.find_trailers(out) + ltrs = [t for t in trailers if t.name.lower() == 'link'] + if ltrs: + logger.info('---') + logger.info('Try following these Link trailers:') + for ltr in ltrs: + logger.info(' %s', ltr.as_string()) + sys.exit(1) + + # Grab the latest series and see if we have a change_id + revs = list(lmbx.series.keys()) + revs.sort(key=lambda r: lmbx.series[r].submission_date or 0) + + change_id: Optional[str] = None + lser = lmbx.get_series(codereview_trailers=False) + for rev in revs: + change_id = lmbx.series[rev].change_id + if not change_id: + continue + logger.info('Backfilling any missing series by change-id') + logger.debug('change_id=%s', change_id) + # Fill in the rest of the series by change_id + q = f'nq:"change-id:{change_id}"' + q_msgs = b4.get_pi_search_results(q, full_threads=True) + if q_msgs: + for q_msg in q_msgs: + lmbx.add_message(q_msg) + break + + logger.debug('Number of series in the mbox: %d', len(lmbx.series)) + logger.info('---') + logger.info('This patch is present in the following series:') + logger.info('---') + firstmsg: Optional[b4.LoreMessage] = None + for rev in revs: + pref = f' v{rev}: ' + lser = lmbx.series[rev] + lmsg: Optional[b4.LoreMessage] = None + if lser.has_cover: + firstmsg = lser.patches[0] + for lmsg in lser.patches[1:]: + if lmsg is None: + continue + if firstmsg is None: + firstmsg = lmsg + if lmsg.git_patch_id == patch_id: + logger.debug('Matched by exact patch-id') + break + if lmsg.subject == csubj: + logger.debug('Matched by subject') + break + + if firstmsg is None: + logger.error('Internal error: no patches in the series?') + sys.exit(1) + if lmsg is None: + # Use the first patch in the series as a fallback + lmsg = firstmsg + logger.info('%s%s', pref, lmsg.full_subject) + logger.info('%s%s', ' ' * len(pref), linkmask % firstmsg.msgid) + + +def main(cmdargs: argparse.Namespace) -> None: + if cmdargs.commit_id: + dig_commit(cmdargs) -- 2.51.0 ++++++ 0001-ez-do-a-patch-id-match-when-pulling-in-trailer-updat.patch ++++++ From: Konstantin Ryabitsev <[email protected]> Date: Thu, 3 Oct 2024 11:07:20 -0400 Subject: ez: do a patch-id match when pulling in trailer updates References: dig-support Git-repo: https://git.kernel.org/pub/scm/utils/b4/b4.git Git-commit: a9f99a4bda313e21116e7d06c012d2aa35840745 Patch-mainline: yes Address two important limitations of trailers -u: - trailer updates were applied even if the local patch has changed - trailers sent to the cover letter were applied to all patches in the current series, even if they were sent to a previous revision and the commits no longer matched Reported-by: Mark Brown <[email protected]> Closes: https://bugzilla.kernel.org/show_bug.cgi?id=219342 Signed-off-by: Konstantin Ryabitsev <[email protected]> Signed-off-by: Jiri Slaby <[email protected]> --- .gitignore | 1 + pyproject.toml | 4 ++ src/b4/__init__.py | 36 ++++++++++-- src/b4/ez.py | 57 ++++++++----------- ...lers-thread-with-followups-no-match.verify | 42 ++++++++++++++ src/tests/test_ez.py | 2 +- 6 files changed, 101 insertions(+), 41 deletions(-) create mode 100644 src/tests/samples/trailers-thread-with-followups-no-match.verify diff --git a/.gitignore b/.gitignore index 85421da2716c..d5e578b42347 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ __pycache__ .venv qodana.yaml *.ipynb +pytest.log diff --git a/pyproject.toml b/pyproject.toml index b15873172937..00f8cbdae296 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,10 @@ b4 = "b4.command:cmd" [tool.pytest.ini_options] filterwarnings = "ignore:.*(pyopenssl|invalid escape sequence).*:DeprecationWarning" norecursedirs = ["tests/helpers", "patatt"] +log_file = "pytest.log" +log_file_level = "DEBUG" +log_file_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)" +log_file_date_format = "%Y-%m-%d %H:%M:%S" [tool.bumpversion] current_version = "0.14.3" diff --git a/src/b4/__init__.py b/src/b4/__init__.py index 2b7ba1d59797..80a54e2df4ea 100644 --- a/src/b4/__init__.py +++ b/src/b4/__init__.py @@ -1286,9 +1286,11 @@ class LoreMessage: self.references = list() if self.msg.get('References'): - for pair in email.utils.getaddresses([str(x) for x in self.msg.get_all('references', [])]): - if pair and pair[1].strip() and pair[1] not in self.references: - self.references.append(pair[1]) + for rbunch in self.msg.get_all('references', list()): + for rchunk in LoreMessage.clean_header(rbunch).split(): + rmsgid = rchunk.strip('<>') + if rmsgid not in self.references: + self.references.append(rmsgid) try: fromdata = email.utils.getaddresses([LoreMessage.clean_header(str(x)) @@ -1298,7 +1300,8 @@ class LoreMessage: if not len(self.fromname.strip()): self.fromname = self.fromemail except IndexError: - pass + self.fromname = '' + self.fromemail = '' msgdate = self.msg.get('Date') if msgdate: @@ -1331,7 +1334,7 @@ class LoreMessage: trailers, others = LoreMessage.find_trailers(self.body, followup=True) # We only pay attention to trailers that are sent in reply - if trailers and self.in_reply_to and not self.has_diff and not self.reply: + if trailers and self.references and not self.has_diff and not self.reply: logger.debug('A follow-up missing a Re: but containing a trailer with no patch diff') self.reply = True if self.reply: @@ -4424,7 +4427,9 @@ def map_codereview_trailers(qmsgs: List[email.message.Message], qmid_map = dict() ref_map = dict() patchid_map = dict() - seen_msgids = set(ignore_msgids) + seen_msgids = set() + if ignore_msgids is not None: + seen_msgids.update(ignore_msgids) for qmsg in qmsgs: qmsgid = LoreMessage.get_clean_msgid(qmsg) if qmsgid in seen_msgids: @@ -4438,6 +4443,7 @@ def map_codereview_trailers(qmsgs: List[email.message.Message], ref_map[qref].append(qlmsg.msgid) logger.info('Analyzing %s code-review messages', len(qmid_map)) + covers = dict() for qmid, qlmsg in qmid_map.items(): logger.debug(' new message: %s', qmid) if not qlmsg.reply: @@ -4459,6 +4465,9 @@ def map_codereview_trailers(qmsgs: List[email.message.Message], if (_qmsg.counter == 0 and (not _qmsg.counters_inferred or _qmsg.has_diffstat) and _qmsg.msgid in ref_map): logger.debug(' stopping: found the cover letter for %s', qlmsg.full_subject) + if _qmsg.msgid not in covers: + covers[_qmsg.msgid] = set() + covers[_qmsg.msgid].add(qlmsg.msgid) break elif _qmsg.has_diff: pqpid = _qmsg.git_patch_id @@ -4479,4 +4488,19 @@ def map_codereview_trailers(qmsgs: List[email.message.Message], # Does it have 'patch-id: ' in the body? # TODO: once we have that functionality in b4 cr + if not covers: + return patchid_map + + # find all patches directly below these covers + for cmsgid, fwmsgids in covers.items(): + logger.debug('Looking at cover: %s', cmsgid) + for qmid, qlmsg in qmid_map.items(): + if qlmsg.in_reply_to == cmsgid and qlmsg.git_patch_id: + pqpid = qlmsg.git_patch_id + for fwmsgid in fwmsgids: + logger.debug('Adding cover follow-up %s to patch-id %s', fwmsgid, pqpid) + if pqpid not in patchid_map: + patchid_map[pqpid] = list() + patchid_map[pqpid].append(qmid_map[fwmsgid]) + return patchid_map diff --git a/src/b4/ez.py b/src/b4/ez.py index 7c708e52cecd..958f41aa1c7c 100644 --- a/src/b4/ez.py +++ b/src/b4/ez.py @@ -1038,7 +1038,7 @@ def update_trailers(cmdargs: argparse.Namespace) -> None: sys.exit(1) ignore_commits = None - changeid = None + tracking = None cover = None msgid = None end = b4.git_revparse_obj('HEAD') @@ -1050,7 +1050,6 @@ def update_trailers(cmdargs: argparse.Namespace) -> None: limit_committer = None start = get_series_start() cover, tracking = load_cover(strip_comments=True) - changeid = tracking['series'].get('change-id') if cmdargs.trailers_from: msgid = cmdargs.trailers_from else: @@ -1145,15 +1144,16 @@ def update_trailers(cmdargs: argparse.Namespace) -> None: bbox.add_message(msg) commit_map = dict() - by_subject = dict() + by_patchid = dict() for lmsg in bbox.series[1].patches: if not lmsg: continue - by_subject[lmsg.subject] = lmsg.msgid + by_patchid[lmsg.git_patch_id] = lmsg.msgid commit_map[lmsg.msgid] = lmsg list_msgs = list() - if changeid and b4.can_network: + if tracking and b4.can_network: + changeid = tracking['series'].get('change-id') logger.info('Checking change-id "%s"', changeid) query = f'"change-id: {changeid}"' smsgs = b4.get_pi_search_results(query, nocache=True) @@ -1171,44 +1171,33 @@ def update_trailers(cmdargs: argparse.Namespace) -> None: if tmsgs is not None: list_msgs += tmsgs - for list_msg in list_msgs: - llmsg = b4.LoreMessage(list_msg) - if not llmsg.trailers: + mismatches = set() + patchid_map = b4.map_codereview_trailers(list_msgs) + for patchid, llmsgs in patchid_map.items(): + if patchid not in by_patchid: + logger.debug('Skipping patch-id %s: not found in the current series', patchid) + logger.debug('Ignoring follow-ups: %s', [x.subject for x in llmsgs]) continue - if llmsg.subject in by_subject: - # Reparent to the commit and add to followups - commit = by_subject[llmsg.subject] - logger.debug('Mapped "%s" to commit %s', llmsg.subject, commit) - plmsg = commit_map[commit] - llmsg.in_reply_to = plmsg.msgid - bbox.followups.append(llmsg) - elif llmsg.counter == 0 and changeid: - logger.debug('Mapped "%s" to the cover letter', llmsg.subject) - # Reparent to the cover and add to followups - llmsg.in_reply_to = 'cover' - bbox.followups.append(llmsg) - else: - # Match by patch-id? - logger.debug('No match for %s', llmsg.subject) - - if msgid or changeid: + for llmsg in llmsgs: + ltrailers, lmismatches = llmsg.get_trailers(sloppy=cmdargs.sloppytrailers) + for ltr in lmismatches: + mismatches.add((ltr.name, ltr.value, llmsg.fromname, llmsg.fromemail)) + commit = by_patchid[patchid] + lmsg = commit_map[commit] + logger.debug('Adding %s to %s', [x.as_string() for x in ltrailers], lmsg.msgid) + lmsg.followup_trailers += ltrailers + + if msgid or tracking: logger.debug('Will query by change-id') codereview_trailers = False else: codereview_trailers = True lser = bbox.get_series(sloppytrailers=cmdargs.sloppytrailers, codereview_trailers=codereview_trailers) - mismatches = list(lser.trailer_mismatches) + mismatches.update(lser.trailer_mismatches) config = b4.get_main_config() seen_froms = set() logger.info('---') - # Do we have follow-up tralers sent to the cover? - if lser.patches[0] and lser.patches[0].followup_trailers: - logger.debug('Applying follow-up trailers from cover to all patches') - for pmsg in lser.patches[1:]: - logger.debug(' %s (%s)', pmsg.subject, pmsg.msgid) - logger.debug(' + %s', [x.as_string() for x in lser.patches[0].followup_trailers]) - pmsg.followup_trailers += lser.patches[0].followup_trailers updates = dict() for lmsg in lser.patches[1:]: @@ -1246,7 +1235,7 @@ def update_trailers(cmdargs: argparse.Namespace) -> None: if len(mismatches): logger.critical('---') logger.critical('NOTE: some trailers ignored due to from/email mismatches:') - for tname, tvalue, fname, femail in lser.trailer_mismatches: + for tname, tvalue, fname, femail in mismatches: logger.critical(' ! Trailer: %s: %s', tname, tvalue) logger.critical(' Msg From: %s <%s>', fname, femail) logger.critical('NOTE: Rerun with -S to apply them anyway') diff --git a/src/tests/samples/trailers-thread-with-followups-no-match.verify b/src/tests/samples/trailers-thread-with-followups-no-match.verify new file mode 100644 index 000000000000..e4f2f3cce278 --- /dev/null +++ b/src/tests/samples/trailers-thread-with-followups-no-match.verify @@ -0,0 +1,42 @@ [email protected] +Minor typo changes imitation +Life imitatus artem. + +Signed-off-by: Konstantin Ryabitsev <[email protected]> +Signed-off-by: Test Override <[email protected]> +--- [email protected] +Add some paragraphs to lipsum +Mostly junk. As expected. + +Signed-off-by: Konstantin Ryabitsev <[email protected]> +Signed-off-by: Test Override <[email protected]> +--- [email protected] +Add more lines to file 1 +This is a second patch in the series. It needed a paragraph with the +words of wisdom. + +Signed-off-by: Konstantin Ryabitsev <[email protected]> +Signed-off-by: Test Override <[email protected]> +--- [email protected] +Remove line 2 from file2 +Etiam in rhoncus lacus. Ut velit nisl, mollis ac commodo vitae, ultrices +quis felis. Proin varius hendrerit volutpat. Pellentesque nec laoreet +quam, eu ullamcorper mi. Donec ut purus ac sapien dignissim elementum eu +ac ante. Mauris sed faucibus orci. + +Vivamus eleifend accumsan ultricies. Cras at erat nec mauris iaculis +eleifend sit amet eu libero. Suspendisse auctor a erat at vestibulum. +Nullam efficitur quis turpis quis sodales. + +Nunc elementum hendrerit arcu eget feugiat. Nulla placerat pellentesque +metus, nec rutrum nulla porttitor vel. Ut tristique commodo sem, ac +sollicitudin enim pharetra et. Mauris sed tellus vitae nunc sollicitudin +fermentum. Phasellus dui elit, malesuada quis metus vel, blandit +tristique felis. Aenean quis tempus enim. + +Signed-off-by: Konstantin Ryabitsev <[email protected]> +Signed-off-by: Test Override <[email protected]> +--- diff --git a/src/tests/test_ez.py b/src/tests/test_ez.py index 1b02e7bede55..7351f7992079 100644 --- a/src/tests/test_ez.py +++ b/src/tests/test_ez.py @@ -25,7 +25,7 @@ def prepdir(gitdir): {'shazam-am-flags': '--signoff'}), # Test matching trailer updates by subject when patch-id changes ('trailers-thread-with-followups', None, (b'vivendum', b'addendum'), [], - ['log', '--format=%ae%n%s%n%b---', 'HEAD~4..'], 'trailers-thread-with-followups', + ['log', '--format=%ae%n%s%n%b---', 'HEAD~4..'], 'trailers-thread-with-followups-no-match', {'shazam-am-flags': '--signoff'}), # Test that we properly perserve commits with --- in them ('trailers-thread-with-followups', 'trailers-with-tripledash', None, [], -- 2.51.0 ++++++ 0002-dig-fix-wrong-msgid-output-for-matches.patch ++++++ From: Konstantin Ryabitsev <[email protected]> Date: Fri, 10 Oct 2025 16:42:54 -0400 Subject: dig: fix wrong msgid output for matches References: dig-support Git-commit: 965071ee174d55969110de4cae739e76a6467ea8 Patch-mainline: yes Due to a logical bug, we were always outputting the msgid of the first message in the first matching series. Signed-off-by: Konstantin Ryabitsev <[email protected]> Signed-off-by: Jiri Slaby <[email protected]> --- src/b4/dig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/b4/dig.py b/src/b4/dig.py index 8b49b90c8e05..e5577a2f0a3a 100644 --- a/src/b4/dig.py +++ b/src/b4/dig.py @@ -154,8 +154,8 @@ def dig_commit(cmdargs: argparse.Namespace) -> None: logger.info('---') logger.info('This patch is present in the following series:') logger.info('---') - firstmsg: Optional[b4.LoreMessage] = None for rev in revs: + firstmsg: Optional[b4.LoreMessage] = None pref = f' v{rev}: ' lser = lmbx.series[rev] lmsg: Optional[b4.LoreMessage] = None @@ -180,7 +180,7 @@ def dig_commit(cmdargs: argparse.Namespace) -> None: # Use the first patch in the series as a fallback lmsg = firstmsg logger.info('%s%s', pref, lmsg.full_subject) - logger.info('%s%s', ' ' * len(pref), linkmask % firstmsg.msgid) + logger.info('%s%s', ' ' * len(pref), linkmask % lmsg.msgid) def main(cmdargs: argparse.Namespace) -> None: -- 2.51.0 ++++++ 0003-dig-actually-handle-commitish-strings.patch ++++++ From: Konstantin Ryabitsev <[email protected]> Date: Tue, 14 Oct 2025 10:59:17 -0400 Subject: dig: actually handle commitish strings References: dig-support Git-repo: https://git.kernel.org/pub/scm/utils/b4/b4.git Git-commit: e8937ac7791dfdba5b788ae128588409bb16778c Patch-mainline: yes We claim to handle commitish strings, but we actually don't convert them into commits. Parse the objects into actual commitid's before we treat them as such. Suggested-by: Linus Torvalds <[email protected]> Link: https://lore.kernel.org/CAHk-=wgNveeyKbMth9d9_vUer4P7=vyarlqqrfhh12watth...@mail.gmail.com Signed-off-by: Konstantin Ryabitsev <[email protected]> Signed-off-by: Jiri Slaby <[email protected]> --- src/b4/command.py | 2 +- src/b4/dig.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/b4/command.py b/src/b4/command.py index c90705ce9c5e..80a96c365b3e 100644 --- a/src/b4/command.py +++ b/src/b4/command.py @@ -390,7 +390,7 @@ def setup_parser() -> argparse.ArgumentParser: # b4 dig sp_dig = subparsers.add_parser('dig', help='Dig into the details of a specific commit') - sp_dig.add_argument('-c', '--commit', dest='commit_id', metavar='COMMITISH', + sp_dig.add_argument('-c', '--commitish', dest='commitish', metavar='COMMITISH', help='Commit-ish object to dig into') sp_dig.set_defaults(func=cmd_dig) diff --git a/src/b4/dig.py b/src/b4/dig.py index e5577a2f0a3a..03ca3211c37b 100644 --- a/src/b4/dig.py +++ b/src/b4/dig.py @@ -25,7 +25,7 @@ try_diff_algos: List[str] = [ ] -def dig_commit(cmdargs: argparse.Namespace) -> None: +def dig_commitish(cmdargs: argparse.Namespace) -> None: config = b4.get_main_config() cfg_llval = config.get('linkmask', '') if isinstance(cfg_llval, str) and '%s' in cfg_llval: @@ -39,9 +39,9 @@ def dig_commit(cmdargs: argparse.Namespace) -> None: sys.exit(1) # Can we resolve this commit to an object? - commit = b4.git_revparse_obj(cmdargs.commit_id, topdir) + commit = b4.git_revparse_obj(f'{cmdargs.commitish}^0', topdir) if not commit: - logger.error('Cannot find a commit matching %s', cmdargs.commit_id) + logger.error('Cannot find a commit matching %s', cmdargs.commitish) sys.exit(1) logger.info('Digging into commit %s', commit) @@ -184,5 +184,5 @@ def dig_commit(cmdargs: argparse.Namespace) -> None: def main(cmdargs: argparse.Namespace) -> None: - if cmdargs.commit_id: - dig_commit(cmdargs) + if cmdargs.commitish: + dig_commitish(cmdargs) -- 2.51.0 ++++++ 0004-dig-first-round-of-refinement-to-dig.patch ++++++ From: Konstantin Ryabitsev <[email protected]> Date: Tue, 14 Oct 2025 16:55:05 -0400 Subject: dig: first round of refinement to dig References: dig-support Git-repo: https://git.kernel.org/pub/scm/utils/b4/b4.git Git-commit: 3ae277e9c7dd3e1df61a14884aabdd5834ad1201 Patch-mainline: yes Lots of changes: - filter search result to only match what is before the commit date, because after that date we're likely to get false-positives from cherry-picks and backports - output a single "most likely came from here" link to stdout, so people can pass this to pipes - add a -a switch that will dig deeper and try to find all previous revisions of that series, whether or not the patch showed up there or not Signed-off-by: Konstantin Ryabitsev <[email protected]> Signed-off-by: Jiri Slaby <[email protected]> --- src/b4/__init__.py | 49 ++++++++++- src/b4/command.py | 4 + src/b4/dig.py | 203 ++++++++++++++++++++++++++++++++------------- 3 files changed, 197 insertions(+), 59 deletions(-) diff --git a/src/b4/__init__.py b/src/b4/__init__.py index b6eab255103f..2b7ba1d59797 100644 --- a/src/b4/__init__.py +++ b/src/b4/__init__.py @@ -37,6 +37,8 @@ from pathlib import Path from contextlib import contextmanager from typing import Optional, Tuple, Set, List, BinaryIO, Union, Sequence, Literal, Iterator, Dict +from email.message import EmailMessage + from email import charset charset.add_charset('utf-8', None) @@ -505,7 +507,9 @@ class LoreSeries: complete: bool = False has_cover: bool = False partial_reroll: bool = False - subject: str + subject: Optional[str] = None + fromname: Optional[str] = None + fromemail: Optional[str] = None indexes: Optional[List[Tuple[str, str]]] = None base_commit: Optional[str] = None change_id: Optional[str] = None @@ -524,6 +528,7 @@ class LoreSeries: def __repr__(self): out = list() out.append('- Series: [v%s] %s' % (self.revision, self.subject)) + out.append(' author: %s <%s>' % (self.fromname, self.fromemail)) out.append(' revision: %s' % self.revision) out.append(' expected: %s' % self.expected) out.append(' complete: %s' % self.complete) @@ -542,6 +547,27 @@ class LoreSeries: return '\n'.join(out) + def __eq__(self, other: object) -> bool: + if not isinstance(other, LoreSeries): + return NotImplemented + # We are the same series if all patch-id's are exactly the same + my_patchids: List[Optional[str]] = list() + for patch in self.patches[1:]: + if patch is not None: + my_patchids.append(patch.git_patch_id) + other_patchids: List[Optional[str]] = list() + for patch in other.patches[1:]: + if patch is not None: + other_patchids.append(patch.git_patch_id) + return my_patchids == other_patchids + + def get_patch_by_msgid(self, msgid: str) -> Optional['LoreMessage']: + for lmsg in self.patches: + if lmsg is not None and lmsg.msgid == msgid: + return lmsg + raise IndexError('No such patch in series') + + def add_patch(self, lmsg: 'LoreMessage') -> None: while len(self.patches) < lmsg.expected + 1: self.patches.append(None) @@ -587,8 +613,12 @@ class LoreSeries: if self.patches[0] is not None: self.subject = self.patches[0].subject + self.fromname = self.patches[0].fromname + self.fromemail = self.patches[0].fromemail elif self.patches[1] is not None: self.subject = self.patches[1].subject + self.fromname = self.patches[1].fromname + self.fromemail = self.patches[1].fromemail def get_slug(self, extended: bool = False) -> str: # Find the first non-None entry @@ -3386,9 +3416,22 @@ def get_series_by_change_id(change_id: str, nocache: bool = False) -> Optional[' return lmbx -def get_series_by_patch_id(patch_id: str, nocache: bool = False) -> Optional['LoreMailbox']: +def get_msgs_by_patch_id(patch_id: str, extra_query: Optional[str] = None, + nocache: bool = False, full_threads: bool = False + ) -> Optional[List[EmailMessage]]: q = f'patchid:{patch_id}' - q_msgs = get_pi_search_results(q, nocache=nocache) + if extra_query: + q = f'{q} {extra_query}' + logger.debug('Full query: %s (nocache=%s)', q, nocache) + q_msgs = get_pi_search_results(q, nocache=nocache, full_threads=full_threads) + if not q_msgs: + return None + + return q_msgs + + +def get_series_by_patch_id(patch_id: str, nocache: bool = False) -> Optional['LoreMailbox']: + q_msgs = get_msgs_by_patch_id(patch_id, full_threads=True, nocache=nocache) if not q_msgs: return None lmbx = LoreMailbox() diff --git a/src/b4/command.py b/src/b4/command.py index 80a96c365b3e..4568b2089868 100644 --- a/src/b4/command.py +++ b/src/b4/command.py @@ -392,6 +392,10 @@ def setup_parser() -> argparse.ArgumentParser: sp_dig = subparsers.add_parser('dig', help='Dig into the details of a specific commit') sp_dig.add_argument('-c', '--commitish', dest='commitish', metavar='COMMITISH', help='Commit-ish object to dig into') + sp_dig.add_argument('-C', '--no-cache', dest='nocache', action='store_true', default=False, + help='Do not use local cache') + sp_dig.add_argument('-a', '--all-series', action='store_true', default=False, + help='Show all series, not just the latest matching') sp_dig.set_defaults(func=cmd_dig) return parser diff --git a/src/b4/dig.py b/src/b4/dig.py index 03ca3211c37b..5d19d59261e7 100644 --- a/src/b4/dig.py +++ b/src/b4/dig.py @@ -9,10 +9,11 @@ import os import sys import b4 import argparse -import email.parser +import re +import urllib.parse from email.message import EmailMessage -from typing import List, Set, Optional +from typing import List, Set, Optional, Union logger = b4.logger @@ -25,6 +26,28 @@ try_diff_algos: List[str] = [ ] +def try_links(links: Set[str]) -> None: + logger.info('Try following these Link trailers:') + for link in links: + logger.info(' Link: %s', link) + + +def print_one_match(subject: str, link: str) -> None: + logger.info('---') + logger.info(subject) + sys.stdout.write(f'{link}\n') + + +def get_all_msgids_from_urls(urls: Set[str]) -> Set[str]: + msgids: Set[str] = set() + for url in urls: + matches = re.search(r'^https?://[^@]+/([^/]+@[^/]+)', url, re.IGNORECASE) + if matches: + chunks = matches.groups() + msgids.add(urllib.parse.unquote(chunks[0])) + return msgids + + def dig_commitish(cmdargs: argparse.Namespace) -> None: config = b4.get_main_config() cfg_llval = config.get('linkmask', '') @@ -56,15 +79,30 @@ def dig_commitish(cmdargs: argparse.Namespace) -> None: logger.error('Merge commit detected, please specify a single-parent commit.') sys.exit(1) + # Look at the commit message and find any Link: trailers + links: Set[str] = set() + ecode, out = b4.git_run_command( + topdir, ['show', '--no-patch', '--format=%B', commit], + ) + if ecode > 0: + logger.error('Could not get commit message for %s', commit) + sys.exit(1) + trailers, _ = b4.LoreMessage.find_trailers(out) + ltrs = [t for t in trailers if t.name.lower() == 'link'] + if ltrs: + links = set(ltr.value for ltr in ltrs) + + msgids = get_all_msgids_from_urls(links) + # Find commit's author and subject from git ecode, out = b4.git_run_command( - topdir, ['show', '--no-patch', '--format=%ae %s', commit], + topdir, ['show', '--no-patch', '--format=%as %ae %s', commit], ) if ecode > 0: logger.error('Could not get commit info for %s', commit) sys.exit(1) - fromeml, csubj = out.strip().split(maxsplit=1) - logger.debug('fromeml=%s, csubj=%s', fromeml, csubj) + cdate, fromeml, csubj = out.strip().split(maxsplit=2) + logger.debug('cdate=%s, fromeml=%s, csubj=%s', cdate, fromeml, csubj) logger.info('Attempting to match by exact patch-id...') showargs = [ '--format=email', @@ -74,7 +112,7 @@ def dig_commitish(cmdargs: argparse.Namespace) -> None: ] # Keep a record so we don't try git-patch-id on identical patches bpatches: Set[bytes] = set() - lmbx: Optional[b4.LoreMailbox] = None + msgs: Optional[List[EmailMessage]] = None for algo in try_diff_algos: logger.debug('Trying with diff-algorithm=%s', algo) algoarg = f'--diff-algorithm={algo}' @@ -97,68 +135,121 @@ def dig_commitish(cmdargs: argparse.Namespace) -> None: sys.exit(1) patch_id = out.split(maxsplit=1)[0] logger.debug('Patch-id for commit %s is %s', commit, patch_id) - logger.info('Trying to find matching series by patch-id %s', patch_id) - lmbx = b4.get_series_by_patch_id(patch_id) - if lmbx: + logger.info('Trying to find matching series by patch-id %s (%s)', patch_id, algo) + # Limit lookup by date prior to the commit date, to weed out any false-positives from + # backports or from erroneously resent series + extra_query = f'AND rt:..{cdate}' + logger.debug('extra_query=%s', extra_query) + msgs = b4.get_msgs_by_patch_id(patch_id, nocache=cmdargs.nocache, extra_query=extra_query) + if msgs: logger.info('Found matching series by patch-id') + for msg in msgs: + msgid = b4.LoreMessage.get_clean_msgid(msg) + if msgid: + logger.debug('Adding from patch-id matches: %s', msgid) + msgids.add(msgid) break - if not lmbx: + if not msgs: logger.info('Attempting to match by author and subject...') - q = '(s:"%s" AND f:"%s")' % (csubj.replace('"', ''), fromeml) - msgs = b4.get_pi_search_results(q) + q = '(s:"%s" AND f:"%s" AND rt:..%s)' % (csubj.replace('"', ''), fromeml, cdate) + msgs = b4.get_pi_search_results(q, nocache=cmdargs.nocache, full_threads=False) if msgs: - logger.info('Found %s matching messages', len(msgs)) - lmbx = b4.LoreMailbox() for msg in msgs: - lmbx.add_message(msg) - else: + msgid = b4.LoreMessage.get_clean_msgid(msg) + if msgid: + logger.debug('Adding from author+subject matches: %s', msgid) + msgids.add(msgid) + if not msgs and not msgids: logger.error('Could not find anything matching commit %s', commit) - # Look at the commit message and find any Link: trailers - ecode, out = b4.git_run_command( - topdir, ['show', '--no-patch', '--format=%B', commit], - ) - if ecode > 0: - logger.error('Could not get commit message for %s', commit) - sys.exit(1) - trailers, _ = b4.LoreMessage.find_trailers(out) - ltrs = [t for t in trailers if t.name.lower() == 'link'] - if ltrs: - logger.info('---') - logger.info('Try following these Link trailers:') - for ltr in ltrs: - logger.info(' %s', ltr.as_string()) + if links: + try_links(links) sys.exit(1) - # Grab the latest series and see if we have a change_id - revs = list(lmbx.series.keys()) - revs.sort(key=lambda r: lmbx.series[r].submission_date or 0) - - change_id: Optional[str] = None - lser = lmbx.get_series(codereview_trailers=False) - for rev in revs: - change_id = lmbx.series[rev].change_id - if not change_id: + logger.info('Will consider promising messages: %s', len(msgids)) + logger.debug('msgids: %s', msgids) + # Go one by one and grab threads by message-id + seen_msgids: Set[str] = set() + lmbxs: List[b4.LoreMailbox] = list() + for msgid in msgids: + if not msgid or msgid in seen_msgids: + logger.debug('Skipping duplicate or invalid msgid %s', msgid) + continue + seen_msgids.add(msgid) + logger.debug('Fetching thread by msgid %s', msgid) + lmbx = b4.get_series_by_msgid(msgid) + if not lmbx: + logger.error('Could not fetch thread for msgid %s, skipping', msgid) continue - logger.info('Backfilling any missing series by change-id') - logger.debug('change_id=%s', change_id) - # Fill in the rest of the series by change_id - q = f'nq:"change-id:{change_id}"' - q_msgs = b4.get_pi_search_results(q, full_threads=True) - if q_msgs: - for q_msg in q_msgs: - lmbx.add_message(q_msg) - break - - logger.debug('Number of series in the mbox: %d', len(lmbx.series)) + if not lmbx.series: + logger.debug('No series found in this mailbox, skipping') + continue + lmbxs.append(lmbx) + + if not lmbxs: + logger.error('Could not fetch any threads for the matching messages!') + sys.exit(1) + + lsers: List[b4.LoreSeries] = list() + for lmbx in lmbxs: + maxrev = max(lmbx.series.keys()) + if cmdargs.all_series and len(lmbx.series) < maxrev: + logger.debug('Fetching prior series') + # Do we have a change-id in this series? + lser = lmbx.get_series(codereview_trailers=False) + fillin_q: str = '' + if lser and lser.change_id: + logger.debug('Found change-id %s in the series', lser.change_id) + fillin_q = f'nq:"change-id:{lser.change_id}"' + elif lser and lser.subject and lser.fromemail: + # We're going to match by first patch/cover letter subject and author. + # It's not perfect, but it's the best we can do without a change-id. + fillin_q = '(s:"%s" AND f:"%s")' % (lser.subject.replace('"', ''), lser.fromemail) + if fillin_q: + fillin_q += f' AND rt:..{cdate}' + logger.debug('fillin_q=%s', fillin_q) + q_msgs = b4.get_pi_search_results(fillin_q, nocache=cmdargs.nocache, full_threads=True) + if q_msgs: + for q_msg in q_msgs: + lmbx.add_message(q_msg) + q_msgid = b4.LoreMessage.get_clean_msgid(q_msg) + if q_msgid: + seen_msgids.add(q_msgid) + + for lser in lmbx.series.values(): + if lser and lser not in lsers: + lsers.append(lser) + + if not len(lsers): + logger.error('Could not find any series containing this patch!') + if links: + try_links(links) + sys.exit(1) + + lsers.sort(key=lambda r: r.submission_date or 0) + logger.debug('Number of matching series: %d', len(lsers)) + lmsg: Optional[b4.LoreMessage] = None + if not cmdargs.all_series: + # Go backwards in time and find the first matching patch + for lser in reversed(lsers): + for lmsg in lser.patches[1:]: + if lmsg is None: + continue + if lmsg.git_patch_id == patch_id: + logger.debug('matched by exact patch-id') + print_one_match(lmsg.full_subject, linkmask % lmsg.msgid) + return + if lmsg.subject == csubj: + logger.debug('matched by subject') + print_one_match(lmsg.full_subject, linkmask % lmsg.msgid) + return + logger.info('---') - logger.info('This patch is present in the following series:') + logger.info('This patch belongs in the following series:') logger.info('---') - for rev in revs: + for lser in lsers: firstmsg: Optional[b4.LoreMessage] = None - pref = f' v{rev}: ' - lser = lmbx.series[rev] - lmsg: Optional[b4.LoreMessage] = None + pref = f' v{lser.revision}: ' if lser.has_cover: firstmsg = lser.patches[0] for lmsg in lser.patches[1:]: @@ -179,7 +270,7 @@ def dig_commitish(cmdargs: argparse.Namespace) -> None: if lmsg is None: # Use the first patch in the series as a fallback lmsg = firstmsg - logger.info('%s%s', pref, lmsg.full_subject) + logger.info('%s%s', pref, firstmsg.full_subject) logger.info('%s%s', ' ' * len(pref), linkmask % lmsg.msgid) -- 2.51.0 ++++++ _scmsync.obsinfo ++++++ --- /var/tmp/diff_new_pack.befGRs/_old 2025-10-15 12:47:43.386775926 +0200 +++ /var/tmp/diff_new_pack.befGRs/_new 2025-10-15 12:47:43.390776093 +0200 @@ -1,4 +1,4 @@ -mtime: 1759335021 -commit: 4ea1653c0a82e128a0a85343b9cfef77f76f7081aa84d3e88b90acfb51c818f7 +mtime: 1760505825 +commit: 1848bfda697b9e55f6d27d18d7b2a4069b7829bc24bf6ddffd46896cf36db548 url: https://src.opensuse.org/jirislaby/d-t-b4.git ++++++ build.specials.obscpio ++++++ ++++++ build.specials.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/.gitignore new/.gitignore --- old/.gitignore 1970-01-01 01:00:00.000000000 +0100 +++ new/.gitignore 2025-10-15 07:24:00.000000000 +0200 @@ -0,0 +1 @@ +.osc
