This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10_1
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_10_1 by this push:
new 91b2ee562b7 releaseWizard: summarize dependency changes from
solr/licenses (#4966)
91b2ee562b7 is described below
commit 91b2ee562b79ddcee89ecf52f08c457f596c3697
Author: David Smiley <[email protected]>
AuthorDate: Fri Oct 9 00:44:53 2026 -0400
releaseWizard: summarize dependency changes from solr/licenses (#4966)
Replaces addDepsToChanges.py, which bulk-converted solrbot commits into
changelog entries (duplicating the entries solrbot PRs already create), with
dev-tools/scripts/dependencyChanges.py.
The new script diffs the jar checksum files in solr/licenses/ between the
previous release tag and HEAD (git only; no build needed):
Backport deltas: validate-changelog.yml unchanged (branch_10x lacks the
solrbot skip / renovate-changelog workflows, and the comment-only change there
does not apply); changelog.adoc section numbered 6.5 to follow 6.4 on this
branch.
Co-authored-by: Claude Opus 5.5 <[email protected]>
(cherry picked from commit 28aeacffa57e143b273ba091f21628950e244f01)
(cherry picked from commit e11d59f5b0731a36f0cc5356a023672ac995a1cb)
---
dev-docs/changelog.adoc | 7 +
dev-tools/scripts/addDepsToChanges.py | 257 ---------------------------------
dev-tools/scripts/dependencyChanges.py | 202 ++++++++++++++++++++++++++
dev-tools/scripts/releaseWizard.yaml | 10 +-
4 files changed, 216 insertions(+), 260 deletions(-)
diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc
index d8b767c81d7..bd8d3e5cd4a 100644
--- a/dev-docs/changelog.adoc
+++ b/dev-docs/changelog.adoc
@@ -272,6 +272,13 @@ Example report output (Json or Markdown):
and the final post-vote forward-porting. It is integrated into the Release
Wizard but can also be run standalone. Run with `--help` for usage.
+=== 6.5 Dependency changes summary
+
+`dev-tools/scripts/dependencyChanges.py` summarizes the third-party dependency
changes
+since the previous release by diffing `solr/licenses/`. By default it prints
to stdout;
+with `--write`, the Release Wizard uses it to write `dependency_update`
changelog entries
+(one each for added, upgraded, and removed).
+
== 7. Further Reading
* https://github.com/logchange/logchange[Logchange web page]
diff --git a/dev-tools/scripts/addDepsToChanges.py
b/dev-tools/scripts/addDepsToChanges.py
deleted file mode 100755
index 3286d03cac4..00000000000
--- a/dev-tools/scripts/addDepsToChanges.py
+++ /dev/null
@@ -1,257 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Script to create changelog YAML entries for solrbot dependency updates
-"""
-import os
-import sys
-
-sys.path.append(os.path.dirname(__file__))
-from scriptutil import *
-
-import argparse
-import re
-import yaml
-from pathlib import Path
-
-line_re = re.compile(r"(.*?) (\(branch_\d+x\) )?\(#(\d+)\)$")
-
-
-class ChangeEntry:
- """
- Represents one dependency change entry.
- Fields:
- - pr_num: string PR number (e.g., '3605')
- - message: cleaned commit message text to be shown in CHANGES (noise
removed)
- - author: Git author (e.g., 'solrbot')
- """
-
- def __init__(self, pr_num: str, message: str, author: str):
- self.pr_num = pr_num
- self.message = message
- self.author = author
-
- def dep_key(self) -> str:
- """
- Extract a dependency key from the message after 'Update ' and before '
to', '(' or end.
- This is used for de-duplication and sorting. Case-insensitive.
- """
- m = re.search(r"(?i)update\s+(.+?)(?:\s+to\b|\s*\(|$)", self.message)
- if m:
- return m.group(1).strip()
- return self.message.strip()
-
- def __str__(self) -> str:
- # Keep trailing newline to preserve existing blank-line formatting by
update_changes
- return f"* PR#{self.pr_num}: {self.message} ({self.author})\n"
-
- def to_yaml_dict(self) -> dict:
- """
- Convert to a dictionary suitable for YAML serialization.
- Extracts JIRA IDs from the title and adds them to links.
- """
- # Extract JIRA IDs from the message
- title, jira_links = extract_jira_issues_from_title(self.message)
-
- # Build links: JIRA issues first, then PR
- links = jira_links.copy() # Start with JIRA links
- links.append({
- 'name': f'PR#{self.pr_num}',
- 'url': f'https://github.com/apache/solr/pull/{self.pr_num}'
- })
-
- return {
- 'title': title,
- 'type': 'dependency_update',
- 'authors': [
- {
- 'name': self.author
- }
- ],
- 'links': links
- }
-
- def yaml_filename(self) -> str:
- """
- Generate a filesystem-safe filename for this entry.
- Format: PR#####-slug.yaml
- Truncates slug on whitespace boundaries, allowing up to 255 chars
total.
- """
- # Clean message for slug
- slug = self.message.lower()
- # Replace whitespace with single space
- slug = re.sub(r'\s+', ' ', slug)
- # Remove non-alphanumeric except dashes
- slug = re.sub(r'[^a-z0-9-._ ]', '', slug)
-
- # Calculate available space for slug
- # Format: "PR" + pr_num + "-" + slug + ".yaml"
- # Typical PR#1234 = 8 chars + "-" = 9 chars, ".yaml" = 5 chars, total
overhead = 14 chars
- # Most filesystems limit filenames to 255 chars
- max_filename_length = 255
- overhead = len(f"PR{self.pr_num}-.yaml")
- max_slug_length = max_filename_length - overhead
-
- # Truncate to max length on word boundaries if necessary
- if len(slug) > max_slug_length:
- # Find the last space within the limit
- truncated = slug[:max_slug_length]
- last_dash = truncated.rfind(' ')
- if last_dash > max_slug_length // 2: # Keep at least half the
available space
- slug = truncated[:last_dash]
- else:
- # If no good word boundary, use hard limit and clean up
trailing spaces
- slug = truncated.rstrip(' ')
- else:
- # Remove trailing spaces
- slug = slug.rstrip(' ')
-
- return f"PR{self.pr_num}-{slug}.yaml"
-
-
-def get_prev_release_tag(ver):
- """
- Based on a given version, compute the git tag for the "previous" version
to calculate changes since.
- For a major version, we want all solrbot commits since last release, i.e.
X-1.Y.Z
- For a minor version, we want all solrbot commits since X.Y-1.0
- For a patch version, we want all solrbot commits since X.Y.Z-1
- """
- releases_arr = run('git tag |grep "releases/solr" | cut -c
15-').strip().split("\n")
- releases = list(map(lambda x: Version.parse(x), releases_arr))
- if ver.is_major_release():
- last = releases.pop()
- return "releases/solr/%s" % last.dot
- if ver.is_minor_release():
- return "releases/solr/%s.%s.0" % (ver.major, ver.minor - 1)
- if ver.is_bugfix_release():
- return "releases/solr/%s.%s.%s" % (ver.major, ver.minor, ver.bugfix -
1)
- return None
-
-
-def read_config():
- parser = argparse.ArgumentParser(description='Adds changelog entries in
changelog/ folder')
- parser.add_argument('--version', type=Version.parse, help='Solr version to
add changes to', required=True)
- parser.add_argument('--user', default='solrbot', help='Git user to get
changes for. Defaults to solrbot')
- newconf = parser.parse_args()
- return newconf
-
-
-def get_gitlog_lines(user: str, prev_tag: str):
- """
- Run git log and return a list of raw subject lines (%s), newest first.
- """
- output = run('git log --author=' + user + ' --oneline --no-merges
--pretty=format:"%s" ' + prev_tag + '..')
- return list(filter(None, output.split("\n")))
-
-
-def parse_gitlog_lines(lines, author: str):
- """
- Parse raw git log subject lines into ChangeEntry objects.
- - Extract pr_num and message
- - Strip '(branch_Nx)' noise
- - Remove optional leading 'chore(deps): '
- - Normalize 'update dependency X' to 'Update X' (case-insensitive)
- """
- entries = []
- for line in lines:
- match = line_re.search(line)
- if not match:
- print("Skipped un-parsable line: %s" % line)
- continue
- text = match.group(1)
- pr_num = match.group(3)
- # Clean message noise
- msg = text
- msg = re.sub(r"^chore\(deps\):\s*", "", msg, flags=re.IGNORECASE)
- # Normalize 'update dependency' to 'Update '
- msg = re.sub(r"(?i)^update\s+dependenc(y|ies)\s+", "Update ", msg)
- entries.append(ChangeEntry(pr_num=pr_num, message=msg, author=author))
- return entries
-
-
-def write_changelog_yaml(entries):
- """
- Write each ChangeEntry to a YAML file in changelog/unreleased/
- """
- changelog_dir = Path('changelog/unreleased')
-
- # Create directory if it doesn't exist
- changelog_dir.mkdir(parents=True, exist_ok=True)
-
- count = 0
- for entry in entries:
- filename = changelog_dir / entry.yaml_filename()
- yaml_data = entry.to_yaml_dict()
-
- # Write YAML file with proper formatting
- with open(filename, 'w') as f:
- yaml.dump(yaml_data, f, default_flow_style=False, sort_keys=False,
allow_unicode=True)
-
- print(f"Created: {filename}")
- count += 1
-
- return count
-
-
-def dedupe_entries(entries):
- """
- De-duplicate dependency update entries (newest first input) by dep_key.
- Keeps the first occurrence for each dependency key.
- """
- seen = set()
- result = []
- for e in entries:
- key = e.dep_key().lower()
- if key not in seen:
- seen.add(key)
- result.append(e)
- return result
-
-
-def sort_entries(entries):
- """Return a new list sorted alphabetically by dependency key
(case-insensitive)."""
- return sorted(entries, key=lambda e: e.dep_key().lower())
-
-
-def main():
- if not os.path.exists('CHANGELOG.md'):
- sys.exit("Tool must be run from the root of a source checkout.")
- newconf = read_config()
- prev_tag = get_prev_release_tag(newconf.version)
- print("Creating changelog YAML entries for dependency updates since git
tag %s" % prev_tag)
- try:
- gitlog_lines = get_gitlog_lines(newconf.user, prev_tag)
- entries = parse_gitlog_lines(gitlog_lines, author=newconf.user)
- if entries:
- deduped = dedupe_entries(entries)
- sorted_entries = sort_entries(deduped)
- count = write_changelog_yaml(sorted_entries)
- print(f"Successfully created {count} changelog YAML entries")
- else:
- print("No changes found for version %s" % newconf.version.dot)
- print("Done")
- except subprocess.CalledProcessError:
- print("Error running git log - check your --version")
- sys.exit(1)
-
-
-if __name__ == '__main__':
- try:
- main()
- except KeyboardInterrupt:
- print('\nReceived Ctrl-C, exiting early')
diff --git a/dev-tools/scripts/dependencyChanges.py
b/dev-tools/scripts/dependencyChanges.py
new file mode 100644
index 00000000000..c5d71a381f4
--- /dev/null
+++ b/dev-tools/scripts/dependencyChanges.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Summarizes changes to the third-party dependencies Solr ships, by diffing the
+jar checksum files in solr/licenses/ between two git refs (by default: the
+previous release tag and HEAD). Jars that moved between the same two versions
+are listed together, since they are typically from the same project.
+
+Prints the summary to stdout, or with --write, writes changelog YAML entries.
+
+This is a stopgap until Solr publishes an SBOM per release, which would be the
+better source to diff.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from collections import defaultdict
+
+sys.path.append(os.path.dirname(__file__))
+from scriptutil import Version, find_current_version
+
+LICENSES_DIR = 'solr/licenses'
+DEFAULT_OUTPUT_DIR = 'changelog/unreleased'
+
+# A version starts at the last '-'-delimited segment that looks like a dotted
number.
+# e.g. log4j-1.2-api-2.25.3, zstd-jni-1.5.6-10, guava-33.4.8-jre
+VERSION_SEG_RE = re.compile(r'^\d+\.\d')
+# Classifiers that distinguish separate jars of the same artifact and version
+CLASSIFIER_RE =
re.compile(r'-(tests?|linux-[\w-]+|osx-[\w-]+|windows-[\w-]+)$')
+# Test artifacts sharing a LICENSE file with shipped ones (e.g.
lucene-test-framework)
+TEST_ARTIFACT_RE = re.compile(r'-(tests?|testing|test-framework)$')
+
+
+def git(*args):
+ return subprocess.run(['git'] + list(args), capture_output=True, text=True,
check=True).stdout
+
+
+def parse_jar(jar_name):
+ """Returns (artifact, version) for a jar name without the '.jar'
extension."""
+ classifier = ''
+ m = CLASSIFIER_RE.search(jar_name)
+ if m:
+ classifier = m.group(0)
+ jar_name = jar_name[:m.start()]
+ segs = jar_name.split('-')
+ idx = max((i for i, s in enumerate(segs) if i > 0 and
VERSION_SEG_RE.match(s)), default=None)
+ if idx is None:
+ idx = max((i for i, s in enumerate(segs) if i > 0 and s[:1].isdigit()),
default=len(segs))
+ return '-'.join(segs[:idx]) + classifier, '-'.join(segs[idx:])
+
+
+def read_licenses_dir(ref):
+ """Returns (artifact -> version, set of LICENSE file prefixes) at the given
ref."""
+ jars = {}
+ license_prefixes = set()
+ for name in git('ls-tree', '--name-only', f'{ref}:{LICENSES_DIR}').split():
+ if name.endswith('.jar.sha1'):
+ artifact, version = parse_jar(name[:-len('.jar.sha1')])
+ jars[artifact] = version
+ else:
+ m = re.match(r'(.+)-LICENSE-.+\.txt$', name)
+ if m:
+ license_prefixes.add(m.group(1))
+ return jars, license_prefixes
+
+
+def has_license(artifact, license_prefixes):
+ """Whether some '-'-delimited prefix of the artifact has a LICENSE file
(mirrors jar-checks.gradle)."""
+ name = CLASSIFIER_RE.sub('', artifact)
+ while True:
+ if name in license_prefixes:
+ return True
+ prefix = re.sub(r'-[^-]+$', '', name)
+ if prefix == name:
+ return False
+ name = prefix
+
+
+def find_previous_release_tag(current):
+ cur = Version.parse(current)
+ best = None
+ for tag in git('tag', '-l', 'releases/solr/*').split():
+ m = re.fullmatch(r'releases/solr/(\d+)\.(\d+)\.(\d+)', tag)
+ if not m:
+ continue
+ v = tuple(int(x) for x in m.groups())
+ if v < (cur.major, cur.minor, cur.bugfix) and (best is None or v >
best[0]):
+ best = (v, tag)
+ if best is None:
+ sys.exit(f'No release tag found before {current}; pass --from')
+ return best[1]
+
+
+def describe_artifacts(artifacts):
+ """e.g. 'asm, asm-tree', or 'jetty-* (23 jars)' when all share the first
name segment."""
+ if len(artifacts) > 3:
+ first = {a.split('-')[0] for a in artifacts}
+ if len(first) == 1:
+ return f'{first.pop()}-* ({len(artifacts)} jars)'
+ return ', '.join(artifacts)
+
+
+CATEGORIES = ('added', 'upgraded', 'removed')
+
+
+def compute_changes(from_ref, to_ref):
+ """Returns category ('added', 'upgraded', 'removed') -> sorted list of
human-readable changes."""
+ old_jars, old_license_prefixes = read_licenses_dir(from_ref)
+ new_jars, new_license_prefixes = read_licenses_dir(to_ref)
+
+ # Only jars with a LICENSE file count: *.sha1 files also cover jars we don't
ship (e.g. test
+ # dependencies), whereas LICENSE files are only required for shipped ones
(since SOLR-15465; older
+ # releases have them for non-shipped jars too, making removals from such a
release unreliable).
+ def shipped(artifact, license_prefixes):
+ return has_license(artifact, license_prefixes) and not
TEST_ARTIFACT_RE.search(artifact)
+
+ by_transition = defaultdict(list) # (old_version or None, new_version or
None) -> artifacts
+ for artifact in old_jars.keys() | new_jars.keys():
+ old, new = old_jars.get(artifact), new_jars.get(artifact)
+ if old == new:
+ continue
+ if shipped(artifact, new_license_prefixes if new else
old_license_prefixes):
+ by_transition[(old, new)].append(artifact)
+
+ changes = {c: [] for c in CATEGORIES}
+ for (old, new), artifacts in by_transition.items():
+ names = describe_artifacts(sorted(artifacts, key=str.lower))
+ if old is None:
+ changes['added'].append(f'{names} {new}')
+ elif new is None:
+ changes['removed'].append(f'{names} {old}')
+ else:
+ changes['upgraded'].append(f'{names} {old} → {new}')
+ return {c: sorted(lines, key=str.lower) for c, lines in changes.items()}
+
+
+def to_yaml(title, version):
+ def quote(s):
+ return "'" + s.replace("'", "''") + "'"
+ return (f'# Generated by dev-tools/scripts/dependencyChanges.py\n'
+ f'title: {quote(title)}\n'
+ f'type: dependency_update\n'
+ f'authors:\n'
+ f' - name: various contributors\n'
+ f'links:\n'
+ f' - name: solr/licenses\n'
+ f' url:
https://github.com/apache/solr/tree/releases/solr/{version}/solr/licenses\n')
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('--from', dest='from_ref',
+ help='Git ref to compare from (default: the release tag
preceding the current version)')
+ parser.add_argument('--to', dest='to_ref', default='HEAD', help='Git ref to
compare to (default: HEAD)')
+ parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT_DIR,
metavar='DIR',
+ help='Write changelog YAML entries (one per category)
instead of printing'
+ f' (default dir: {DEFAULT_OUTPUT_DIR})')
+ args = parser.parse_args()
+
+ version = find_current_version()
+ from_ref = args.from_ref or find_previous_release_tag(version)
+ changes = compute_changes(from_ref, args.to_ref)
+ since = from_ref.rsplit('/', 1)[-1]
+
+ if not args.write:
+ print(f'Dependency changes from {from_ref} to {args.to_ref}:')
+ for category in CATEGORIES:
+ print(f'{category.capitalize()}: ' + ('; '.join(changes[category]) or
'(none)'))
+ return
+
+ for i, category in enumerate(CATEGORIES, 1): # numbered so the changelog
lists them in this order
+ path = os.path.join(args.write, f'dependency-changes-{i}-{category}.yml')
+ if changes[category]:
+ title = f'Third-party dependencies {category} since {since}: ' + ';
'.join(changes[category])
+ with open(path, 'w', encoding='utf-8') as f:
+ f.write(to_yaml(title, version))
+ print(f'Wrote {len(changes[category])} {category} dependency changes to
{path}')
+ elif os.path.exists(path):
+ os.remove(path)
+ print(f'Removed {path}; no {category} dependency changes')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/dev-tools/scripts/releaseWizard.yaml
b/dev-tools/scripts/releaseWizard.yaml
index ec83c020ce0..1e13c326851 100644
--- a/dev-tools/scripts/releaseWizard.yaml
+++ b/dev-tools/scripts/releaseWizard.yaml
@@ -674,18 +674,22 @@ groups:
- !Todo
id: dependency_updates_changes
title: Add dependency updates to changelog
- description: Bulk add all 'solrbot' dependency updates since last release
+ description: |
+ Add changelog entries summarizing third-party dependencies added,
upgraded, and removed since the
+ previous release, derived from the jar checksum files in
`solr/licenses/`.
+ Re-running it overwrites the entries, e.g. if dependencies changed
before a respin.
+ Run the script without `--write` to just print the summary.
depends: clean_git_checkout
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
- commands_text: We call out to a helper script that compiles SolrBot
changes into `changelog/unreleased`
+ commands_text: We call out to a helper script that writes
`changelog/unreleased/dependency-changes-*.yml`
confirm_each_command: true
commands:
- !Command
cmd: git checkout {{ release_branch }}
stdout: true
- !Command
- cmd: python3 -u dev-tools/scripts/addDepsToChanges.py --user solrbot
--version {{ release_version }}
+ cmd: python3 -u dev-tools/scripts/dependencyChanges.py --write
tee: true
- !Command
cmd: git add changelog && git commit -m "Add dependency updates to
changelog for {{ release_version }}" && git push
