This is an automated email from the ASF dual-hosted git repository. ebenizzy pushed a commit to branch asf-release-scripts in repository https://gitbox.apache.org/repos/asf/burr.git
commit c2e0027d269afbcb05f30ff5fef5bfce62d23ea9 Author: Elijah ben Izzy <[email protected]> AuthorDate: Sun Nov 16 21:07:31 2025 -0800 Adds scripts for releasing burr - uses flit as the build system - one script for helping with the release - the other for helping with the build of artifacts - tested out fairly extensively --- CONTRIBUTING.rst | 19 ++ __init__.py | 0 burr/cli/__main__.py | 31 ++- pyproject.toml | 107 +++++++--- scripts/README.md | 203 +++++++++++++++++++ scripts/build_artifacts.py | 290 +++++++++++++++++++++++++++ scripts/release_helper.py | 442 +++++++++++++++++++++++++++++++++++++++++ scripts/setup_keys.sh | 95 +++++++++ telemetry/ui/package-lock.json | 2 + telemetry/ui/package.json | 2 + tests/test_release_config.py | 127 ++++++++++++ 11 files changed, 1290 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c22b7996..db48538f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,3 +1,22 @@ +.. + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + ============ Contributing ============ diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/burr/cli/__main__.py b/burr/cli/__main__.py index cc89b06f..109eb835 100644 --- a/burr/cli/__main__.py +++ b/burr/cli/__main__.py @@ -27,7 +27,9 @@ import time import webbrowser from contextlib import contextmanager from importlib.resources import files +from pathlib import Path from types import ModuleType +from typing import Optional from burr import system, telemetry from burr.core.persistence import PersistedStateData @@ -54,7 +56,7 @@ def _telemetry_if_enabled(event: str): telemetry.create_and_send_cli_event(event) -def _command(command: str, capture_output: bool, addl_env: dict = None) -> str: +def _command(command: str, capture_output: bool, addl_env: dict | None = None) -> str: """Runs a simple command""" if addl_env is None: addl_env = {} @@ -78,7 +80,27 @@ def _command(command: str, capture_output: bool, addl_env: dict = None) -> str: def _get_git_root() -> str: - return _command("git rev-parse --show-toplevel", capture_output=True) + env_root = os.environ.get("BURR_PROJECT_ROOT") + if env_root: + return env_root + try: + return _command("git rev-parse --show-toplevel", capture_output=True) + except subprocess.CalledProcessError: + package_root = _locate_package_root() + if package_root is not None: + logger.warning("Not inside a git repository; using package root %s.", package_root) + return package_root + logger.warning("Not inside a git repository; defaulting to current directory.") + return os.getcwd() + + +def _locate_package_root() -> Optional[str]: + path = Path(__file__).resolve() + for candidate in (path.parent,) + tuple(path.parents): + telemetry_dir = candidate / "telemetry" / "ui" + if telemetry_dir.exists(): + return str(candidate) + return None def open_when_ready(check_url: str, open_url: str): @@ -118,13 +140,16 @@ def _build_ui(): # create a symlink so we can get packages inside it... cmd = "rm -rf burr/tracking/server/build" _command(cmd, capture_output=False) - cmd = "cp -R telemetry/ui/build burr/tracking/server/build" + cmd = "mkdir -p burr/tracking/server/build" + _command(cmd, capture_output=False) + cmd = "cp -a telemetry/ui/build/. burr/tracking/server/build/" _command(cmd, capture_output=False) @cli.command() def build_ui(): git_root = _get_git_root() + logger.info("UI build: using project root %s", git_root) with cd(git_root): _build_ui() diff --git a/pyproject.toml b/pyproject.toml index 7839fb3e..d1c3c878 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [build-system] -requires = ["setuptools >= 61.0"] -build-backend = "setuptools.build_meta" +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" [project] name = "burr" @@ -216,21 +233,6 @@ ray = [ "ray[default]" ] -[tool.setuptools] -include-package-data = true - -[tool.setuptools.packages.find] -include = ["burr", "burr.*"] - -# we need to ensure this is there... -[tool.setuptools.package-data] -burr = [ - "burr/tracking/server/build/**/*", - "burr/tracking/server/demo_data/**/*", - "py.typed", -] - - [tool.aerich] tortoise_orm = "burr.tracking.server.s3.settings.TORTOISE_ORM" location = "./burr/tracking/server/s3/migrations" @@ -254,11 +256,66 @@ burr-test-case = "burr.cli.__main__:cli_test_case" name = "burr" [tool.flit.sdist] -include = ["LICENSE", "NOTICE", "DISCLAIMER", "burr/tracking/server/build/**/*", - "burr/tracking/server/demo_data/**/*", - "examples/email-assistant/*", - "examples/multi-modal-chatbot/*", - "examples/streaming-fastapi/*", - "examples/deep-researcher/*", - "py.typed",] -exclude = [] +include = [ + "LICENSE", + "NOTICE", + "DISCLAIMER", + "scripts/**", + "telemetry/ui/**", + "examples/__init__.py", + "examples/email-assistant/**", + "examples/multi-modal-chatbot/**", + "examples/streaming-fastapi/**", + "examples/deep-researcher/**", +] +exclude = [ + "telemetry/ui/node_modules/**", + "telemetry/ui/build/**", + "telemetry/ui/dist/**", + "telemetry/ui/.cache/**", + "telemetry/ui/.next/**", + "**/__pycache__/**", + "**/*.pyc", + ".git/**", + ".github/**", + "docs/**", + # Exclude unwanted examples - flit auto-includes examples/ as a package because of __init__.py, + # and exclude patterns take precedence over include patterns. We must explicitly exclude each + # unwanted example directory. Only email-assistant, multi-modal-chatbot, streaming-fastapi, + # and deep-researcher should be included (see include list above). + # NOTE: If you add/remove examples, update this list AND tests/test_release_config.py + "examples/README.md", + "examples/validate_examples.py", + "examples/adaptive-crag/**", + "examples/conversational-rag/**", + "examples/custom-serde/**", + "examples/deployment/**", + "examples/hamilton-integration/**", + "examples/haystack-integration/**", + "examples/hello-world-counter/**", + "examples/image-telephone/**", + "examples/instructor-gemini-flash/**", + "examples/integrations/**", + "examples/llm-adventure-game/**", + "examples/ml-training/**", + "examples/multi-agent-collaboration/**", + "examples/openai-compatible-agent/**", + "examples/opentelemetry/**", + "examples/other-examples/**", + "examples/parallelism/**", + "examples/pytest/**", + "examples/rag-lancedb-ingestion/**", + "examples/ray/**", + "examples/recursive/**", + "examples/simple-chatbot-intro/**", + "examples/simulation/**", + "examples/streaming-overview/**", + "examples/talks/**", + "examples/templates/**", + "examples/test-case-creation/**", + "examples/tool-calling/**", + "examples/tracing-and-spans/**", + "examples/typed-state/**", + "examples/web-server/**", + "examples/youtube-to-social-media-post/**", +] diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..7c759be2 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,203 @@ +# Burr Release Scripts + +This directory contains helper scripts to automate the Apache release workflow. + +## Overview + +The release process has two phases: + +1. **Source-only release** (for Apache voting): Contains source code, build scripts, and UI source—but NO pre-built artifacts +2. **Wheel build** (for PyPI): Built from the source release, includes pre-built UI assets + +All packaging configuration lives in `pyproject.toml`: +- `[build-system]` uses `flit_core` as the build backend +- `[tool.flit.sdist]` controls what goes in the source tarball +- Wheel contents are controlled by what exists in `burr/` when `flit build --format wheel` runs + +## 1. Create the Source Release Candidate + +From the repo root: + +```bash +python scripts/release_helper.py --apache-id <your-id> --rc-num 0 [--dry-run] +``` + +Example: + +```bash +# Dry run (no git tag or SVN upload) +python scripts/release_helper.py --apache-id myid --rc-num 0 --dry-run + +# Real release +python scripts/release_helper.py --apache-id myid --rc-num 0 +``` + +**What it does:** +1. Reads version from `pyproject.toml` +2. Cleans `dist/` directory +3. **Removes `burr/tracking/server/build/`** to ensure no pre-built UI in source tarball +4. Runs `flit build --format sdist` + - Includes files specified in `[tool.flit.sdist] include` + - Excludes files specified in `[tool.flit.sdist] exclude` +5. Creates Apache-branded tarball with GPG signatures and SHA512 checksums +6. Tags git as `v{version}-incubating-RC{num}` (unless `--dry-run`) +7. Uploads to Apache SVN (unless `--dry-run`) + +**Output:** +- `dist/burr-<version>.tar.gz` — source-only tarball +- `dist/apache-burr-<version>-incubating.tar.gz` — ASF-branded tarball +- Signature (`.asc`) and checksum (`.sha512`) files + +## 2. Test the Source Release (Voter Simulation) + +This simulates what Apache voters and release managers will do when validating the release. + +**Automated testing:** + +```bash +bash scripts/simulate_release.sh +``` + +This script: +1. Cleans `/tmp/burr-release-test/` +2. Extracts the Apache tarball +3. Creates a fresh virtual environment +4. Builds UI artifacts and wheel (next step) +5. Verifies both packages and prints their locations + +**Manual testing:** + +```bash +cd /tmp +tar -xzf /path/to/dist/apache-burr-<version>-incubating.tar.gz +cd burr-<version> + +# Verify source contents +ls scripts/ # Build scripts should be present +ls telemetry/ui/ # UI source should be present +ls examples/ # Example directories should be present +ls burr/tracking/server/build/ # Should NOT exist (no pre-built UI) + +# Create clean environment +python -m venv venv && source venv/bin/activate +pip install -e . +pip install flit + +# Build artifacts and wheel (see step 3) +python scripts/build_artifacts.py all --clean +ls dist/*.whl +deactivate +``` + +## 3. Build Artifacts and Wheel + +The `build_artifacts.py` script has three subcommands: + +### Build everything (recommended): + +```bash +python scripts/build_artifacts.py all --clean +``` + +This runs both `artifacts` and `wheel` subcommands in sequence. + +### Build UI artifacts only: + +```bash +python scripts/build_artifacts.py artifacts [--skip-install] +``` + +**What it does:** +1. Checks for Node.js and npm +2. **Cleans `burr/tracking/server/build/`** to ensure fresh UI build +3. Installs burr from source: `pip install -e .` (unless `--skip-install`) +4. Runs `burr-admin-build-ui`: + - `npm install --prefix telemetry/ui` + - `npm run build --prefix telemetry/ui` + - **Creates `burr/tracking/server/build/`** and copies built UI into it +5. Verifies UI assets exist in `burr/tracking/server/build/` + +### Build wheel only (assumes artifacts exist): + +```bash +python scripts/build_artifacts.py wheel [--clean] +``` + +**What it does:** +1. Checks for `flit` +2. Verifies `burr/tracking/server/build/` contains UI assets +3. Optionally cleans `dist/` (with `--clean`) +4. Runs `flit build --format wheel` + - **Packages all files in `burr/` directory, including `burr/tracking/server/build/`** + - Does NOT include files outside `burr/` (e.g., `telemetry/ui/`, `scripts/`, `examples/`) +5. Verifies `.whl` file was created + +**Output:** `dist/burr-<version>-py3-none-any.whl` (includes bundled UI) + +## 4. Upload to PyPI + +After building the wheel: + +```bash +twine upload dist/burr-<version>-py3-none-any.whl +``` + +## Package Contents Reference + +Understanding what goes in each package type: + +### Source tarball (`apache-burr-{version}-incubating.tar.gz`) + +**Controlled by:** `[tool.flit.sdist]` in `pyproject.toml` + `release_helper.py` cleanup + +**Includes:** +- ✅ `burr/` — Full package source code +- ✅ `scripts/` — Build helper scripts (this directory!) +- ✅ `telemetry/ui/` — UI source code (package.json, src/, public/, etc.) +- ✅ `examples/email-assistant/`, `examples/multi-modal-chatbot/`, etc. — Selected example directories +- ✅ `LICENSE`, `NOTICE`, `DISCLAIMER` — Apache license files + +**Excludes:** +- ❌ `burr/tracking/server/build/` — Deleted by `release_helper.py` before build +- ❌ `telemetry/ui/node_modules/` — Excluded by `[tool.flit.sdist]` +- ❌ `telemetry/ui/build/`, `telemetry/ui/dist/` — Excluded by `[tool.flit.sdist]` +- ❌ `docs/`, `.git/`, `.github/` — Excluded by `[tool.flit.sdist]` + +**How it's built:** +```bash +rm -rf burr/tracking/server/build # Ensure no pre-built UI +flit build --format sdist # Build from [tool.flit.sdist] config +``` + +--- + +### Wheel (`burr-{version}-py3-none-any.whl`) + +**Controlled by:** What exists in `burr/` directory when `flit build --format wheel` runs + +**Includes:** +- ✅ `burr/` — Complete package (all `.py` files, `py.typed`, etc.) +- ✅ `burr/tracking/server/build/` — **Pre-built UI assets** (created by `build_artifacts.py`) +- ✅ `burr/tracking/server/demo_data/` — Demo data files + +**Excludes:** +- ❌ `telemetry/ui/` — Not in `burr/` package +- ❌ `examples/` — Not in `burr/` package (sdist-only) +- ❌ `scripts/` — Not in `burr/` package (sdist-only) +- ❌ `LICENSE`, `NOTICE`, `DISCLAIMER` — Not needed in wheel (sdist-only) + +**How it's built:** +```bash +burr-admin-build-ui # Creates burr/tracking/server/build/ +flit build --format wheel # Packages everything in burr/ +``` + +--- + +### Key Insight + +The **same `burr/` source directory** produces different outputs based on **when you build** and **what format**: + +1. **sdist (source tarball):** Includes external files (`scripts/`, `telemetry/ui/`, `examples/`) via `[tool.flit.sdist]` config, but excludes `burr/tracking/server/build/` because we delete it first. + +2. **wheel (binary distribution):** Only packages `burr/` directory contents, but includes `burr/tracking/server/build/` because we create it before building the wheel. diff --git a/scripts/build_artifacts.py b/scripts/build_artifacts.py new file mode 100644 index 00000000..9a62027a --- /dev/null +++ b/scripts/build_artifacts.py @@ -0,0 +1,290 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Build artifacts/wheels helper with subcommands: + + python scripts/build_artifacts.py artifacts [--skip-install] + python scripts/build_artifacts.py wheel [--clean] + python scripts/build_artifacts.py all [--skip-install] [--clean] + +Subcommands: + artifacts -> Build UI artifacts only + wheel -> Build wheel (requires artifacts to exist) + all -> Run both steps (artifacts then wheel) +""" + +import argparse +import os +import shutil +import subprocess +import sys + + +def _ensure_project_root() -> bool: + if not os.path.exists("pyproject.toml"): + print("Error: pyproject.toml not found.") + print("Please run this script from the root of the Burr source directory.") + return False + return True + + +def _check_node_prereqs() -> bool: + print("Checking for required tools...") + required_tools = ["node", "npm"] + missing_tools = [] + + for tool in required_tools: + if shutil.which(tool) is None: + missing_tools.append(tool) + print(f" ✗ '{tool}' not found") + else: + print(f" ✓ '{tool}' found") + + if missing_tools: + print(f"\nError: Missing required tools: {', '.join(missing_tools)}") + print("Please install Node.js and npm to build the UI.") + return False + + print("All required tools found.\n") + return True + + +def _require_flit() -> bool: + if shutil.which("flit") is None: + print("✗ flit CLI not found. Please install it with: pip install flit") + return False + print("✓ flit CLI found.\n") + return True + + +def _install_burr(skip_install: bool) -> bool: + if skip_install: + print("Skipping burr installation as requested.\n") + return True + + print("Installing burr from source...") + try: + subprocess.run( + [sys.executable, "-m", "pip", "install", "-e", "."], + check=True, + cwd=os.getcwd(), + ) + print("✓ Burr installed successfully.\n") + return True + except subprocess.CalledProcessError as exc: + print(f"✗ Error installing burr: {exc}") + return False + + +def _build_ui() -> bool: + print("Building UI assets...") + try: + env = os.environ.copy() + env["BURR_PROJECT_ROOT"] = os.getcwd() + subprocess.run(["burr-admin-build-ui"], check=True, env=env) + print("✓ UI build completed successfully.\n") + return True + except subprocess.CalledProcessError as exc: + print(f"✗ Error building UI: {exc}") + return False + + +def _verify_artifacts() -> bool: + build_dir = "burr/tracking/server/build" + print(f"Verifying build output in {build_dir}...") + + if not os.path.exists(build_dir): + print(f"Build directory missing, creating placeholder at {build_dir}...") + os.makedirs(build_dir, exist_ok=True) + + if not os.listdir(build_dir): + print(f"✗ Build directory is empty: {build_dir}") + return False + + print("✓ Build output verified.\n") + return True + + +def _clean_dist(): + if os.path.exists("dist"): + print("Cleaning dist/ directory...") + shutil.rmtree("dist") + print("✓ dist/ directory cleaned.\n") + + +def _clean_ui_build(): + """Remove any existing UI build directory to ensure clean state.""" + ui_build_dir = "burr/tracking/server/build" + if os.path.exists(ui_build_dir): + print(f"Cleaning existing UI build directory: {ui_build_dir}") + shutil.rmtree(ui_build_dir) + print("✓ UI build directory cleaned.\n") + + +def _build_wheel() -> bool: + print("Building wheel distribution with 'flit build --format wheel'...") + try: + env = os.environ.copy() + env["FLIT_USE_VCS"] = "0" + subprocess.run(["flit", "build", "--format", "wheel"], check=True, env=env) + print("✓ Wheel build completed successfully.\n") + return True + except subprocess.CalledProcessError as exc: + print(f"✗ Error building wheel: {exc}") + return False + + +def _verify_wheel() -> bool: + print("Verifying wheel output...") + + if not os.path.exists("dist"): + print("✗ dist/ directory not found") + return False + + wheel_files = [f for f in os.listdir("dist") if f.endswith(".whl")] + if not wheel_files: + print("✗ No wheel files found in dist/") + if os.listdir("dist"): + print("Contents of dist/ directory:") + for item in os.listdir("dist"): + print(f" - {item}") + return False + + print(f"✓ Found {len(wheel_files)} wheel file(s):") + for wheel_file in wheel_files: + wheel_path = os.path.join("dist", wheel_file) + size = os.path.getsize(wheel_path) + print(f" - {wheel_file} ({size:,} bytes)") + + print() + return True + + +def create_artifacts(skip_install: bool) -> bool: + if not _ensure_project_root(): + print("Failed to confirm project root.") + return False + if not _check_node_prereqs(): + print("Node/npm prerequisite check failed.") + return False + # Clean any existing UI build to ensure fresh state + _clean_ui_build() + if not _install_burr(skip_install): + print("Installing burr from source failed.") + return False + if not _build_ui(): + print("UI build failed.") + return False + if not _verify_artifacts(): + print("UI artifact verification failed.") + return False + return True + + +def create_wheel(clean: bool) -> bool: + if not _ensure_project_root(): + print("Failed to confirm project root.") + return False + if not _require_flit(): + print("Missing flit CLI.") + return False + if not _verify_artifacts(): + print("Please run the 'artifacts' subcommand first.") + return False + if clean: + _clean_dist() + if not _build_wheel(): + return False + if not _verify_wheel(): + return False + return True + + +def build_all(skip_install: bool, clean: bool) -> bool: + if not create_artifacts(skip_install=skip_install): + return False + if not create_wheel(clean=clean): + return False + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Build artifacts/wheels for Burr using subcommands." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + artifacts_parser = subparsers.add_parser("artifacts", help="Build UI artifacts only.") + artifacts_parser.add_argument( + "--skip-install", + action="store_true", + help="Skip reinstalling burr when building artifacts", + ) + + wheel_parser = subparsers.add_parser( + "wheel", help="Build wheel distribution (requires artifacts)." + ) + wheel_parser.add_argument( + "--clean", + action="store_true", + help="Clean dist/ directory before building wheel", + ) + + all_parser = subparsers.add_parser("all", help="Build artifacts and wheel in sequence.") + all_parser.add_argument( + "--skip-install", + action="store_true", + help="Skip reinstalling burr when building artifacts", + ) + all_parser.add_argument( + "--clean", + action="store_true", + help="Clean dist/ directory before building wheel", + ) + + args = parser.parse_args() + + print("=" * 80) + print(f"Burr Build Helper - command: {args.command}") + print("=" * 80) + print() + + success = False + if args.command == "artifacts": + success = create_artifacts(skip_install=args.skip_install) + elif args.command == "wheel": + success = create_wheel(clean=args.clean) + elif args.command == "all": + success = build_all(skip_install=args.skip_install, clean=args.clean) + + if success: + print("=" * 80) + print("✅ Build Complete!") + print("=" * 80) + if args.command in {"wheel", "all"}: + print("\nWheel files are in the dist/ directory.") + print("You can now upload to PyPI with:") + print(" twine upload dist/*.whl") + print() + else: + print("\n❌ Build failed.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/release_helper.py b/scripts/release_helper.py new file mode 100644 index 00000000..d620a13d --- /dev/null +++ b/scripts/release_helper.py @@ -0,0 +1,442 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import glob +import hashlib +import os +import re +import shutil +import subprocess +import sys + +# --- Configuration --- +# You need to fill these in for your project. +# The name of your project's short name (e.g., 'myproject'). +PROJECT_SHORT_NAME = "burr" +# The file where you want to update the version number. +VERSION_FILE = "pyproject.toml" +# A regular expression pattern to find the version string in the VERSION_FILE. +VERSION_PATTERN = r'version\s*=\s*"(\d+\.\d+\.\d+)"' + + +def _fail(message: str) -> None: + print(f"\n❌ {message}") + sys.exit(1) + + +def get_version_from_file(file_path: str) -> str: + """Get the version from a file.""" + with open(file_path, encoding="utf-8") as f: + content = f.read() + match = re.search(VERSION_PATTERN, content) + if match: + version = match.group(1) + return version + raise ValueError(f"Could not find version in {file_path}") + + +def check_prerequisites(): + """Checks for necessary command-line tools and Python modules.""" + print("Checking for required tools...") + required_tools = ["git", "gpg", "svn", "flit"] + for tool in required_tools: + if shutil.which(tool) is None: + _fail( + f"Required tool '{tool}' not found. Please install it and ensure it's in your PATH." + ) + + print("All required tools found.") + + +def update_version(version, _rc_num): + """Updates the version number in the specified file.""" + print(f"Updating version in {VERSION_FILE} to {version}...") + try: + with open(VERSION_FILE, "r", encoding="utf-8") as f: + content = f.read() + # For pyproject.toml, we just update the version string directly + new_version_string = f'version = "{version}"' + new_content = re.sub(VERSION_PATTERN, new_version_string, content) + if new_content == content: + print("Error: Could not find or replace version string. Check your VERSION_PATTERN.") + return False + + with open(VERSION_FILE, "w", encoding="utf-8") as f: + f.write(new_content) + + print("Version updated successfully.") + return True + + except FileNotFoundError: + _fail(f"{VERSION_FILE} not found.") + except (OSError, re.error) as e: + _fail(f"An error occurred while updating the version: {e}") + + +def sign_artifacts(archive_name: str) -> list[str]: + """Creates signed files for the designated artifact.""" + files = [] + # Sign the tarball with GPG. The user must have a key configured. + try: + subprocess.run( + ["gpg", "--armor", "--output", f"{archive_name}.asc", "--detach-sig", archive_name], + check=True, + ) + files.append(f"{archive_name}.asc") + print(f"Created GPG signature: {archive_name}.asc") + except subprocess.CalledProcessError as e: + _fail(f"Error signing tarball {archive_name}: {e}") + + # Generate SHA512 checksum. + sha512_hash = hashlib.sha512() + with open(archive_name, "rb") as f: + while True: + data = f.read(65536) + if not data: + break + sha512_hash.update(data) + + with open(f"{archive_name}.sha512", "w", encoding="utf-8") as f: + f.write(f"{sha512_hash.hexdigest()}\n") + print(f"Created SHA512 checksum: {archive_name}.sha512") + files.append(f"{archive_name}.sha512") + return files + + +def create_release_artifacts(version, build_wheel=False) -> list[str]: + """Creates the source tarball, GPG signatures, and checksums using flit. + + Args: + version: The version string for the release + build_wheel: If True, also build and sign a wheel distribution + """ + print("\n[Step 1/1] Creating source release artifacts with 'flit build'...") + + # Clean the dist directory before building. + if os.path.exists("dist"): + shutil.rmtree("dist") + # Ensure no pre-built UI assets slip into the source package. + ui_build_dir = os.path.join("burr", "tracking", "server", "build") + if os.path.exists(ui_build_dir): + print("Removing previously built UI artifacts...") + shutil.rmtree(ui_build_dir) + + # Warn if git working tree is dirty/untracked + try: + dirty = ( + subprocess.check_output(["git", "status", "--porcelain"], stderr=subprocess.DEVNULL) + .decode() + .strip() + ) + if dirty: + print( + "⚠️ Detected untracked or modified files. flit may refuse to build; " + "consider committing/stashing or verify FLIT_USE_VCS=0." + ) + print(" Git status summary:") + for line in dirty.splitlines(): + print(f" {line}") + except subprocess.CalledProcessError: + pass + + # Use flit to create the source distribution. + try: + env = os.environ.copy() + env["FLIT_USE_VCS"] = "0" + subprocess.run(["flit", "build", "--format", "sdist"], check=True, env=env) + print("✓ flit sdist created successfully.") + except subprocess.CalledProcessError as e: + _fail(f"Error creating source distribution: {e}") + + # Find the created tarball in the dist directory. + expected_tar_ball = f"dist/burr-{version.lower()}.tar.gz" + tarball_path = glob.glob(expected_tar_ball) + + if not tarball_path: + details = [] + if os.path.exists("dist"): + details.append("Contents of 'dist':") + for item in os.listdir("dist"): + details.append(f"- {item}") + else: + details.append("'dist' directory not found.") + _fail( + "Could not find the generated source tarball in the 'dist' directory.\n" + + "\n".join(details) + ) + + # Rename the tarball to apache-burr-{version.lower()}-incubating.tar.gz + apache_tar_ball = f"dist/apache-burr-{version.lower()}-incubating.tar.gz" + shutil.move(tarball_path[0], apache_tar_ball) + print(f"✓ Created source tarball: {apache_tar_ball}") + + # Sign the Apache tarball + signed_files = sign_artifacts(apache_tar_ball) + all_files = [apache_tar_ball] + signed_files + + # Optionally build the wheel (without built UI artifacts) + if build_wheel: + print("\n[Step 2/2] Creating wheel distribution with 'flit build --format wheel'...") + try: + env = os.environ.copy() + env["FLIT_USE_VCS"] = "0" + subprocess.run(["flit", "build", "--format", "wheel"], check=True, env=env) + print("✓ flit wheel created successfully.") + except subprocess.CalledProcessError as e: + _fail(f"Error creating wheel distribution: {e}") + + # Find the created wheel in the dist directory. + expected_wheel = f"dist/burr-{version.lower()}-*.whl" + wheel_path = glob.glob(expected_wheel) + + if not wheel_path: + details = [] + if os.path.exists("dist"): + details.append("Contents of 'dist':") + for item in os.listdir("dist"): + details.append(f"- {item}") + else: + details.append("'dist' directory not found.") + _fail( + "Could not find the generated wheel in the 'dist' directory.\n" + "\n".join(details) + ) + + # Rename the wheel to apache-burr-{version.lower()}-incubating-{rest}.whl + # Extract the wheel tags (e.g., py3-none-any.whl) + original_wheel = os.path.basename(wheel_path[0]) + # Pattern: burr-{version}-{tags}.whl -> apache-burr-{version}-incubating-{tags}.whl + wheel_tags = original_wheel.replace(f"burr-{version.lower()}-", "") + apache_wheel = f"dist/apache-burr-{version.lower()}-incubating-{wheel_tags}" + shutil.move(wheel_path[0], apache_wheel) + print(f"✓ Created wheel: {apache_wheel}") + + # Sign the Apache wheel + wheel_signed_files = sign_artifacts(apache_wheel) + all_files.extend([apache_wheel] + wheel_signed_files) + + return all_files + + +def svn_upload(version, rc_num, archive_files, apache_id): + """Uploads the artifacts to the ASF dev distribution repository.""" + print("Uploading artifacts to ASF SVN...") + svn_path = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}/apache-burr/{version}-incubating-RC{rc_num}"; + + try: + # Create a new directory for the release candidate. + subprocess.run( + [ + "svn", + "mkdir", + "-m", + f"Creating directory for {version}-incubating-RC{rc_num}", + svn_path, + ], + check=True, + ) + + # Get the files to import (tarball, asc, sha512). + files_to_import = archive_files + + # Use svn import for the new directory. + for file_path in files_to_import: + subprocess.run( + [ + "svn", + "import", + file_path, + f"{svn_path}/{os.path.basename(file_path)}", + "-m", + f"Adding {os.path.basename(file_path)}", + "--username", + apache_id, + ], + check=True, + ) + + print(f"Artifacts successfully uploaded to: {svn_path}") + return svn_path + + except subprocess.CalledProcessError as e: + print(f"Error during SVN upload: {e}") + print("Make sure you have svn access configured for your Apache ID.") + return None + + +def generate_email_template(version, rc_num, svn_url): + """Generates the content for the [VOTE] email.""" + print("Generating email template...") + version_with_incubating = f"{version}-incubating" + tag = f"v{version}" + + email_content = f"""[VOTE] Release Apache {PROJECT_SHORT_NAME} {version_with_incubating} (release candidate {rc_num}) + +Hi all, + +This is a call for a vote on releasing Apache {PROJECT_SHORT_NAME} {version_with_incubating}, +release candidate {rc_num}. + +This release includes the following changes (see CHANGELOG for details): +- [List key changes here] + +The artifacts for this release candidate can be found at: +{svn_url} + +The Git tag to be voted upon is: +{tag} + +The release hash is: +[Insert git commit hash here] + + +Release artifacts are signed with the following key: +[Insert your GPG key ID here] +The KEYS file is available at: +https://downloads.apache.org/incubator/{PROJECT_SHORT_NAME}/KEYS + +Please download, verify, and test the release candidate. + +For testing, please run some of the examples, scripts/qualify.sh has +a sampling of them to run. + +The vote will run for a minimum of 72 hours. +Please vote: + +[ ] +1 Release this package as Apache {PROJECT_SHORT_NAME} {version_with_incubating} +[ ] +0 No opinion +[ ] -1 Do not release this package because... (Please provide a reason) + +Checklist for reference: +[ ] Download links are valid. +[ ] Checksums and signatures. +[ ] LICENSE/NOTICE files exist +[ ] No unexpected binary files +[ ] All source files have ASF headers +[ ] Can compile from source + +On behalf of the Apache {PROJECT_SHORT_NAME} PPMC, +[Your Name] +""" + print("\n" + "=" * 80) + print("EMAIL TEMPLATE (COPY AND PASTE TO YOUR MAILING LIST)") + print("=" * 80) + print(email_content) + print("=" * 80) + + +def main(): + """ + ### How to Use the Updated Script + + 1. **Install flit**: + ```bash + pip install flit + ``` + 2. **Configure the Script**: Open `apache_release_helper.py` in a text editor and update the three variables at the top of the file with your project's details: + * `PROJECT_SHORT_NAME` + * `VERSION_FILE` and `VERSION_PATTERN` + 3. **Prerequisites**: + * You must have `git`, `gpg`, `svn`, and `flit` installed. + * Your GPG key and SVN access must be configured for your Apache ID. + 4. **Run the Script**: + Open your terminal, navigate to the root of your project directory, and run the script with the desired version, release candidate number, and Apache ID. + + + python apache_release_helper.py 1.2.3 0 your_apache_id + """ + parser = argparse.ArgumentParser(description="Automates parts of the Apache release process.") + parser.add_argument("version", help="The new release version (e.g., '1.0.0').") + parser.add_argument("rc_num", help="The release candidate number (e.g., '0' for RC0).") + parser.add_argument("apache_id", help="Your apache user ID.") + parser.add_argument( + "--dry-run", + action="store_true", + help="Run in dry-run mode (skip git tag creation and SVN upload)", + ) + parser.add_argument( + "--build-wheel", + action="store_true", + help="Also build and sign a wheel distribution (in addition to the source tarball)", + ) + args = parser.parse_args() + + version = args.version + rc_num = args.rc_num + apache_id = args.apache_id + dry_run = args.dry_run + build_wheel = args.build_wheel + + if dry_run: + print("\n*** DRY RUN MODE - No git tags or SVN uploads will be performed ***\n") + + check_prerequisites() + + current_version = get_version_from_file(VERSION_FILE) + print(current_version) + if current_version != version: + _fail( + "Version mismatch. Update pyproject.toml to the requested version before running the script." + ) + + tag_name = f"v{version}-incubating-RC{rc_num}" + if dry_run: + print(f"\n[DRY RUN] Would create git tag '{tag_name}'") + else: + print(f"\nChecking for git tag '{tag_name}'...") + try: + # Check if the tag already exists + existing_tag = subprocess.check_output(["git", "tag", "-l", tag_name]).decode().strip() + if existing_tag == tag_name: + print(f"Git tag '{tag_name}' already exists.") + response = ( + input("Do you want to continue without creating a new tag? (y/n): ") + .lower() + .strip() + ) + if response != "y": + print("Aborting.") + sys.exit(1) + else: + # Tag does not exist, create it + print(f"Creating git tag '{tag_name}'...") + subprocess.run(["git", "tag", tag_name], check=True) + print(f"Git tag {tag_name} created.") + except subprocess.CalledProcessError as e: + _fail(f"Error checking or creating Git tag: {e}") + + # Create artifacts + archive_files = create_release_artifacts(version, build_wheel=build_wheel) + + # Upload artifacts + # NOTE: You MUST have your SVN client configured to use your Apache ID and have permissions. + if dry_run: + svn_url = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}/apache-burr/{version}-incubating-RC{rc_num}"; + print(f"\n[DRY RUN] Would upload artifacts to: {svn_url}") + else: + svn_url = svn_upload(version, rc_num, archive_files, apache_id) + if not svn_url: + _fail("SVN upload failed.") + + # Generate email + generate_email_template(version, rc_num, svn_url) + + print("\nProcess complete. Please copy the email template to your mailing list.") + + +if __name__ == "__main__": + main() diff --git a/scripts/setup_keys.sh b/scripts/setup_keys.sh new file mode 100755 index 00000000..a4f12615 --- /dev/null +++ b/scripts/setup_keys.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script helps new Apache committers set up their GPG keys for releases. +# It guides you through creating a new key, exports the public key, and +# provides instructions on how to add it to your project's KEYS file. + +echo "========================================================" +echo " Apache GPG Key Setup Script" +echo "========================================================" +echo " " +echo "Step 1: Generating a new GPG key." +echo " " +echo "Please be aware of Apache's best practices for GPG keys:" +echo "- **Key Type:** Select **(1) RSA and RSA**." +echo "- **Key Size:** Enter **4096**." +echo "- **Email Address:** Use your official **@apache.org** email address." +echo "- **Passphrase:** Use a strong, secure passphrase." +echo " " +read -p "Press [Enter] to start the GPG key generation..." + +# Generate a new GPG key +# The --batch and --passphrase-fd 0 options are used for automation, +# but the script will still require interactive input. +gpg --full-gen-key + +if [ $? -ne 0 ]; then + echo "Error: GPG key generation failed. Please check your GPG installation." + exit 1 +fi + +echo " " +echo "Step 2: Listing your GPG keys to find the new key ID." +echo "Your new key is listed under 'pub' with a string of 8 or 16 characters after the '/'." + +# List all GPG keys +gpg --list-keys + +echo " " +read -p "Please copy and paste your new key ID here (e.g., A1B2C3D4 or 1234ABCD5678EF01): " KEY_ID + +if [ -z "$KEY_ID" ]; then + echo "Error: Key ID cannot be empty. Exiting." + exit 1 +fi + +echo " " +echo "Step 3: Exporting your public key to a file." + +# Export the public key in ASCII armored format +gpg --armor --export "$KEY_ID" > "$KEY_ID.asc" + +if [ $? -ne 0 ]; then + echo "Error: Public key export failed. Please ensure the Key ID is correct." + rm -f "$KEY_ID.asc" + exit 1 +fi + +echo "Checking out dist repository to update KEYS file" +svn checkout --depth immediates https://dist.apache.org/repos/dist dist +cd dist/release +svn checkout https://dist.apache.org/repos/dist/release/incubator/burr incubator/burr + +cd ../../ +gpg --list-keys "$KEY_ID" >> dist/release/incubator/burr/KEYS +cat "$KEY_ID.asc" >> dist/release/incubator/burr/KEYS +cd dist/release/incubator/burr + +echo " " +echo "========================================================" +echo " Setup Complete!" +echo "========================================================" +echo "Your public key has been saved to: $KEY_ID.asc" +echo " " +echo "NEXT STEPS (VERY IMPORTANT):" +echo "1. Please inspect the KEYS file to ensure the new key is added correctly. It should be in the current directory." +echo "2. If all good run: svn update KEYS && svn commit -m \"Adds new key $KEY_ID for YOUR NAME\"" +echo "3. Inform the mailing list that you've updated the KEYS file." +echo " The updated KEYS file is essential for others to verify your release signatures." +echo " " diff --git a/telemetry/ui/package-lock.json b/telemetry/ui/package-lock.json index b71fbefc..21461ed4 100644 --- a/telemetry/ui/package-lock.json +++ b/telemetry/ui/package-lock.json @@ -25,6 +25,7 @@ "@uiw/react-json-view": "^2.0.0-alpha.12", "clsx": "^2.1.0", "dagre": "^0.8.5", + "es-abstract": "^1.22.4", "fuse.js": "^7.0.0", "heroicons": "^2.1.1", "react": "^18.2.0", @@ -38,6 +39,7 @@ "react-syntax-highlighter": "^15.5.0", "reactflow": "^11.10.4", "remark-gfm": "^4.0.0", + "string.prototype.matchall": "^4.0.10", "tailwindcss-question-mark": "^0.4.0", "typescript": "^4.9.5", "web-vitals": "^2.1.4" diff --git a/telemetry/ui/package.json b/telemetry/ui/package.json index 1c492bf3..1a391fcc 100644 --- a/telemetry/ui/package.json +++ b/telemetry/ui/package.json @@ -20,6 +20,7 @@ "@uiw/react-json-view": "^2.0.0-alpha.12", "clsx": "^2.1.0", "dagre": "^0.8.5", + "es-abstract": "^1.22.4", "fuse.js": "^7.0.0", "heroicons": "^2.1.1", "react": "^18.2.0", @@ -33,6 +34,7 @@ "react-syntax-highlighter": "^15.5.0", "reactflow": "^11.10.4", "remark-gfm": "^4.0.0", + "string.prototype.matchall": "^4.0.10", "tailwindcss-question-mark": "^0.4.0", "typescript": "^4.9.5", "web-vitals": "^2.1.4" diff --git a/tests/test_release_config.py b/tests/test_release_config.py new file mode 100644 index 00000000..e97aa628 --- /dev/null +++ b/tests/test_release_config.py @@ -0,0 +1,127 @@ +""" +Tests to validate release configuration in pyproject.toml. + +This ensures the examples include/exclude lists stay in sync with the actual +examples directory structure. +""" + +import tomllib +from pathlib import Path + + +def test_examples_include_exclude_coverage(): + """ + Verify that pyproject.toml's [tool.flit.sdist] include/exclude lists cover + all example directories. + + WHY THIS TEST EXISTS: + Flit automatically includes the examples/ directory in the release tarball because + it's a Python package (has __init__.py). Without explicit include/exclude rules, + ALL examples would be shipped in the Apache release, which is not intended. + + For Apache releases, we only want to include 4 specific examples for voters to test: + - email-assistant + - multi-modal-chatbot + - streaming-fastapi + - deep-researcher + + All other examples must be explicitly excluded. This test ensures the configuration + stays in sync with the filesystem when examples are added/removed. + + If this test fails, you need to update pyproject.toml: + - To INCLUDE an example: add it to [tool.flit.sdist] include list + - To EXCLUDE an example: add it to [tool.flit.sdist] exclude list + """ + # Load pyproject.toml + project_root = Path(__file__).parent.parent + pyproject_path = project_root / "pyproject.toml" + + with open(pyproject_path, "rb") as f: + config = tomllib.load(f) + + flit_sdist = config.get("tool", {}).get("flit", {}).get("sdist", {}) + include_patterns = flit_sdist.get("include", []) + exclude_patterns = flit_sdist.get("exclude", []) + + # Extract example directories from include patterns + included_examples = set() + for pattern in include_patterns: + if pattern.startswith("examples/") and pattern.endswith("/**"): + # Extract directory name from patterns like "examples/email-assistant/**" + dir_name = pattern.removeprefix("examples/").removesuffix("/**") + included_examples.add(dir_name) + + # Extract example directories from exclude patterns + excluded_examples = set() + excluded_files = set() + for pattern in exclude_patterns: + if pattern.startswith("examples/"): + if pattern.endswith("/**"): + # Directory pattern like "examples/adaptive-crag/**" + dir_name = pattern.removeprefix("examples/").removesuffix("/**") + excluded_examples.add(dir_name) + else: + # File pattern like "examples/__init__.py" + file_name = pattern.removeprefix("examples/") + excluded_files.add(file_name) + + # Get actual example directories from filesystem + examples_dir = project_root / "examples" + actual_dirs = set() + actual_files = set() + + if examples_dir.exists(): + for item in examples_dir.iterdir(): + if item.name.startswith(".") or item.name == "__pycache__": + continue + if item.is_dir(): + actual_dirs.add(item.name) + else: + actual_files.add(item.name) + + # Check coverage + configured_dirs = included_examples | excluded_examples + missing_from_config = actual_dirs - configured_dirs + extra_in_config = configured_dirs - actual_dirs + + # Build error message if mismatch found + errors = [] + + if missing_from_config: + errors.append( + f"\n❌ Example directories exist but are NOT in pyproject.toml config:\n" + f" {sorted(missing_from_config)}\n" + f"\n WHY THIS MATTERS:\n" + f" Flit auto-discovers examples/ as a package (it has __init__.py) and will\n" + f" include ALL subdirectories in the release tarball unless explicitly excluded.\n" + f" Every example directory MUST be either included or excluded to ensure the\n" + f" Apache release contains only the intended examples for voters to test.\n" + f"\n To fix: Add to pyproject.toml [tool.flit.sdist]:\n" + f" - To INCLUDE in Apache release: add 'examples/<name>/**' to 'include' list\n" + f" - To EXCLUDE from Apache release: add 'examples/<name>/**' to 'exclude' list\n" + f"\n Currently only these 4 examples should be included:\n" + f" email-assistant, multi-modal-chatbot, streaming-fastapi, deep-researcher\n" + ) + + if extra_in_config: + errors.append( + f"\n❌ Example directories in pyproject.toml but NOT in filesystem:\n" + f" {sorted(extra_in_config)}\n" + f"\n WHY THIS MATTERS:\n" + f" These entries reference examples that no longer exist and should be removed\n" + f" to keep the configuration accurate and maintainable.\n" + f"\n To fix: Remove these entries from pyproject.toml [tool.flit.sdist]\n" + ) + + # Report what's currently configured (for debugging) + if errors: + summary = ( + f"\n📋 Current configuration:\n" + f" Included examples ({len(included_examples)}): {sorted(included_examples)}\n" + f" Excluded examples ({len(excluded_examples)}): {sorted(excluded_examples)}\n" + f" Excluded files ({len(excluded_files)}): {sorted(excluded_files)}\n" + f" Actual directories ({len(actual_dirs)}): {sorted(actual_dirs)}\n" + ) + errors.append(summary) + + assert not errors, "\n".join(errors)
