This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git
The following commit(s) were added to refs/heads/main by this push:
new 1a11573 Add a PMC updater and release candidate processing code
1a11573 is described below
commit 1a11573c714aa93c274fc1af4342af5104131ae1
Author: Sean B. Palmer <[email protected]>
AuthorDate: Thu Feb 13 15:46:58 2025 +0200
Add a PMC updater and release candidate processing code
---
atr/models.py | 10 +-
atr/routes.py | 191 +++++++++++++++++++++++++++++--
atr/server.py | 5 +
atr/static/root.css | 4 +-
atr/templates/add-release-candidate.html | 17 ++-
atr/templates/update-pmcs.html | 30 +++++
poetry.lock | 49 +++++++-
pyproject.toml | 2 +
uv.lock | 30 +++++
9 files changed, 318 insertions(+), 20 deletions(-)
diff --git a/atr/models.py b/atr/models.py
index 8552402..61c9dd4 100644
--- a/atr/models.py
+++ b/atr/models.py
@@ -120,11 +120,16 @@ class DistributionChannel(SQLModel, table=True):
product_line: Optional[ProductLine] =
Relationship(back_populates="distribution_channels")
-class Package(BaseModel):
+class Package(SQLModel, table=True):
+ id: Optional[int] = Field(default=None, primary_key=True)
file: str
signature: str
checksum: str
+ # Many-to-one: A package belongs to one release
+ release_key: Optional[str] = Field(default=None,
foreign_key="release.storage_key")
+ release: Optional["Release"] = Relationship(back_populates="packages")
+
class VoteEntry(BaseModel):
result: bool
@@ -172,7 +177,8 @@ class Release(SQLModel, table=True):
package_managers: List[str] = Field(default_factory=list,
sa_column=Column(JSON))
version: str
- packages: List[Package] = Field(default_factory=list,
sa_column=Column(JSON))
+ # One-to-many: A release can have multiple packages
+ packages: List[Package] = Relationship(back_populates="release")
sboms: List[str] = Field(default_factory=list, sa_column=Column(JSON))
# Many-to-one: A release can have one vote policy, a vote policy can be
used by multiple releases
diff --git a/atr/routes.py b/atr/routes.py
index 680376f..eb01b55 100644
--- a/atr/routes.py
+++ b/atr/routes.py
@@ -17,7 +17,10 @@
"routes.py"
-from typing import List
+import hashlib
+import json
+from pathlib import Path
+from typing import List, Tuple
from asfquart import APP
from asfquart.auth import Requirements as R, require
@@ -26,13 +29,49 @@ from asfquart.session import read as session_read
from quart import current_app, render_template, request
from sqlmodel import Session, select
from sqlalchemy.exc import IntegrityError
+import httpx
-from .models import PMC
+from .models import PMC, Release, ReleaseStage, ReleasePhase, Package
if APP is ...:
raise ValueError("APP is not set")
+def compute_sha3_256(file_data: bytes) -> str:
+ "Compute SHA3-256 hash of file data."
+ return hashlib.sha3_256(file_data).hexdigest()
+
+
+def compute_sha512(file_path: Path) -> str:
+ "Compute SHA-512 hash of a file."
+ sha512 = hashlib.sha512()
+ with open(file_path, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ sha512.update(chunk)
+ return sha512.hexdigest()
+
+
+async def save_file_by_hash(file, base_dir: Path) -> Tuple[Path, str]:
+ """
+ Save a file using its SHA3-256 hash as the filename.
+ Returns the path where the file was saved and its hash.
+ """
+ # FileStorage.read() returns bytes directly, no need to await
+ data = file.read()
+ file_hash = compute_sha3_256(data)
+
+ # Create path with hash as filename
+ path = base_dir / file_hash
+ path.parent.mkdir(parents=True, exist_ok=True)
+
+ # Only write if file doesn't exist
+ # If it does exist, it'll be the same content anyway
+ if not path.exists():
+ path.write_bytes(data)
+
+ return path, file_hash
+
+
@APP.route("/add-release-candidate", methods=["GET", "POST"])
@require(R.committer)
async def add_release_candidate() -> str:
@@ -43,10 +82,7 @@ async def add_release_candidate() -> str:
# For POST requests, handle the file upload
if request.method == "POST":
- # We'll implement the actual file handling later
- # For now just return a message about what we would do
form = await request.form
- files = await request.files
project_name = form.get("project_name")
if not project_name:
@@ -58,12 +94,67 @@ async def add_release_candidate() -> str:
f"You must be a PMC member of {project_name} to submit a
release candidate", errorcode=403
)
- release_file = files.get("release_file")
- if not release_file:
- raise ASFQuartException("Release file is required", errorcode=400)
+ # Get all uploaded files
+ files = await request.files
+
+ # Get the release artifact and signature files
+ artifact_file = files.get("release_artifact")
+ signature_file = files.get("release_signature")
+
+ if not artifact_file:
+ raise ASFQuartException("Release artifact file is required",
errorcode=400)
+ if not signature_file:
+ raise ASFQuartException("Detached GPG signature file is required",
errorcode=400)
+ if not signature_file.filename.endswith(".asc"):
+ # TODO: Could also check that it's artifact name + ".asc"
+ # And at least warn if it's not
+ raise ASFQuartException("Signature file must have .asc extension",
errorcode=400)
+
+ # Save files using their hashes as filenames
+ storage_dir = Path(current_app.config["RELEASE_STORAGE_DIR"]) /
project_name
+ artifact_path, artifact_hash = await save_file_by_hash(artifact_file,
storage_dir)
+ # TODO: Do we need to do anything with the signature hash?
+ # These should be identical, but path might be absolute?
+ # TODO: Need to check, ideally. Could have a data browser
+ signature_path, _ = await save_file_by_hash(signature_file,
storage_dir)
+
+ # Compute SHA-512 checksum of the artifact for the package record
+ # We're using SHA-3-256 for the filename, so we need to use SHA-3-512
for the checksum
+ checksum_512 = compute_sha512(artifact_path)
+
+ # Store in database
+ with Session(current_app.config["engine"]) as db_session:
+ # Get PMC
+ statement = select(PMC).where(PMC.project_name == project_name)
+ pmc = db_session.exec(statement).first()
+ if not pmc:
+ raise ASFQuartException("PMC not found", errorcode=404)
+
+ # Create release record using artifact hash as storage key
+ # At some point this presumably won't work, because we can have
many artifacts
+ # But meanwhile it's fine
+ # TODO: Extract version from filename or add to form
+ release = Release(
+ storage_key=artifact_hash,
+ stage=ReleaseStage.CANDIDATE,
+ phase=ReleasePhase.RELEASE_CANDIDATE,
+ pmc_id=pmc.id,
+ version="",
+ )
+ db_session.add(release)
+
+ # Create package record
+ package = Package(
+
file=str(artifact_path.relative_to(current_app.config["RELEASE_STORAGE_DIR"])),
+
signature=str(signature_path.relative_to(current_app.config["RELEASE_STORAGE_DIR"])),
+ checksum=checksum_512,
+ release_key=release.storage_key,
+ )
+ db_session.add(package)
+
+ db_session.commit()
- # TODO: Implement actual file handling
- return f"Would process release candidate for {project_name} from file
{release_file.filename}"
+ return f"Successfully uploaded release candidate for
{project_name}"
# For GET requests, show the form
return await render_template(
@@ -74,6 +165,86 @@ async def add_release_candidate() -> str:
)
[email protected]("/admin/update-pmcs", methods=["GET", "POST"])
+async def admin_update_pmcs() -> str:
+ "Update PMCs from remote, authoritative committee-info.json."
+ # Check authentication
+ session = await session_read()
+ if session is None:
+ raise ASFQuartException("Not authenticated", errorcode=401)
+
+ # List of users allowed to update PMCs
+ ALLOWED_USERS = {"cwells", "fluxo", "gmcdonald", "humbedooh", "sbp", "tn",
"wave"}
+
+ if session.uid not in ALLOWED_USERS:
+ raise ASFQuartException("You are not authorized to update PMCs",
errorcode=403)
+
+ if request.method == "POST":
+ # TODO: We should probably lift this branch
+ # Or have the "GET" in a branch, and then we can happy path this POST
branch
+ # Fetch committee-info.json from Whimsy
+ WHIMSY_URL = "https://whimsy.apache.org/public/committee-info.json";
+ async with httpx.AsyncClient() as client:
+ try:
+ response = await client.get(WHIMSY_URL)
+ response.raise_for_status()
+ data = response.json()
+ except (httpx.RequestError, json.JSONDecodeError) as e:
+ raise ASFQuartException(f"Failed to fetch committee data:
{str(e)}", errorcode=500)
+
+ committees = data.get("committees", {})
+ updated_count = 0
+
+ with Session(current_app.config["engine"]) as db_session:
+ for committee_id, info in committees.items():
+ # Skip non-PMC committees
+ if not info.get("pmc", False):
+ continue
+
+ # Get or create PMC
+ statement = select(PMC).where(PMC.project_name == committee_id)
+ pmc = db_session.exec(statement).first()
+ if not pmc:
+ pmc = PMC(project_name=committee_id)
+ db_session.add(pmc)
+
+ # Update PMC data
+ roster = info.get("roster", {})
+ # All roster members are PMC members
+ pmc.pmc_members = list(roster.keys())
+ # All PMC members are also committers
+ pmc.committers = list(roster.keys())
+
+ # Mark chairs as release managers
+ # TODO: Who else is a release manager? How do we know?
+ chairs = [m["id"] for m in info.get("chairs", [])]
+ pmc.release_managers = chairs
+
+ updated_count += 1
+
+ # Add special entry for Tooling PMC
+ # Not clear why, but it's not in the Whimsy data
+ statement = select(PMC).where(PMC.project_name == "tooling")
+ tooling_pmc = db_session.exec(statement).first()
+ if not tooling_pmc:
+ tooling_pmc = PMC(project_name="tooling")
+ db_session.add(tooling_pmc)
+ updated_count += 1
+
+ # Update Tooling PMC data
+ # Could put this in the "if not tooling_pmc" block, perhaps
+ tooling_pmc.pmc_members = ["wave", "tn", "sbp"]
+ tooling_pmc.committers = ["wave", "tn", "sbp"]
+ tooling_pmc.release_managers = ["wave"]
+
+ db_session.commit()
+
+ return f"Successfully updated {updated_count} PMCs from Whimsy"
+
+ # For GET requests, show the update form
+ return await render_template("update-pmcs.html")
+
+
@APP.route("/pmc/create/<project_name>")
async def pmc_create_arg(project_name: str) -> dict:
"Create a new PMC with some sample data."
diff --git a/atr/server.py b/atr/server.py
index 79d98e4..7282064 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -53,6 +53,11 @@ def create_app() -> QuartApp:
os.chdir(state_dir)
print(f"Working directory changed to: {os.getcwd()}")
+ # Set up release storage directory
+ release_storage = os.path.join(state_dir, "releases")
+ os.makedirs(release_storage, exist_ok=True)
+ app.config["RELEASE_STORAGE_DIR"] = release_storage
+
sqlite_url = "sqlite:///./atr.db"
engine = create_engine(
sqlite_url,
diff --git a/atr/static/root.css b/atr/static/root.css
index 1dbc347..8da932b 100644
--- a/atr/static/root.css
+++ b/atr/static/root.css
@@ -5,7 +5,7 @@ body {
}
h1 {
- color: #303284;
+ color: #036;
}
.pmc-list {
@@ -32,7 +32,7 @@ h1 {
.pmc-name {
font-weight: bold;
font-size: 1.2em;
- color: #303284;
+ color: #036;
margin-bottom: 0.5em;
}
diff --git a/atr/templates/add-release-candidate.html
b/atr/templates/add-release-candidate.html
index 871fd15..e58e0a6 100644
--- a/atr/templates/add-release-candidate.html
+++ b/atr/templates/add-release-candidate.html
@@ -76,12 +76,19 @@
</div>
<div class="form-group">
- <label for="release_file">Release Candidate Archive:</label>
- <input type="file" id="release_file" name="release_file" required
+ <label for="release_artifact">Release Candidate Archive:</label>
+ <input type="file" id="release_artifact" name="release_artifact" required
accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/java-archive,.tar.gz,.tgz,.zip,.jar"
- aria-describedby="file-help">
- <span id="file-help">Upload the release candidate archive (tar.gz, zip,
or jar)</span>
- <!-- TODO: What file names should we support here? Just any file name?-->
+ aria-describedby="artifact-help">
+ <span id="artifact-help">Upload the release candidate archive (tar.gz,
zip, or jar)</span>
+ </div>
+
+ <div class="form-group">
+ <label for="release_signature">Detached GPG Signature:</label>
+ <input type="file" id="release_signature" name="release_signature"
required
+ accept="application/pgp-signature,.asc"
+ aria-describedby="signature-help">
+ <span id="signature-help">Upload the detached GPG signature (.asc) file
for the release candidate</span>
</div>
<button type="submit" {% if not pmc_memberships %}disabled{% endif %}>
diff --git a/atr/templates/update-pmcs.html b/atr/templates/update-pmcs.html
new file mode 100644
index 0000000..fd01175
--- /dev/null
+++ b/atr/templates/update-pmcs.html
@@ -0,0 +1,30 @@
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width,initial-scale=1.0">
+ <title>ATR | Update PMCs</title>
+ <link rel="stylesheet" href="{{ url_for('static', filename='root.css') }}">
+ <style>
+ .form-group {
+ margin-bottom: 1rem;
+ }
+ button {
+ margin-top: 1rem;
+ padding: 0.5rem 1rem;
+ }
+ </style>
+</head>
+<body>
+ <h1>Update PMCs</h1>
+ <p class="intro">This page allows you to update the PMC information in the
database from committee-info.json.</p>
+
+ <div class="warning">
+ <p><strong>Note:</strong> This operation will update all PMC information,
including member lists and release manager assignments.</p>
+ </div>
+
+ <form method="post">
+ <button type="submit">Update PMCs</button>
+ </form>
+</body>
+</html>
diff --git a/poetry.lock b/poetry.lock
index eff2582..3503f30 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -958,6 +958,53 @@ files = [
{file = "hpack-4.1.0.tar.gz", hash =
"sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca"},
]
+[[package]]
+name = "httpcore"
+version = "1.0.7"
+description = "A minimal low-level HTTP client."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+ {file = "httpcore-1.0.7-py3-none-any.whl", hash =
"sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"},
+ {file = "httpcore-1.0.7.tar.gz", hash =
"sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"},
+]
+
+[package.dependencies]
+certifi = "*"
+h11 = ">=0.13,<0.15"
+
+[package.extras]
+asyncio = ["anyio (>=4.0,<5.0)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+trio = ["trio (>=0.22.0,<1.0)"]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+description = "The next generation HTTP client."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+ {file = "httpx-0.28.1-py3-none-any.whl", hash =
"sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"},
+ {file = "httpx-0.28.1.tar.gz", hash =
"sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"},
+]
+
+[package.dependencies]
+anyio = "*"
+certifi = "*"
+httpcore = "==1.*"
+idna = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+zstd = ["zstandard (>=0.18.0)"]
+
[[package]]
name = "hypercorn"
version = "0.17.3"
@@ -2293,4 +2340,4 @@ propcache = ">=0.2.0"
[metadata]
lock-version = "2.1"
python-versions = "~=3.13"
-content-hash =
"0f091d55acfd7f3269b025aa80eadad13ff6b281f8c72c7cdd229d207f0b609d"
+content-hash =
"6be50693b57621af6fa35e34f54c89113075e48ee3c13ec418e92586fcc1c141"
diff --git a/pyproject.toml b/pyproject.toml
index 82b1887..0323bd2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,7 @@ dependencies = [
"alembic~=1.14",
"asfquart",
"cryptography~=44.0",
+ "httpx~=0.27",
"hypercorn~=0.17",
"sqlmodel~=0.0",
]
@@ -39,6 +40,7 @@ mypy = "^1.15.0"
alembic = "~=1.14"
asfquart = { path = "./asfquart", develop = true }
cryptography = "~=44.0"
+httpx = "~=0.27"
hypercorn = "~=0.17"
sqlmodel = "~=0.0"
diff --git a/uv.lock b/uv.lock
index 597cff8..ee6dade 100644
--- a/uv.lock
+++ b/uv.lock
@@ -451,6 +451,34 @@ wheels = [
{ url =
"https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl";,
hash =
"sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size
= 34357 },
]
+[[package]]
+name = "httpcore"
+version = "1.0.7"
+source = { registry = "https://pypi.org/simple"; }
+dependencies = [
+ { name = "certifi" },
+ { name = "h11" },
+]
+sdist = { url =
"https://files.pythonhosted.org/packages/6a/41/d7d0a89eb493922c37d343b607bc1b5da7f5be7e383740b4753ad8943e90/httpcore-1.0.7.tar.gz";,
hash =
"sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c", size
= 85196 }
+wheels = [
+ { url =
"https://files.pythonhosted.org/packages/87/f5/72347bc88306acb359581ac4d52f23c0ef445b57157adedb9aee0cd689d2/httpcore-1.0.7-py3-none-any.whl";,
hash =
"sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd", size
= 78551 },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple"; }
+dependencies = [
+ { name = "anyio" },
+ { name = "certifi" },
+ { name = "httpcore" },
+ { name = "idna" },
+]
+sdist = { url =
"https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz";,
hash =
"sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size
= 141406 }
+wheels = [
+ { url =
"https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl";,
hash =
"sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size
= 73517 },
+]
+
[[package]]
name = "hypercorn"
version = "0.17.3"
@@ -943,6 +971,7 @@ dependencies = [
{ name = "alembic" },
{ name = "asfquart" },
{ name = "cryptography" },
+ { name = "httpx" },
{ name = "hypercorn" },
{ name = "sqlmodel" },
]
@@ -960,6 +989,7 @@ requires-dist = [
{ name = "alembic", specifier = "~=1.14" },
{ name = "asfquart", editable = "asfquart" },
{ name = "cryptography", specifier = "~=44.0" },
+ { name = "httpx", specifier = "~=0.27" },
{ name = "hypercorn", specifier = "~=0.17" },
{ name = "sqlmodel", specifier = "~=0.0" },
]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
