This is an automated email from the ASF dual-hosted git repository. tn pushed a commit to branch add-project-model in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
commit 7df875cf9fb112df80b96f5c4f0be5f24ffc7ed5 Author: Thomas Neidhart <[email protected]> AuthorDate: Tue Mar 11 10:56:28 2025 +0100 adding project object --- atr/blueprints/admin/admin.py | 36 ++++- atr/db/models.py | 121 +++++++++++--- atr/db/service.py | 36 +++-- atr/routes/candidate.py | 69 ++++---- atr/routes/keys.py | 8 +- atr/routes/{project.py => pmc.py} | 29 ++-- atr/routes/project.py | 10 +- atr/routes/release.py | 17 +- atr/server.py | 6 +- atr/templates/candidate-create.html | 8 +- atr/templates/candidate-review.html | 1 + atr/templates/includes/sidebar.html | 5 + .../{project-directory.html => pmc-directory.html} | 23 ++- atr/templates/pmc-view.html | 175 +++++++++++++++++++++ atr/templates/project-directory.html | 46 ++---- atr/util.py | 9 +- docs/conventions.html | 3 - docs/conventions.md | 5 - 18 files changed, 441 insertions(+), 166 deletions(-) diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py index 47dac4b..ac42f68 100644 --- a/atr/blueprints/admin/admin.py +++ b/atr/blueprints/admin/admin.py @@ -14,12 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import logging from collections import defaultdict from collections.abc import Mapping from pathlib import Path from statistics import mean, median, stdev -from typing import Any +from typing import Any, Final import aiofiles.os import httpx @@ -33,6 +33,7 @@ from atr.datasources.apache import ( get_current_podlings_data, get_groups_data, get_ldap_projects_data, + get_projects_data, ) from atr.db import create_async_db_session from atr.db.models import ( @@ -41,6 +42,7 @@ from atr.db.models import ( Package, PMCKeyLink, ProductLine, + Project, PublicSigningKey, Release, Task, @@ -50,6 +52,8 @@ from atr.db.service import get_pmc_by_name from . import blueprint +_LOGGER: Final = logging.getLogger(__name__) + @blueprint.route("/performance") async def admin_performance() -> str: @@ -140,6 +144,7 @@ async def admin_data(model: str = "PMC") -> str: # Map of model names to their classes models = { "PMC": PMC, + "Project": Project, "Release": Release, "Package": Package, "VotePolicy": VotePolicy, @@ -206,6 +211,7 @@ async def _update_pmcs() -> int: ldap_projects = await get_ldap_projects_data() podlings_data = await get_current_podlings_data() groups_data = await get_groups_data() + projects_data = await get_projects_data() updated_count = 0 @@ -221,7 +227,7 @@ async def _update_pmcs() -> int: # Get or create PMC pmc = await get_pmc_by_name(name, db_session) if not pmc: - pmc = PMC(project_name=name) + pmc = PMC(name=name) db_session.add(pmc) # Update PMC data from groups.json @@ -243,10 +249,10 @@ async def _update_pmcs() -> int: # Then add PPMCs (podlings) for podling_name, podling_data in podlings_data: # Get or create PPMC - statement = select(PMC).where(PMC.project_name == podling_name) + statement = select(PMC).where(PMC.name == podling_name) ppmc = (await db_session.execute(statement)).scalar_one_or_none() if not ppmc: - ppmc = PMC(project_name=podling_name) + ppmc = PMC(name=podling_name) db_session.add(ppmc) # Update PPMC data from groups.json @@ -262,10 +268,10 @@ async def _update_pmcs() -> int: # Add special entry for Tooling PMC # Not clear why, but it's not in the Whimsy data - statement = select(PMC).where(PMC.project_name == "tooling") + statement = select(PMC).where(PMC.name == "tooling") tooling_pmc = (await db_session.execute(statement)).scalar_one_or_none() if not tooling_pmc: - tooling_pmc = PMC(project_name="tooling") + tooling_pmc = PMC(name="tooling") db_session.add(tooling_pmc) updated_count += 1 @@ -276,6 +282,22 @@ async def _update_pmcs() -> int: tooling_pmc.release_managers = ["wave"] tooling_pmc.is_podling = False + for project_name, project_status in projects_data: + # Get or create Project + my_statement = select(Project).where(Project.name == project_name) + aproject = (await db_session.execute(my_statement)).scalar_one_or_none() + if not aproject: + aproject = Project(name=project_name) + db_session.add(aproject) + + pmc = await get_pmc_by_name(project_status.pmc) + if pmc is None: + _LOGGER.error(f"project {project_name} has unknown PMC {project_status.pmc}") + continue + else: + aproject.pmc = pmc + aproject.full_name = project_status.name + return updated_count diff --git a/atr/db/models.py b/atr/db/models.py index ac0b481..fdea586 100644 --- a/atr/db/models.py +++ b/atr/db/models.py @@ -78,6 +78,8 @@ class VotePolicy(sqlmodel.SQLModel, table=True): # One-to-many: A vote policy can be used by multiple PMCs pmcs: list["PMC"] = sqlmodel.Relationship(back_populates="vote_policy") + # One-to-many: A vote policy can be used by multiple Projects + projects: list["Project"] = sqlmodel.Relationship(back_populates="vote_policy") # One-to-many: A vote policy can be used by multiple product lines product_lines: list["ProductLine"] = sqlmodel.Relationship(back_populates="vote_policy") # One-to-many: A vote policy can be used by multiple releases @@ -86,12 +88,25 @@ class VotePolicy(sqlmodel.SQLModel, table=True): class PMC(ATRSQLModel, table=True): id: int | None = sqlmodel.Field(default=None, primary_key=True) - project_name: str = sqlmodel.Field(unique=True) - # True if this is an incubator podling with a PPMC, otherwise False + name: str = sqlmodel.Field(unique=True) + full_name: str | None = sqlmodel.Field(default=None) + # True if this a podling PPMC is_podling: bool = sqlmodel.Field(default=False) - # One-to-many: A PMC can have multiple product lines, each product line belongs to one PMC - product_lines: list["ProductLine"] = sqlmodel.Relationship(back_populates="pmc") + # One-to-many: A PMC can have parent PMC, e.g. in the case of a PPMC + child_pmcs: list["PMC"] = sqlmodel.Relationship( + sa_relationship_kwargs=dict( + backref=sqlalchemy.orm.backref("parent_pmc", remote_side="PMC.id"), + ), + ) + parent_pmc_id: int | None = sqlmodel.Field(default=None, foreign_key="pmc.id") + + # One-to-many: A PMC can have multiple projects, each project belongs to one PMC + _projects: list["Project"] = sqlmodel.Relationship(back_populates="_pmc") + + @property + async def projects(self) -> list["Project"]: + return await self.awaitable_attrs._projects # type: ignore pmc_members: list[str] = sqlmodel.Field(default_factory=list, sa_column=sqlalchemy.Column(sqlalchemy.JSON)) committers: list[str] = sqlmodel.Field(default_factory=list, sa_column=sqlalchemy.Column(sqlalchemy.JSON)) @@ -108,23 +123,61 @@ class PMC(ATRSQLModel, table=True): vote_policy_id: int | None = sqlmodel.Field(default=None, foreign_key="votepolicy.id") vote_policy: VotePolicy | None = sqlmodel.Relationship(back_populates="pmcs") - # One-to-many: A PMC can have multiple releases - releases: list["Release"] = sqlmodel.Relationship(back_populates="pmc") - @property def display_name(self) -> str: """Get the display name for the PMC/PPMC.""" - if self.is_podling: - return f"{self.project_name} (podling)" - return self.project_name + name = self.name if self.full_name is None else self.full_name + return f"{name} (PPMC)" if self.is_podling else name -class ProductLine(sqlmodel.SQLModel, table=True): +class Project(ATRSQLModel, table=True): id: int | None = sqlmodel.Field(default=None, primary_key=True) + name: str = sqlmodel.Field(unique=True) + full_name: str | None = sqlmodel.Field(default=None) + + # True if this a podling PPMC + is_podling: bool = sqlmodel.Field(default=False) # Many-to-one: A product line belongs to one PMC, a PMC can have multiple product lines pmc_id: int | None = sqlmodel.Field(default=None, foreign_key="pmc.id") - pmc: PMC | None = sqlmodel.Relationship(back_populates="product_lines") + _pmc: PMC | None = sqlmodel.Relationship(back_populates="_projects") + + @property + async def pmc(self) -> PMC | None: + return await self.awaitable_attrs._pmc # type: ignore + + @pmc.setter + def pmc(self, value: PMC) -> None: + self._pmc = value + + # One-to-many: A PMC can have multiple product lines, each product line belongs to one PMC + _product_lines: list["ProductLine"] = sqlmodel.Relationship(back_populates="_project") + + @property + async def product_lines(self) -> list["ProductLine"]: + return await self.awaitable_attrs._product_lines # type: ignore + + # Many-to-one: A Project can have one vote policy, a vote policy can be used by multiple entities + vote_policy_id: int | None = sqlmodel.Field(default=None, foreign_key="votepolicy.id") + vote_policy: VotePolicy | None = sqlmodel.Relationship(back_populates="projects") + + @property + def display_name(self) -> str: + """Get the display name for the Project.""" + name = self.name if self.full_name is None else self.full_name + return f"{name} (podling)" if self.is_podling else name + + +class ProductLine(ATRSQLModel, table=True): + id: int | None = sqlmodel.Field(default=None, primary_key=True) + + # Many-to-one: A product line belongs to one PMC, a PMC can have multiple product lines + project_id: int | None = sqlmodel.Field(default=None, foreign_key="project.id") + _project: Project | None = sqlmodel.Relationship(back_populates="_product_lines") + + @property + async def project(self) -> Project | None: + return await self.awaitable_attrs._project # type: ignore product_name: str latest_version: str @@ -137,7 +190,11 @@ class ProductLine(sqlmodel.SQLModel, table=True): vote_policy: VotePolicy | None = sqlmodel.Relationship(back_populates="product_lines") # One-to-many: A product line can have multiple releases, each release belongs to one product line - releases: list["Release"] = sqlmodel.Relationship(back_populates="product_line") + _releases: list["Release"] = sqlmodel.Relationship(back_populates="_product_line") + + @property + async def releases(self) -> list["Release"]: + return await self.awaitable_attrs._releases # type: ignore class DistributionChannel(sqlmodel.SQLModel, table=True): @@ -153,7 +210,7 @@ class DistributionChannel(sqlmodel.SQLModel, table=True): product_line: ProductLine | None = sqlmodel.Relationship(back_populates="distribution_channels") -class Package(sqlmodel.SQLModel, table=True): +class Package(ATRSQLModel, table=True): # The SHA3-256 hash of the file, used as filename in storage # TODO: We should discuss making this unique artifact_sha3: str = sqlmodel.Field(primary_key=True) @@ -172,7 +229,11 @@ class Package(sqlmodel.SQLModel, table=True): # Many-to-one: A package belongs to one release release_key: str | None = sqlmodel.Field(default=None, foreign_key="release.storage_key") - release: Optional["Release"] = sqlmodel.Relationship(back_populates="packages") + _release: Optional["Release"] = sqlmodel.Relationship(back_populates="_packages") + + @property + async def release(self) -> Optional["Release"]: + return await self.awaitable_attrs._release # type: ignore # One-to-many: A package can have multiple tasks tasks: list["Task"] = sqlmodel.Relationship( @@ -263,24 +324,40 @@ class Task(sqlmodel.SQLModel, table=True): ) -class Release(sqlmodel.SQLModel, table=True): +class Release(ATRSQLModel, table=True): storage_key: str = sqlmodel.Field(primary_key=True) stage: ReleaseStage phase: ReleasePhase created: datetime.datetime - # Many-to-one: A release belongs to one PMC, a PMC can have multiple releases - pmc_id: int | None = sqlmodel.Field(default=None, foreign_key="pmc.id") - pmc: PMC | None = sqlmodel.Relationship(back_populates="releases") - # Many-to-one: A release belongs to one product line, a product line can have multiple releases product_line_id: int | None = sqlmodel.Field(default=None, foreign_key="productline.id") - product_line: ProductLine | None = sqlmodel.Relationship(back_populates="releases") + _product_line: ProductLine | None = sqlmodel.Relationship(back_populates="_releases") + + @property + async def product_line(self) -> ProductLine | None: + return await self.awaitable_attrs._product_line # type: ignore + + @property + async def pmc(self) -> PMC | None: + product_line = await self.product_line + if product_line is not None: + project = await product_line.project + if project is not None: + return await project.pmc + + return None package_managers: list[str] = sqlmodel.Field(default_factory=list, sa_column=sqlalchemy.Column(sqlalchemy.JSON)) version: str + # One-to-many: A release can have multiple packages - packages: list[Package] = sqlmodel.Relationship(back_populates="release") + _packages: list[Package] = sqlmodel.Relationship(back_populates="_release") + + @property + async def packages(self) -> list[Package]: + return await self.awaitable_attrs._packages # type: ignore + sboms: list[str] = sqlmodel.Field(default_factory=list, sa_column=sqlalchemy.Column(sqlalchemy.JSON)) # Many-to-one: A release can have one vote policy, a vote policy can be used by multiple releases diff --git a/atr/db/service.py b/atr/db/service.py index 16a5116..28a938f 100644 --- a/atr/db/service.py +++ b/atr/db/service.py @@ -25,26 +25,42 @@ from sqlalchemy.orm import selectinload from sqlalchemy.orm.attributes import InstrumentedAttribute from sqlmodel import select -from atr.db.models import PMC, ProductLine, Release, Task +from atr.db.models import PMC, ProductLine, Project, Release, Task from . import create_async_db_session -async def get_pmc_by_name(project_name: str, session: AsyncSession | None = None) -> PMC | None: - """Returns a PMC object by name.""" +async def get_pmcs(session: AsyncSession | None = None) -> Sequence[PMC]: + """Returns a sequence of all PMCs sorted by their name in ascending order.""" + async with create_async_db_session() if session is None else nullcontext(session) as db_session: + # Get all PMCs + statement = select(PMC).order_by(PMC.name) + pmcs = (await db_session.execute(statement)).scalars().all() + return pmcs + + +async def get_pmc_by_name(name: str, session: AsyncSession | None = None) -> PMC | None: + """Returns a PMC identified by its name.""" async with create_async_db_session() if session is None else nullcontext(session) as db_session: - statement = select(PMC).where(PMC.project_name == project_name) + statement = select(PMC).where(PMC.name == name) pmc = (await db_session.execute(statement)).scalar_one_or_none() return pmc -async def get_pmcs(session: AsyncSession | None = None) -> Sequence[PMC]: - """Returns a list of PMC objects.""" +async def get_projects(session: AsyncSession | None = None) -> Sequence[Project]: + """Returns a sequence of all Projects sorted by their name in ascending order.""" async with create_async_db_session() if session is None else nullcontext(session) as db_session: - # Get all PMCs and their latest releases - statement = select(PMC).order_by(PMC.project_name) - pmcs = (await db_session.execute(statement)).scalars().all() - return pmcs + statement = select(Project).order_by(Project.name) + projects = (await db_session.execute(statement)).scalars().all() + return projects + + +async def get_project_by_name(name: str, session: AsyncSession | None = None) -> Project | None: + """Returns a PMC identified by its name.""" + async with create_async_db_session() if session is None else nullcontext(session) as db_session: + statement = select(Project).where(Project.name == name) + project = (await db_session.execute(statement)).scalar_one_or_none() + return project async def get_release_by_key(storage_key: str) -> Release | None: diff --git a/atr/routes/candidate.py b/atr/routes/candidate.py index 0934c50..9f4b659 100644 --- a/atr/routes/candidate.py +++ b/atr/routes/candidate.py @@ -18,12 +18,12 @@ """candidate.py""" import datetime +import logging import secrets -from typing import cast +from typing import Final, cast from quart import Request, redirect, render_template, request, url_for -from sqlalchemy.orm import selectinload -from sqlalchemy.orm.attributes import InstrumentedAttribute +from sqlalchemy.sql.expression import ColumnElement from sqlmodel import select from werkzeug.wrappers.response import Response @@ -35,18 +35,20 @@ from asfquart.session import read as session_read from atr.db import create_async_db_session from atr.db.models import ( PMC, - Package, ProductLine, + Project, Release, ReleasePhase, ReleaseStage, - Task, ) from atr.routes import app_route, format_file_size, get_form +from atr.util import flatten_list if APP is ...: raise RuntimeError("APP is not set") +_LOGGER: Final = logging.getLogger(__name__) + def format_artifact_name(project_name: str, product_name: str, version: str, is_podling: bool = False) -> str: """Format an artifact name according to Apache naming conventions. @@ -82,19 +84,24 @@ async def release_add_post(session: ClientSession, request: Request) -> Response # Create the release record in the database async with create_async_db_session() as db_session: async with db_session.begin(): - statement = select(PMC).where(PMC.project_name == project_name) - pmc = (await db_session.execute(statement)).scalar_one_or_none() + statement = select(Project).where(Project.name == project_name) + project = (await db_session.execute(statement)).scalar_one_or_none() + if not project: + _LOGGER.error(f"Project not found for name {project_name}") + _LOGGER.debug(f"Available committees: {session.committees}") + _LOGGER.debug(f"Available projects: {session.projects}") + raise ASFQuartException("Project not found", errorcode=404) + + pmc = await project.pmc if not pmc: - APP.logger.error(f"PMC not found for project {project_name}") - APP.logger.debug(f"Available committees: {session.committees}") - APP.logger.debug(f"Available projects: {session.projects}") + _LOGGER.error(f"PMC not found for project with name {project_name}") raise ASFQuartException("PMC not found", errorcode=404) # Verify user is a PMC member or committer of the project # We use pmc.display_name, so this must come within the transaction - if project_name not in session.committees and project_name not in session.projects: + if pmc.name not in session.committees + session.projects: raise ASFQuartException( - f"You must be a PMC member or committer of {pmc.display_name} to submit a release candidate", + f"You must be a PMC member or committer of '{project.display_name}' to submit a release candidate", errorcode=403, ) @@ -104,13 +111,13 @@ async def release_add_post(session: ClientSession, request: Request) -> Response # Create or get existing product line statement = select(ProductLine).where( - ProductLine.pmc_id == pmc.id, ProductLine.product_name == product_name + ProductLine.project_id == project.id, ProductLine.product_name == product_name ) product_line = (await db_session.execute(statement)).scalar_one_or_none() if not product_line: # Create new product line if it doesn't exist - product_line = ProductLine(pmc_id=pmc.id, product_name=product_name, latest_version=version) + product_line = ProductLine(project_id=project.id, product_name=product_name, latest_version=version) db_session.add(product_line) # Flush to get the product_line.id await db_session.flush() @@ -120,7 +127,6 @@ async def release_add_post(session: ClientSession, request: Request) -> Response storage_key=storage_key, stage=ReleaseStage.CANDIDATE, phase=ReleasePhase.RELEASE_CANDIDATE, - pmc_id=pmc.id, product_line_id=product_line.id, version=version, created=datetime.datetime.now(datetime.UTC), @@ -148,18 +154,17 @@ async def root_candidate_create() -> Response | str: # Get PMC objects for all projects the user is a member of async with create_async_db_session() as db_session: - from sqlalchemy.sql.expression import ColumnElement - - project_list = session.committees + session.projects - project_name: ColumnElement[str] = cast(ColumnElement[str], PMC.project_name) - statement = select(PMC).where(project_name.in_(project_list)) + committee_list = session.committees + session.projects + pmc_name: ColumnElement[str] = cast(ColumnElement[str], PMC.name) + statement = select(PMC).where(pmc_name.in_(committee_list)) user_pmcs = (await db_session.execute(statement)).scalars().all() + user_projects = flatten_list([await pmc.projects for pmc in user_pmcs]) # For GET requests, show the form return await render_template( "candidate-create.html", asf_id=session.uid, - user_pmcs=user_pmcs, + user_projects=user_projects, ) @@ -178,26 +183,18 @@ async def root_candidate_review() -> str: # TODO: We don't actually record who uploaded the release candidate # We should probably add that information! # TODO: This duplicates code in root_package_add - release_pmc = selectinload(cast(InstrumentedAttribute[PMC], Release.pmc)) - release_packages = selectinload(cast(InstrumentedAttribute[list[Package]], Release.packages)) - package_tasks = release_packages.selectinload(cast(InstrumentedAttribute[list[Task]], Package.tasks)) - release_product_line = selectinload(cast(InstrumentedAttribute[ProductLine], Release.product_line)) + committee_list = session.committees + session.projects + pmc_name: ColumnElement[str] = cast(ColumnElement[str], PMC.name) + statement = ( select(Release) - .options(release_pmc, release_packages, package_tasks, release_product_line) + .join(ProductLine) + .join(Project) .join(PMC) + .where(pmc_name.in_(committee_list)) .where(Release.stage == ReleaseStage.CANDIDATE) ) - releases = (await db_session.execute(statement)).scalars().all() - - # Filter to only show releases for PMCs or PPMCs where the user is a member or committer - user_releases = [] - for r in releases: - if r.pmc is None: - continue - # For PPMCs the "members" are stored in the committers field - if session.uid in r.pmc.pmc_members or session.uid in r.pmc.committers: - user_releases.append(r) + user_releases = (await db_session.execute(statement)).scalars().all() # time.sleep(0.37) # await asyncio.sleep(0.73) diff --git a/atr/routes/keys.py b/atr/routes/keys.py index 5392dee..e9cacae 100644 --- a/atr/routes/keys.py +++ b/atr/routes/keys.py @@ -157,7 +157,7 @@ async def key_user_session_add( # Link key to selected PMCs for pmc_name in selected_pmcs: - pmc_statement = select(PMC).where(PMC.project_name == pmc_name) + pmc_statement = select(PMC).where(PMC.name == pmc_name) pmc = (await db_session.execute(pmc_statement)).scalar_one_or_none() if pmc and pmc.id: link = PMCKeyLink(pmc_id=pmc.id, key_fingerprint=key_record.fingerprint) @@ -190,9 +190,9 @@ async def root_keys_add() -> str: async with create_async_db_session() as db_session: from sqlalchemy.sql.expression import ColumnElement - project_list = session.committees + session.projects - project_name = cast(ColumnElement[str], PMC.project_name) - pmc_statement = select(PMC).where(project_name.in_(project_list)) + committee_list = session.committees + session.projects + pmc_name = cast(ColumnElement[str], PMC.name) + pmc_statement = select(PMC).where(pmc_name.in_(committee_list)) user_pmcs = (await db_session.execute(pmc_statement)).scalars().all() if request.method == "POST": diff --git a/atr/routes/project.py b/atr/routes/pmc.py similarity index 62% copy from atr/routes/project.py copy to atr/routes/pmc.py index cbd1991..867167d 100644 --- a/atr/routes/project.py +++ b/atr/routes/pmc.py @@ -26,19 +26,20 @@ from atr.db.service import get_pmc_by_name, get_pmcs from atr.routes import algorithms, app_route -@app_route("/projects") -async def root_project_directory() -> str: - """Main project directory page.""" - async with create_async_db_session() as session: - projects = await get_pmcs(session) - return await render_template("project-directory.html", projects=projects) - - -@app_route("/projects/<project_name>") -async def root_project_view(project_name: str) -> str: - async with create_async_db_session() as session: - project = await get_pmc_by_name(project_name, session=session) - if not project: +@app_route("/pmcs") +async def root_pmc_directory() -> str: + """Display the list of PMCs.""" + async with create_async_db_session() as db_session: + pmcs = await get_pmcs(db_session) + return await render_template("pmc-directory.html", pmcs=pmcs) + + +@app_route("/pmcs/<name>") +async def root_pmc_view(name: str) -> str: + """Display a specific PMC.""" + async with create_async_db_session() as db_session: + pmc = await get_pmc_by_name(name, db_session) + if not pmc: raise HTTPException(404) - return await render_template("project-view.html", project=project, algorithms=algorithms) + return await render_template("pmc-view.html", pmc=pmc, algorithms=algorithms) diff --git a/atr/routes/project.py b/atr/routes/project.py index cbd1991..a3bc97d 100644 --- a/atr/routes/project.py +++ b/atr/routes/project.py @@ -22,7 +22,7 @@ from http.client import HTTPException from quart import render_template from atr.db import create_async_db_session -from atr.db.service import get_pmc_by_name, get_pmcs +from atr.db.service import get_project_by_name, get_projects from atr.routes import algorithms, app_route @@ -30,14 +30,14 @@ from atr.routes import algorithms, app_route async def root_project_directory() -> str: """Main project directory page.""" async with create_async_db_session() as session: - projects = await get_pmcs(session) + projects = await get_projects(session) return await render_template("project-directory.html", projects=projects) -@app_route("/projects/<project_name>") -async def root_project_view(project_name: str) -> str: +@app_route("/projects/<name>") +async def root_project_view(name: str) -> str: async with create_async_db_session() as session: - project = await get_pmc_by_name(project_name, session=session) + project = await get_project_by_name(name, session) if not project: raise HTTPException(404) diff --git a/atr/routes/release.py b/atr/routes/release.py index de2659d..2778838 100644 --- a/atr/routes/release.py +++ b/atr/routes/release.py @@ -20,12 +20,9 @@ import logging import logging.handlers from pathlib import Path -from typing import cast from quart import flash, redirect, render_template, request, url_for from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload -from sqlalchemy.orm.attributes import InstrumentedAttribute from sqlmodel import select from werkzeug.wrappers.response import Response @@ -35,7 +32,6 @@ from asfquart.base import ASFQuartException from asfquart.session import read as session_read from atr.db import create_async_db_session from atr.db.models import ( - PMC, Release, Task, TaskStatus, @@ -56,8 +52,7 @@ async def release_delete_validate(db_session: AsyncSession, release_key: str, se # if Release.pmc is None: # raise FlashError("Release has no associated PMC") - rel_pmc = cast(InstrumentedAttribute[PMC], Release.pmc) - statement = select(Release).options(selectinload(rel_pmc)).where(Release.storage_key == release_key) + statement = select(Release).where(Release.storage_key == release_key) result = await db_session.execute(statement) release = result.scalar_one_or_none() @@ -65,19 +60,19 @@ async def release_delete_validate(db_session: AsyncSession, release_key: str, se raise FlashError("Release not found") # Check permissions - if release.pmc: - if (session_uid not in release.pmc.pmc_members) and (session_uid not in release.pmc.committers): + pmc = await release.pmc + if pmc: + if session_uid not in pmc.pmc_members + pmc.committers: raise FlashError("You don't have permission to delete this release") - return release async def release_files_delete(release: Release, uploads_path: Path) -> None: """Delete all files associated with a release.""" - if not release.packages: + if not await release.packages: return - for package in release.packages: + for package in await release.packages: await package_files_delete(package, uploads_path) diff --git a/atr/server.py b/atr/server.py index 6552a59..1b7adec 100644 --- a/atr/server.py +++ b/atr/server.py @@ -30,7 +30,7 @@ from werkzeug.routing import Rule import asfquart import asfquart.generics import asfquart.session -from asfquart.base import QuartApp +from asfquart.base import QuartApp, ASFQuartException from atr.blueprints import register_blueprints from atr.config import AppConfig, ConfigMode, get_config, get_config_mode from atr.db import create_database @@ -52,10 +52,11 @@ class ApiOnlyOpenAPIProvider(OpenAPIProvider): def register_routes(app: QuartApp) -> tuple[str, ...]: - from atr.routes import candidate, dev, docs, download, keys, package, project, release, root + from atr.routes import candidate, dev, docs, download, keys, package, pmc, project, release, root # Add a global error handler to show helpful error messages with tracebacks. @app.errorhandler(Exception) + @app.errorhandler(ASFQuartException) async def handle_any_exception(error: Exception) -> Any: import traceback @@ -76,6 +77,7 @@ def register_routes(app: QuartApp) -> tuple[str, ...]: download.__name__, keys.__name__, package.__name__, + pmc.__name__, project.__name__, release.__name__, root.__name__, diff --git a/atr/templates/candidate-create.html b/atr/templates/candidate-create.html index 5922607..a221cea 100644 --- a/atr/templates/candidate-create.html +++ b/atr/templates/candidate-create.html @@ -83,11 +83,11 @@ <td> <select id="project_name" name="project_name" required> <option value="">Select a project...</option> - {% for pmc in user_pmcs|sort(attribute='project_name') %} - <option value="{{ pmc.project_name }}">{{ pmc.display_name }}</option> + {% for project in user_projects|sort(attribute='name') %} + <option value="{{ project.name }}">{{ project.display_name }}</option> {% endfor %} </select> - {% if not user_pmcs %} + {% if not user_projects %} <p class="error-message">You must be a PMC member or committer to submit a release candidate.</p> {% endif %} </td> @@ -115,7 +115,7 @@ <tr> <td></td> <td> - <button type="submit" {% if not user_pmcs %}disabled{% endif %}>Create release</button> + <button type="submit" {% if not user_projects %}disabled{% endif %}>Create release</button> </td> </tr> </tbody> diff --git a/atr/templates/candidate-review.html b/atr/templates/candidate-review.html index d0d8fb4..0f44375 100644 --- a/atr/templates/candidate-review.html +++ b/atr/templates/candidate-review.html @@ -213,6 +213,7 @@ {% if releases %} {% for release in releases %} + {% set product_line = release.product_line %} <div class="candidate-header"> <h3>{{ release.pmc.display_name }}</h3> <div class="candidate-meta"> diff --git a/atr/templates/includes/sidebar.html b/atr/templates/includes/sidebar.html index 9c9cabc..17d7fff 100644 --- a/atr/templates/includes/sidebar.html +++ b/atr/templates/includes/sidebar.html @@ -34,6 +34,11 @@ <a href="{{ url_for('root') }}" {% if request.endpoint == 'root' %}class="active"{% endif %}>About</a> </li> + <li> + <i class="fa-solid fa-sitemap"></i> + <a href="{{ url_for('root_pmc_directory') }}" + {% if request.endpoint == 'root_pmc_directory' %}class="active"{% endif %}>Committees</a> + </li> <li> <i class="fa-solid fa-diagram-project"></i> <a href="{{ url_for('root_project_directory') }}" diff --git a/atr/templates/project-directory.html b/atr/templates/pmc-directory.html similarity index 81% copy from atr/templates/project-directory.html copy to atr/templates/pmc-directory.html index 3aae192..2fbde53 100644 --- a/atr/templates/project-directory.html +++ b/atr/templates/pmc-directory.html @@ -1,11 +1,11 @@ {% extends "layouts/base.html" %} {% block title %} - Project directory ~ ATR + PMC directory ~ ATR {% endblock title %} {% block description %} - List of all ASF projects and their latest releases. + List of all ASF committees. {% endblock description %} {% block head_extra %} @@ -68,8 +68,8 @@ {% endblock head_extra %} {% block content %} - <h1>Project directory</h1> - <p class="intro">Current ASF projects and their releases:</p> + <h1>PMC directory</h1> + <p class="intro">Current ASF committees:</p> <input type="text" id="pmc-filter" @@ -77,11 +77,10 @@ <button type="button" onclick="filter()">Filter</button> <div class="pmc-grid"> - {% for pmc in projects %} - <div class="pmc-card"> - <a href="{{ url_for('root_project_view', project_name=pmc.project_name) }}"> - <div class="pmc-name">{{ pmc.display_name }}</div> - </a> + {% for pmc in pmcs %} + <div class="pmc-card" + onclick="location.href='{{ url_for("root_pmc_view", name=pmc.name) }}';"> + <div class="pmc-name">{{ pmc.display_name }}</div> <div class="pmc-stats"> <div class="stat-item"> <span class="stat-label">PMC Members</span> @@ -92,8 +91,8 @@ <span class="stat-value">{{ pmc.committers|length }}</span> </div> <div class="stat-item"> - <span class="stat-label">Release Managers</span> - <span class="stat-value">{{ pmc.release_managers|length }}</span> + <span class="stat-label">Projects</span> + <span class="stat-value">{{ pmc.projects|length }}</span> </div> </div> </div> @@ -110,7 +109,7 @@ for (let card of cards) { const nameElement = card.getElementsByClassName("pmc-name"); const name = nameElement[0].innerHTML; - card.hidden = !!(pmcFilter && !name.startsWith(pmcFilter)); + card.hidden = !pmcFilter || name.search(new RegExp(pmcFilter, "i")) < 0; } } </script> diff --git a/atr/templates/pmc-view.html b/atr/templates/pmc-view.html new file mode 100644 index 0000000..867d05e --- /dev/null +++ b/atr/templates/pmc-view.html @@ -0,0 +1,175 @@ +{% extends "layouts/base.html" %} + +{% block title %} + Project ~ ATR +{% endblock title %} + +{% block description %} + Release candidates to which you have access. +{% endblock description %} + +{% block stylesheets %} + {{ super() }} + <style> + .card-header { + border: 1px solid #ddd; + border-radius: 4px; + padding: 1rem; + margin-bottom: 1rem; + background-color: #f8f8f8; + } + + .card-header h3 { + margin: 0 0 0.5rem 0; + } + + .card-meta { + color: #666; + font-size: 0.9em; + display: flex; + flex-wrap: wrap; + gap: 1rem; + margin-bottom: .5rem; + padding-bottom: 1rem; + border-bottom: 1px solid #ddd; + } + + .card-meta-item::after { + content: "•"; + margin-left: 1rem; + color: #ccc; + } + + .card-meta-item:last-child::after { + content: none; + } + + .card-body { + display: flex; + gap: 1rem; + align-items: center; + } + + .keys-grid { + display: grid; + gap: 1.5rem; + } + + .key-card { + background: white; + border: 1px solid #d1d2d3; + border-radius: 4px; + overflow: hidden; + padding: 1rem; + } + + .key-card table { + margin: 0; + } + + .key-card td { + word-break: break-all; + } + + .key-card h3 { + margin-top: 0; + margin-bottom: 1rem; + } + </style> +{% endblock stylesheets %} + +{% block content %} + <h1>{{ pmc.display_name | capitalize }}</h1> + + <div class="card-header"> + <h3>PMC members</h3> + <div class="card-meta"></div> + <div class="card-body"> + <p> + {% for user in pmc.pmc_members %}{{ user }},{% endfor %} + </p> + </div> + </div> + + <div class="card-header"> + <h3>Committers</h3> + <div class="card-meta"></div> + <div class="card-body"> + <p> + {% for user in pmc.committers %}{{ user }},{% endfor %} + </p> + </div> + </div> + + <div class="card-header"> + <h3>Signing Keys</h3> + <div class="card-meta"></div> + <div class="keys-grid"> + {% for key in pmc.public_signing_keys %} + <div class="key-card"> + <table> + <tbody> + <tr> + <th>Fingerprint</th> + <td>{{ key.fingerprint }}</td> + </tr> + <tr> + <th>Key Type</th> + <td>{{ algorithms[key.algorithm] }} ({{ key.length }} bits)</td> + </tr> + <tr> + <th>Created</th> + <td>{{ key.created.strftime("%Y-%m-%d %H:%M:%S") }}</td> + </tr> + <tr> + <th>Expires</th> + <td> + {% if key.expires %} + {% set days_until_expiry = (key.expires - now).days %} + {% if days_until_expiry < 0 %} + <span class="expiry-error"> + {{ key.expires.strftime("%Y-%m-%d %H:%M:%S") }} + <span class="notice-badge error">Expired</span> + </span> + {% elif days_until_expiry <= 30 %} + <span class="expiry-warning"> + {{ key.expires.strftime("%Y-%m-%d %H:%M:%S") }} + <span class="notice-badge warning">Expires in {{ days_until_expiry }} days</span> + </span> + {% else %} + {{ key.expires.strftime("%Y-%m-%d %H:%M:%S") }} + {% endif %} + {% else %} + Never + {% endif %} + </td> + </tr> + <tr> + <th>User ID</th> + <td>{{ key.declared_uid or 'Not specified' }}</td> + </tr> + </tbody> + </table> + </div> + {% endfor %} + </div> + </div> + + <div class="card-header"> + <h3>Voting Policy</h3> + <div class="card-meta"></div> + <div class="card-body"></div> + </div> + + <div class="card-header"> + <h3>Projects</h3> + <div class="card-meta"></div> + <div class="card-body"></div> + {% for project in pmc.projects %}<div class="key-card">{{ project.display_name }}</div>{% endfor %} + </div> + +{% endblock content %} + +{% block javascripts %} + {{ super() }} +{% endblock javascripts %} diff --git a/atr/templates/project-directory.html b/atr/templates/project-directory.html index 3aae192..aca5812 100644 --- a/atr/templates/project-directory.html +++ b/atr/templates/project-directory.html @@ -10,14 +10,14 @@ {% block head_extra %} <style> - .pmc-grid { + .project-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 1.5rem; margin: 2rem 0; } - .pmc-card { + .project-card { background: white; border: 1px solid #e0e0e0; border-radius: 8px; @@ -26,19 +26,19 @@ transition: transform 0.2s ease, box-shadow 0.2s ease; } - .pmc-card:hover { + .project-card:hover { transform: translateY(-2px); box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1); } - .pmc-name { + .project-name { font-size: 1.5rem; font-weight: 500; color: #333; margin-bottom: 1rem; } - .pmc-stats { + .project-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 1rem; @@ -72,30 +72,16 @@ <p class="intro">Current ASF projects and their releases:</p> <input type="text" - id="pmc-filter" + id="project-filter" onkeydown="if (event.key == 'Enter'){ filter() }" /> <button type="button" onclick="filter()">Filter</button> - <div class="pmc-grid"> - {% for pmc in projects %} - <div class="pmc-card"> - <a href="{{ url_for('root_project_view', project_name=pmc.project_name) }}"> - <div class="pmc-name">{{ pmc.display_name }}</div> - </a> - <div class="pmc-stats"> - <div class="stat-item"> - <span class="stat-label">PMC Members</span> - <span class="stat-value">{{ pmc.pmc_members|length }}</span> - </div> - <div class="stat-item"> - <span class="stat-label">Committers</span> - <span class="stat-value">{{ pmc.committers|length }}</span> - </div> - <div class="stat-item"> - <span class="stat-label">Release Managers</span> - <span class="stat-value">{{ pmc.release_managers|length }}</span> - </div> - </div> + <div class="project-grid"> + {% for project in projects %} + <div class="project-card" + onclick="location.href='{{ url_for("root_project_view", name=project.name) }}';"> + <div class="project-name">{{ project.display_name }}</div> + <div class="project-stats"></div> </div> {% endfor %} </div> @@ -105,12 +91,12 @@ {{ super() }} <script> function filter() { - const pmcFilter = document.getElementById("pmc-filter").value; - const cards = document.getElementsByClassName("pmc-card"); + const projectFilter = document.getElementById("project-filter").value; + const cards = document.getElementsByClassName("project-card"); for (let card of cards) { - const nameElement = card.getElementsByClassName("pmc-name"); + const nameElement = card.getElementsByClassName("project-name"); const name = nameElement[0].innerHTML; - card.hidden = !!(pmcFilter && !name.startsWith(pmcFilter)); + card.hidden = !projectFilter || name.search(new RegExp(projectFilter, "i")) < 0; } } </script> diff --git a/atr/util.py b/atr/util.py index a49819b..230e5ce 100644 --- a/atr/util.py +++ b/atr/util.py @@ -16,16 +16,19 @@ # under the License. import hashlib +import itertools from collections.abc import Mapping from dataclasses import dataclass from functools import cache from pathlib import Path -from typing import Annotated, Any +from typing import Annotated, Any, TypeVar import aiofiles from pydantic import GetCoreSchemaHandler, TypeAdapter, create_model from pydantic_core import CoreSchema, core_schema +T = TypeVar("T") + @cache def get_admin_users() -> set[str]: @@ -125,3 +128,7 @@ class DictToList: _get_dict_to_list_validator(adapter, self.key), handler(source_type), ) + + +def flatten_list(list_of_lists: list[list[T]]) -> list[T]: + return list(itertools.chain.from_iterable(list_of_lists)) diff --git a/docs/conventions.html b/docs/conventions.html index 68df1c1..ac697b1 100644 --- a/docs/conventions.html +++ b/docs/conventions.html @@ -18,11 +18,8 @@ <h3>Use the <code>Final</code> type with all constants</h3> <p>This pattern must be followed for top level constants, and should be followed for function and method level constants too. The longer the function, the more important the use of <code>Final</code>.</p> <h3>Prefix global variables with <code>global_</code></h3> -<p>TBD: imho global variables are already implicitly defined by the rules above, uppercase name and no underscore as prefix so no need to add an additional <code>global_</code> prefix, that would just be redundant. -Any variable at module level that is written in uppercase and does not have an underscore is global by definition.</p> <p>Top level variables should be avoided. When their use is necessary, prefix them with <code>global_</code>, using lowercase letters, to ensure clear identification of their scope. Use an underscore prefix too, <code>_global_</code>, when the variable is private.</p> <h3>Import modules as their least significant name part</h3> -<p>TBD: this rule should have some exceptions imho, e.g. <code>typing</code>. They are quite commonly known and used a lot, like <code>typing.Any</code>.</p> <p>Import modules using their least significant name component:</p> <pre><code class="language-python"># Preferred import a.b.c as c diff --git a/docs/conventions.md b/docs/conventions.md index 2b2d69e..61ac911 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -32,15 +32,10 @@ This pattern must be followed for top level constants, and should be followed fo ### Prefix global variables with `global_` -TBD: imho global variables are already implicitly defined by the rules above, uppercase name and no underscore as prefix so no need to add an additional `global_` prefix, that would just be redundant. -Any variable at module level that is written in uppercase and does not have an underscore is global by definition. - Top level variables should be avoided. When their use is necessary, prefix them with `global_`, using lowercase letters, to ensure clear identification of their scope. Use an underscore prefix too, `_global_`, when the variable is private. ### Import modules as their least significant name part -TBD: this rule should have some exceptions imho, e.g. `typing`. They are quite commonly known and used a lot, like `typing.Any`. - Import modules using their least significant name component: ```python --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
