This is an automated email from the ASF dual-hosted git repository. arm pushed a commit to branch gha-distributions in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit cabce79158ee5a551c5a7a032239036b0da9534e Author: Alastair McFarlane <[email protected]> AuthorDate: Mon Jan 12 14:24:37 2026 +0000 Rename GHA task to be more specific, and work in extra required parameters. UI work to trigger distributions and endpoint to register an SSH key --- atr/api/__init__.py | 30 ++++++++- atr/db/interaction.py | 18 +++++ atr/form.py | 4 ++ atr/get/distribution.py | 85 ++++++++++++++++++----- atr/get/finish.py | 126 ++++++++++++++++++++++++++++------- atr/models/api.py | 15 +++++ atr/models/results.py | 6 +- atr/models/sql.py | 10 ++- atr/post/distribution.py | 107 ++++++++++++++++++++++++++--- atr/server.py | 3 +- atr/shared/distribution.py | 13 ++-- atr/storage/writers/distributions.py | 50 +++++++++++++- atr/tasks/__init__.py | 2 +- atr/tasks/gha.py | 96 +++++++++++++++++++++----- atr/templates/check-selected.html | 12 +++- 15 files changed, 492 insertions(+), 85 deletions(-) diff --git a/atr/api/__init__.py b/atr/api/__init__.py index 06c932e..e543b43 100644 --- a/atr/api/__init__.py +++ b/atr/api/__init__.py @@ -251,6 +251,32 @@ async def committees_list() -> DictResponse: ).model_dump(), 200 [email protected]("/distribute/ssh/register", methods=["POST"]) +@quart_schema.validate_request(models.api.DistributeSshRegisterArgs) +async def distribute_ssh_register(data: models.api.DistributeSshRegisterArgs) -> DictResponse: + """ + Register an SSH key sent with a corroborating Trusted Publisher JWT, + validating the requested version is in the correct phase. + """ + payload, asf_uid, project = await interaction.trusted_jwt_for_version( + data.publisher, data.jwt, interaction.TrustedProjectPhase(data.phase), data.version + ) + async with storage.write_as_committee_member(util.unwrap(project.committee).name, asf_uid) as wacm: + fingerprint, expires = await wacm.ssh.add_workflow_key( + payload["actor"], + payload["actor_id"], + project.name, + data.ssh_key, + ) + + return models.api.DistributeSshRegisterResults( + endpoint="/distribute/ssh/register", + fingerprint=fingerprint, + project=project.name, + expires=expires, + ).model_dump(), 200 + + @api.route("/distribution/record", methods=["POST"]) @jwtoken.require @quart_schema.security_scheme([{"BearerAuth": []}]) @@ -279,7 +305,7 @@ async def distribution_record(data: models.api.DistributionRecordArgs) -> DictRe async with storage.write(asf_uid) as write: wacm = write.as_committee_member(release.committee.name) await wacm.distributions.record_from_data( - release, + release.name, data.staging, dd, ) @@ -656,7 +682,7 @@ async def publisher_distribution_record(data: models.api.PublisherDistributionRe async with storage.write(asf_uid) as write: wacm = write.as_committee_member(release.committee.name) await wacm.distributions.record_from_data( - release, + release.name, data.staging, dd, ) diff --git a/atr/db/interaction.py b/atr/db/interaction.py index f8026f8..1bd097c 100644 --- a/atr/db/interaction.py +++ b/atr/db/interaction.py @@ -179,6 +179,24 @@ async def trusted_jwt(publisher: str, jwt: str, phase: TrustedProjectPhase) -> t return payload, asf_uid, project +async def trusted_jwt_for_version( + publisher: str, jwt: str, phase: TrustedProjectPhase, version_name: str +) -> tuple[dict[str, Any], str, sql.Project]: + payload, asf_uid, project = await trusted_jwt(publisher, jwt, phase) + async with db.session() as db_data: + release = await db_data.release(project_name=project.name, version=version_name).get() + if not release: + raise InteractionError(f"Release {version} does not exist in project {project.name}") + if phase == TrustedProjectPhase.COMPOSE and release.phase != sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT: + raise InteractionError(f"Release {version} is not in compose phase") + if phase == TrustedProjectPhase.VOTE and release.phase != sql.ReleasePhase.RELEASE_CANDIDATE: + raise InteractionError(f"Release {version} is not in vote phase") + if phase == TrustedProjectPhase.FINISH and release.phase != sql.ReleasePhase.RELEASE_PREVIEW: + raise InteractionError(f"Release {version} is not in finish phase") + + return payload, asf_uid, project + + async def has_failing_checks(release: sql.Release, revision_number: str, caller_data: db.Session | None = None) -> bool: async with db.ensure_session(caller_data) as data: query = ( diff --git a/atr/form.py b/atr/form.py index 6f40775..909b0e6 100644 --- a/atr/form.py +++ b/atr/form.py @@ -517,6 +517,10 @@ Email = pydantic.EmailStr class Enum[EnumType: enum.Enum]: + # These exist for type checkers - at runtime, the actual type is the enum + name: str + value: str | int + @staticmethod def __class_getitem__(enum_class: type[EnumType]): def validator(v: Any) -> EnumType: diff --git a/atr/get/distribution.py b/atr/get/distribution.py index 4f166a9..9a284d7 100644 --- a/atr/get/distribution.py +++ b/atr/get/distribution.py @@ -29,30 +29,30 @@ import atr.util as util import atr.web as web [email protected]("/distributions/list/<project>/<version>") -async def list_get(session: web.Committer, project: str, version: str) -> str: [email protected]("/distributions/list/<project_name>/<version_name>") +async def list_get(session: web.Committer, project_name: str, version_name: str) -> str: async with db.session() as data: distributions = await data.distribution( - release_name=sql.release_name(project, version), + release_name=sql.release_name(project_name, version_name), ).all() block = htm.Block() - release = await shared.distribution.release_validated(project, version, staging=None) + release = await shared.distribution.release_validated(project_name, version_name, staging=None) staging = release.phase == sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT - render.html_nav_phase(block, project, version, staging) + render.html_nav_phase(block, project_name, version_name, staging) record_a_distribution = htm.a( ".btn.btn-primary", href=util.as_url( - stage if staging else record, - project=project, - version=version, + stage_record if staging else record, + project=project_name, + version=version_name, ), )["Record a distribution"] # Distribution list for project-version - block.h1["Distribution list for ", htm.em[f"{project}-{version}"]] + block.h1["Distribution list for ", htm.em[f"{project_name}-{version_name}"]] if not distributions: block.p["No distributions found."] block.p[record_a_distribution] @@ -89,7 +89,7 @@ async def list_get(session: web.Committer, project: str, version: str) -> str: delete_form = form.render( model_cls=shared.distribution.DeleteForm, - action=util.as_url(post.distribution.delete, project=project, version=version), + action=util.as_url(post.distribution.delete, project=project_name, version=version_name), form_classes=".d-inline-block.m-0", submit_classes="btn-danger btn-sm", submit_label="Delete", @@ -105,20 +105,71 @@ async def list_get(session: web.Committer, project: str, version: str) -> str: ) block.append(htm.div(".mb-3")[delete_form]) - title = f"Distribution list for {project} {version}" + title = f"Distribution list for {project_name} {version_name}" return await template.blank(title, content=block.collect()) [email protected]("/distribution/automate/<project>/<version>") +async def automate(session: web.Committer, project: str, version: str) -> str: + return await _automate_form_page(project, version, staging=False) + + [email protected]("/distribution/stage/automate/<project>/<version>") +async def stage_automate(session: web.Committer, project: str, version: str) -> str: + return await _automate_form_page(project, version, staging=True) + + @get.committer("/distribution/record/<project>/<version>") async def record(session: web.Committer, project: str, version: str) -> str: return await _record_form_page(project, version, staging=False) [email protected]("/distribution/stage/<project>/<version>") -async def stage(session: web.Committer, project: str, version: str) -> str: [email protected]("/distribution/stage/record/<project>/<version>") +async def stage_record(session: web.Committer, project: str, version: str) -> str: return await _record_form_page(project, version, staging=True) +async def _automate_form_page(project: str, version: str, staging: bool) -> str: + """Helper to render the distribution automation form page.""" + await shared.distribution.release_validated(project, version, staging=staging) + + block = htm.Block() + render.html_nav_phase(block, project, version, staging=staging) + + title = "Create a staging distribution" if staging else "Create a distribution" + block.h1[title] + + block.p[ + "Create a distribution of ", + htm.strong[f"{project}-{version}"], + " using the form below.", + ] + block.p[ + "You can also ", + htm.a(href=util.as_url(list_get, project_name=project, version_name=version))["view the distribution list"], + ".", + ] + + # Determine the action based on staging + action = ( + util.as_url(post.distribution.stage_automate_selected, project=project, version=version) + if staging + else util.as_url(post.distribution.automate_selected, project=project, version=version) + ) + + # TODO: Reuse the same form for now - maybe we can combine this and the function below adding an automate=True arg + # Render the distribution form + form_html = form.render( + model_cls=shared.distribution.DistributeForm, + submit_label="Distribute", + action=action, + defaults={"package": project, "version": version}, + ) + block.append(form_html) + + return await template.blank(title, content=block.collect()) + + async def _record_form_page(project: str, version: str, staging: bool) -> str: """Helper to render the distribution recording form page.""" await shared.distribution.release_validated(project, version, staging=staging) @@ -126,23 +177,23 @@ async def _record_form_page(project: str, version: str, staging: bool) -> str: block = htm.Block() render.html_nav_phase(block, project, version, staging=staging) - title = "Record a staging distribution" if staging else "Record a manual distribution" + title = "Record a manual staging distribution" if staging else "Record a manual distribution" block.h1[title] block.p[ - "Record a distribution of ", + "Record a manual distribution of ", htm.strong[f"{project}-{version}"], " using the form below.", ] block.p[ "You can also ", - htm.a(href=util.as_url(list_get, project=project, version=version))["view the distribution list"], + htm.a(href=util.as_url(list_get, project_name=project, version_name=version))["view the distribution list"], ".", ] # Determine the action based on staging action = ( - util.as_url(post.distribution.stage_selected, project=project, version=version) + util.as_url(post.distribution.stage_record_selected, project=project, version=version) if staging else util.as_url(post.distribution.record_selected, project=project, version=version) ) diff --git a/atr/get/finish.py b/atr/get/finish.py index 921bf83..3526853 100644 --- a/atr/get/finish.py +++ b/atr/get/finish.py @@ -19,6 +19,7 @@ import dataclasses import json import pathlib +from collections.abc import Sequence import aiofiles.os import asfquart.base as base @@ -42,6 +43,7 @@ import atr.mapping as mapping import atr.models.sql as sql import atr.render as render import atr.shared as shared +import atr.tasks.gha as gha import atr.template as template import atr.util as util import atr.web as web @@ -60,13 +62,9 @@ async def selected( ) -> tuple[web.QuartResponse, int] | web.WerkzeugResponse | str: """Finish a release preview.""" try: - ( - release, - source_files_rel, - target_dirs, - deletable_dirs, - rc_analysis, - ) = await _get_page_data(project_name, version_name) + (release, source_files_rel, target_dirs, deletable_dirs, rc_analysis, tasks) = await _get_page_data( + project_name, version_name + ) except ValueError: async with db.session() as data: release_fallback = await data.release( @@ -89,6 +87,7 @@ async def selected( target_dirs=target_dirs, deletable_dirs=deletable_dirs, rc_analysis=rc_analysis, + distribution_tasks=tasks, ) @@ -134,14 +133,31 @@ async def _deletable_choices( async def _get_page_data( project_name: str, version_name: str -) -> tuple[sql.Release, list[pathlib.Path], set[pathlib.Path], list[tuple[str, str]], RCTagAnalysisResult]: +) -> tuple[ + sql.Release, list[pathlib.Path], set[pathlib.Path], list[tuple[str, str]], RCTagAnalysisResult, Sequence[sql.Task] +]: """Get all the data needed to render the finish page.""" async with db.session() as data: + via = sql.validate_instrumented_attribute release = await data.release( project_name=project_name, version=version_name, _committee=True, ).demand(base.ASFQuartException("Release does not exist", errorcode=404)) + tasks = [ + t + for t in ( + await data.task( + project_name=project_name, + version_name=version_name, + revision_number=release.latest_revision_number, + task_type=sql.TaskType.DISTRIBUTION_WORKFLOW, + ) + .order_by(sql.sqlmodel.desc(via(sql.Task.started))) + .all() + ) + if t.status in [sql.TaskStatus.QUEUED, sql.TaskStatus.ACTIVE, sql.TaskStatus.FAILED] + ] if release.phase != sql.ReleasePhase.RELEASE_PREVIEW: raise ValueError("Release is not in preview phase") @@ -151,7 +167,7 @@ async def _get_page_data( deletable_dirs = await _deletable_choices(latest_revision_dir, target_dirs) rc_analysis_result = await _analyse_rc_tags(latest_revision_dir) - return release, source_files_rel, target_dirs, deletable_dirs, rc_analysis_result + return release, source_files_rel, target_dirs, deletable_dirs, rc_analysis_result, tasks def _render_delete_directory_form(deletable_dirs: list[tuple[str, str]]) -> htm.Element: @@ -247,6 +263,7 @@ async def _render_page( target_dirs: set, deletable_dirs: list[tuple[str, str]], rc_analysis: RCTagAnalysisResult, + distribution_tasks: Sequence[sql.Task], ) -> str: """Render the finish page using htm.py.""" page = htm.Block() @@ -275,8 +292,11 @@ async def _render_page( "such as Maven Central, PyPI, or Docker Hub." ] - # TODO alert - page.append(_render_todo_alert(release)) + if len(distribution_tasks) > 0: + page.append(_render_distribution_tasks(release, distribution_tasks)) + + page.append(_render_dist_warning()) + page.append(_render_distribution_buttons(release)) # Move files section page.append(_render_move_section(max_files_to_show=10)) @@ -401,7 +421,7 @@ def _render_release_card(release: sql.Release) -> htm.Element: version_name=release.version, ), )[ - htpy.i(".bi.bi-download"), + htm.icon("download"), " Download all files", ], htm.a( @@ -413,7 +433,7 @@ def _render_release_card(release: sql.Release) -> htm.Element: version_name=release.version, ), )[ - htpy.i(".bi.bi-archive"), + htm.icon("archive"), " Show files", ], htm.a( @@ -425,7 +445,7 @@ def _render_release_card(release: sql.Release) -> htm.Element: version_name=release.version, ), )[ - htpy.i(".bi.bi-clock-history"), + htm.icon("clock-history"), " Show revisions", ], htm.a( @@ -437,7 +457,7 @@ def _render_release_card(release: sql.Release) -> htm.Element: version_name=release.version, ), )[ - htpy.i(".bi.bi-check-circle"), + htm.icon("check-circle"), " Announce and distribute", ], ], @@ -446,25 +466,83 @@ def _render_release_card(release: sql.Release) -> htm.Element: return card -def _render_todo_alert(release: sql.Release) -> htm.Element: - """Render the TODO alert about distribution tools.""" - return htm.div(".alert.alert-warning.mb-4", role="alert")[ - htm.p(".fw-semibold.mb-1")["TODO"], +def _render_distribution_buttons(release: sql.Release) -> htm.Element: + """Render the distribution tool buttons.""" + return htm.div()[ htm.p(".mb-1")[ - "We plan to add tools to help release managers to distribute release artifacts on distribution networks. " - "Currently you must do this manually. Once you've distributed your release artifacts, you can ", htm.a( + ".btn.btn-primary.me-2", + href=util.as_url( + distribution.automate, + project=release.project.name, + version=release.version, + ), + )["Distribute"], + htm.a( + ".btn.btn-secondary.me-2", href=util.as_url( distribution.record, project=release.project.name, version=release.version, - ) - )["record them on the ATR"], - ".", + ), + )["Record a manual distribution"], + ], + ] + + +def _render_distribution_tasks(release: sql.Release, tasks: Sequence[sql.Task]) -> htm.Element: + """Render current and failed distribution tasks.""" + failed_tasks = [t for t in tasks if t.status == sql.TaskStatus.FAILED] + in_progress_tasks = [t for t in tasks if t.status in [sql.TaskStatus.QUEUED, sql.TaskStatus.ACTIVE]] + + block = htm.Block() + + if len(failed_tasks) > 0: + summary = f"{len(failed_tasks)} distribution{'s' if len(failed_tasks) > 1 else ''} failed for this release" + block.append( + htm.div(".alert.alert-danger.mb-3")[ + htm.h3["Failed distributions"], + htm.details[ + htm.summary[summary], + htm.div[*[_render_task(f) for f in failed_tasks]], + ], + ] + ) + if len(in_progress_tasks) > 0: + block.append( + htm.div(".alert.alert-info.mb-3")[ + htm.h3["In-progress distributions"], + htm.p["One or more automatic distributions are still in-progress:"], + *[_render_task(f) for f in in_progress_tasks], + htm.button( + ".btn.btn-success.me-2", + {"onclick": "window.location.reload()"}, + )["Refresh"], + ] + ) + return block.collect() + + +def _render_dist_warning() -> htm.Element: + """Render the alert about distribution tools.""" + return htm.div(".alert.alert-warning.mb-4", role="alert")[ + htm.p(".fw-semibold.mb-1")["NOTE:"], + htm.p(".mb-1")[ + "Tools to distribute automatically are still being developed, " + "you must do this manually at present. Please use the manual record function below to do so.", ], ] +def _render_task(task: sql.Task) -> htm.Element: + """Render a distribution task's details.""" + args: gha.DistributionWorkflow = gha.DistributionWorkflow.model_validate(task.task_args) + status = task.status.value + return htm.p[ + f"{args.platform} ({args.package} {args.version}): {task.error if task.error else status.capitalize()}" + ] + + async def _sources_and_targets(latest_revision_dir: pathlib.Path) -> tuple[list[pathlib.Path], set[pathlib.Path]]: source_items_rel: list[pathlib.Path] = [] target_dirs: set[pathlib.Path] = {pathlib.Path(".")} diff --git a/atr/models/api.py b/atr/models/api.py index d06c5d3..6df6a14 100644 --- a/atr/models/api.py +++ b/atr/models/api.py @@ -67,6 +67,21 @@ class CommitteesListResults(schema.Strict): committees: Sequence[sql.Committee] +class DistributeSshRegisterArgs(schema.Strict): + publisher: str = schema.example("user") + jwt: str = schema.example("eyJhbGciOiJIUzI1[...]mMjLiuyu5CSpyHI=") + ssh_key: str = schema.example("ssh-ed25519 AAAAC3NzaC1lZDI1NTEgH5C9okWi0dh25AAAAIOMqqnkVzrm0SdG6UOoqKLsabl9GKJl") + phase: str = schema.Field(strict=False, default="compose", json_schema_extra={"examples": ["compose", "finish"]}) + version: str = schema.example("0.0.1") + + +class DistributeSshRegisterResults(schema.Strict): + endpoint: Literal["/distribute/ssh/register"] = schema.alias("endpoint") + fingerprint: str = schema.example("SHA256:0123456789abcdef0123456789abcdef01234567") + project: str = schema.example("example") + expires: int = schema.example(1713547200) + + class DistributionRecordArgs(schema.Strict): project: str = schema.example("example") version: str = schema.example("0.0.1") diff --git a/atr/models/results.py b/atr/models/results.py index 713291d..5dc1ce2 100644 --- a/atr/models/results.py +++ b/atr/models/results.py @@ -24,10 +24,10 @@ import atr.sbom.models.osv as osv from . import schema -class GithubActionsWorkflow(schema.Strict): +class DistributionWorkflow(schema.Strict): """Result of the task to run a Github workflow.""" - kind: Literal["github_actions_workflow"] = schema.Field(alias="kind") + kind: Literal["distribution_workflow"] = schema.Field(alias="kind") name: str = schema.description("The name of the action being performed") run_id: int = schema.description("The ID of the workflow run") url: str = schema.description("The URL of the workflow run") @@ -193,7 +193,7 @@ class MetadataUpdate(schema.Strict): Results = Annotated[ - GithubActionsWorkflow + DistributionWorkflow | HashingCheck | MessageSend | MetadataUpdate diff --git a/atr/models/sql.py b/atr/models/sql.py index 8957123..8938aeb 100644 --- a/atr/models/sql.py +++ b/atr/models/sql.py @@ -53,6 +53,7 @@ sqlmodel.SQLModel.metadata = sqlalchemy.MetaData( @dataclasses.dataclass(frozen=True) class DistributionPlatformValue: name: str + gh_slug: str template_url: str template_staging_url: str | None = None requires_owner_namespace: bool = False @@ -95,12 +96,14 @@ class CheckResultStatusIgnore(str, enum.Enum): class DistributionPlatform(enum.Enum): ARTIFACT_HUB = DistributionPlatformValue( name="Artifact Hub", + gh_slug="artifacthub", template_url="https://artifacthub.io/api/v1/packages/helm/{owner_namespace}/{package}/{version}";, template_staging_url="https://staging.artifacthub.io/api/v1/packages/helm/{owner_namespace}/{package}/{version}";, requires_owner_namespace=True, ) DOCKER_HUB = DistributionPlatformValue( name="Docker Hub", + gh_slug="dockerhub", template_url="https://hub.docker.com/v2/namespaces/{owner_namespace}/repositories/{package}/tags/{version}";, # TODO: Need to use staging tags? # template_staging_url="https://hub.docker.com/v2/namespaces/{owner_namespace}/repositories/{package}/tags/{version}";, @@ -108,6 +111,7 @@ class DistributionPlatform(enum.Enum): ) # GITHUB = DistributionPlatformValue( # name="GitHub", + # gh_slug="github", # template_url="https://api.github.com/repos/{owner_namespace}/{package}/releases/tags/v{version}";, # # Combine with {"prerelease": true} # template_staging_url="https://api.github.com/repos/{owner_namespace}/{package}/releases";, @@ -115,6 +119,7 @@ class DistributionPlatform(enum.Enum): # ) MAVEN = DistributionPlatformValue( name="Maven Central", + gh_slug="maven", template_url="https://search.maven.org/solrsearch/select?q=g:{owner_namespace}+AND+a:{package}+AND+v:{version}&core=gav&rows=20&wt=json";, # Java ASF projects use staging URLs along the lines of # https://repository.apache.org/content/repositories/orgapachePROJECT-NNNN/ @@ -123,17 +128,20 @@ class DistributionPlatform(enum.Enum): ) NPM = DistributionPlatformValue( name="npm", + gh_slug="npm", # TODO: Need to parse dist-tags template_url="https://registry.npmjs.org/{package}";, ) NPM_SCOPED = DistributionPlatformValue( name="npm (scoped)", + gh_slug="npm", # TODO: Need to parse dist-tags template_url="https://registry.npmjs.org/@{owner_namespace}/{package}";, requires_owner_namespace=True, ) PYPI = DistributionPlatformValue( name="PyPI", + gh_slug="pypi", template_url="https://pypi.org/pypi/{package}/{version}/json";, template_staging_url="https://test.pypi.org/pypi/{package}/{version}/json";, ) @@ -179,7 +187,7 @@ class TaskStatus(str, enum.Enum): class TaskType(str, enum.Enum): - GITHUB_ACTION_WORKFLOW = "github_action_workflow" + DISTRIBUTION_WORKFLOW = "distribution_workflow" HASHING_CHECK = "hashing_check" KEYS_IMPORT_FILE = "keys_import_file" LICENSE_FILES = "license_files" diff --git a/atr/post/distribution.py b/atr/post/distribution.py index 15fa395..c5f3428 100644 --- a/atr/post/distribution.py +++ b/atr/post/distribution.py @@ -25,6 +25,9 @@ import atr.shared as shared import atr.storage as storage import atr.web as web +_AUTOMATED_PLATFORMS = [shared.distribution.DistributionPlatform.MAVEN] +_AUTOMATED_PLATFORMS_STAGE = [shared.distribution.DistributionPlatform.MAVEN] + @post.committer("/distribution/delete/<project>/<version>") @post.form(shared.distribution.DeleteForm) @@ -53,12 +56,84 @@ async def delete( ) return await session.redirect( get.distribution.list_get, - project=project, - version=version, + project_name=project, + version_name=version, success="Distribution deleted", ) +async def automate_form_process_page( + session: web.Committer, + form_data: shared.distribution.DistributeForm, + project: str, + version: str, + /, + staging: bool = False, +) -> web.WerkzeugResponse: + allowed_platforms = _AUTOMATED_PLATFORMS_STAGE if staging else _AUTOMATED_PLATFORMS + if form_data.platform not in allowed_platforms: + platform_str = form_data.platform.value + return await session.redirect( + get.distribution.stage_automate if staging else get.distribution.automate, + project=project, + version=version, + error=f"Platform {platform_str} is not supported for automated distribution", + ) + sql_platform = form_data.platform.to_sql() # type: ignore[attr-defined] + dd = distribution.Data( + platform=sql_platform, + owner_namespace=form_data.owner_namespace, + package=form_data.package, + version=form_data.version, + details=form_data.details, + ) + release, committee = await shared.distribution.release_validated_and_committee( + project, version, staging=staging, release_policy=True + ) + if release.release_policy is None or release.release_policy.github_repository_name == "": + return await session.redirect( + get.distribution.stage_automate if staging else get.distribution.automate, + project=project, + version=version, + error="Project does not have a release policy configured, or no GitHub repository is specified", + ) + repo = release.release_policy.github_repository_name + workflow = f"distribute-{sql_platform.value.gh_slug}.yml" + async with storage.write_as_committee_member(committee_name=committee.name) as w: + try: + await w.distributions.automate( + release.name, + dd.platform, + dd.owner_namespace, + "apache", + repo, + workflow, + project, + version, + release.latest_revision_number, + dd.package, + dd.version, + staging, + ) + except storage.AccessError as e: + # Instead of calling record_form_page_new, redirect with error message + return await session.redirect( + get.distribution.stage_automate if staging else get.distribution.automate, + project=project, + version=version, + error=str(e), + ) + + # Success - redirect to distribution list with success message + message = "Distribution queued successfully." + return await session.redirect( + get.distribution.list_get if staging else get.finish.selected, + project_name=project, + version_name=version, + success=message, + ) + + async def record_form_process_page( session: web.Committer, form_data: shared.distribution.DistributeForm, @@ -84,14 +159,14 @@ async def record_form_process_page( async with storage.write_as_committee_member(committee_name=committee.name) as w: try: _dist, added, _metadata = await w.distributions.record_from_data( - release=release, + release=release.name, staging=staging, dd=dd, ) except storage.AccessError as e: # Instead of calling record_form_page_new, redirect with error message return await session.redirect( - get.distribution.stage if staging else get.distribution.record, + get.distribution.stage_record if staging else get.distribution.record, project=project, version=version, error=str(e), @@ -101,12 +176,28 @@ async def record_form_process_page( message = "Distribution recorded successfully." if added else "Distribution was already recorded." return await session.redirect( get.distribution.list_get, - project=project, - version=version, + project_name=project, + version_name=version, success=message, ) [email protected]("/distribution/automate/<project>/<version>") [email protected](shared.distribution.DistributeForm) +async def automate_selected( + session: web.Committer, distribute_form: shared.distribution.DistributeForm, project: str, version: str +) -> web.WerkzeugResponse: + return await automate_form_process_page(session, distribute_form, project, version, staging=False) + + [email protected]("/distribution/stage/automate/<project>/<version>") [email protected](shared.distribution.DistributeForm) +async def stage_automate_selected( + session: web.Committer, distribute_form: shared.distribution.DistributeForm, project: str, version: str +) -> web.WerkzeugResponse: + return await automate_form_process_page(session, distribute_form, project, version, staging=True) + + @post.committer("/distribution/record/<project>/<version>") @post.form(shared.distribution.DistributeForm) async def record_selected( @@ -115,9 +206,9 @@ async def record_selected( return await record_form_process_page(session, distribute_form, project, version, staging=False) [email protected]("/distribution/stage/<project>/<version>") [email protected]("/distribution/stage/record/<project>/<version>") @post.form(shared.distribution.DistributeForm) -async def stage_selected( +async def stage_record_selected( session: web.Committer, distribute_form: shared.distribution.DistributeForm, project: str, version: str ) -> web.WerkzeugResponse: return await record_form_process_page(session, distribute_form, project, version, staging=True) diff --git a/atr/server.py b/atr/server.py index e39ffff..3f13739 100644 --- a/atr/server.py +++ b/atr/server.py @@ -335,9 +335,10 @@ def _app_setup_security_headers(app: base.QuartApp) -> None: # Both object-src 'none' and base-uri 'none' are required by ASVS v5 3.4.3 (L2) # The frame-ancestors 'none' directive is required by ASVS v5 3.4.6 (L2) # Bootstrap uses data: URLs extensively, so we need to include that in img-src + # The script hash allows window.location.reload() and nothing else csp_directives = [ "default-src 'self'", - "script-src 'self'", + "script-src 'self' 'sha256-4TpZ3Tx5SLybDXPQaSHGuP1RU4D+pzck+02JLVY61BY=' 'unsafe-hashes'", "style-src 'self' 'unsafe-inline'", "img-src 'self' https://apache.org https://incubator.apache.org https://www.apache.org data:", "font-src 'self'", diff --git a/atr/shared/distribution.py b/atr/shared/distribution.py index 444d066..743c781 100644 --- a/atr/shared/distribution.py +++ b/atr/shared/distribution.py @@ -132,10 +132,7 @@ def html_tr_a(label: str, value: str | None) -> htm.Element: async def release_validated( - project: str, - version: str, - committee: bool = False, - staging: bool | None = None, + project: str, version: str, committee: bool = False, staging: bool | None = None, release_policy: bool = False ) -> sql.Release: match staging: case True: @@ -149,6 +146,7 @@ async def release_validated( project_name=project, version=version, _committee=committee, + _release_policy=release_policy, ).demand(RuntimeError(f"Release {project} {version} not found")) if release.phase not in phase: raise RuntimeError(f"Release {project} {version} is not in {phase}") @@ -158,12 +156,9 @@ async def release_validated( async def release_validated_and_committee( - project: str, - version: str, - *, - staging: bool | None = None, + project: str, version: str, *, staging: bool | None = None, release_policy: bool = False ) -> tuple[sql.Release, sql.Committee]: - release = await release_validated(project, version, committee=True, staging=staging) + release = await release_validated(project, version, committee=True, staging=staging, release_policy=release_policy) committee = release.committee if committee is None: raise RuntimeError(f"Release {project} {version} has no committee") diff --git a/atr/storage/writers/distributions.py b/atr/storage/writers/distributions.py index 2db5d85..a9efd84 100644 --- a/atr/storage/writers/distributions.py +++ b/atr/storage/writers/distributions.py @@ -31,6 +31,8 @@ import atr.models.distribution as distribution import atr.models.sql as sql import atr.storage as storage import atr.storage.outcome as outcome +import atr.tasks.gha as gha +import atr.util as util class GeneralPublic: @@ -95,6 +97,50 @@ class CommitteeMember(CommitteeParticipant): self.__asf_uid = asf_uid self.__committee_name = committee_name + async def automate( + self, + release_name: str, + platform: sql.DistributionPlatform, + owner_namespace: str | None, + owner: str, + repo: str, + workflow: str, + project_name: str, + version_name: str, + revision_number: str | None, + package: str, + version: str, + staging: bool, + ) -> sql.Task: + dist_task = sql.Task( + task_type=sql.TaskType.DISTRIBUTION_WORKFLOW, + task_args=gha.DistributionWorkflow( + name=release_name, + namespace=owner_namespace or "", + package=package, + version=version, + project_name=project_name, + version_name=version_name, + platform=platform.name, + owner=owner, + repo=repo, + ref="main", # TODO: Un-hardcode + workflow=workflow, + staging=staging, + arguments={}, + ).model_dump(), + asf_uid=util.unwrap(self.__asf_uid), + added=datetime.datetime.now(datetime.UTC), + status=sql.TaskStatus.QUEUED, + project_name=project_name, + version_name=version_name, + revision_number=revision_number, + ) + self.__data.add(dist_task) + await self.__data.commit() + await self.__data.refresh(dist_task) + return dist_task + async def record( self, release_name: str, @@ -148,7 +194,7 @@ class CommitteeMember(CommitteeParticipant): async def record_from_data( self, - release: sql.Release, + release: str, staging: bool, dd: distribution.Data, ) -> tuple[sql.Distribution, bool, distribution.Metadata]: @@ -176,7 +222,7 @@ class CommitteeMember(CommitteeParticipant): web_url=web_url, ) dist, added = await self.record( - release_name=release.name, + release_name=release, platform=dd.platform, owner_namespace=dd.owner_namespace, package=dd.package, diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py index dde92ed..9d89e31 100644 --- a/atr/tasks/__init__.py +++ b/atr/tasks/__init__.py @@ -189,7 +189,7 @@ def queued( def resolve(task_type: sql.TaskType) -> Callable[..., Awaitable[results.Results | None]]: # noqa: C901 match task_type: - case sql.TaskType.GITHUB_ACTION_WORKFLOW: + case sql.TaskType.DISTRIBUTION_WORKFLOW: return gha.trigger_workflow case sql.TaskType.HASHING_CHECK: return hashing.check diff --git a/atr/tasks/gha.py b/atr/tasks/gha.py index f64b2cc..7b9123f 100644 --- a/atr/tasks/gha.py +++ b/atr/tasks/gha.py @@ -24,48 +24,79 @@ import aiohttp import atr.config as config import atr.log as log +import atr.models.distribution as distribution import atr.models.results as results import atr.models.schema as schema +import atr.models.sql as sql + +# import atr.shared as shared +import atr.storage as storage import atr.tasks.checks as checks _BASE_URL: Final[str] = "https://api.github.com/repos"; _IN_PROGRESS_STATUSES: Final[list[str]] = ["in_progress", "queued", "requested", "waiting", "pending", "expected"] _COMPLETED_STATUSES: Final[list[str]] = ["completed"] _FAILED_STATUSES: Final[list[str]] = ["failure", "startup_failure"] -_TIMEOUT_S = 5 +_TIMEOUT_S = 60 -class GithubActionsWorkflow(schema.Strict): +class DistributionWorkflow(schema.Strict): """Arguments for the task to start a Github Actions workflow.""" owner: str = schema.description("Github owner of the repository") repo: str = schema.description("Repository in which to start the workflow") - workflow_id: str = schema.description("Workflow ID") ref: str = schema.description("Git ref to trigger the workflow") + workflow: str = schema.description("Workflow to trigger") + namespace: str = schema.description("Namespace to distribute to") + package: str = schema.description("Package to distribute") + version: str = schema.description("Version to distribute") + staging: bool = schema.description("Whether this is a staging distribution") + project_name: str = schema.description("Project name in ATR") + version_name: str = schema.description("Version name in ATR") + platform: str = schema.description("Distribution platform") arguments: dict[str, str] = schema.description("Workflow arguments") name: str = schema.description("Name of the run") [email protected]_model(GithubActionsWorkflow) -async def trigger_workflow(args: GithubActionsWorkflow) -> results.Results | None: [email protected]_model(DistributionWorkflow) +async def trigger_workflow(args: DistributionWorkflow) -> results.Results | None: unique_id = f"{args.name}-{uuid.uuid4()}" - payload = {"ref": args.ref, "inputs": {"atr-id": unique_id, **args.arguments}} + # release, committee = await shared.distribution.release_validated_and_committee( + # args.project, + # args.version, + # staging=True, # TODO: Un-hardcode + # ) + try: + sql_platform = sql.DistributionPlatform[args.platform] + except KeyError: + _fail(f"Invalid platform: {args.platform}") + payload = { + "ref": args.ref, + "inputs": { + "atr-id": unique_id, + "platform": args.platform, + "distribution-package": args.package, + "distribution-version": args.version, + "staging": "true" if args.staging else "false", + **args.arguments, + }, + } headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {config.get().GITHUB_TOKEN}"} log.info( - f"Triggering Github workflow {args.owner}/{args.repo}/{args.workflow_id} with args: { + f"Triggering Github workflow {args.owner}/{args.repo}/{args.workflow} with args: { json.dumps(args.arguments, indent=2) }" ) async with aiohttp.ClientSession() as session: try: async with session.post( - f"{_BASE_URL}/{args.owner}/{args.repo}/actions/workflows/{args.workflow_id}/dispatches", + f"{_BASE_URL}/{args.owner}/{args.repo}/actions/workflows/{args.workflow}/dispatches", headers=headers, json=payload, ) as response: response.raise_for_status() except aiohttp.ClientResponseError as e: - _fail(f"Failed to trigger workflow run: {e.message} ({e.status})") + _fail(f"Failed to trigger GitHub workflow: {e.message} ({e.status})") run, run_id = await _find_triggered_run(session, args, headers, unique_id) @@ -73,13 +104,46 @@ async def trigger_workflow(args: GithubActionsWorkflow) -> results.Results | Non run = await _wait_for_completion(session, args, headers, run_id, unique_id) if run.get("status") in _FAILED_STATUSES: - _fail(f"Github workflow {args.owner}/{args.repo}/{args.workflow_id} run {run_id} failed with error") + _fail(f"Github workflow {args.owner}/{args.repo}/{args.workflow} run {run_id} failed with error") if run.get("status") in _COMPLETED_STATUSES: - log.info(f"Workflow {args.owner}/{args.repo}/{args.workflow_id} run {run_id} completed successfully") - return results.GithubActionsWorkflow( - kind="github_actions_workflow", name=args.name, run_id=run_id, url=run.get("html_url", "") + log.info(f"Workflow {args.owner}/{args.repo}/{args.workflow} run {run_id} completed successfully") + await _record_distribution( + "committee.name", + "release", + sql_platform, + namespace=args.namespace, + package=args.package, + version=args.version, + staging=True, + ) + return results.DistributionWorkflow( + kind="distribution_workflow", name=args.name, run_id=run_id, url=run.get("html_url", "") ) - _fail(f"Timed out waiting for workflow {args.owner}/{args.repo}/{args.workflow_id}") + _fail(f"Timed out waiting for GitHub workflow {args.owner}/{args.repo}/{args.workflow}") + + +async def _record_distribution( + committee_name: str, + release: str, + platform: sql.DistributionPlatform, + namespace: str, + package: str, + version: str, + staging: bool, +): + log.info("Creating distribution record") + dd = distribution.Data( + platform=platform, + owner_namespace=namespace, + package=package, + version=version, + details=False, + ) + async with storage.write_as_committee_member(committee_name=committee_name) as w: + try: + _dist, _added, _metadata = await w.distributions.record_from_data(release=release, staging=staging, dd=dd) + except storage.AccessError as e: + _fail(f"Failed to record distribution: {e}") def _fail(message: str) -> NoReturn: @@ -89,7 +153,7 @@ def _fail(message: str) -> NoReturn: async def _find_triggered_run( session: aiohttp.ClientSession, - args: GithubActionsWorkflow, + args: DistributionWorkflow, headers: dict[str, str], unique_id: str, ) -> tuple[dict[str, Any], int]: @@ -140,7 +204,7 @@ async def _request_and_retry( async def _wait_for_completion( session: aiohttp.ClientSession, - args: GithubActionsWorkflow, + args: DistributionWorkflow, headers: dict[str, str], run_id: int, unique_id: str, diff --git a/atr/templates/check-selected.html b/atr/templates/check-selected.html index 0bb1534..77b793d 100644 --- a/atr/templates/check-selected.html +++ b/atr/templates/check-selected.html @@ -125,9 +125,19 @@ </div> </div> {% if phase == "release_candidate_draft" %} + <h3 id="distribution" class="mt-4">Distribution</h3> + <p> + While this release is in draft, you can create a staging distribution. Use the buttons below to either create one automatically (where supported) or record a manual distribution performed outside of ATR. + </p> + <div class="alert alert-warning mb-4"> + <p class="fw-semibold mb-1">NOTE:</p> + <p>At present, automated distributions are being developed. Please use the manual record button if you need to record a distribution.</p> + </div> <p> <a class="btn btn-primary" - href="{{ as_url(get.distribution.stage, project=release.project.name, version=release.version) }}">Record a distribution</a> + href="{{ as_url(get.distribution.stage_automate, project=release.project.name, version=release.version) }}">Distribute</a> + <a class="btn btn-secondary" + href="{{ as_url(get.distribution.stage_record, project=release.project.name, version=release.version) }}">Record a manual distribution</a> </p> <h2 id="more-actions">More actions</h2> <h3 id="ignored-checks" class="mt-4">Ignored checks</h3> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
