This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/main by this push:
new 8b88fe1 Restore the link exporter and link checker lint
8b88fe1 is described below
commit 8b88fe14f07274c2d0b7cd551f716c7d50e5ff6a
Author: Sean B. Palmer <[email protected]>
AuthorDate: Tue Oct 28 17:05:56 2025 +0000
Restore the link exporter and link checker lint
---
.pre-commit-config.yaml | 14 +-
atr/blueprints/__init__.py | 28 +-
atr/blueprints/admin.py | 4 +-
atr/blueprints/api.py | 4 +-
atr/blueprints/get.py | 11 +-
atr/blueprints/post.py | 11 +-
atr/docs/overview-of-the-code.html | 2 +-
atr/docs/overview-of-the-code.md | 2 +-
atr/docs/storage-interface.html | 4 +-
atr/docs/storage-interface.md | 4 +-
atr/docs/user-interface.html | 2 +-
atr/docs/user-interface.md | 2 +-
atr/get/keys.py | 11 +-
atr/get/revisions.py | 7 +-
atr/principal.py | 7 +-
atr/route.py | 496 ------------------------------------
atr/shared/keys.py | 3 +-
atr/shared/start.py | 3 +-
scripts/README.md | 2 +-
scripts/lint/jinja_route_checker.py | 6 +-
20 files changed, 76 insertions(+), 547 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 8161197..83a6aa5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -86,10 +86,10 @@ repos:
types: [python]
exclude: ^tests/
- # - id: jinja-route-check
- # name: Jinja Route Checker
- # description: Check whether routes used in Jinja2 templates actually
exist
- # entry: uv run python scripts/lint/jinja_route_checker.py
- # language: system
- # pass_filenames: false
- # always_run: true
+ - id: jinja-route-check
+ name: Jinja Route Checker
+ description: Check whether routes used in Jinja2 templates actually exist
+ entry: uv run python scripts/lint/jinja_route_checker.py
+ language: system
+ pass_filenames: false
+ always_run: true
diff --git a/atr/blueprints/__init__.py b/atr/blueprints/__init__.py
index cf5f68a..a0f1318 100644
--- a/atr/blueprints/__init__.py
+++ b/atr/blueprints/__init__.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+import json
+import pathlib
from types import ModuleType
from typing import Protocol, runtime_checkable
@@ -26,23 +28,35 @@ import atr.blueprints.get as get
import atr.blueprints.icons as icons
import atr.blueprints.post as post
+_all_routes: list[str] = []
+
@runtime_checkable
class RoutesModule(Protocol):
ROUTES_MODULE: bool = True
-def check_module(module: ModuleType) -> None:
+def register(app: base.QuartApp) -> None:
+ import atr.config as config
+
+ _check_blueprint(*admin.register(app))
+ _check_blueprint(*api.register(app))
+ _check_blueprint(*get.register(app))
+ app.register_blueprint(icons.BLUEPRINT)
+ _check_blueprint(*post.register(app))
+
+ _export_routes(pathlib.Path(config.get().STATE_DIR))
+
+
+def _check_blueprint(module: ModuleType, routes: list[str]) -> None:
# We need to know that the routes were actually imported
# Otherwise ASFQuart will not know about them, even if the blueprint is
registered
# In other words, registering a blueprint does not automatically import
its routes
if not isinstance(module, RoutesModule):
raise ValueError(f"Module {module} is not a RoutesModule")
+ _all_routes.extend(routes)
-def register(app: base.QuartApp) -> None:
- check_module(admin.register(app))
- check_module(api.register(app))
- check_module(get.register(app))
- app.register_blueprint(icons.BLUEPRINT)
- check_module(post.register(app))
+def _export_routes(state_dir: pathlib.Path) -> None:
+ routes_file = state_dir / "routes.json"
+ routes_file.write_text(json.dumps(sorted(_all_routes), indent=2))
diff --git a/atr/blueprints/admin.py b/atr/blueprints/admin.py
index f0106c7..baccd93 100644
--- a/atr/blueprints/admin.py
+++ b/atr/blueprints/admin.py
@@ -42,11 +42,11 @@ async def _check_admin_access() -> None:
quart.g.session = web.Committer(web_session)
-def register(app: base.QuartApp) -> ModuleType:
+def register(app: base.QuartApp) -> tuple[ModuleType, list[str]]:
import atr.admin as admin
app.register_blueprint(_BLUEPRINT)
- return admin
+ return admin, []
def get(path: str) -> Callable[[web.CommitterRouteFunction[Any]],
web.RouteFunction[Any]]:
diff --git a/atr/blueprints/api.py b/atr/blueprints/api.py
index d40d6d5..d5c1491 100644
--- a/atr/blueprints/api.py
+++ b/atr/blueprints/api.py
@@ -31,11 +31,11 @@ _BLUEPRINT = quart.Blueprint("api_blueprint", __name__,
url_prefix="/api")
route = _BLUEPRINT.route
-def register(app: base.QuartApp) -> ModuleType:
+def register(app: base.QuartApp) -> tuple[ModuleType, list[str]]:
import atr.api as api
app.register_blueprint(_BLUEPRINT)
- return api
+ return api, []
def _exempt_blueprint(app: base.QuartApp) -> None:
diff --git a/atr/blueprints/get.py b/atr/blueprints/get.py
index 628f148..c4bddc3 100644
--- a/atr/blueprints/get.py
+++ b/atr/blueprints/get.py
@@ -30,13 +30,14 @@ import atr.web as web
_BLUEPRINT_NAME = "get_blueprint"
_BLUEPRINT = quart.Blueprint(_BLUEPRINT_NAME, __name__)
+_routes: list[str] = []
-def register(app: base.QuartApp) -> ModuleType:
+def register(app: base.QuartApp) -> tuple[ModuleType, list[str]]:
import atr.get as get
app.register_blueprint(_BLUEPRINT)
- return get
+ return get, _routes
def committer(path: str) -> Callable[[web.CommitterRouteFunction[Any]],
web.RouteFunction[Any]]:
@@ -75,6 +76,9 @@ def committer(path: str) ->
Callable[[web.CommitterRouteFunction[Any]], web.Rout
decorated = auth.require(auth.Requirements.committer)(wrapper)
_BLUEPRINT.add_url_rule(path, endpoint=endpoint, view_func=decorated,
methods=["GET"])
+ module_name = func.__module__.split(".")[-1]
+ _routes.append(f"get.{module_name}.{func.__name__}")
+
return decorated
return decorator
@@ -94,6 +98,9 @@ def public(path: str) -> Callable[[Callable[...,
Awaitable[Any]]], web.RouteFunc
_BLUEPRINT.add_url_rule(path, endpoint=endpoint, view_func=wrapper,
methods=["GET"])
+ module_name = func.__module__.split(".")[-1]
+ _routes.append(f"get.{module_name}.{func.__name__}")
+
return wrapper
return decorator
diff --git a/atr/blueprints/post.py b/atr/blueprints/post.py
index 622c680..144bcdf 100644
--- a/atr/blueprints/post.py
+++ b/atr/blueprints/post.py
@@ -30,13 +30,14 @@ import atr.web as web
_BLUEPRINT_NAME = "post_blueprint"
_BLUEPRINT = quart.Blueprint(_BLUEPRINT_NAME, __name__)
+_routes: list[str] = []
-def register(app: base.QuartApp) -> ModuleType:
+def register(app: base.QuartApp) -> tuple[ModuleType, list[str]]:
import atr.post as post
app.register_blueprint(_BLUEPRINT)
- return post
+ return post, _routes
def committer(path: str) -> Callable[[web.CommitterRouteFunction[Any]],
web.RouteFunction[Any]]:
@@ -75,6 +76,9 @@ def committer(path: str) ->
Callable[[web.CommitterRouteFunction[Any]], web.Rout
decorated = auth.require(auth.Requirements.committer)(wrapper)
_BLUEPRINT.add_url_rule(path, endpoint=endpoint, view_func=decorated,
methods=["POST"])
+ module_name = func.__module__.split(".")[-1]
+ _routes.append(f"post.{module_name}.{func.__name__}")
+
return decorated
return decorator
@@ -94,6 +98,9 @@ def public(path: str) -> Callable[[Callable[...,
Awaitable[Any]]], web.RouteFunc
_BLUEPRINT.add_url_rule(path, endpoint=endpoint, view_func=wrapper,
methods=["POST"])
+ module_name = func.__module__.split(".")[-1]
+ _routes.append(f"post.{module_name}.{func.__name__}")
+
return wrapper
return decorator
diff --git a/atr/docs/overview-of-the-code.html
b/atr/docs/overview-of-the-code.html
index 615f22d..8823da0 100644
--- a/atr/docs/overview-of-the-code.html
+++ b/atr/docs/overview-of-the-code.html
@@ -39,7 +39,7 @@
<p>The ATR <a href="/ref/atr/worker.py"><code>worker</code></a> module
implements the workers. Each worker process runs in a loop. It claims the
oldest queued task from the database, executes it, records the result, and then
claims the next task atomically using an <code>UPDATE ... WHERE</code>
statement. After a worker has processed a fixed number of tasks, it exits
voluntarily to help to avoid memory leaks. The manager then spawns a fresh
worker to replace it. Task execution happens in [...]
<p>Tasks themselves are defined in the ATR <a
href="/ref/atr/tasks/"><code>tasks</code></a> directory. The <a
href="/ref/atr/tasks/__init__.py"><code>tasks</code></a> module contains
functions for queueing tasks and resolving task types to their handler
functions. Task types include operations such as importing keys, generating
SBOMs, sending messages, and importing files from SVN. The most common category
of task is automated checks on release artifacts. These checks are implemented
in [...]
<h2 id="api">API</h2>
-<p>The ATR API provides programmatic access to most ATR functionality. API
endpoints are defined in <a
href="/ref/atr/bps/api/api.py"><code>bps.api.api</code></a>, and their URL
paths are prefixed with <code>/api/</code>. The API uses <a
href="https://www.openapis.org/";>OpenAPI</a> for documentation, which is
automatically generated from the endpoint definitions and served at
<code>/api/docs</code>. Users send requests with a <a
href="https://en.wikipedia.org/wiki/JSON_Web_Token";>JWT</a> [...]
+<p>The ATR API provides programmatic access to most ATR functionality. API
endpoints are defined in <a
href="/ref/atr/api/routes.py"><code>api.routses</code></a>, and their URL paths
are prefixed with <code>/api/</code>. The API uses <a
href="https://www.openapis.org/";>OpenAPI</a> for documentation, which is
automatically generated from the endpoint definitions and served at
<code>/api/docs</code>. Users send requests with a <a
href="https://en.wikipedia.org/wiki/JSON_Web_Token";>JWT</a> [...]
<p>API request and response models are defined in <a
href="/ref/atr/models/api.py"><code>models.api</code></a> using Pydantic. Each
endpoint has an associated request model that validates incoming data, and a
response model that validates outgoing data. The API returns JSON in all cases,
with appropriate HTTP status codes.</p>
<h2 id="other-important-interfaces">Other important interfaces</h2>
<p>ATR uses ASF OAuth for user login, and then determines what actions each
user can perform based on their committee memberships. The ATR <a
href="/ref/atr/principal.py"><code>principal</code></a> module handles
authorization by checking whether users are members of relevant committees. It
queries and caches LDAP to get committee membership information. The <a
href="/ref/atr/principal.py:Authorisation"><code>Authorisation</code></a> class
provides methods to check whether a user is a me [...]
diff --git a/atr/docs/overview-of-the-code.md b/atr/docs/overview-of-the-code.md
index c5d568e..935330f 100644
--- a/atr/docs/overview-of-the-code.md
+++ b/atr/docs/overview-of-the-code.md
@@ -70,7 +70,7 @@ Tasks themselves are defined in the ATR
[`tasks`](/ref/atr/tasks/) directory. Th
## API
-The ATR API provides programmatic access to most ATR functionality. API
endpoints are defined in [`bps.api.api`](/ref/atr/bps/api/api.py), and their
URL paths are prefixed with `/api/`. The API uses
[OpenAPI](https://www.openapis.org/) for documentation, which is automatically
generated from the endpoint definitions and served at `/api/docs`. Users send
requests with a [JWT](https://en.wikipedia.org/wiki/JSON_Web_Token) created
from a [PAT](https://en.wikipedia.org/wiki/Personal_access_t [...]
+The ATR API provides programmatic access to most ATR functionality. API
endpoints are defined in [`api.routses`](/ref/atr/api/routes.py), and their URL
paths are prefixed with `/api/`. The API uses
[OpenAPI](https://www.openapis.org/) for documentation, which is automatically
generated from the endpoint definitions and served at `/api/docs`. Users send
requests with a [JWT](https://en.wikipedia.org/wiki/JSON_Web_Token) created
from a [PAT](https://en.wikipedia.org/wiki/Personal_access_to [...]
API request and response models are defined in
[`models.api`](/ref/atr/models/api.py) using Pydantic. Each endpoint has an
associated request model that validates incoming data, and a response model
that validates outgoing data. The API returns JSON in all cases, with
appropriate HTTP status codes.
diff --git a/atr/docs/storage-interface.html b/atr/docs/storage-interface.html
index 184e070..54dd0ef 100644
--- a/atr/docs/storage-interface.html
+++ b/atr/docs/storage-interface.html
@@ -24,9 +24,9 @@
new_release, _project = await wacp.release.start(project_name, version)
</code></pre>
<p>The <code>wacp</code> object, short for <code>w</code>rite <code>a</code>s
<code>c</code>ommittee <code>p</code>articipant, provides access to
domain-specific writers: <code>announce</code>, <code>checks</code>,
<code>distributions</code>, <code>keys</code>, <code>policy</code>,
<code>project</code>, <code>release</code>, <code>sbom</code>,
<code>ssh</code>, <code>tokens</code>, and <code>vote</code>.</p>
-<p>The write session takes an optional <a
href="/ref/atr/route.py:CommitterSession"><code>CommitterSession</code></a> or
ASF UID, typically <code>session.uid</code> from the logged-in user. If you
omit the UID, the session determines it automatically from the current request
context. The write object checks LDAP memberships and raises <a
href="/ref/atr/storage/__init__.py:AccessError"><code>storage.AccessError</code></a>
if the user is not authorized for the requested permission level.</p>
+<p>The write session takes an optional <a
href="/ref/atr/web.py:Committer"><code>Committer</code></a> or ASF UID,
typically <code>session.uid</code> from the logged-in user. If you omit the
UID, the session determines it automatically from the current request context.
The write object checks LDAP memberships and raises <a
href="/ref/atr/storage/__init__.py:AccessError"><code>storage.AccessError</code></a>
if the user is not authorized for the requested permission level.</p>
<p>Because projects belong to committees, we provide <a
href="/ref/atr/storage/__init__.py:as_project_committee_member"><code>write.as_project_committee_member(project_name)</code></a>
and <a
href="/ref/atr/storage/__init__.py:as_project_committee_participant"><code>write.as_project_committee_participant(project_name)</code></a>,
which look up the project's committee and authenticate the user as a member or
participant of that committee. This is convenient when, for example, the URL
prov [...]
-<p>Here is a more complete example from <a
href="/ref/atr/bps/api/api.py"><code>bps/api/api.py</code></a> that shows the
classic three step pattern:</p>
+<p>Here is a more complete example from <a
href="/ref/atr/api/routes.py"><code>api/routes.py</code></a> that shows the
classic three step pattern:</p>
<pre><code class="language-python">async with storage.write(asf_uid) as write:
# 1. Request permissions
wafc = write.as_foundation_committer()
diff --git a/atr/docs/storage-interface.md b/atr/docs/storage-interface.md
index 97b508e..eff6110 100644
--- a/atr/docs/storage-interface.md
+++ b/atr/docs/storage-interface.md
@@ -39,11 +39,11 @@ async with storage.write(session) as write:
The `wacp` object, short for `w`rite `a`s `c`ommittee `p`articipant, provides
access to domain-specific writers: `announce`, `checks`, `distributions`,
`keys`, `policy`, `project`, `release`, `sbom`, `ssh`, `tokens`, and `vote`.
-The write session takes an optional
[`CommitterSession`](/ref/atr/route.py:CommitterSession) or ASF UID, typically
`session.uid` from the logged-in user. If you omit the UID, the session
determines it automatically from the current request context. The write object
checks LDAP memberships and raises
[`storage.AccessError`](/ref/atr/storage/__init__.py:AccessError) if the user
is not authorized for the requested permission level.
+The write session takes an optional [`Committer`](/ref/atr/web.py:Committer)
or ASF UID, typically `session.uid` from the logged-in user. If you omit the
UID, the session determines it automatically from the current request context.
The write object checks LDAP memberships and raises
[`storage.AccessError`](/ref/atr/storage/__init__.py:AccessError) if the user
is not authorized for the requested permission level.
Because projects belong to committees, we provide
[`write.as_project_committee_member(project_name)`](/ref/atr/storage/__init__.py:as_project_committee_member)
and
[`write.as_project_committee_participant(project_name)`](/ref/atr/storage/__init__.py:as_project_committee_participant),
which look up the project's committee and authenticate the user as a member or
participant of that committee. This is convenient when, for example, the URL
provides a project name.
-Here is a more complete example from
[`bps/api/api.py`](/ref/atr/bps/api/api.py) that shows the classic three step
pattern:
+Here is a more complete example from [`api/routes.py`](/ref/atr/api/routes.py)
that shows the classic three step pattern:
```python
async with storage.write(asf_uid) as write:
diff --git a/atr/docs/user-interface.html b/atr/docs/user-interface.html
index ceb2577..f5a0bf0 100644
--- a/atr/docs/user-interface.html
+++ b/atr/docs/user-interface.html
@@ -106,7 +106,7 @@ async def add(session: route.CommitterSession) -> str:
form=form,
)
</code></pre>
-<p>The route is decorated with <code>@route.committer</code>, which ensures
that the route fails before the function is even entered if authentication
fails. The function receives a <code>session</code> object, which is an
instance of <a
href="/ref/atr/route.py:CommitterSession"><code>route.CommitterSession</code></a>
with a range of useful properties and methods. The function then loads data,
creates a form, checks if the request is a POST, and either processes the form
or displays it. [...]
+<p>The route is decorated with <code>@route.committer</code>, which ensures
that the route fails before the function is even entered if authentication
fails. The function receives a <code>session</code> object, which is an
instance of <a href="/ref/atr/web.py:Committer"><code>web.Committer</code></a>
with a range of useful properties and methods. The function then loads data,
creates a form, checks if the request is a POST, and either processes the form
or displays it. After successful p [...]
<p>The template receives the form object and renders it by passing it to one
of the <code>forms.render_*</code> functions. We previously used Jinja2 macros
for this, but are migrating to the new rendering functions in Python (e.g. in
<a href="/ref/atr/get/distribution.py"><code>get/distribution.py</code></a> and
<a href="/ref/atr/get/ignores.py"><code>get/ignores.py</code></a>). The
template also receives other data like <code>asf_id</code> and
<code>user_committees</code>, which it uses [...]
<p>If you use the programmatic rendering functions from <a
href="/ref/atr/forms.py"><code>forms</code></a>, you can skip the template
entirely. These functions return htpy elements, which you can combine with
other htpy elements and return directly from the route, which is often useful
for admin routes, for example. You can also use <a
href="/ref/atr/template.py:blank"><code>template.blank</code></a>, which
renders a minimal template with just a title and content area. This is useful
for [...]
<p>Bootstrap CSS classes are applied automatically by the form rendering
functions. The functions use classes like <code>form-control</code>,
<code>form-select</code>, <code>btn-primary</code>, <code>is-invalid</code>,
and <code>invalid-feedback</code>. We currently use Bootstrap 5. If you
generate HTML manually with htpy, you can apply Bootstrap classes yourself by
using the CSS selector syntax like <code>htpy.div(".container")</code> or the
class attribute like <code>htpy.div(class_="c [...]
diff --git a/atr/docs/user-interface.md b/atr/docs/user-interface.md
index d6e1aec..a5424d4 100644
--- a/atr/docs/user-interface.md
+++ b/atr/docs/user-interface.md
@@ -148,7 +148,7 @@ async def add(session: route.CommitterSession) -> str:
)
```
-The route is decorated with `@route.committer`, which ensures that the route
fails before the function is even entered if authentication fails. The function
receives a `session` object, which is an instance of
[`route.CommitterSession`](/ref/atr/route.py:CommitterSession) with a range of
useful properties and methods. The function then loads data, creates a form,
checks if the request is a POST, and either processes the form or displays it.
After successful processing, it creates a fresh [...]
+The route is decorated with `@route.committer`, which ensures that the route
fails before the function is even entered if authentication fails. The function
receives a `session` object, which is an instance of
[`web.Committer`](/ref/atr/web.py:Committer) with a range of useful properties
and methods. The function then loads data, creates a form, checks if the
request is a POST, and either processes the form or displays it. After
successful processing, it creates a fresh form to clear the [...]
The template receives the form object and renders it by passing it to one of
the `forms.render_*` functions. We previously used Jinja2 macros for this, but
are migrating to the new rendering functions in Python (e.g. in
[`get/distribution.py`](/ref/atr/get/distribution.py) and
[`get/ignores.py`](/ref/atr/get/ignores.py)). The template also receives other
data like `asf_id` and `user_committees`, which it uses to display information
or make decisions about what to show.
diff --git a/atr/get/keys.py b/atr/get/keys.py
index d3a5a54..0cbd946 100644
--- a/atr/get/keys.py
+++ b/atr/get/keys.py
@@ -15,8 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-"""keys.py"""
-
import datetime
import asfquart as asfquart
@@ -25,7 +23,6 @@ import werkzeug.wrappers.response as response
import atr.blueprints.get as get
import atr.db as db
-import atr.route as route
import atr.shared as shared
import atr.storage as storage
import atr.template as template
@@ -45,8 +42,8 @@ async def details(session: web.Committer, fingerprint: str)
-> str | response.Re
return await shared.keys.details(session, fingerprint)
[email protected]("/keys/export/<committee_name>")
-async def export(session: route.CommitterSession, committee_name: str) ->
web.TextResponse:
[email protected]("/keys/export/<committee_name>")
+async def export(session: web.Committer, committee_name: str) ->
web.TextResponse:
"""Export a KEYS file for a specific committee."""
async with storage.write() as write:
wafc = write.as_foundation_committer()
@@ -55,8 +52,8 @@ async def export(session: route.CommitterSession,
committee_name: str) -> web.Te
return web.TextResponse(keys_file_text)
[email protected]("/keys")
-async def keys(session: route.CommitterSession) -> str:
[email protected]("/keys")
+async def keys(session: web.Committer) -> str:
"""View all keys associated with the user's account."""
committees_to_query = list(set(session.committees + session.projects))
diff --git a/atr/get/revisions.py b/atr/get/revisions.py
index 4a7922b..ebcf31a 100644
--- a/atr/get/revisions.py
+++ b/atr/get/revisions.py
@@ -23,17 +23,18 @@ import asfquart.base as base
import sqlalchemy.orm as orm
import sqlmodel
+import atr.blueprints.get as get
import atr.db as db
import atr.forms as forms
import atr.models.schema as schema
import atr.models.sql as sql
-import atr.route as route
import atr.template as template
import atr.util as util
+import atr.web as web
[email protected]("/revisions/<project_name>/<version_name>")
-async def selected(session: route.CommitterSession, project_name: str,
version_name: str) -> str:
[email protected]("/revisions/<project_name>/<version_name>")
+async def selected(session: web.Committer, project_name: str, version_name:
str) -> str:
"""Show the revision history for a release candidate draft or release
preview."""
await session.check_access(project_name)
diff --git a/atr/principal.py b/atr/principal.py
index 45421db..66c7d99 100644
--- a/atr/principal.py
+++ b/atr/principal.py
@@ -28,7 +28,6 @@ import asfquart.session
import atr.config as config
import atr.ldap as ldap
import atr.log as log
-import atr.route as route
import atr.util as util
import atr.web as web
@@ -61,7 +60,7 @@ class ArgumentNoneType:
ArgumentNone = ArgumentNoneType()
-type UID = route.CommitterSession | web.Committer | str | None |
ArgumentNoneType
+type UID = web.Committer | str | None | ArgumentNoneType
def attr_to_list(attr):
@@ -354,9 +353,9 @@ class AsyncObject:
class Authorisation(AsyncObject):
async def __init__(self, asf_uid: UID = ArgumentNone):
match asf_uid:
- case ArgumentNoneType() | route.CommitterSession() |
web.Committer():
+ case ArgumentNoneType() | web.Committer():
match asf_uid:
- case route.CommitterSession() | web.Committer():
+ case web.Committer():
asfquart_session = asf_uid._session
case _:
asfquart_session = await asfquart.session.read()
diff --git a/atr/route.py b/atr/route.py
deleted file mode 100644
index db30a3f..0000000
--- a/atr/route.py
+++ /dev/null
@@ -1,496 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import annotations
-
-import asyncio
-import functools
-import logging
-import time
-from typing import TYPE_CHECKING, Any, Concatenate, Final, NoReturn,
ParamSpec, Protocol, TypeVar
-
-import aiofiles
-import asfquart
-import asfquart.auth as auth
-import asfquart.base as base
-import asfquart.session as session
-import quart
-
-import atr.config as config
-import atr.db as db
-import atr.models.sql as sql
-import atr.user as user
-import atr.util as util
-
-if TYPE_CHECKING:
- from collections.abc import Awaitable, Callable, Coroutine, Sequence
-
- import werkzeug.datastructures as datastructures
- import werkzeug.wrappers.response as response
-
-if asfquart.APP is ...:
- raise RuntimeError("APP is not set")
-
-P = ParamSpec("P")
-R = TypeVar("R", covariant=True)
-T = TypeVar("T")
-
-# TODO: Should get this from config, checking debug there
-_MEASURE_PERFORMANCE: Final[bool] = True
-
-
-class AsyncFileHandler(logging.Handler):
- """A logging handler that writes logs asynchronously using aiofiles."""
-
- def __init__(self, filename, mode="w", encoding=None):
- super().__init__()
- self.filename = filename
-
- if mode != "w":
- raise RuntimeError("Only write mode is supported")
-
- self.encoding = encoding
- self.queue = asyncio.Queue()
- self.our_worker_task = None
-
- def our_worker_task_ensure(self):
- """Lazily create the worker task if it doesn't exist and there's an
event loop."""
- if self.our_worker_task is None:
- try:
- loop = asyncio.get_running_loop()
- self.our_worker_task = loop.create_task(self.our_worker())
- except RuntimeError:
- # No event loop running yet, try again on next emit
- ...
-
- async def our_worker(self):
- """Background task that writes queued log messages to file."""
- # Use a binary mode literal with aiofiles.open
- #
https://github.com/Tinche/aiofiles/blob/main/src/aiofiles/threadpool/__init__.py
- # We should be able to use any mode, but pyright requires a binary mode
- async with aiofiles.open(self.filename, "wb+") as f:
- while True:
- record = await self.queue.get()
- if record is None:
- break
-
- try:
- # Format the log record first
- formatted_message = self.format(record) + "\n"
- message_bytes = formatted_message.encode(self.encoding or
"utf-8")
- await f.write(message_bytes)
- await f.flush()
- except Exception:
- self.handleError(record)
- finally:
- self.queue.task_done()
-
- def emit(self, record):
- """Queue the record for writing by the worker task."""
- try:
- # Ensure worker task is running
- self.our_worker_task_ensure()
-
- # Queue the record, but handle the case where no event loop is
running yet
- try:
- self.queue.put_nowait(record)
- except RuntimeError:
- ...
- except Exception:
- self.handleError(record)
-
- def close(self):
- """Shut down the worker task cleanly."""
- if self.our_worker_task is not None and not
self.our_worker_task.done():
- try:
- self.queue.put_nowait(None)
- except RuntimeError:
- # No running event loop, no need to clean up
- ...
- super().close()
-
-
-# This is the type of functions to which we apply @committer_get
-# In other words, functions which accept CommitterSession as their first arg
-class CommitterRouteHandler(Protocol[R]):
- """Protocol for @committer_get decorated functions."""
-
- __name__: str
- __doc__: str | None
-
- def __call__(self, session: CommitterSession, *args: Any, **kwargs: Any)
-> Awaitable[R]: ...
-
-
-class CommitterSession:
- """Session with extra information about committers."""
-
- def __init__(self, web_session: session.ClientSession) -> None:
- self._projects: list[sql.Project] | None = None
- self._session = web_session
-
- @property
- def asf_uid(self) -> str:
- if self._session.uid is None:
- raise base.ASFQuartException("Not authenticated", errorcode=401)
- return self._session.uid
-
- def __getattr__(self, name: str) -> Any:
- # TODO: Not type safe, should subclass properly if possible
- # For example, we can access session.no_such_attr and the type
checkers won't notice
- return getattr(self._session, name)
-
- async def check_access(self, project_name: str) -> None:
- if not any((p.name == project_name) for p in (await
self.user_projects)):
- if user.is_admin(self.uid):
- # Admins can view all projects
- # But we must warn them when the project is not one of their
own
- # TODO: This code is difficult to test locally
- # TODO: This flash sometimes displays after deleting a
project, which is a bug
- await quart.flash("This is not your project, but you have
access as an admin", "warning")
- return
- raise base.ASFQuartException("You do not have access to this
project", errorcode=403)
-
- async def check_access_committee(self, committee_name: str) -> None:
- if committee_name not in self.committees:
- if user.is_admin(self.uid):
- # Admins can view all committees
- # But we must warn them when the committee is not one of their
own
- # TODO: As above, this code is difficult to test locally
- await quart.flash("This is not your committee, but you have
access as an admin", "warning")
- return
- raise base.ASFQuartException("You do not have access to this
committee", errorcode=403)
-
- @property
- def app_host(self) -> str:
- return config.get().APP_HOST
-
- @property
- def host(self) -> str:
- request_host = quart.request.host
- if ":" in request_host:
- domain, port = request_host.split(":")
- # Could be an IPv6 address, so need to check whether port is a
valid integer
- if port.isdigit():
- return domain
- return request_host
-
- def only_user_releases(self, releases: Sequence[sql.Release]) ->
list[sql.Release]:
- return util.user_releases(self.uid, releases)
-
- async def redirect(
- self, route: CommitterRouteHandler[R], success: str | None = None,
error: str | None = None, **kwargs: Any
- ) -> response.Response:
- """Redirect to a route with a success or error message."""
- return await redirect(route, success, error, **kwargs)
-
- async def release(
- self,
- project_name: str,
- version_name: str,
- phase: sql.ReleasePhase | db.NotSet | None = db.NOT_SET,
- latest_revision_number: str | db.NotSet | None = db.NOT_SET,
- data: db.Session | None = None,
- with_committee: bool = True,
- with_project: bool = True,
- with_release_policy: bool = False,
- with_project_release_policy: bool = False,
- with_revisions: bool = False,
- ) -> sql.Release:
- # We reuse db.NOT_SET as an entirely different sentinel
- # TODO: We probably shouldn't do that, or should make it clearer
- if phase is None:
- phase_value = db.NOT_SET
- elif phase is db.NOT_SET:
- phase_value = sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT
- else:
- phase_value = phase
- release_name = sql.release_name(project_name, version_name)
- if data is None:
- async with db.session() as data:
- release = await data.release(
- name=release_name,
- phase=phase_value,
- latest_revision_number=latest_revision_number,
- _committee=with_committee,
- _project=with_project,
- _release_policy=with_release_policy,
- _project_release_policy=with_project_release_policy,
- _revisions=with_revisions,
- ).demand(base.ASFQuartException("Release does not exist",
errorcode=404))
- else:
- release = await data.release(
- name=release_name,
- phase=phase_value,
- latest_revision_number=latest_revision_number,
- _committee=with_committee,
- _project=with_project,
- _release_policy=with_release_policy,
- _project_release_policy=with_project_release_policy,
- _revisions=with_revisions,
- ).demand(base.ASFQuartException("Release does not exist",
errorcode=404))
- return release
-
- @property
- async def user_candidate_drafts(self) -> list[sql.Release]:
- return await user.candidate_drafts(self.uid,
user_projects=self._projects)
-
- # @property
- # async def user_committees(self) -> list[models.Committee]:
- # return ...
-
- @property
- async def user_projects(self) -> list[sql.Project]:
- if self._projects is None:
- self._projects = await user.projects(self.uid)
- return self._projects[:]
-
-
-class FlashError(RuntimeError): ...
-
-
-class MicrosecondsFormatter(logging.Formatter):
- # Answers on a postcard if you know why Python decided to use a comma by
default
- default_msec_format = "%s.%03d"
-
-
-# Setup a dedicated logger for route performance metrics
-# NOTE: This code block must come after AsyncFileHandler and
MicrosecondsFormatter
-route_logger: Final = logging.getLogger("route.performance")
-# Use custom formatter that properly includes microseconds
-# TODO: Is this actually UTC?
-route_logger_handler: Final[AsyncFileHandler] =
AsyncFileHandler("deprecated-route-performance.log")
-route_logger_handler.setFormatter(MicrosecondsFormatter("%(asctime)s -
%(message)s"))
-route_logger.addHandler(route_logger_handler)
-route_logger.setLevel(logging.INFO)
-# If we don't set propagate to False then it logs to the term as well
-route_logger.propagate = False
-
-
-# This is the type of functions to which we apply @app_route
-# In other words, functions which accept no session
-class RouteHandler(Protocol[R]):
- """Protocol for @app_route decorated functions."""
-
- __name__: str
- __doc__: str | None
-
- def __call__(self, *args: Any, **kwargs: Any) -> Awaitable[R]: ...
-
-
-def app_route(
- path: str, methods: list[str] | None = None, endpoint: str | None = None,
measure_performance: bool = True
-) -> Callable:
- """Register a route with the Flask app with built-in performance
logging."""
-
- def decorator(f: Callable[P, Coroutine[Any, Any, T]]) -> Callable[P,
Awaitable[T]]:
- # First apply our performance measuring decorator
- if _MEASURE_PERFORMANCE and measure_performance:
- measured_func = app_route_performance_measure(path, methods)(f)
- else:
- measured_func = f
- # Then apply the original route decorator
- return asfquart.APP.route(path, methods=methods,
endpoint=endpoint)(measured_func)
-
- return decorator
-
-
-def app_route_performance_measure(route_path: str, http_methods: list[str] |
None = None) -> Callable:
- """Decorator that measures and logs route performance with path and method
information."""
-
- # def format_time(seconds: float) -> str:
- # """Format time in appropriate units (µs or ms)."""
- # microseconds = seconds * 1_000_000
- # if microseconds < 1000:
- # return f"{microseconds:.2f} µs"
- # else:
- # milliseconds = microseconds / 1000
- # return f"{milliseconds:.2f} ms"
-
- def decorator(f: Callable[P, Coroutine[Any, Any, T]]) -> Callable[P,
Awaitable[T]]:
- @functools.wraps(f)
- async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
- # This wrapper is based on an outstanding idea by Mostafa Farzán
- # Farzán realised that we can step the event loop manually
- # That way, we can also divide it into synchronous and
asynchronous parts
- # The synchronous part is done using coro.send(None)
- # The asynchronous part is done using asyncio.sleep(0)
- # We use two methods for measuring the async part, and take the
largest
- # This performance measurement adds a bit of overhead, about
10-20ms
- # Therefore it should be avoided in production, or made more
efficient
- # We could perhaps use for a small portion of requests
- blocking_time = 0.0
- async_time = 0.0
- loop_time = 0.0
- total_start = time.perf_counter()
- coro = f(*args, **kwargs)
- try:
- while True:
- # Measure the synchronous part
- sync_start = time.perf_counter()
- future = coro.send(None)
- sync_end = time.perf_counter()
- blocking_time += sync_end - sync_start
-
- # Measure the asynchronous part in two different ways
- loop = asyncio.get_running_loop()
- wait_start = time.perf_counter()
- loop_start = loop.time()
- if future is not None:
- done = asyncio.Event()
- future.add_done_callback(lambda _: done.set())
- await done.wait()
- wait_end = time.perf_counter()
- loop_end = loop.time()
- async_time += wait_end - wait_start
- loop_time += loop_end - loop_start
-
- # Raise exception if any
- # future.result()
- except StopIteration as e:
- total_end = time.perf_counter()
- total_time = total_end - total_start
-
- methods_str = ",".join(http_methods) if http_methods else "GET"
-
- nonblocking_time = max(async_time, loop_time)
- # If async time is more than 10% different from loop time, log
it
- delta_symbol = "="
- nonblocking_delta = abs(async_time - loop_time)
- # Must check that nonblocking_time is not 0 to avoid division
by zero
- if nonblocking_time and ((nonblocking_delta /
nonblocking_time) > 0.1):
- delta_symbol = "!"
- route_logger.info(
- "%s %s %s %s %s %s %s",
- methods_str,
- route_path,
- f.__name__,
- delta_symbol,
- int(blocking_time * 1000),
- int(nonblocking_time * 1000),
- int(total_time * 1000),
- )
-
- return e.value
-
- return wrapper
-
- return decorator
-
-
-# This decorator is an adaptor between @committer_get and @app_route functions
-def committer(
- path: str, methods: list[str] | None = None, measure_performance: bool =
True
-) -> Callable[[CommitterRouteHandler[R]], RouteHandler[R]]:
- """Decorator for committer GET routes that provides an enhanced session
object."""
-
- def decorator(func: CommitterRouteHandler[R]) -> RouteHandler[R]:
- async def wrapper(*args: Any, **kwargs: Any) -> R:
- web_session = await session.read()
- if web_session is None:
- _authentication_failed()
-
- enhanced_session = CommitterSession(web_session)
- return await func(enhanced_session, *args, **kwargs)
-
- # Generate a unique endpoint name
- endpoint = func.__module__ + "_" + func.__name__
-
- # Set the name before applying decorators
- wrapper.__name__ = func.__name__
- wrapper.__doc__ = func.__doc__
- wrapper.__annotations__["endpoint"] = endpoint
-
- # Apply decorators in reverse order
- decorated = auth.require(auth.Requirements.committer)(wrapper)
- decorated = app_route(
- path, methods=methods or ["GET"], endpoint=endpoint,
measure_performance=measure_performance
- )(decorated)
-
- return decorated
-
- return decorator
-
-
-async def get_form(request: quart.Request) -> datastructures.MultiDict:
- # The request.form() method in Quart calls a synchronous tempfile method
- # It calls quart.wrappers.request.form _load_form_data
- # Which calls quart.formparser parse and parse_func and parser.parse
- # Which calls _write which calls tempfile, which is synchronous
- # It's getting a tempfile back from some prior call
- # We can't just make blockbuster ignore the call because then it ignores
it everywhere
- app = asfquart.APP
-
- if app is ...:
- raise RuntimeError("APP is not set")
-
- # Or quart.current_app?
- blockbuster = app.extensions.get("blockbuster")
-
- # Turn blockbuster off
- if blockbuster is not None:
- blockbuster.deactivate()
- form = await request.form
- # Turn blockbuster on
- if blockbuster is not None:
- blockbuster.activate()
- return form
-
-
-def public(
- path: str, methods: list[str] | None = None, measure_performance: bool =
True
-) -> Callable[[Callable[Concatenate[CommitterSession | None, P],
Awaitable[R]]], RouteHandler[R]]:
- """Decorator for public GET routes that provides an enhanced session
object."""
-
- def decorator(func: Callable[Concatenate[CommitterSession | None, P],
Awaitable[R]]) -> RouteHandler[R]:
- async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- web_session = await session.read()
- enhanced_session = CommitterSession(web_session) if web_session
else None
- return await func(enhanced_session, *args, **kwargs)
-
- # Generate a unique endpoint name
- endpoint = func.__module__ + "_" + func.__name__
-
- # Set the name before applying decorators
- wrapper.__name__ = func.__name__
- wrapper.__doc__ = func.__doc__
- wrapper.__annotations__["endpoint"] = endpoint
-
- # Apply decorators in reverse order
- decorated = app_route(
- path, methods=methods or ["GET"], endpoint=endpoint,
measure_performance=measure_performance
- )(wrapper)
-
- return decorated
-
- return decorator
-
-
-async def redirect[R](
- route: RouteHandler[R], success: str | None = None, error: str | None =
None, **kwargs: Any
-) -> response.Response:
- """Redirect to a route with a success or error message."""
- if success is not None:
- await quart.flash(success, "success")
- elif error is not None:
- await quart.flash(error, "error")
- return quart.redirect(util.as_url(route, **kwargs))
-
-
-def _authentication_failed() -> NoReturn:
- """Handle authentication failure with an exception."""
- # NOTE: This is a separate function to fix a problem with analysis flow in
mypy
- raise base.ASFQuartException("Not authenticated", errorcode=401)
diff --git a/atr/shared/keys.py b/atr/shared/keys.py
index b9d50b5..e3f74d8 100644
--- a/atr/shared/keys.py
+++ b/atr/shared/keys.py
@@ -34,7 +34,6 @@ import atr.forms as forms
import atr.get as get
import atr.log as log
import atr.models.sql as sql
-import atr.route as route
import atr.shared as shared
import atr.storage as storage
import atr.storage.outcome as outcome
@@ -173,7 +172,7 @@ async def add(session: web.Committer) -> str:
form = await AddOpenPGPKeyForm.create_form()
forms.choices(form.selected_committees, committee_choices)
- except (route.FlashError, web.FlashError) as e:
+ except web.FlashError as e:
log.warning("FlashError adding OpenPGP key: %s", e)
await quart.flash(str(e), "error")
except Exception as e:
diff --git a/atr/shared/start.py b/atr/shared/start.py
index 307dc2c..cb00313 100644
--- a/atr/shared/start.py
+++ b/atr/shared/start.py
@@ -25,7 +25,6 @@ import atr.db.interaction as interaction
import atr.forms as forms
import atr.get.compose as compose
import atr.models.sql as sql
-import atr.route as route
import atr.storage as storage
import atr.template as template
import atr.web as web
@@ -71,7 +70,7 @@ async def selected(session: web.Committer, project_name: str)
-> response.Respon
version_name=new_release.version,
success="Release candidate draft created successfully",
)
- except (route.FlashError, web.FlashError, base.ASFQuartException) as e:
+ except (web.FlashError, base.ASFQuartException) as e:
# Flash the error and let the code fall through to render the
template below
await quart.flash(str(e), "error")
diff --git a/scripts/README.md b/scripts/README.md
index eabac84..8864779 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -52,7 +52,7 @@ Imports OpenPGP public keys from ASF committee KEYS files
into the ATR database.
## lint/jinja\_route\_checker.py
-Validates that Jinja templates only reference routes that exist. Scans all
templates in `atr/templates/` for `as_url(routes.<name>)` calls and reports any
references to routes not found in `state/routes.json`.
+Validates that Jinja templates only reference routes that exist. Scans all
templates in `atr/templates/` for `as_url(get.<name>)` and
`as_url(post.<name>)` calls and reports any references to routes not found in
`state/routes.json`. The routes file is automatically generated when the
application starts by collecting routes from each blueprint's decorators.
## release\_path\_parse.py
diff --git a/scripts/lint/jinja_route_checker.py
b/scripts/lint/jinja_route_checker.py
index bc93d83..b4e4e05 100755
--- a/scripts/lint/jinja_route_checker.py
+++ b/scripts/lint/jinja_route_checker.py
@@ -27,7 +27,7 @@ import re
import sys
from typing import Final
-_AS_URL_PATTERN: Final = re.compile(r"as_url\(routes\.([a-zA-Z0-9_.]+)")
+_AS_URL_PATTERN: Final = re.compile(r"as_url\((get|post)\.([a-zA-Z0-9_.]+)")
class JinjaRouteChecker:
@@ -73,7 +73,9 @@ class JinjaRouteChecker:
# Find all as_url calls with routes
for match in _AS_URL_PATTERN.finditer(content):
- route_path = match.group(1)
+ module_prefix = match.group(1)
+ route_suffix = match.group(2)
+ route_path = f"{module_prefix}.{route_suffix}"
if route_path not in self.available_routes:
# Get approximate line number
line_number = content[: match.start()].count("\n") + 1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
