This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch adopt-magpie-and-upgrade-hooks in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit b40f0826da6c6e8dbb1ea8974cb6ac769060dbcf Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Oct 25 11:40:45 2026 +0100 Adopt Apache Magpie and upgrade the pre-commit hooks Commit the Magpie floor (.apache-magpie.lock), the Claude Code plugin wiring derived from it, and the project's configuration store in .apache-magpie-overrides/. Upgrade every pre-commit hook, replace flake8 with ruff, and add a Markdown license hook. Add Dependabot for pre-commit, GitHub Actions, pip and Terraform, with a seven-day cooldown. Generated-by: Claude Opus 5 --- .apache-magpie-overrides/README.md | 42 ++ .../commit-attribution.toml | 28 +- .apache-magpie-overrides/project.md | 723 +++++++++++++++++++++ .apache-magpie-overrides/repo-health-config.md | 191 ++++++ .flake8 => .apache-magpie.lock | 26 +- .asf.yaml | 17 + .claude/settings.json | 17 + .github/dependabot.yml | 67 ++ .gitignore | 20 + .pre-commit-config.yaml | 41 +- .../packer/files/runner-supervisor.py | 7 +- pyproject.toml | 8 + requirements-test.txt | 17 + scripts/list_committers | 16 +- 14 files changed, 1181 insertions(+), 39 deletions(-) diff --git a/.apache-magpie-overrides/README.md b/.apache-magpie-overrides/README.md new file mode 100644 index 0000000..0972dd6 --- /dev/null +++ b/.apache-magpie-overrides/README.md @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + --> + +# apache-magpie overrides + +Agent-readable instructions that override specific steps or +behaviours of apache-magpie framework skills, scoped to +this adopter repo. Each override file is named after the +framework skill it modifies (e.g. `pr-management-triage.md` +overrides the `pr-management-triage` skill). + +The framework skills consult this directory at run-time +before executing default behaviour. See +[`docs/setup/agentic-overrides.md`](https://github.com/apache/magpie/blob/main/docs/setup/agentic-overrides.md) +in the framework for the full contract. + +**Hard rule**: never modify the snapshot under +`<repo-root>/.apache-magpie/`. Local mods go here. +Framework changes go via PR to `apache/magpie`. + +**Personal (non-shared) overrides** belong in +`<repo-root>/.apache-magpie-local/` (gitignored; a linked worktree +without its own falls back to the main checkout's). Use that +directory for per-developer paths, local tooling, or role- +specific capability enablements you do not want committed +to this repo. diff --git a/pyproject.toml b/.apache-magpie-overrides/commit-attribution.toml similarity index 50% copy from pyproject.toml copy to .apache-magpie-overrides/commit-attribution.toml index 0333bb0..20aca87 100644 --- a/pyproject.toml +++ b/.apache-magpie-overrides/commit-attribution.toml @@ -14,15 +14,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -[tool.black] -line-length = 110 -target-version = ['py36', 'py37', 'py38'] -skip-string-normalization = true -[tool.isort] -line_length = 110 -combine_as_imports = true -default_section = 'THIRDPARTY' -# Need to be consistent with the exclude config defined in pre-commit-config.yaml -skip = ['build','.tox','venv'] -profile = 'black' +# Which trailer an agent-assisted commit carries. +# Resolution rules and the default wording of each convention: +# docs/setup/commit-attribution.md. +# +# In .apache-magpie-overrides/ (committed) this is the project's policy, +# and it wins. In the personal layer (.apache-magpie-local/ when the +# project adopted Magpie, <git-common-dir>/apache-magpie/ otherwise) it is +# one contributor's preference, used only when the project's file is absent +# or says contributor-choice. + +# generated-by | assisted-by | co-authored-by | none | custom +# The project's file may also say contributor-choice. +convention = "generated-by" + +# Optional; required when convention = "custom". +# Placeholders: {agent}, {model}, {email}. +# template = "Generated-by: {agent} ({model})" diff --git a/.apache-magpie-overrides/project.md b/.apache-magpie-overrides/project.md new file mode 100644 index 0000000..8e5c886 --- /dev/null +++ b/.apache-magpie-overrides/project.md @@ -0,0 +1,723 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + --> + +<!-- SPDX-License-Identifier: Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0 --> + +<!-- START doctoc generated TOC please keep comment here to allow auto update --> +<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE --> +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Apache Airflow CI infrastructure — project manifest](#apache-airflow-ci-infrastructure--project-manifest) + - [Identity](#identity) + - [Repositories](#repositories) + - [Mailing lists](#mailing-lists) + - [Tools enabled](#tools-enabled) + - [Source control](#source-control) + - [CVE tooling](#cve-tooling) + - [GitHub project board](#github-project-board) + - [Project chat](#project-chat) + - [Mail sources](#mail-sources) + - [Backend declaration](#backend-declaration) + - [Drafting backend](#drafting-backend) + - [Per-backend config](#per-backend-config) + - [Issue-template fields](#issue-template-fields) + - [Security workflow configuration](#security-workflow-configuration) + - [CVE authority](#cve-authority) + - [Governance](#governance) + - [Security inbox](#security-inbox) + - [Forwarders](#forwarders) + - [Mail provider](#mail-provider) + - [Archive system](#archive-system) + - [Project metadata](#project-metadata) + - [Security cross-reference](#security-cross-reference) + - [Tracker](#tracker) + - [Advisory records](#advisory-records) + - [Scope detection](#scope-detection) + - [Release process](#release-process) + - [Roster](#roster) + - [Product](#product) + - [Pointers to sibling files](#pointers-to-sibling-files) + +<!-- END doctoc generated TOC please keep comment here to allow auto update --> + +<!-- SPDX-License-Identifier: Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0 --> + +# Apache Airflow CI infrastructure — project manifest + +This is the **project configuration** for `apache/airflow-ci-infra`. +Every skill under [`../../.claude/skills/`](../../../skills/) +reads the project name from `<project-config>/project.md` and then loads this manifest to resolve project-specific identity, +repositories, mailing lists, and references to the other files in +this directory. + +**Note on Auto-sourcing:** Stable fields under Identity and Repositories +(`upstream_repo`, `upstream_default_branch`, `product_family_url`, labels) +can be automatically derived from GitHub repository metadata via +`gh repo view` during `/magpie-setup:setup adopt` or `/magpie-setup:setup upgrade`, for +any adopter. The **Mailing lists** are auto-sourced **only for +`organization: ASF`** projects (from `.asf.yaml`, with `*.apache.org` +defaults); a non-ASF `organization` fills them in by hand. Hand-editing is +only required when a source is absent, incomplete, or you wish to override +a derived value. + +Grep for `TODO` to see every field you still need to fill in: + +```bash +grep -n TODO projects/<name>/project.md +``` + +## Identity + +| Key | Value | +|---|---| +| `organization` | `ASF` | +| `project_name` | `Apache Airflow` | +| `vendor` | `Apache Software Foundation` | +| `short_name` | `airflow` | +| `product_family_url` | `https://airflow.apache.org/` | + +The `vendor` / `project_name` pair is what lands in the `vendor` and +`product` fields of the CVE 5.x record the CVE-JSON generator +produces. + +## Repositories + +| Key | Value | Purpose | +|---|---|---| +| `tracker_repo` | TODO: e.g. `foo-s/foo-s` | Private security tracker (this repo) | +| `tracker_repo_url` | TODO | | +| `tracker_default_branch` | TODO: e.g. `main` | Default PR target for the tracker repo | +| `tracker_project_board_url` | TODO: URL of the GitHub Project V2 board, if any | Security board | +| `upstream_repo` | `apache/airflow-ci-infra` | Public codebase where fixes land | +| `upstream_repo_url` | `https://github.com/apache/airflow-ci-infra` | | +| `upstream_default_branch` | `main` | Upstream's default branch — what `<default-branch>` resolves to. Distinct from `tracker_default_branch` | +| `upstream_agents_md_url` | TODO: `https://github.com/<upstream>/blob/main/AGENTS.md` | Conventions this repo mirrors | +| `upstream_contributing_docs_url` | TODO | | +| `upstream_genai_disclosure_anchor` | TODO: URL + anchor for the project's Gen-AI disclosure guideline | | +| `upstream_security_policy_url` | TODO: `https://github.com/<upstream>/security/policy` | | +| `security_pages_url` | TODO: e.g. `https://foo.apache.org/security` | Public security pages on the project website — updated by the release manager after each advisory (ASF security-committers policy step); read by `security-issue-sync` for the security-pages checklist reminder. Leave unset if the project has no security pages — the skill then uses the plain phrase without a link. To have sync propose the edit itself, also configure [Advisory records](#advisory-records) | + +## Mailing lists + +*(For `organization: ASF` projects only: auto-sourced from the `.asf.yaml` +`notifications:` block if present, else `*.apache.org` defaults. A non-ASF +`organization` has no `.asf.yaml` — fill these in by hand.)* + +| Key | Value | Notes | +|---|---|---| +| `security_list` | `[email protected]` | Inbound reports; **not** publicly archived (asked on adopt; with no dedicated list, request one from ASF Infra, and use `private@<project>.apache.org` only as a flagged fallback) | +| `private_list` | `[email protected]` | PMC escalation; **not** publicly archived (auto-sourced default) | +| `users_list` | `[email protected]` | Public advisories end up here; publicly archived (auto-sourced from `.asf.yaml` or default) | +| `dev_list` | `[email protected]` | Release `[RESULT][VOTE]` threads; publicly archived (auto-sourced from `.asf.yaml` or default) | +| `announce_list` | `[email protected]` | Cross-project announcement list; publicly archived | +| `commits_list` | `[email protected]` | Publicly archived (auto-sourced from `.asf.yaml`) | + +Only URLs on publicly archived lists may appear in CVE `references[]` as +`vendor-advisory`; see `../../AGENTS.md` and +[`security-model.md`](security-model.md). + +The foundation-wide security forwarding address (e.g. `[email protected]` +for ASF projects) is org-level — inherited from your organization's +`security_inbox.foundation_security_address`. Do not declare it here. + +## Tools enabled + +| Capability | Tool | Adapter directory | Config knobs declared here | +|---|---|---|---| +| Issue tracking + project board | `github` | [`../../tools/github/`](../../../tools/github/) | `tracker_repo`, `upstream_repo`, `github_project_board_*`, `issue_template_fields` | +| Security tracker backend | `github` (default), `local-files`, or `jira` — see [Tracker](#tracker) | [`../../tools/security-tracker/`](../../../tools/security-tracker/) (contract), [`tools/local-tracker/`](../../../tools/local-tracker/), [`tools/jira/`](../../../tools/jira/) | `tracker.platform`, `tracker.local.*`, `tracker.jira.*` | +| Source control (VCS) | `github` (Git, default), or `svn` / `asf-svn` (Subversion) — see [Source control](#source-control); replaceable with another non-Git VCS | [`../../tools/github/source-control.md`](../../../tools/github/source-control.md), [`tools/asf-svn/source-control.md`](../../../tools/asf-svn/source-control.md), [`tools/vcs/`](../../../tools/vcs/) | `vcs_backend`, `upstream_repo`, `upstream_default_branch`; for `svn` also `svn_url`, `svn_repository_root`. On Subversion `secur [...] +| Inbound email / drafts | `<one or more mail-source backends>` | [`../../tools/mail-source/contract.md`](../../../tools/mail-source/contract.md) (abstract) + per-backend adapter dirs (`tools/gmail/`, `tools/ponymail/`, `tools/mail-source/imap/`, `tools/mail-source/mbox/`, ...) | See [Mail sources](#mail-sources) below — declare each backend's role (primary / preferred-for-`<op>` / fallback / optional) and `mandatory` flag | +| CVE allocation + record mgmt | *org-level* — inherited from `organizations/<org>/organization.md → cve_authority.tool`; for ASF: `vulnogram` ([`tools/cve-tool-vulnogram/`](../../../tools/cve-tool-vulnogram/)), for independent: `mitre-form` | — | override in [CVE authority](#cve-authority) only if this project differs from its org | +| Security cross-ref | `osv` | [`../../tools/osv/`](../../../tools/osv/) | `security_cross_ref.tool`, `security_cross_ref.ecosystem` | +| Project metadata (rosters / people / releases) | *org-level* — inherited from `organizations/<org>/organization.md → project_metadata.kind`; for ASF: `apache-projects` ([`tools/apache-projects/`](../../../tools/apache-projects/)), for independent: `none` | — | override in [Project metadata](#project-metadata) only if this project differs from its org | +| Project chat | TODO: `slack`, `discord`, or `none` | [`../../tools/chat/`](../../../tools/chat/) (abstract) + adapter dirs (`tools/chat-slack/`, `tools/chat-discord/`) | `chat.kind`, `chat.channels`, `chat.guild_id` — see [Project chat](#project-chat) below; read-only, public channels only | +| Release comms | TODO: the backend that carries release announcements — for ASF: `dev_list` / `announce_list` / `users_list`; for GitHub Releases leave blank | — | whichever release-comms keys the org default or per-project override declares | + +To replace a tool (e.g. swap GitHub issues for JIRA), declare an +alternate tool in the table above, add a `tools/<name>/` adapter +directory, and make sure the values the generic skills need are still +reachable from this manifest. + +### Source control + +Which VCS the project's code lives in. +A skill that runs [`magpie-vcs`](../../../tools/vcs/README.md) passes `vcs_backend` as `--backend`, so a checkout of the wrong kind fails loudly instead of silently using another backend. +`/magpie-setup config` and `adopt` detect a Subversion-hosted project with `magpie-vcs probe` and propose these values; a Git project gets `git`. + +```yaml +vcs_backend: git # git | svn | hg | fossil; unset means git + +# Subversion only (vcs_backend: svn). +svn_url: TODO # project base URL, above trunk/branches/tags, e.g. https://svn.apache.org/repos/asf/foo +svn_repository_root: TODO # repository root, e.g. https://svn.apache.org/repos/asf +``` + +With `vcs_backend: svn`: + +- `upstream_default_branch` is `trunk`, the line of work `magpie-vcs branch` reports. +- The `svn` backend supports only the `trunk` / `branches/<name>` / `tags/<name>` layout below `svn_url`. + For a repository organised any other way, leave `upstream_default_branch` `TODO`: `branch` then reports the repository-relative path, and `new-branch` and name-based `switch` refuse. +- `upstream_repo` names a GitHub repository only when the project really has one, such as a mirror it takes pull requests on. + Otherwise leave it `TODO`: the skills that act on GitHub pull requests and issues do not apply + (see [`tools/vcs/` → Subversion-hosted projects](../../../tools/vcs/README.md#subversion-hosted-projects)). + +## CVE tooling + +The CNA tool, allocation URL, record URL template, and allocation gate are +**org-level** — inherited from your organization's `cve_authority` and +`governance` blocks (see +[`organizations/ASF/organization.md`](../../../organizations/ASF/organization.md) +for ASF defaults, +[`organizations/independent/organization.md`](../../../organizations/independent/organization.md) +for the GitHub-native baseline). Override a key in the +[Security workflow configuration → CVE authority](#cve-authority) block only +if this project's CNA setup genuinely differs from its organization. + +The CNA's org UUID (`asf_org_id`, the `assignerOrgId` of every record) is +org-level too: the ASF CNA uses one UUID for every Apache project, and the +ASF organization supplies it. +It is not needed to allocate a CVE; only the generated CVE JSON carries it. + +For **ASF adopters**, fill in the per-project CNA queue fields below: + +| Key | Value | +|---|---| +| `cna_private_owner` | TODO: e.g. `foo` (CNA_private.owner — the PMC's name in Vulnogram, the lower-case project slug) | +| `cna_private_projecturl` | TODO: e.g. `https://foo.apache.org/` | +| `cna_private_userslist` | TODO: e.g. `[email protected]` | + +## GitHub project board + +If the project uses a Projects V2 board for its security-issue view, +declare the node IDs below. Fetch with the introspection query in +[`../../tools/github/project-board.md`](../../../tools/github/project-board.md#introspection--re-fetch-the-option-ids). +If the project does not run a board, leave the table blank — skills +treat missing board config as *"no board reconciliation"*. + +| Key | Value | +|---|---| +| `project_board_url` | TODO | +| `project_board_number` | TODO | +| `project_board_node_id` | TODO | +| `status_field_node_id` | TODO | + +**`Status` column → option-ID mapping** (re-fetch if any write +returns `not found`): + +| Column | Option ID | +|---|---| +| `Needs triage` | TODO | +| `Assessed` | TODO | +| `CVE allocated` | TODO | +| `PR created` | TODO | +| `PR merged` | TODO | +| `Fix released` | TODO | +| `Announced` | TODO | + +## Project chat + +The contributor-growth skills read public chat channels for community signals, through the [`tools/chat/`](../../../tools/chat/) contract. +Leave `kind: none` if the project has no public chat, or does not want it read. + +```yaml +chat: + kind: none # slack | discord | none + # guild_id: "" # Discord only, optional: server (guild) ID when the bot joins several servers + channels: [] # channel names (Discord also accepts IDs) to read; empty = every public channel — + # list them explicitly in a workspace shared by several projects +``` + +## Mail sources + +The skills treat every supported mail backend the same way — +through the abstract operations defined in +[`../../tools/mail-source/contract.md`](../../../tools/mail-source/contract.md). +The adopter declares which backends are configured, what *role* +each plays, and whether any are *mandatory*. The skill's resolution +rule (see the contract) then picks the right backend per operation +at run time. + +### Backend declaration + +One row per configured backend. **Exactly one** row carries +`role: primary`. Multiple rows may carry `preferred for <op>` to +override the primary for specific operations. `fallback` rows are +tried in order when no preferred / primary backend supports the op. +`mandatory: yes` means the skill **refuses to run** when that +backend is unavailable; `no` means the skill continues with the +remaining backends (and skips ops that no available backend supports). + +| Backend | Role | Mandatory | Notes | +|---|---|---|---| +| TODO: `gmail` | TODO: e.g. `primary` | TODO: `yes` / `no` | TODO: e.g. "Triager Gmail account subscribed to `<security-list>` and `<private-list>`" | +| TODO: `ponymail` | TODO: e.g. `fallback` or `preferred for thread_url` | TODO: `yes` / `no` | TODO: e.g. "Read-only archive backstop; install per [`tools/ponymail/tool.md`](../../../tools/ponymail/tool.md)" | +| TODO: *(add more rows as needed — `imap`, `mbox`, `mailman3`, project-specific adapter)* | | | | + +> **Mail backend selection is org-level.** The `mail_provider` block in +> your organization manifest sets the primary and fallback backends +> (for ASF: `primary: gmail-mcp`, `fallback: ponymail`; for independent: +> `primary: none`, `fallback: none`). Only declare backends here that +> override or extend what the organization manifest provides. For ASF +> projects where PonyMail is the inherited fallback, declare it here only +> to change its `mandatory` flag or role for this specific project; the +> backend itself is already wired by the organization. Install PonyMail per +> [`../../tools/ponymail/tool.md`](../../../tools/ponymail/tool.md#keeping-the-checkout-current). + +Reference adapter docs: +[`tools/gmail/tool.md`](../../../tools/gmail/tool.md) (full read+write), +[`tools/ponymail/tool.md`](../../../tools/ponymail/tool.md) (read-only ASF archive), +[`tools/mail-source/imap/README.md`](../../../tools/mail-source/imap/README.md) (stub), +[`tools/mail-source/mbox/README.md`](../../../tools/mail-source/mbox/README.md) (read-only offline archive — stub), +[`tools/mail-source/mailman3/README.md`](../../../tools/mail-source/mailman3/README.md) (read-only public Mailman 3 / Hyperkitty archive). + +### Drafting backend + +Where the skills file the reply drafts they write. +The project sets it here; a contributor overrides it for themselves with +`tools.mail_create` in their own `user.md`. + +```yaml +mail_create: + backend: gmail # gmail (default) or maildir + maildir: # only read when backend: maildir + drafts_path: ~/Mail/<project>/Drafts + from_address: TODO +``` + +`gmail` drafts in Gmail +([`tools/gmail/draft-backends.md`](../../../tools/gmail/draft-backends.md)); +`maildir` files the draft into a local Maildir for Thunderbird or another +mail client ([`tools/maildir/README.md`](../../../tools/maildir/README.md)). +Resolution and the per-person override: +[`draft-backends.md` § *How the skills pick a backend*](../../../tools/gmail/draft-backends.md#how-the-skills-pick-a-backend). + +### Per-backend config + +Per-backend values the generic recipes substitute in. Only fill in +the rows for backends declared above; leave the rest blank or +remove the row. + +| Key | Backend | Value | +|---|---|---| +| `security_list_domain` | `gmail` | TODO: e.g. `security.foo.apache.org` — Gmail `list:` operator uses the domain form | +| `ponymail_private_search_url_template` | `ponymail` | TODO | +| `ponymail_public_search_url_template` | `ponymail` | TODO | +| `ponymail_api_url_template` | `ponymail` | TODO | +| `ponymail_thread_url_template` | `ponymail` | `https://lists.apache.org/thread/<hash>?<list>` | +| `imap_host` | `imap` | TODO: e.g. `imap.example.org` | +| `imap_account` | `imap` | TODO: e.g. `[email protected]` | +| `imap_security_list_folder` | `imap` | TODO: e.g. `INBOX.security-list` | +| `imap_drafts_folder` | `imap` | TODO: e.g. `Drafts` (or leave blank to declare `create_draft` unsupported on this adapter) | +| `mbox_archive_path` | `mbox` | TODO: e.g. `/srv/audit/security-list-2024.mbox` | +| `mailman3_archive_url` | `mailman3` | TODO: e.g. `https://mail.example.org/archives` (the Hyperkitty root, without `/api`) | + +## Issue-template fields + +The skills' body-field roles map to the following concrete `###` +headings in the project's issue template (the concrete YAML file lives in the +adopter's `<upstream>` repo; the generic role → field contract is in The generic role → GitHub-field +contract lives in +[`../../tools/github/issue-template.md`](../../../tools/github/issue-template.md); +the concrete names below are what skills read and write for this +project. + +| Role (generic) | Field name | Template type | Required? | +|---|---|---|---| +| `issue-description` | TODO | `textarea` | TODO | +| `public-summary` | TODO | `textarea` | TODO | +| `affected-versions` | TODO | `input` | TODO | +| `security-thread` | TODO | `input` | TODO | +| `public-advisory-url` | TODO | `input` | TODO | +| `reporter-credit` | TODO | `input` | TODO | +| `pr-with-fix` | TODO | `input` | TODO | +| `cwe` | TODO | `input` | TODO | +| `severity` | TODO | `dropdown` | TODO | +| `cve-tool-link` | TODO | `input` | TODO | + +## Security workflow configuration + +Skills resolve every workflow knob from the three-layer chain +`project.md → organizations/<org>/organization.md → framework default` +(see [`AGENTS.md` § Configuration resolution order](../../../AGENTS.md#configuration-resolution-order)). +The **organization** you named under *Identity* supplies the org-wide +defaults — the CNA tool, the governance gate, the mail / forwarder / +archive backends, the project-metadata source, and the release-manager +lookup cascade. For an ASF project these resolve to the Vulnogram / +PonyMail / `apache-projects-mcp` / ASF-security values in +[`organizations/ASF/organization.md`](../../../organizations/ASF/organization.md); +`organization: independent` inherits the GitHub-native baseline in +[`organizations/independent/organization.md`](../../../organizations/independent/organization.md). + +**Declare below only what is specific to this project.** Each block that +is purely org-level says so and shows nothing — copy the matching key +from the organization manifest down into that block only to override it +(the project value wins). The blocks that every adopter fills in keep +their per-project keys. + +### CVE authority + +Org-level — inherited from your organization's `cve_authority` block +(CNA tool, allocate / record / source-tab URLs, state mapping, +propagation, allocation-email flag, reviewer channel). Override a single +key here only if this project's CNA setup differs from its organization. + +### Governance + +Org-level except the escalation contact: the allocation gate, gate +label, release-vote gating, private governance list, and roster URL are +inherited. Declare this project's escalation contact: + +```yaml +governance: + # GitHub handle (or external contact) the skills cc / @-mention when + # escalating beyond the security team. + # Consumed by: security-issue-sync, pr-management-triage. + escalation_contact: "@<escalation-contact>" +``` + +### Security inbox + +Org-level except the concrete address: the inbox `kind`, the foundation +security address, the `has_forwarder_relay` flag, the list filter, and +the reporter-response timeout are inherited. Declare this project's +inbound address: + +```yaml +security_inbox: + # The concrete inbound address / channel ID / form URL for this project. + # Consumed by: security-issue-import, security-issue-sync, canned-responses. + address: <security-list> + # Optional override: addresses that send CVE-tool bookkeeping mail. + # ASF projects inherit [email protected]; if overriding this list, repeat + # that address and add the active CNA reviewers. + # Consumed by: security-issue-import's Step 1 pre-filter. + # cve_tool_senders: [[email protected], [email protected]] +``` + +**Optional override — `reporter_response_timeout_days`.** Days of +reporter silence, counted from the security team's most recent outbound +message on the reporter thread, before `security-issue-sync` proposes +proceeding with the fix and announcement without further reporter +sign-off. Per +[ASF policy](https://www.apache.org/security/committers.html), an +unresponsive reporter must not block the project team, particularly for +a high-severity or high-impact issue. Your organization declares the +default (14 days for both `ASF` and `independent`); add the key here only +to depart from it — a project handling especially sensitive reports may +want a longer window, one under heavy report volume a shorter one. +Consumed by `security-issue-sync` (Step 1c staleness check / Step 2b +proposal). + +```yaml +security_inbox: + reporter_response_timeout_days: 21 # only if your project differs from its organization +``` + +### Forwarders + +Org-level — inherited from your organization's `forwarders` block (which +relay adapters are enabled and their per-adapter detect / credit rules). +Override here only to enable an extra relay this project uses. + +### Mail provider + +Org-level — inherited from your organization's `mail_provider` block +(primary + fallback mail backends). Override here only if this project +reads mail from a different backend than its organization. + +### Archive system + +Org-level — inherited from your organization's `archive_system` block +(public-archive backend + URL templates). Override here only if this +project's advisories surface on a different archive than its organization. + +### Project metadata + +Org-level — inherited from your organization's `project_metadata` block +(roster / people / releases backend + whether it is mandatory). Override +here only if this project uses a different metadata source. + +### Security cross-reference + +Vulnerability cross-referencing and alias lookup configuration (OSV.dev / NVD). +Declare this project's default package ecosystem and tool adapter: + +```yaml +security_cross_ref: + # Tool adapter: osv (tools/osv/) | none + tool: osv + # Default package ecosystem for vulnerability queries (e.g. PyPI, Maven, npm, Go, crates.io) + ecosystem: PyPI +``` + +### Tracker + +The tracker `board`, `visibility`, and `skill_url_template` +are org-level defaults (`platform` defaults to `github`) — override them here only if this project differs +(e.g. a project that runs its security tracker publicly sets +`visibility: public`). Declare this project's per-tracker vocabulary: + +```yaml +tracker: + # Where security reports live — the security-tracker contract + # (tools/security-tracker/contract.md). `github` (default): the private + # `tracker_repo`. `local-files`: markdown files in a folder of a private + # Git or SVN working copy (tools/local-tracker/). `jira`: a Jira project + # whose issues carry an issue-security level (tools/jira/). + # `/magpie-setup config` asks and, for local-files, learns the format of + # the reports already in the folder. `TODO` (not yet asked) reads as + # `github`; the question writes the confirmed choice here. + platform: TODO + + # local-files only. + local: + root: TODO # tracker folder, relative to the repository root, e.g. `security/reports` + + # jira only. + jira: + url: TODO # e.g. https://issues.example.org/jira + project: TODO # project key, e.g. FOOSEC + security_level: TODO # issue-security level every tracker must carry + issue_type: Bug + deployment: server # server | cloud + + # Whether the reporter can see the tracker issue once opened. + reporter_has_access: false + + # Whether the tracker drives a board / kanban view. + project_board_enabled: true + + # Body-field heading names — role -> the concrete `###` heading in this + # project's issue template. Skills refer to these by role. + # Consumed by: every skill that reads/writes the issue body. + body_fields: + cve_link: "CVE tool link" + mailing_thread: "Mailing list thread URL" + affected_versions: "Affected versions" + + # Tracker labels — role -> the concrete label name in this project. + # Consumed by: security-issue-triage, security-issue-sync, pr-management-triage. + labels: + security_marker: "security" + needs_triage: "needs triage" + pr_open: "pr created" + pr_merged: "pr merged" + cve_allocated: "cve allocated" + not_cve_worthy: "not cve worthy" + # Label on the single open "rejected without tracker" ledger issue + # (see below). NOT the security_marker label. + rejections_ledger: "rejections-ledger" +``` + +#### Rejected-without-tracker ledger + +The `security-issue-import` skill sometimes rejects a report with a +canned reply **without creating a tracker** (the disposition lives only +on the mail thread). To keep those rejections countable, the team +records each one as a comment on a single dedicated **ledger issue** in +`tracker_repo`: one **open** issue, labelled with the +`tracker.labels.rejections_ledger` value (default `rejections-ledger`) +and **not** carrying the security-marker label. + +Adopters who want the *rejected without tracker* dashboard stat must: + +1. **Create the ledger issue** once in `tracker_repo` and label it + `rejections-ledger` (keep it open; the skills resolve it via + `gh issue list --repo <tracker> --state open --label + rejections-ledger`). +2. **Set the dashboard knob.** Point + `security-tracker-stats.md → rejections_ledger_label` (or the + `rejections_ledger_label:` key in the renderer's YAML overlay) at the + same label. Set it to `null` to disable the stat — then no ledger + issue is needed. + +Each rejection comment carries a machine-parseable block +(`<!-- rejection v1 -->` with `date:` / `reporter:` / `canned:` / +`thread:` / `summary:` lines); a one-time historical backfill is a +single `<!-- rejection-backfill v1 count: N -->` comment. The +`security-tracker-stats-dashboard` renderer parses these and excludes the +ledger issue from all tracker classification. Closes handled by +`security-issue-invalidate` are **not** ledger entries. + +### Advisory records + +Optional, per-project — where the project records each published CVE in its own published security records +(the site repository its security pages are built from), +so that `security-issue-sync` can propose the change once cve.org shows the CVE published. +Leave the block out and the step is skipped with a one-line note; the security-pages checklist reminder stays as it is. +The deterministic part is `advisory-records` in +[`tools/cve-tool-vulnogram/generate-cve-json/`](../../../tools/cve-tool-vulnogram/generate-cve-json/README.md#record-a-published-cve-in-the-projects-security-records--advisory-records); +the step is described in [`docs/security/advisory-records.md`](../../../docs/security/advisory-records.md). + +```yaml +advisory_records: + # git | svn. Default: the project's source-control backend (Tools enabled → Source control). + vcs: git + # Repository URL, used only to propose the first checkout. + repository: https://github.com/<org>/<project>-site.git + # Local working copy. A per-user path: `user.md → environment.advisory_records_clone` wins over this. + working_copy: ~/code/<project>-site + # Git only: the branch and remote the commit is pushed to. + branch: main + remote: origin + # One line; placeholder {cve_id}. + commit_message: "Add {cve_id} to the security pages" + + # cve-json-file — one CVE 5.x JSON file per CVE, the record generate-cve-json emits + # (e.g. Apache HTTP Server keeps these under content/security/json/). + # release-page — an entry in a per-release-line page, one section per fixed release, newest first + # (e.g. Apache Tomcat's xdocs/security-<line>.xml). + layout: cve-json-file + + cve_json_file: + path: content/security/json/{cve_id}.json # placeholders {cve_id}, {year} + indent: 4 + keep_cna_private: false # drop the CNA tool's internal workflow block + + release_page: + format: xml # xml | markdown + path: xdocs/security-{line}.xml # placeholder {line} + release_line_parts: 1 # version components that name a line: 1 → "9" for 9.0.99 + section_name: "Fixed in {product} {version}" + # Template files, relative to <project-config>; omit to use the shipped examples + # (tools/cve-tool-vulnogram/generate-cve-json/src/generate_cve_json/advisory_templates/). + # section_template: advisory-records/section.xml + # entry_template: advisory-records/entry.xml + # fix_commit_template: '<hashlink hash="{hash}"/>' + # section_parent: body # xml: where the first release section goes +``` + +### Scope detection + +Per-project — whether this project distinguishes scope sub-products, and +the label → sub-product map: + +```yaml +scope_detection: + # When false, every issue maps to the single product in the `product` block. + # Consumed by: security-issue-triage, generate-cve-json, security-issue-sync. + enabled: true + + # Scope label -> sub-product: tracker label -> CVE product / packageName + # / upstream path-prefix the skill uses to confirm a PR touches that scope. + labels: + <scope-label>: + product: "<Product Name>" + packageName: "<package-name>" + path_prefix: "<path-prefix-regex>" + <secondary-scope-label>: + product: "<Secondary Product Name>" + packageName: "<secondary-package-name>" + path_prefix: "<secondary-path-prefix-regex>" +``` + +### Release process + +The release-manager lookup cascade and artifact registries are org-level +and inherited. Declare this project's stale milestones and changelog +fragment tool: + +```yaml +release_process: + # Milestones the skills treat as "stale" (overdue for re-targeting) — + # exact milestone-name matches. + # Consumed by: security-issue-sync, pr-management-triage. + stale_milestones: + - "<stale-milestone-1>" + - "<stale-milestone-2>" + + # Whether the upstream repo uses a changelog-fragment tool, and which one. + # Consumed by: security-issue-fix, issue-fix-workflow. + newsfragments: + enabled: true + tool: <fragment-tool> +``` + +### Roster + +The roster `source` is org-level and inherited. Declare this project's +bare-name → handle map and release managers: + +```yaml +roster: + # Mailing-list threads reference contributors by first name; this binds + # those to GitHub handles for @-mentions. + # Consumed by: security-issue-sync, pr-management-mentor. + bare_name_handles: + # Example (replace with your project's contributors): + "<FirstName>": "@<handle>" + + # Release-manager handles, current first. Keep in sync with release-trains.md. + # Consumed by: security-issue-sync, security-issue-fix. + release_managers: + # Example (replace with your project's release managers): + - "@<handle>" +``` + +### Product + +Per-project — the product identity used in CVE records, advisories, and +title normalization: + +```yaml +product: + # Human-readable product name — lands in the CVE record's `product` field. + # Consumed by: generate-cve-json, canned-responses templating. + name: <ProjectShortName> + + # Package-name shape for the primary artifact (PyPI / npm / Maven). + # Consumed by: generate-cve-json, canned-responses templating. + package_name: <package-name> + + # Regex matched against changed paths in an upstream PR to confirm + # "this PR really touches the product" — a backstop in the fix flow. + # Consumed by: security-issue-fix, pr-management-triage. + code_pointer_path_prefix: "<code-path-prefix-regex>" + + # Prefixes the title-normalization skill strips from an inbound subject + # when building a CVE title (matched at the start, case-insensitively). + # Consumed by: title-normalization, generate-cve-json, canned-responses. + subject_prefix_strip: + - "[SECURITY]" + - "[Security Report]" + - "Re:" + - "Fwd:" + - "<vendor>:" + - "<vendor> <product>:" + + # Prefix the affected-versions extractor strips to leave the bare version. + # Consumed by: security-issue-sync, generate-cve-json. + affected_version_extract_prefix: "<ProjectShortName>" +``` + +## Pointers to sibling files + +- [`release-trains.md`](release-trains.md) — fast-moving release state, release-manager attribution, security-team roster. +- [`milestones.md`](milestones.md) — milestone naming conventions. +- [`scope-labels.md`](scope-labels.md) — scope label → CVE product mapping. +- [`security-model.md`](security-model.md) — Security-Model URL + anchors. +- [`title-normalization.md`](title-normalization.md) — CVE title strip cascade. +- [`fix-workflow.md`](fix-workflow.md) — fork / toolchain / commit-trailer specifics. +- [`naming-conventions.md`](naming-conventions.md) — project-specific editorial rules. +- [`canned-responses.md`](canned-responses.md) — reporter-facing reply templates. +- [`skill-sources.md`](skill-sources.md) — trusted external skill sources this project pulls skills from (the install gate). +- [`README.md`](README.md) — project file index + onboarding checklist. diff --git a/.apache-magpie-overrides/repo-health-config.md b/.apache-magpie-overrides/repo-health-config.md new file mode 100644 index 0000000..9b97730 --- /dev/null +++ b/.apache-magpie-overrides/repo-health-config.md @@ -0,0 +1,191 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + --> + +<!-- START doctoc generated TOC please keep comment here to allow auto update --> +<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE --> +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Repo-health audit configuration](#repo-health-audit-configuration) + +<!-- END doctoc generated TOC please keep comment here to allow auto update --> + +<!-- SPDX-License-Identifier: Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0 --> + +# Repo-health audit configuration + +Per-skill switches for the repo-health audit family. Copy this file into +your `<project-config>/` directory and fill in the `TODO` values. Skills +in this family (`ci-runner-audit`, `workflow-security-audit`, +`dependency-audit`, `license-compliance-audit`, `flaky-test-triage`, and +`dependency-license-audit`) read from this file at run time. + +See `docs/repo-health/README.md` for a full description of each skill +and adopter-contract details. + +--- + +```yaml +repo_health: + + # --------------------------------------------------------------------------- + # ci-runner-audit — obsolete GitHub-hosted runner labels and macOS arch + # mismatches. Consumed by: ci-runner-audit (experimental). + # --------------------------------------------------------------------------- + ci_runner_audit: + + # Runner label names the skill treats as obsolete/retired. + # Default: the GitHub-deprecated label list below. + # Override when: your project uses custom runners whose labels should also + # be flagged (add them here), or you want to suppress a label from the + # default list (remove it). + deprecated_runner_labels: + - ubuntu-18.04 + - ubuntu-20.04 + - windows-2019 + - macos-11 + - macos-12 + + # Additional repositories to audit alongside the project's primary repos. + # Use `owner/repo` form. Leave empty to audit only the repos listed in + # project.md (upstream_repo, tracker_repo). + extra_repos: [] + + # --------------------------------------------------------------------------- + # workflow-security-audit — GitHub Actions security issues surfaced by + # zizmor. Consumed by: workflow-security-audit (experimental). + # --------------------------------------------------------------------------- + workflow_security_audit: + + # zizmor finding classes to enable. All four are on by default. + # Remove a class to suppress that finding category for this project. + # ASF default: all enabled (injection and fork-secrets are high-severity; + # excessive-permissions and unpinned-actions are medium-severity). + # Override when: the project intentionally uses floating action references + # (e.g. in non-production automation) or has a justified write permission. + enabled_rules: + - injection # untrusted event data interpolated into run: steps + - excessive-permissions # write-all or unneeded write scopes + - unpinned-actions # @tag or @branch references instead of SHA pins + - fork-secrets # secrets accessible from pull_request_target + + # Additional repositories to audit beyond project.md → upstream_repo. + extra_repos: [] + + # --------------------------------------------------------------------------- + # dependency-audit — known-vulnerability check on direct + transitive deps. + # Consumed by: dependency-audit (experimental). + # --------------------------------------------------------------------------- + dependency_audit: + + # Dependency managers in use. Selects the audit tool adapter. + # Allowed values: pip, npm, cargo, maven, gradle + managers: + - pip + + # Minimum severity level to include in the report. + # Allowed values: low, medium, high, critical + # Default: medium (suppress low-signal informational hits). + min_severity: medium + + # --------------------------------------------------------------------------- + # license-compliance-audit — SPDX header and NOTICE compliance check. + # Consumed by: license-compliance-audit (experimental). + # --------------------------------------------------------------------------- + license_compliance_audit: + + # Required SPDX license expression for every source file under + # source_paths. Must be a valid SPDX expression string. + required_spdx_expression: "Apache-2.0" + + # Source paths to audit for license headers (relative to upstream repo root). + source_paths: + - ".github/" + - "arm-docker-ami/" + - "github-runner-ami/" + - "lambdas/" + - "scripts/" + - "terraform/" + - "tests/" + + # Paths to skip — test fixtures, vendored code, generated files, etc. + # The header templates themselves. + skip_paths: + - "license-templates/" + + # --------------------------------------------------------------------------- + # flaky-test-triage — intermittent test failure detection from CI run history. + # Consumed by: flaky-test-triage (experimental). + # --------------------------------------------------------------------------- + flaky_test_triage: + + # Audit window in days. The skill scans CI run history over this period. + # Default: 30 days (enough signal without fetching unbounded history). + window_days: 30 + + # Minimum failure-rate fraction to flag a job as candidate flaky. + # A job that fails 10% of the time or more is flagged by default. + # Override when: your project has a higher noise floor and 10% is too noisy. + failure_rate_threshold: 0.10 + + # Job-name glob patterns to include in the analysis. + # Leave empty to analyse all jobs in the CI run. + include_patterns: [] + + # Job-name glob patterns to exclude (known-always-failing or skipped jobs). + exclude_patterns: [] + + # --------------------------------------------------------------------------- + # dependency-license-audit — license classification of direct + transitive + # dependencies against a policy. Consumed by: dependency-license-audit. + # --------------------------------------------------------------------------- + dependency_license_audit: + + # License policy model. + # asf — apply the ASF three-category model (A allowed, B allowed + # in binary/convenience-binary form only, X forbidden). + # allowlist — allow only the SPDX expressions in allowed_licenses below. + # Default: asf. + policy: asf + + # SPDX expressions always treated as allowed, regardless of policy. + # Override when: your project permits additional permissive licenses. + allowed_licenses: + - Apache-2.0 + - MIT + - BSD-2-Clause + - BSD-3-Clause + - ISC + + # SPDX expressions always treated as forbidden (ASF category X). + # Override when: your project has an exception for a specific dependency. + forbidden_licenses: + - GPL-2.0-only + - GPL-3.0-only + - AGPL-3.0-only + - LGPL-3.0-only + + # Audit transitive dependencies, not just direct ones. Default: true. + include_transitive: true + + # How to treat a dependency whose license cannot be resolved. + # Allowed values: flag (report as unknown), ignore. + # Default: flag. + unknown_license_action: flag +``` diff --git a/.flake8 b/.apache-magpie.lock similarity index 58% rename from .flake8 rename to .apache-magpie.lock index 6e7651c..88c68ae 100644 --- a/.flake8 +++ b/.apache-magpie.lock @@ -14,7 +14,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -[flake8] -max-line-length = 110 -ignore = E203,E231,E731,W504,I001,W503 -exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.eggs,*.egg + +# .apache-magpie.lock — committed; the project's floor. + +method: marketplace +url: apache/magpie +min_version: 0.9.0.dev202610250024 + +plugins: + - magpie-setup + - magpie-utilities + - magpie-agent-guard + - magpie-vetted-ops + - magpie-repo-health + +reconciled: + version: 0.9.0.dev202610250024 + at: 2026-10-25 + skills: + dependency-audit: sha256:19ba62ec34a55604 + flaky-test-triage: sha256:eda3c3891897f08d + license-compliance-audit: sha256:b40b2993e5f18f54 + workflow-security-audit: sha256:e50eafff464d131a diff --git a/.asf.yaml b/.asf.yaml index 7cd4a1b..dd7aa9d 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + github: rulesets: - name: "Default Branch Protection" diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..89da45c --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,17 @@ +{ + "enabledPlugins": { + "magpie-setup@apache-magpie": true, + "magpie-utilities@apache-magpie": true, + "magpie-agent-guard@apache-magpie": true, + "magpie-vetted-ops@apache-magpie": true, + "magpie-repo-health@apache-magpie": true + }, + "extraKnownMarketplaces": { + "apache-magpie": { + "source": { + "source": "github", + "repo": "apache/magpie" + } + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ebc5af7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# One grouped pull request per ecosystem and week. The cooldown keeps a release out until it has +# been public for a week, so a compromised or yanked release is usually caught before it lands. +version: 2 +updates: + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + pre-commit: + patterns: ["*"] + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + github-actions: + patterns: ["*"] + + - package-ecosystem: pip + directories: + - / + - /lambdas/scale_out_runner + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + pip: + patterns: ["*"] + + # Updates the providers in .terraform.lock.hcl and the version constraints in version.tf. A + # terraform/runs-on change is planned on its pull request and applied after merge, behind the + # runs-on-prod approval. + - package-ecosystem: terraform + directories: + - /terraform/runs-on + - /terraform/runs-on-access + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + terraform: + patterns: ["*"] diff --git a/.gitignore b/.gitignore index ae24629..994f4d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + .cache __pycache__/ .mypy_cache @@ -46,3 +63,6 @@ session-manager-plugin.* # End of https://www.toptal.com/developers/gitignore/api/terraform github-runner-ami/packer/session-manager-plugin.deb github-runner-ami/packer/session-manager-plugin.rpm + +# Apache Magpie personal layer +/.apache-magpie-local/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa32fe1..d0bacdc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. --- -default_stages: [commit, push] +default_stages: [pre-commit, pre-push] default_language_version: # force all unspecified python hooks to run python3 python: python3 @@ -26,12 +26,14 @@ repos: - id: identity - id: check-hooks-apply - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.9 + rev: v1.5.6 hooks: - id: forbid-tabs - id: insert-license name: Add license - exclude: ^\.github/.*$|^license-templates/|\.json$ + # LICENSE is the license itself, lock files are generated, and cloud-init.yml must start with + # #cloud-config. + exclude: ^\.github/.*$|^license-templates/|\.json$|\.md$|\.rst$|^LICENSE$|\.terraform\.lock\.hcl$|^cloud-init\.yml$ args: - --comment-style - "|#|" @@ -48,13 +50,23 @@ repos: - license-templates/LICENSE.rst - --fuzzy-match-generates-todo files: \.rst$ + - id: insert-license + name: Add license for all Markdown files + exclude: ^\.github/.*$ + args: + - --comment-style + - "<!--|| -->" + - --license-filepath + - license-templates/LICENSE.txt + - --fuzzy-match-generates-todo + files: \.md$ - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 26.10.1 hooks: - id: black args: [--config=./pyproject.toml] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v6.0.0 hooks: - id: check-merge-conflict - id: debug-statements @@ -63,29 +75,26 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending - id: trailing-whitespace - - id: fix-encoding-pragma - args: - - --remove - repo: https://github.com/asottile/pyupgrade - rev: v2.7.4 + rev: v3.22.0 hooks: - id: pyupgrade args: ["--py36-plus"] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.7.0 + rev: v1.10.0 hooks: - id: rst-backticks - id: python-no-log-warn - - repo: https://github.com/timothycrosley/isort - rev: 5.7.0 + - repo: https://github.com/PyCQA/isort + rev: 9.0.2 hooks: - id: isort name: Run isort to sort imports files: \.py$ # To keep consistent with the global isort skip config defined in setup.cfg exclude: ^build/.*$|^.tox/.*$|^venv/.*$ - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.2 hooks: - - id: flake8 - name: Run flake8 + - id: ruff-check + name: Run ruff diff --git a/github-runner-ami/packer/files/runner-supervisor.py b/github-runner-ami/packer/files/runner-supervisor.py index 6bc31d3..fe6e8e5 100755 --- a/github-runner-ami/packer/files/runner-supervisor.py +++ b/github-runner-ami/packer/files/runner-supervisor.py @@ -72,6 +72,7 @@ The order of operations is: then allow the termination lifecycle to continue """ + import ctypes import datetime import enum @@ -614,7 +615,7 @@ class ProcessWatcher: check_call(['systemctl', 'stop', 'actions.runner', '--no-block']) def protect_from_scale_in(self, protect: bool = True): - """ Set (or unset) ProtectedFromScaleIn on our instance""" + """Set (or unset) ProtectedFromScaleIn on our instance""" if not OWN_ASG: # Not part of an ASG return @@ -723,7 +724,7 @@ class ProcessWatcher: """ class bpf_insn(ctypes.Structure): - """"The BPF instruction data structure""" + """The BPF instruction data structure""" _fields_ = [ ("code", ctypes.c_ushort), @@ -733,7 +734,7 @@ class ProcessWatcher: ] class bpf_program(ctypes.Structure): - """"Structure for BIOCSETF""" + """Structure for BIOCSETF""" _fields_ = [("bf_len", ctypes.c_uint), ("bf_insns", ctypes.POINTER(bpf_insn))] diff --git a/pyproject.toml b/pyproject.toml index 0333bb0..76e05b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,3 +26,11 @@ default_section = 'THIRDPARTY' # Need to be consistent with the exclude config defined in pre-commit-config.yaml skip = ['build','.tox','venv'] profile = 'black' + +[tool.ruff] +line-length = 110 + +[tool.ruff.lint] +# pycodestyle errors and warnings plus pyflakes, as flake8 ran them. +select = ['E', 'W', 'F'] +ignore = ['E731'] diff --git a/requirements-test.txt b/requirements-test.txt index 4e2748b..1dbcd4f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + pytest~=6.0 moto diff --git a/scripts/list_committers b/scripts/list_committers index e9697b4..d0b0d09 100755 --- a/scripts/list_committers +++ b/scripts/list_committers @@ -21,12 +21,17 @@ import rich_click as click from github import Github from rich.console import Console + console = Console(color_system="standard", width=200) + @click.command(short_help='List committer logins - used to sync list of committers in CI configuration') [email protected]('--github-token', envvar='GITHUB_TOKEN', - help="You can generate the token with readOrg permissions: " - "https://github.com/settings/tokens/new?description=Read%20Org&scopes=read:org";) [email protected]( + '--github-token', + envvar='GITHUB_TOKEN', + help="You can generate the token with readOrg permissions: " + "https://github.com/settings/tokens/new?description=Read%20Org&scopes=read:org";, +) def main(github_token): gh = Github(github_token) org = gh.get_organization('apache') @@ -37,7 +42,8 @@ def main(github_token): click.echo(" - update the `/runners/apache/airflow/configOverlay` parameter in AWS SSM ParameterStore") click.echo(" - restart the self-hosted runners") click.echo( - " - Inform the new committer, that it's time to open PR to update list of committers in dev/breeze/src/airflow_breeze/global_constants.py (COMMITTERS variable)" + " - Inform the new committer, that it's time to open PR to update list of committers in " + "dev/breeze/src/airflow_breeze/global_constants.py (COMMITTERS variable)" ) click.echo(',\n'.join(committer_usernames)) click.echo(""" @@ -53,7 +59,7 @@ AWS spot instances that are started on-demand. Those instances are way bigger than the Public Runners available for all ASF projects. They have 8 cores and 64 GB memory and all the builds happen in-memory, and our builds are optimized to use parallelism and memory so that -often what you will get is 3-4 times speed-up vs the Public instances. +often what you will get is 3-4 times speed-up vs the Public instances. We currently have up to 35 such instances running at a time, so there is a possibility - if there are many committers pushing their changes in a short
