This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch runs-on-template-wildcard in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit e5603738c37eba04abeb5c6dce76a1028f1f3089 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Oct 27 23:01:11 2026 +0100 Let RunsOn upgrades deploy without an admin step Generated-by: Claude Opus 5 --- terraform/runs-on-access/README.rst | 7 +++-- terraform/runs-on-access/main.tf | 13 ++++----- terraform/runs-on/README.rst | 27 ++++++++++-------- terraform/runs-on/runs-on.yml.example | 53 ++++++++++++++++++++++++++--------- terraform/runs-on/variables.tf | 6 ++++ 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/terraform/runs-on-access/README.rst b/terraform/runs-on-access/README.rst index 04979cd..f4a6ac5 100644 --- a/terraform/runs-on-access/README.rst +++ b/terraform/runs-on-access/README.rst @@ -36,9 +36,10 @@ Roles Assumed only by jobs in the ``runs-on-prod`` environment, which ``.asf.yaml`` limits to ``main`` and to deployments approved by a required reviewer other than the person who triggered them. It can call ``cloudformation:UpdateStack`` on the ``runs-on`` stack only with - a template from RunsOn's published bucket, only for the versions listed in - ``runs_on_versions`` (exact template URLs), so a RunsOn upgrade means adding the version there - and applying this module before bumping ``runs_on_version`` in ``terraform/runs-on``. It can + one of the versioned templates RunsOn publishes + (``https://runs-on.s3.eu-west-1.amazonaws.com/cloudformation/template-v*.yaml``), so a RunsOn + upgrade or rollback needs only the reviewed ``runs_on_version`` change in + ``terraform/runs-on``. It can also pass the service role below and write the state and its lock file. It holds no infrastructure permissions. ``runs-on-cloudformation`` diff --git a/terraform/runs-on-access/main.tf b/terraform/runs-on-access/main.tf index 771c71a..bfc50bf 100644 --- a/terraform/runs-on-access/main.tf +++ b/terraform/runs-on-access/main.tf @@ -20,9 +20,9 @@ locals { stack_arn = "arn:aws:cloudformation:us-east-2:${local.account_id}:stack/runs-on/*" state = "arn:aws:s3:::airflow-ci-tfstate/tf-state/runs-on.tfstate" - # RunsOn releases the apply role may deploy; older published templates stay reachable otherwise. - # Upgrading RunsOn means adding the new version here and applying this module first. - runs_on_versions = ["v3.3.2", "v3.5.1"] + # Any versioned template RunsOn publishes, so an upgrade (or a rollback) is a reviewed change of + # runs_on_version in terraform/runs-on alone. Templates from anywhere else stay out of reach. + runs_on_template_urls = "https://runs-on.s3.eu-west-1.amazonaws.com/cloudformation/template-v*.yaml"; # The repository is pinned by its immutable IDs (repository_owner_id and repository_id claims), # so a renamed or re-created repository cannot match. Fork pull requests get no OIDC token at all, @@ -163,12 +163,9 @@ data "aws_iam_policy_document" "apply" { actions = ["cloudformation:UpdateStack"] resources = [local.stack_arn] condition { - test = "StringEquals" + test = "StringLike" variable = "cloudformation:TemplateUrl" - values = [ - for v in local.runs_on_versions : - "https://runs-on.s3.eu-west-1.amazonaws.com/cloudformation/template-${v}.yaml"; - ] + values = [local.runs_on_template_urls] } } # The provider sends the default tags on every UpdateStack. diff --git a/terraform/runs-on/README.rst b/terraform/runs-on/README.rst index e59290a..b869704 100644 --- a/terraform/runs-on/README.rst +++ b/terraform/runs-on/README.rst @@ -149,15 +149,16 @@ Bump ``runs_on_version`` (or any parameter) and ``terraform plan`` / ``apply``: always resubmits the template URL, which is what RunsOn's `upgrade guide <https://runs-on.com/docs/maintenance/upgrades/>`_ requires (a reused processed template can silently skip parameter changes). Minor and patch releases have no downtime; major releases may. -Rollback is the previous ``runs_on_version``. The apply role may only deploy the templates listed -in ``runs_on_versions`` in `terraform/runs-on-access <../runs-on-access/README.rst>`_, so a -``runs_on_version`` bump first needs the new version added there and applied by an admin. For the -same reason, rollback works only while the previous version is still listed: keep it during an -upgrade (for example ``["v3.3.2", "v3.4.0"]``) and drop it later. Template SHA-256s: -``v3.3.2`` ``3017805d77c1d717fdd229a1e1242db12c985944699d3d8dca01f6f28b35c16d``, ``v3.5.1`` -``9acd6fd154e2bb921f95466c45380d2e3d7bd718553a12af4e5c76d15915bd57``. Both templates create the -same IAM roles, so the service role's guardrails in ``terraform/runs-on-access`` need no change -for this upgrade. +Rollback is the previous ``runs_on_version``. The apply role may deploy any versioned template from +RunsOn's bucket (``template-v*.yaml``, `terraform/runs-on-access <../runs-on-access/README.rst>`_) +and ``runs_on_version`` must look like ``v3.5.1``, so an upgrade or rollback needs no admin step. +What still needs an admin is IAM: the CloudFormation service role may only manage the IAM roles +named in ``stack_role_prefixes`` and attach the two managed policies listed in +``terraform/runs-on-access``. Before a bump compare the new template's ``AWS::IAM::Role``, +``AWS::IAM::InstanceProfile`` and ``ManagedPolicyArns`` with those lists; anything new fails the +stack update until an admin adds it there and applies that module. +Template SHA-256s: ``v3.3.2`` ``3017805d77c1d717fdd229a1e1242db12c985944699d3d8dca01f6f28b35c16d``, +``v3.5.1`` ``9acd6fd154e2bb921f95466c45380d2e3d7bd718553a12af4e5c76d15915bd57``. Every plan shows ``"LicenseKey" = "****" -> "arn:...:parameter/runs-on/license-key"``: CloudFormation masks ``NoEcho`` parameters when Terraform reads the stack, and the AWS provider @@ -244,8 +245,12 @@ of it was the Spot quota (peak 2332 of 3000 vCPU); the on-demand quota filled up Of 2156 Spot capacity errors in a day, 1959 were x64 (``m7a+m7i+m6a+m6i``) and 197 arm64. More instance families give the Spot allocator more pools, so the x64 presets in ``runs-on.yml.example`` also allow ``m8a``, ``m8i``, ``m5a`` and ``m5`` (Spot ``2xlarge`` on -2026-10-27: 0.08-0.16 USD/hour against 0.12-0.15 for the existing families). Jobs that spell -the family out in their label (``apache/airflow`` workflows) need the same change there. +2026-10-27: 0.08-0.16 USD/hour against 0.12-0.15 for the existing families). Since +``apache/airflow#75822`` the jobs (which spell their labels out instead of using these presets) +also take Spot from capacity-optimized pools (``spot=co``) and accept up to twice their size +(``cpu=8+16/ram=32+64``). Capacity-optimized ignores price, so a job can land on the larger size +at up to twice the hourly price; the weekly report and the daily cost report show whether the net +cost goes down. The CloudFormation template places runners in two Availability Zones (us-east-2a and 2b, fixed by the template). RunsOn's Spot spread across zones (``RUNS_ON_SPOT_FORCE_ZONE_SPREAD``, diff --git a/terraform/runs-on/runs-on.yml.example b/terraform/runs-on/runs-on.yml.example index 4f8b422..ee45cbd 100644 --- a/terraform/runs-on/runs-on.yml.example +++ b/terraform/runs-on/runs-on.yml.example @@ -25,42 +25,67 @@ # # Classes are the EC2 equivalents of the measured EKS classes (AIP-118 appendix): # small 4 vCPU / 16 GB, medium 8 / 32, xl 32 / 128. The xl class gets 128 GB because xdist -# parallelism follows the host's 32 vCPUs. Spot is RunsOn's default (automatic rerun on -# interruption); `volume` replaces the GitHub-hosted runners' root + /mnt disks. +# parallelism follows the host's vCPUs. Each class also accepts up to twice its size, and Spot comes +# from capacity-optimized pools (`spot: co`), as in the job labels of apache/airflow#75822; RunsOn +# reruns an interrupted job on on-demand. `volume` replaces the GitHub-hosted runners' root + /mnt +# disks. runners: airflow-amd-small: - cpu: 4 - ram: 16 + cpu: [4, 8] + ram: [16, 32] family: ["m8a", "m8i", "m7a", "m7i", "m6a", "m6i", "m5a", "m5"] image: ubuntu22-full-x64 volume: 150gb + spot: co airflow-amd-medium: - cpu: 8 - ram: 32 + cpu: [8, 16] + ram: [32, 64] family: ["m8a", "m8i", "m7a", "m7i", "m6a", "m6i", "m5a", "m5"] image: ubuntu22-full-x64 volume: 150gb + spot: co airflow-amd-xl: - cpu: 32 - ram: 128 + cpu: [32, 64] + ram: [128, 256] family: ["m8a", "m8i", "m7a", "m7i", "m6a", "m6i", "m5a", "m5"] image: ubuntu22-full-x64 volume: 150gb + spot: co airflow-arm-small: - cpu: 4 - ram: 16 + cpu: [4, 8] + ram: [16, 32] family: ["m8g", "m7g", "m6g"] image: ubuntu22-full-arm64 volume: 150gb + spot: co airflow-arm-medium: - cpu: 8 - ram: 32 + cpu: [8, 16] + ram: [32, 64] family: ["m8g", "m7g", "m6g"] image: ubuntu22-full-arm64 volume: 150gb + spot: co airflow-arm-xl: - cpu: 32 - ram: 128 + cpu: [32, 64] + ram: [128, 256] family: ["m8g", "m7g", "m6g"] image: ubuntu22-full-arm64 volume: 150gb + spot: co + airflow-amd-in-memory: + cpu: [8, 16] + ram: [32, 128] + family: ["m8a", "m8i", "m7a", "m7i", "m6a", "m6i", "m5a", "m5", + "r8i", "r7a", "r7i", "r6a", "r6i"] + image: ubuntu22-full-x64 + volume: 150gb + spot: co + extras: ["tmpfs"] + airflow-arm-in-memory: + cpu: [8, 16] + ram: [32, 128] + family: ["m8g", "m7g", "m6g", "r8g", "r7g", "r6g"] + image: ubuntu22-full-arm64 + volume: 150gb + spot: co + extras: ["tmpfs"] diff --git a/terraform/runs-on/variables.tf b/terraform/runs-on/variables.tf index 05d9fff..047ac17 100644 --- a/terraform/runs-on/variables.tf +++ b/terraform/runs-on/variables.tf @@ -28,6 +28,12 @@ variable "runs_on_version" { description = "RunsOn release to deploy; selects the version-pinned CloudFormation template." type = string default = "v3.5.1" + + # The apply role may deploy only template-v*.yaml from RunsOn's bucket (terraform/runs-on-access). + validation { + condition = can(regex("^v[0-9]+\\.[0-9]+\\.[0-9]+$", var.runs_on_version)) + error_message = "runs_on_version must be a RunsOn release tag such as v3.5.1." + } } variable "github_organization" {
