This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20652-ebda190c41d9d9f1d43e43229b672ec037cc3946 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 177665df8f3f937b06ae235834dcf219126bba04 Author: voonhous <[email protected]> AuthorDate: Fri Nov 13 04:41:08 2026 +0000 chore(ci): skip tests on docs-only queue entries (#20652) * chore(ci): skip tests on docs-only queue entries A merge-queue entry that changes only docs and images still ran every test job of Java CI and Java CI Engines on ASF runners. A new `changes` job, run on merge_group only, diffs the entry's base and head trees with the push filter's ignore patterns. Every test job needs it and skips when it reports docs_only, and the two aggregates accept an all-skipped queue run unless the run was cancelled. On every other event the job is skipped, so PR and push runs wait on no extra runner, and a failed job runs the full suite. Part of #20471 (item 6). * chore(ci): pin ALLGREEN and drop a dead cancelled() term The docs-only skip is safe only because the merge queue's grouping_strategy is ALLGREEN; a node test now pins it. In the aggregate step, cancelled() reads the job's own status, which stays success for an always() job started after a cancel, so the term never fired. A queue run cancelled before its test jobs started fails on their cancelled results instead. --- .github/ci/scripts/test-fork-ci-status.cjs | 153 ++++++++++++++++++++++++++--- .github/workflows/README.md | 6 +- .github/workflows/bot.yml | 96 +++++++++++++++--- .github/workflows/java_ci_engines.yml | 92 ++++++++++++++--- 4 files changed, 311 insertions(+), 36 deletions(-) diff --git a/.github/ci/scripts/test-fork-ci-status.cjs b/.github/ci/scripts/test-fork-ci-status.cjs index c13ef9010459..fb58bcefae27 100644 --- a/.github/ci/scripts/test-fork-ci-status.cjs +++ b/.github/ci/scripts/test-fork-ci-status.cjs @@ -18,7 +18,9 @@ // Adapted from apache/arrow-rs .github/ci/scripts/test-fork-ci-status.cjs. const assert = require('node:assert/strict'); const { execFileSync } = require('node:child_process'); -const { readFileSync } = require('node:fs'); +const fs = require('node:fs'); +const { readFileSync } = fs; +const os = require('node:os'); const path = require('node:path'); const { test } = require('node:test'); @@ -403,14 +405,18 @@ function interpolate(value, scope) { } const functions = { always: () => true, + cancelled: () => false, startsWith: (text, prefix) => text.startsWith(prefix), contains: (list, item) => list.includes(item), }; const withName = (ci, github) => ({ ...github, workflow: ci.name }); const group = (ci, github) => interpolate(ci.workflow.concurrency.group, { github: withName(ci, github) }); const cancels = (ci, github) => interpolate(ci.workflow.concurrency['cancel-in-progress'], { github: withName(ci, github) }); -const jobIf = (ci, job, github) => interpolate(ci.workflow.jobs[job].if, { github: withName(ci, github), ...functions }); -const testJobs = (ci) => Object.keys(ci.workflow.jobs).filter((job) => job !== ci.aggregate); +// The `changes` job runs on merge_group only, so it is skipped and has no outputs by default. +const skippedChanges = { changes: { result: 'skipped', outputs: {} } }; +const jobIf = (ci, job, github, needs = skippedChanges) => + interpolate(ci.workflow.jobs[job].if, { github: withName(ci, github), needs, ...functions }); +const testJobs = (ci) => Object.keys(ci.workflow.jobs).filter((job) => job !== ci.aggregate && job !== 'changes'); // GitHub expressions resolve absent properties to an empty string. function context({ event_name, repository = 'apache/hudi', ref = 'refs/heads/feature', sha = 'first', @@ -431,13 +437,17 @@ const upstreamPush = (branch) => context({ event_name: 'push', ref: 'refs/heads/ const FORK_PR = "github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master'"; const UPSTREAM_OTHER_PUSH = "github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x'"; +const DOCS_ONLY = "needs.changes.outputs.docs_only == 'true'"; // PyYAML resolves the `*route` aliases, so every test job shows the expanded condition. test('every test job carries the route condition, and the aggregate job does not', () => { for (const ci of javaCi) { const jobs = testJobs(ci); assert.ok(jobs.length > 1, ci.file); for (const job of jobs) { - assert.equal(ci.workflow.jobs[job].if, '${{ !(' + FORK_PR + ') && !(' + UPSTREAM_OTHER_PUSH + ') }}', ci.file + ' ' + job); + assert.equal(ci.workflow.jobs[job].if, + '${{ !cancelled() && !(' + FORK_PR + ') && !(' + UPSTREAM_OTHER_PUSH + ') && ' + DOCS_ONLY.replace('==', '!=') + ' }}', + ci.file + ' ' + job); + assert.equal(ci.workflow.jobs[job].needs, 'changes', ci.file + ' ' + job); } const aggregate = ci.workflow.jobs[ci.aggregate]; assert.equal(aggregate.if, '${{ always() && !(' + UPSTREAM_OTHER_PUSH + ') }}', ci.file); @@ -447,13 +457,17 @@ test('every test job carries the route condition, and the aggregate job does not } }); -// [case, context, whether the test jobs run, whether the aggregate job runs] -for (const [name, github, tests, aggregate] of [ +// [case, context, whether the test jobs run, whether the aggregate job runs, the `changes` job] +for (const [name, github, tests, aggregate, needs] of [ ['fork PR to master', forkPr, false, true], ['fork PR to release-1.2', { ...forkPr, base_ref: 'release-1.2' }, true, true], ['fork PR to branch-0.x', { ...forkPr, base_ref: 'branch-0.x' }, true, true], ['same-repository PR to master', upstreamPr, true, true], ['merge queue', queue, true, true], + ['merge queue with a docs-only change', queue, false, true, { changes: { result: 'success', outputs: { docs_only: 'true' } } }], + ['merge queue with a code change', queue, true, true, { changes: { result: 'success', outputs: { docs_only: 'false' } } }], + // A failed selector has no outputs, so the queue tests everything. + ['merge queue with a failed selector', queue, true, true, { changes: { result: 'failure', outputs: {} } }], // Only the event_name guard keeps a queue run from matching FORK_PR if base_ref were set. ['merge queue with a base_ref', { ...queue, base_ref: 'master' }, true, true], ['upstream push to master', upstreamPush('master'), true, true], @@ -465,17 +479,19 @@ for (const [name, github, tests, aggregate] of [ ]) { test('routes ' + name, () => { for (const ci of javaCi) { - for (const job of testJobs(ci)) assert.equal(jobIf(ci, job, github), tests, ci.file + ' ' + job); - assert.equal(jobIf(ci, ci.aggregate, github), aggregate, ci.file + ' ' + ci.aggregate); + for (const job of testJobs(ci)) assert.equal(jobIf(ci, job, github, needs), tests, ci.file + ' ' + job); + assert.equal(jobIf(ci, ci.aggregate, github, needs), aggregate, ci.file + ' ' + ci.aggregate); + // The selector runs on merge_group only; GitHub evaluates an `if` without ${{ }} as an expression. + assert.equal(evaluate(ci.workflow.jobs.changes.if, { github }), github.event_name === 'merge_group', ci.file); } }); } -function aggregateFails(ci, github, results) { +function aggregateFails(ci, github, results, overrides = {}) { const job = ci.workflow.jobs[ci.aggregate]; const env = { FORK_PR: String(interpolate(job.env.FORK_PR, { github })) }; const step = job.steps.find((item) => item.name === 'Fail unless every job succeeded'); const needs = Object.fromEntries(results.map((result, index) => ['job-' + index, { result }])); - return interpolate(step.if, { github, env, needs, ...functions }); + return interpolate(step.if, { github, env, needs, ...functions, ...overrides }); } test('the aggregate jobs accept skipped test jobs only on fork PRs to master', () => { for (const ci of javaCi) { @@ -483,7 +499,7 @@ test('the aggregate jobs accept skipped test jobs only on fork PRs to master', ( for (const results of [['skipped', 'failure'], ['cancelled', 'skipped'], ['success', 'skipped'], ['success', 'success']]) { assert.equal(aggregateFails(ci, forkPr, results), true, ci.file + ' ' + results.join()); } - for (const github of [upstreamPr, queue, forkPush, upstreamPush('master'), { ...forkPr, base_ref: 'release-1.2' }]) { + for (const github of [upstreamPr, forkPush, upstreamPush('master'), { ...forkPr, base_ref: 'release-1.2' }]) { assert.equal(aggregateFails(ci, github, ['success', 'success', 'success']), false, ci.file); for (const results of [['skipped', 'skipped'], ['success', 'skipped'], ['failure', 'success'], ['success', 'cancelled']]) { assert.equal(aggregateFails(ci, github, results), true, ci.file + ' ' + results.join()); @@ -491,6 +507,32 @@ test('the aggregate jobs accept skipped test jobs only on fork PRs to master', ( } } }); +// On merge_group the route skips a test job only for DOCS_ONLY, which skips them all. +test('the aggregate jobs accept a merge queue run in which every test job skipped', () => { + for (const ci of javaCi) { + for (const results of [['success', 'success', 'success'], ['skipped', 'skipped', 'skipped']]) { + assert.equal(aggregateFails(ci, queue, results), false, ci.file + ' ' + results.join()); + } + for (const results of [['success', 'skipped'], ['skipped', 'failure'], ['skipped', 'cancelled'], ['failure', 'success'], ['success', 'cancelled']]) { + assert.equal(aggregateFails(ci, queue, results), true, ci.file + ' ' + results.join()); + } + // A queue run cancelled before its test jobs started is not docs-only: the test jobs still + // pending at the cancel report cancelled, not skipped. + assert.equal(aggregateFails(ci, queue, ['cancelled', 'cancelled', 'cancelled']), true, ci.file); + } +}); +// The docs-only skip relies on every queue entry passing its own run. Under HEADGREEN only the +// head of a group must pass, so a docs-only entry there would merge the code entries ahead of it. +test('the merge queue requires every entry to pass, which the docs-only skip relies on', () => { + const asf = JSON.parse(execFileSync('python3', [ + '-c', 'import json, sys, yaml; print(json.dumps(yaml.safe_load(sys.stdin)))', + ], { input: readFileSync(path.join(__dirname, '../../../.asf.yaml')), encoding: 'utf8' })); + const queues = asf.github.rulesets.flatMap((ruleset) => ruleset.rules) + .filter((rule) => rule.type === 'merge_queue'); + assert.equal(queues.length, 1); + assert.equal(queues[0].parameters.grouping_strategy, 'ALLGREEN', + 'bot.yml and java_ci_engines.yml skip the tests of docs-only queue entries, which is safe only under ALLGREEN'); +}); test('each aggregate job links the fork run of its own workflow on fork PRs', () => { for (const ci of javaCi) { const step = ci.workflow.jobs[ci.aggregate].steps.find((item) => item.name === 'Link to fork CI'); @@ -501,6 +543,95 @@ test('each aggregate job links the fork run of its own workflow on fork PRs', () } }); +// The docs-only selector, the `changes` job, is the same in both workflows. +const changesJob = javaCi[0].workflow.jobs.changes; +const changesStep = changesJob.steps.find((item) => item.id === 'diff'); +test('the changes job is identical in both workflows and runs on merge_group only', () => { + for (const ci of javaCi) assert.deepEqual(ci.workflow.jobs.changes, changesJob, ci.file); + assert.equal(changesJob.if, "github.event_name == 'merge_group'"); + assert.equal(changesJob.outputs.docs_only, '${{ steps.diff.outputs.docs_only }}'); + // Expressions reach the script only through env. + assert.doesNotMatch(changesStep.run, /\$\{\{/); +}); +test('the changes job excludes exactly what the push filter of each workflow ignores', () => { + const excluded = [...changesStep.run.matchAll(/':\(([^)]*)\)([^']*)'/g)].map(([pathspec]) => pathspec); + assert.ok(excluded.length > 1); + for (const ci of javaCi) { + // PyYAML reads the key `on` as the boolean true. + const ignored = ci.workflow.true.push.paths.filter((pattern) => pattern.startsWith('!')).map((pattern) => { + const glob = pattern.slice(1); + // GitHub's `**.md` matches at any depth; git's `**/*.md` does too, including the root. + if (/^\*\*\.[a-z]+$/.test(glob)) return "':(exclude,glob)**/*" + glob.slice(2) + "'"; + // Without the glob magic a pathspec without wildcards matches that path from the root only. + assert.match(glob, /^[^*?[]+$/, ci.file + ' ' + pattern); + return "':(exclude)" + glob + "'"; + }); + assert.deepEqual(excluded, ignored, ci.file); + } +}); +// Runs the selector script against a local repository served over file://, as the merge queue +// runs it against apache/hudi. +test('the changes job reports docs_only for diffs of only docs and images', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'hudi-changes-')); + const env = { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: path.join(tmp, 'gitconfig') }; + const git = (...args) => execFileSync('git', [ + '-c', 'user.name=ci', '-c', '[email protected]', '-c', 'init.defaultBranch=master', ...args, + ], { cwd: source, env, encoding: 'utf8', stdio: 'pipe' }).trim(); + const write = (file, content = file) => { + fs.mkdirSync(path.dirname(path.join(source, file)), { recursive: true }); + fs.writeFileSync(path.join(source, file), content + '\n'); + }; + const source = path.join(tmp, 'apache', 'hudi.git'); + const select = (base, head) => { + const work = fs.mkdtempSync(path.join(tmp, 'run-')); + const output = path.join(tmp, path.basename(work) + '.output'); + const summary = path.join(tmp, path.basename(work) + '.summary'); + execFileSync('bash', ['-c', changesStep.run], { + cwd: work, stdio: 'pipe', env: { + ...env, GITHUB_SERVER_URL: 'file://' + tmp, GITHUB_REPOSITORY: 'apache/hudi', + BASE_SHA: base, HEAD_SHA: head, GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, + }, + }); + return { output: readFileSync(output, 'utf8'), summary: readFileSync(summary, 'utf8') }; + }; + try { + fs.mkdirSync(source, { recursive: true }); + git('init', '-q'); + git('config', 'uploadpack.allowFilter', 'true'); + git('config', 'uploadpack.allowAnySHA1InWant', 'true'); + for (const file of ['README.md', 'notes.md', 'sub/A.java', '.gitignore']) write(file); + git('add', '-A'); + git('commit', '-qm', 'base'); + const base = git('rev-parse', 'HEAD'); + // [case, change, whether docs-only, files the summary lists] + for (const [name, change, docsOnly, listed = []] of [ + ['root README.md', () => write('README.md', 'changed'), true], + ['nested markdown', () => write('rfc/x/y.md'), true], + ['image', () => write('docs/images/arch.png'), true], + ['root .gitignore', () => write('.gitignore', 'changed'), true], + ['nested .gitignore', () => write('sub/.gitignore'), false, ['sub/.gitignore']], + ['yml file', () => write('docker/compose.yml'), false, ['docker/compose.yml']], + ['java file', () => write('sub/A.java', 'changed'), false, ['sub/A.java']], + ['rename of .md to .java', () => git('mv', 'notes.md', 'Notes.java'), false, ['Notes.java']], + ['docs and one java file', () => { write('docs/guide.md'); write('sub/A.java', 'changed'); }, false, ['sub/A.java']], + ['more than 50 files', () => { for (let i = 0; i < 52; i++) write('gen/F' + i + '.java'); }, false, ['(52)', 'gen/F0.java', 'gen/F7.java', '... and 2 more']], + ]) { + git('checkout', '-q', '--detach', base); + change(); + git('add', '-A'); + git('commit', '-qm', name); + const { output, summary } = select(base, git('rev-parse', 'HEAD')); + assert.equal(output, 'docs_only=' + docsOnly + '\n', name); + for (const text of listed) assert.ok(summary.includes(text), name + ': ' + text + ' in\n' + summary); + assert.doesNotMatch(summary, /\.md$|notes|guide|gen\/F[89]\.java/m, name); + } + // A failed fetch fails the job, which leaves docs_only empty, so the test jobs run. + assert.throws(() => select(base, 'f'.repeat(40))); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +}); + test('successive fork pushes share a cancellation group, separate branches do not', () => { for (const ci of javaCi) { assert.equal(cancels(ci, forkPush), true); diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6025162a8710..aac53336d93b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -3,8 +3,8 @@ When updating the pr template, you must consider if updates need to be made to scripts/pr_compliance.py ## What are the files in workflows? -- bot.yml: Java CI. Spark lanes (JDK 11 on Spark 3.5, JDK 17 on Spark 4.2). Runs on PRs, merge-queue groups and branch pushes; only push runs are path-filtered. Forks run it on every branch push; upstream it skips every job on pushes to branches other than master, release-* and branch-0.x, and every job but the aggregate on PRs from forks to master (see [Pull requests from forks](#pull-requests-from-forks)). No job depends on another, except the final `Java CI result` job, which aggregat [...] -- java_ci_engines.yml: Java CI Engines. Flink, bundle validation and integration tests. Same triggers, fork routing and layout rules as bot.yml; its aggregate job is `Java CI Engines result`. The two are separate workflows because the ASF GitHub Actions policy caps the jobs running at once in one workflow at 20, and a single workflow for both would exceed it +- bot.yml: Java CI. Spark lanes (JDK 11 on Spark 3.5, JDK 17 on Spark 4.2). Runs on PRs, merge-queue groups and branch pushes; only push runs are path-filtered. Forks run it on every branch push; upstream it skips every job on pushes to branches other than master, release-* and branch-0.x, and every job but the aggregate on PRs from forks to master (see [Pull requests from forks](#pull-requests-from-forks)). Every test job depends only on the `Detect docs-only changes` job (`changes`), w [...] +- java_ci_engines.yml: Java CI Engines. Flink, bundle validation and integration tests. Same triggers, fork routing, docs-only merge-queue skip and layout rules as bot.yml; its aggregate job is `Java CI Engines result`. The two are separate workflows because the ASF GitHub Actions policy caps the jobs running at once in one workflow at 20, and a single workflow for both would exceed it - fork_ci_status.yml: Fork CI status. Mirrors the fork's bot.yml and java_ci_engines.yml runs for the head commit of each open fork PR to master onto the PR as one informational `CI (fork)` check, which reports the worse of the two results. Runs when an upstream Java CI or Java CI Engines run completes and every 15 minutes for late runs and reruns. It never checks out PR code. Its tests, .github/ci/scripts/test-fork-ci-status.cjs, also cover the routing and concurrency of bot.yml and jav [...] - fork_coverage_bridge.yml: Fork coverage bridge. Every 15 minutes, uploads to Codecov the JaCoCo reports that the fork's bot.yml and java_ci_engines.yml runs stored as artifacts (through .github/actions/upload-coverage) for the head commit of each recently updated fork PR to master, and records each uploaded commit as a `Coverage (fork)` check. It never checks out or runs PR code; its tests are .github/ci/scripts/test-fork-coverage-bridge.cjs - validate_source.yml: PR and Source Validation. One job, `validate-pr-and-source`, the only required status check: it checks the PR description with scripts/pr_compliance.py, runs that script's own tests and the fork_ci_status.yml and fork_coverage_bridge.yml tests, validates the PR title, rejects commits with a Claude co-author trailer, and runs the source release checks (RAT, binary files, copyright). Never path-filtered, because a required check must report on every PR. Also runs on [...] @@ -19,4 +19,4 @@ To reduce load on ASF's shared runners, pull requests from forks to `master` run Each push runs two workflows on your fork, `Java CI` and `Java CI Engines`, and both appear in your fork's Actions tab. View results and rerun failed jobs there; the PR's `CI (fork)` check combines both runs and may lag. A push that changes only docs or images creates no fork run, because both workflows path-filter push events. The PR's Codecov report arrives up to about 15 minutes after the fork runs finish, through the coverage bridge (fork_coverage_bridge.yml). -The `CI (fork)` check links to the fork runs and is informational. When your PR is approved and added to the merge queue, both workflows run using ASF's runners against the proposed merge with `master`. Release-branch PRs, and PRs from branches pushed to apache/hudi itself, keep upstream testing. +The `CI (fork)` check links to the fork runs and is informational. When your PR is approved and added to the merge queue, both workflows run using ASF's runners against the proposed merge with `master`, and skip their test jobs if the PR changes only docs and images. Release-branch PRs, and PRs from branches pushed to apache/hudi itself, keep upstream testing. diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index afc02b803a89..420b74336644 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -6,7 +6,9 @@ name: Java CI # decision now costs nothing. The filter applies to push only: pull_request and merge_group runs # must always report the aggregate check at the end of this file, and a required check whose # workflow the filter skipped never reports, so it would stay pending forever. A docs-only PR -# therefore runs the full suite; a fork PR to master runs it in the merge queue. +# therefore runs the full suite on pull_request. In the merge queue, the `changes` job below +# applies the same patterns to the entry's diff instead, and the test jobs of a docs-only entry +# skip; any other entry, including every fork PR to master, runs the full suite there. # yml/yaml files are deliberately NOT ignored, unlike the old filter: docker/compose/*.yml drives # the integration tests, several modules keep yml test resources, and the workflow files themselves # need CI to run when they change. @@ -66,9 +68,9 @@ env: SCALA_TEST_DML_FILTER: -DwildcardSuites=org.apache.spark.sql.hudi.dml SCALA_TEST_OTHERS_FILTER: -DwildcardSuites=org.apache.hudi,org.apache.spark.hudi,org.apache.spark.sql.avro,org.apache.spark.sql.execution,org.apache.spark.sql.hive,org.apache.spark.sql.hudi.analysis,org.apache.spark.sql.hudi.blob,org.apache.spark.sql.hudi.catalog,org.apache.spark.sql.hudi.command,org.apache.spark.sql.hudi.common,org.apache.spark.sql.hudi.ddl,org.apache.spark.sql.hudi.procedure,org.apache.spark.sql.hudi.feature -# Two conditions route the jobs. GitHub expressions cannot be named, so the first test job spells -# them out in full under the YAML anchor `route`, every other test job reuses it through the alias -# `*route`, and the aggregate job at the end spells out the parts it needs. +# Three conditions route the jobs. GitHub expressions cannot be named, so the first test job +# spells them out in full under the YAML anchor `route`, every other test job reuses it through +# the alias `*route`, and the aggregate job at the end spells out the parts it needs. # # FORK_PR, a pull request from a fork to master: # github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master' @@ -82,7 +84,59 @@ env: # UPSTREAM_OTHER_PUSH, a push to apache/hudi other than to master, release-* or branch-0.x: # github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x' # Such branches back same-repository PRs, which test them through pull_request. +# +# DOCS_ONLY, a merge-queue entry that changes only docs and images: +# needs.changes.outputs.docs_only == 'true' +# Every test job needs the `changes` job, which runs on merge_group only. On every other event it +# is skipped, and a job whose need was skipped is skipped too unless its condition calls a status +# function, so the route starts with `!cancelled()`. A failed `changes` job leaves the output +# empty, so the test jobs run. jobs: + # Decides on a merge-queue run whether the entry changes only docs and images, so that the test + # jobs can skip; see DOCS_ONLY above. The excluded patterns are the negated patterns of the push + # filter at the top of this file: `!**.md` becomes `**/*.md`, which also matches at the root, and + # `!.gitignore` stays root only. The job fetches the two commits without blobs instead of + # checking out the repository, and diffs their trees. Keep it identical in bot.yml and + # java_ci_engines.yml; .github/ci/scripts/test-fork-ci-status.cjs checks both and runs it. + changes: + name: Detect docs-only changes + if: github.event_name == 'merge_group' + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + contents: read + outputs: + docs_only: ${{ steps.diff.outputs.docs_only }} + steps: + - name: List changed files other than docs and images + id: diff + env: + BASE_SHA: ${{ github.event.merge_group.base_sha }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + run: | + set -euo pipefail + git init -q + git fetch -q --no-tags --depth=1 --filter=blob:none "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$BASE_SHA" "$HEAD_SHA" + files=$(git diff --no-renames --name-only "$BASE_SHA" "$HEAD_SHA" -- . \ + ':(exclude,glob)**/*.bmp' ':(exclude,glob)**/*.gif' ':(exclude,glob)**/*.jpg' \ + ':(exclude,glob)**/*.jpeg' ':(exclude,glob)**/*.md' ':(exclude,glob)**/*.pdf' \ + ':(exclude,glob)**/*.png' ':(exclude,glob)**/*.svg' ':(exclude).gitignore') + if [ -z "$files" ]; then + echo 'docs_only=true' >> "$GITHUB_OUTPUT" + echo 'Docs-only change: the test jobs are skipped.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'docs_only=false' >> "$GITHUB_OUTPUT" + count=$(grep -c '' <<< "$files") + { + echo "Not docs-only: the test jobs run. Changed files other than docs and images ($count):" + echo + echo '```' + sed -n '1,50p' <<< "$files" + if [ "$count" -gt 50 ]; then echo "... and $((count - 50)) more"; fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + fi + # hudi-cli rides along with hudi-spark-client here rather than joining the catch-all # module lists in test-common-and-other-modules: its functional suite starts a # SparkSession per test class, and this job has the headroom for it while that one does @@ -90,7 +144,8 @@ jobs: # this job's build instead of paying for a second one. It stays listed in the catch-all # job's UT_MODULES/FT_MODULES exclusions so the tests are not run twice. test-spark-client-and-hadoop-common: - if: &route ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master') && !(github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x') }} + if: &route ${{ !cancelled() && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master') && !(github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x') && needs.changes.outputs.docs_only != 'true' }} + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 75 strategy: @@ -166,6 +221,7 @@ jobs: test-utilities: if: *route + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: @@ -217,6 +273,7 @@ jobs: test-common-and-other-modules: if: *route + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 90 strategy: @@ -606,6 +663,7 @@ jobs: test-hudi-hadoop-mr-and-hudi-java-client: if: *route + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 40 strategy: @@ -650,6 +708,7 @@ jobs: test-spark-java17-java-tests-part1: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -706,6 +765,7 @@ jobs: test-spark-java17-java-tests-part2: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -772,6 +832,7 @@ jobs: test-spark-java17-java-tests-part3: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -832,6 +893,7 @@ jobs: test-spark-java17-scala-dml-tests: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -892,6 +954,7 @@ jobs: test-spark-java17-scala-other-tests: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -996,10 +1059,15 @@ jobs: # failed or cancelled one. Each workflow has its own check rather than one for both, because the # ASF GitHub Actions policy caps the jobs running at once in one workflow at 20 (15 recommended), # and a single run of both workflows would exceed it. Adding or resharding a job means adding its - # id to `needs` below and `if: *route` to the job, and nothing else. The one exception to the - # skipped-job rule is FORK_PR, where every test job skips on purpose and the merge queue runs - # them before merge. UPSTREAM_OTHER_PUSH skips this job too, since nothing ran that it could - # aggregate. + # id to `needs` below and `needs: changes` and `if: *route` to the job, and nothing else. The + # skipped-job rule has two exceptions. On FORK_PR every test job skips on purpose and the merge + # queue runs them before merge. On merge_group every test job may skip for DOCS_ONLY: there the + # route skips no test job otherwise and a failed `changes` job makes them run, so a queue run in + # which every test job skipped is docs-only. Skipping the tests of a docs-only entry is safe only + # because .asf.yaml sets the merge queue's grouping_strategy to ALLGREEN, so the entries ahead of + # it must pass their own runs; under HEADGREEN a docs-only entry at the head of a group would + # merge the code entries ahead of it on its own run. test-fork-ci-status.cjs pins ALLGREEN. + # UPSTREAM_OTHER_PUSH skips this job too, since nothing ran that it could aggregate. java-ci-result: name: Java CI result runs-on: ubuntu-slim @@ -1034,12 +1102,18 @@ jobs: printf 'Look for commit `%s`. Enable Actions in your fork and push your branch if no run exists.\n\n' "$PR_HEAD_SHA" printf 'This upstream PR run skips the Java CI test jobs. The merge queue runs them on ASF runners before merge.\n' } >> "$GITHUB_STEP_SUMMARY" + # On merge_group, skipped jobs pass only when every job skipped, which means DOCS_ONLY; see + # the comment above this job. A queue run cancelled before its test jobs started fails on the + # `cancelled` results of the test jobs still pending at the cancel. cancelled() cannot catch + # it: in a step it reads this job's own status, which stays success for an always() job that + # started after the cancel. - name: Fail unless every job succeeded if: >- ${{ env.FORK_PR == 'true' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) - || env.FORK_PR != 'true' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped')) }} + || env.FORK_PR != 'true' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') && !(github.event_name == 'merge_group' && !contains(needs.*.result, 'success'))) }} env: - REQUIREMENT: ${{ env.FORK_PR == 'true' && 'Every job in Java CI must be skipped on a fork PR to master.' || 'Every job in Java CI must succeed.' }} + REQUIREMENT: ${{ env.FORK_PR == 'true' && 'Every job in Java CI must be skipped on a fork PR to master.' || 'Every job in Java CI must succeed, or every job must be skipped on a docs-only merge-queue run.' }} run: | echo "::error::$REQUIREMENT The summary step above lists each result." exit 1 diff --git a/.github/workflows/java_ci_engines.yml b/.github/workflows/java_ci_engines.yml index b6e629b4f936..31695adbd1bb 100644 --- a/.github/workflows/java_ci_engines.yml +++ b/.github/workflows/java_ci_engines.yml @@ -6,7 +6,9 @@ name: Java CI Engines # decision now costs nothing. The filter applies to push only: pull_request and merge_group runs # must always report the aggregate check at the end of this file, and a required check whose # workflow the filter skipped never reports, so it would stay pending forever. A docs-only PR -# therefore runs the full suite; a fork PR to master runs it in the merge queue. +# therefore runs the full suite on pull_request. In the merge queue, the `changes` job below +# applies the same patterns to the entry's diff instead, and the test jobs of a docs-only entry +# skip; any other entry, including every fork PR to master, runs the full suite there. # yml/yaml files are deliberately NOT ignored, unlike the old filter: docker/compose/*.yml drives # the integration tests, several modules keep yml test resources, and the workflow files themselves # need CI to run when they change. @@ -63,9 +65,9 @@ env: FLINK_IT_FILTER1: -Dit.test=ITTestHoodieDataSource FLINK_IT_FILTER2: -Dit.test=!ITTestHoodieDataSource -# Two conditions route the jobs. GitHub expressions cannot be named, so the first test job spells -# them out in full under the YAML anchor `route`, every other test job reuses it through the alias -# `*route`, and the aggregate job at the end spells out the parts it needs. +# Three conditions route the jobs. GitHub expressions cannot be named, so the first test job +# spells them out in full under the YAML anchor `route`, every other test job reuses it through +# the alias `*route`, and the aggregate job at the end spells out the parts it needs. # # FORK_PR, a pull request from a fork to master: # github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master' @@ -79,9 +81,62 @@ env: # UPSTREAM_OTHER_PUSH, a push to apache/hudi other than to master, release-* or branch-0.x: # github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x' # Such branches back same-repository PRs, which test them through pull_request. +# +# DOCS_ONLY, a merge-queue entry that changes only docs and images: +# needs.changes.outputs.docs_only == 'true' +# Every test job needs the `changes` job, which runs on merge_group only. On every other event it +# is skipped, and a job whose need was skipped is skipped too unless its condition calls a status +# function, so the route starts with `!cancelled()`. A failed `changes` job leaves the output +# empty, so the test jobs run. jobs: + # Decides on a merge-queue run whether the entry changes only docs and images, so that the test + # jobs can skip; see DOCS_ONLY above. The excluded patterns are the negated patterns of the push + # filter at the top of this file: `!**.md` becomes `**/*.md`, which also matches at the root, and + # `!.gitignore` stays root only. The job fetches the two commits without blobs instead of + # checking out the repository, and diffs their trees. Keep it identical in bot.yml and + # java_ci_engines.yml; .github/ci/scripts/test-fork-ci-status.cjs checks both and runs it. + changes: + name: Detect docs-only changes + if: github.event_name == 'merge_group' + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + contents: read + outputs: + docs_only: ${{ steps.diff.outputs.docs_only }} + steps: + - name: List changed files other than docs and images + id: diff + env: + BASE_SHA: ${{ github.event.merge_group.base_sha }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + run: | + set -euo pipefail + git init -q + git fetch -q --no-tags --depth=1 --filter=blob:none "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$BASE_SHA" "$HEAD_SHA" + files=$(git diff --no-renames --name-only "$BASE_SHA" "$HEAD_SHA" -- . \ + ':(exclude,glob)**/*.bmp' ':(exclude,glob)**/*.gif' ':(exclude,glob)**/*.jpg' \ + ':(exclude,glob)**/*.jpeg' ':(exclude,glob)**/*.md' ':(exclude,glob)**/*.pdf' \ + ':(exclude,glob)**/*.png' ':(exclude,glob)**/*.svg' ':(exclude).gitignore') + if [ -z "$files" ]; then + echo 'docs_only=true' >> "$GITHUB_OUTPUT" + echo 'Docs-only change: the test jobs are skipped.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'docs_only=false' >> "$GITHUB_OUTPUT" + count=$(grep -c '' <<< "$files") + { + echo "Not docs-only: the test jobs run. Changed files other than docs and images ($count):" + echo + echo '```' + sed -n '1,50p' <<< "$files" + if [ "$count" -gt 50 ]; then echo "... and $((count - 50)) more"; fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + fi + test-flink-1: - if: &route ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master') && !(github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x') }} + if: &route ${{ !cancelled() && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.base_ref == 'master') && !(github.event_name == 'push' && github.repository == 'apache/hudi' && github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/heads/release-') && github.ref != 'refs/heads/branch-0.x') && needs.changes.outputs.docs_only != 'true' }} + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -152,6 +207,7 @@ jobs: test-flink-2: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -247,6 +303,7 @@ jobs: validate-bundles: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -339,6 +396,7 @@ jobs: validate-bundle-spark4: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -451,6 +509,7 @@ jobs: integration-tests: if: *route + needs: changes runs-on: ubuntu-24.04 strategy: matrix: @@ -652,10 +711,15 @@ jobs: # failed or cancelled one. Each workflow has its own check rather than one for both, because the # ASF GitHub Actions policy caps the jobs running at once in one workflow at 20 (15 recommended), # and a single run of both workflows would exceed it. Adding or resharding a job means adding its - # id to `needs` below and `if: *route` to the job, and nothing else. The one exception to the - # skipped-job rule is FORK_PR, where every test job skips on purpose and the merge queue runs - # them before merge. UPSTREAM_OTHER_PUSH skips this job too, since nothing ran that it could - # aggregate. + # id to `needs` below and `needs: changes` and `if: *route` to the job, and nothing else. The + # skipped-job rule has two exceptions. On FORK_PR every test job skips on purpose and the merge + # queue runs them before merge. On merge_group every test job may skip for DOCS_ONLY: there the + # route skips no test job otherwise and a failed `changes` job makes them run, so a queue run in + # which every test job skipped is docs-only. Skipping the tests of a docs-only entry is safe only + # because .asf.yaml sets the merge queue's grouping_strategy to ALLGREEN, so the entries ahead of + # it must pass their own runs; under HEADGREEN a docs-only entry at the head of a group would + # merge the code entries ahead of it on its own run. test-fork-ci-status.cjs pins ALLGREEN. + # UPSTREAM_OTHER_PUSH skips this job too, since nothing ran that it could aggregate. java-ci-engines-result: name: Java CI Engines result runs-on: ubuntu-slim @@ -686,12 +750,18 @@ jobs: printf 'Look for commit `%s`. Enable Actions in your fork and push your branch if no run exists.\n\n' "$PR_HEAD_SHA" printf 'This upstream PR run skips the Java CI Engines test jobs. The merge queue runs them on ASF runners before merge.\n' } >> "$GITHUB_STEP_SUMMARY" + # On merge_group, skipped jobs pass only when every job skipped, which means DOCS_ONLY; see + # the comment above this job. A queue run cancelled before its test jobs started fails on the + # `cancelled` results of the test jobs still pending at the cancel. cancelled() cannot catch + # it: in a step it reads this job's own status, which stays success for an always() job that + # started after the cancel. - name: Fail unless every job succeeded if: >- ${{ env.FORK_PR == 'true' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) - || env.FORK_PR != 'true' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped')) }} + || env.FORK_PR != 'true' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') && !(github.event_name == 'merge_group' && !contains(needs.*.result, 'success'))) }} env: - REQUIREMENT: ${{ env.FORK_PR == 'true' && 'Every job in Java CI Engines must be skipped on a fork PR to master.' || 'Every job in Java CI Engines must succeed.' }} + REQUIREMENT: ${{ env.FORK_PR == 'true' && 'Every job in Java CI Engines must be skipped on a fork PR to master.' || 'Every job in Java CI Engines must succeed, or every job must be skipped on a docs-only merge-queue run.' }} run: | echo "::error::$REQUIREMENT The summary step above lists each result." exit 1
