This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 99aa8839732795038288adcfb031a726700ff930
Author: Jarek Potiuk <jarek.pot...@polidea.com>
AuthorDate: Sun Nov 1 17:54:37 2020 +0100
Adds more aggressive cancelling of duplicate Build Image jobs (#12018)
This change adds even more aggressive cancelling of duplicates of
'Build Image' jobs. it's not an obvious task to know which
Build Image jobs are duplicates, we are matching those duplicates
based on specially crafted "build-info" job names. We add
Event, Branch, Repo to the job names and assume that two
runs with the same event + branch + repo are duplicates.
It also disables self-preservation for this step because
it is perfectly ok to cancel itself in case there is a newer
in-progress Build Image job.
Unfortunately even this will not work perfectly well. Those job
names are resolved only for the jobs that are runnning rather than
the queued ones, so in case we have several duplicates of the
same build image job in the queue, they will not be found/cancelled.
The cancelling will only happen if both duplicates are already
running.
It's good enough for now and we cannot do much more until there
is a missing feature added to GitHub API that allows to link
the workflow_run with the run that triggered it. This issue has
been raised to GitHub Support and internal engineering ticket
has been apparently opened to add this feature.
More detailed status for the missing feature is kept at #11294
(cherry picked from commit 1d14e74e33efbcb17e5553dba92baf9c8b0fc7c8)
---
.github/workflows/build-images-workflow-run.yml | 40 +++++++++++++++----------
.github/workflows/ci.yml | 2 +-
2 files changed, 26 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml
b/.github/workflows/build-images-workflow-run.yml
index 972195a..a3880e7 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -65,7 +65,7 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
sourceRunId: ${{ github.event.workflow_run.id }}
- name: "Cancel duplicated 'CI Build' runs"
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: allDuplicates
@@ -77,12 +77,12 @@ jobs:
#
https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/16
echo "::set-output name=buildImages::${BUILD_IMAGES}"
- name: "Cancel duplicated 'Build Image' runs"
-
# We find duplicates of our own "Build Image" runs - due to a missing
feature
- # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch from the
- # build-info step there to find the duplicates ¯\_(ツ)_/¯.
-
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch matching
+ # trick ¯\_(ツ)_/¯. We name the build-info job appropriately
+ # and then we try to find and cancel all the jobs with the same Event
+ Repo + Branch as the
+ # current Event/Repo/Branch combination.
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
with:
cancelMode: namedJobs
token: ${{ secrets.GITHUB_TOKEN }}
@@ -93,14 +93,12 @@ jobs:
Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}.*"]
if: env.BUILD_IMAGES == 'true'
- name: "Cancel all 'CI Build' runs where some jobs failed"
-
# We find any of the "CI Build" workflow runs, where any of the
important jobs
# failed. The important jobs are selected by the regexp array below.
# We also produce list of canceled "CI Build' runs as output, so that
we
# can cancel all the matching "Build Images" workflow runs in the two
following steps.
# Yeah. Adding to the complexity ¯\_(ツ)_/¯.
-
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
id: cancel-failed
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -111,7 +109,6 @@ jobs:
["^Static checks.*", "^Build docs$", "^Spell check docs$",
"^Backport packages$",
"^Checks: Helm tests$", "^Test OpenAPI*"]
- name: "Extract canceled failed runs"
-
# We use this step to build regexp that will be used to match the
Source Run id in
# the build-info job below. If we cancelled some "CI Build" runs in
the "cancel-failed' step
# above - we want to cancel also the corresponding "Build Images"
runs. Again we have
@@ -133,15 +130,15 @@ jobs:
# We take the extracted regexp array prepared in the previous step and
we use
# it to cancel any jobs that have matching names containing Source Run
Id:
# followed by one of the run ids. Yes I know it's super complex
¯\_(ツ)_/¯.
- if: env.BUILD_IMAGES == 'true' &&
steps.source-run-info-failed.outputs.cancelledRuns != '[]'
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ if: env.BUILD_IMAGES == 'true' &&
steps.cancel-failed.outputs.cancelledRuns != '[]'
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
with:
cancelMode: namedJobs
token: ${{ secrets.GITHUB_TOKEN }}
notifyPRCancel: true
jobNameRegexps: ${{
steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
- name: "Cancel duplicated 'CodeQL' runs"
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
id: cancel
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -164,6 +161,19 @@ jobs:
else
echo "::set-output name=upgradeToLatestConstraints::false"
fi
+ - name: "Cancel all duplicated 'Build Image' runs"
+ # We find duplicates of all "Build Image" runs - due to a missing
feature
+ # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch matching
+ # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then
we try to match
+ # all the jobs with the same Event + Repo + Branch match and cancel
all the duplicates for those
+ # This might cancel own run, so this is the last step in the job
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
+ with:
+ cancelMode: allDuplicatedNamedJobs
+ token: ${{ secrets.GITHUB_TOKEN }}
+ notifyPRCancel: true
+ selfPreservation: false
+ jobNameRegexps: '["Event: \\S* Repo: \\S* Branch: \\S* "]'
build-info:
# The name is such long because we are using it to cancel duplicated
'Build Images' runs
@@ -372,7 +382,7 @@ jobs:
needs: [build-images]
steps:
- name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: self
@@ -387,7 +397,7 @@ jobs:
needs: [build-images]
steps:
- name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses:
potiuk/cancel-workflow-runs@99869d37d982384d18c79539b67df94f17557cbe # v4_1
+ uses:
potiuk/cancel-workflow-runs@f06d03cd576a179ea5169d048dbd8c8d73757b52 # v4_4
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: self
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 630b5d6..a5457c0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -117,7 +117,7 @@ jobs:
uses: actions/checkout@v2
- name: >
Event: ${{ github.event_name }}
- Repo: ${{ github.repository }}
+ Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
Branch: ${{ github.head_ref }}
Run id: ${{ github.run_id }}
Sha: ${{ github.sha }}
