This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b251d22 Remove legacy GitHub Packages (#16776)
b251d22 is described below
commit b251d22fffad63124eec5246b80035408b543704
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jul 4 13:44:51 2021 +0200
Remove legacy GitHub Packages (#16776)
This PR removes the legacy GitHub Packages support:
* removes checking for images in Packages/Registry
* removes output informing about the registry
* hard-codes registry to ghcr.io
* Updaes documentation describing the registries
---
.github/workflows/build-images.yml | 1 -
.github/workflows/ci.yml | 27 ------
BREEZE.rst | 30 ------
CI.rst | 44 +++------
IMAGES.rst | 113 +++++------------------
breeze | 24 -----
breeze-complete | 6 --
scripts/ci/libraries/_initialization.sh | 3 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 36 --------
9 files changed, 38 insertions(+), 246 deletions(-)
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index 9097241..9cbae1d 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -47,7 +47,6 @@ env:
GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
INSTALL_PROVIDERS_FROM_SOURCES: "true"
- GITHUB_REGISTRY: "ghcr.io"
TARGET_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
concurrency:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 701e916..ac3af1f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,7 +37,6 @@ env:
VERBOSE: "true"
DOCKER_CACHE: "pulled"
USE_GITHUB_REGISTRY: "true"
- GITHUB_REGISTRY: "ghcr.io"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your
repo but by default the
@@ -268,8 +267,6 @@ jobs:
BACKEND: sqlite
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }}
- outputs:
- githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -297,9 +294,6 @@ jobs:
# We are utilising single job to wait for all images because this job
merely waits
# for the images to be available.
# The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
- # The job will set the output "githubRegistry" - result of auto-detect
which registry has
- # been used by checking where the image can be downloaded from.
- #
run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
@@ -313,7 +307,6 @@ jobs:
SKIP: "identity"
MOUNT_SELECTED_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -419,7 +412,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
if: needs.build-info.outputs.docs-build == 'true'
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -467,7 +459,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
AIRFLOW_EXTRAS: "all"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
VERSION_SUFFIX_FOR_PYPI: ".dev0"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
NON_INTERACTIVE: "true"
GENERATE_PROVIDERS_ISSUE: "true"
if: needs.build-info.outputs.image-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -516,7 +507,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
AIRFLOW_EXTRAS: "all"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
VERSION_SUFFIX_FOR_PYPI: ".dev0"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
NON_INTERACTIVE: "true"
GENERATE_PROVIDERS_ISSUE: "true"
if: needs.build-info.outputs.image-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -560,7 +550,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
BACKEND: ""
DB_RESET: "false"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name !=
'schedule')
@@ -620,7 +609,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
POSTGRES_VERSION: ${{ matrix.postgres-version }}
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -677,7 +665,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
MYSQL_VERSION: ${{ matrix.mysql-version }}
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -733,7 +720,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
MSSQL_VERSION: ${{ matrix.mssql-version }}
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -787,7 +773,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -839,7 +824,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
TEST_TYPES: "Quarantined"
NUM_RUNS: 10
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -939,8 +923,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{
needs.build-info.outputs.defaultPythonVersion }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
- outputs:
- githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -963,8 +945,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
# "build-images-workflow-run.yml' run as pull_request_target.
# We are utilising single job to wait for all images because this job
merely waits
# For the images to be available. The test jobs wait for it to
complete!
- # The job will set the output "githubRegistry" - result of auto-detect
which registry has
- # been used by checking where the image can be downloaded from.
#
id: wait-for-images
env:
@@ -989,7 +969,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
EXECUTOR: ${{matrix.executor}}
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
- GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
@@ -1055,7 +1034,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
EXECUTOR: "KubernetesExecutor"
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
- GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
@@ -1131,7 +1109,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
- GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -1193,7 +1170,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -1229,7 +1205,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING:
${{needs.build-info.outputs.pythonVersionsListAsString}}
# Only run it for direct pushes
if: >
@@ -1326,8 +1301,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: React UI tests
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, ci-images]
- env:
- GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-ui-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
diff --git a/BREEZE.rst b/BREEZE.rst
index 98af7ac..22f7e0c 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1446,16 +1446,6 @@ This is the current syntax for `./breeze <./breeze>`_:
DockerHub. You need to be logged in to the registry in order to be
able to pull/push from
and you need to be committer to push to Apache Airflow' GitHub
registry.
- --github-registry GITHUB_REGISTRY
- GitHub registry used. GitHub has legacy Packages registry and Public
Beta Container
- registry.
-
- Default: ghcr.io.
-
- If you use this flag, automatically --use-github-registry flag is
enabled.
-
- ghcr.io docker.pkg.github.com
-
-g, --github-repository GITHUB_REPOSITORY
GitHub repository used to pull, push images when cache is used.
Default: apache/airflow.
@@ -1620,16 +1610,6 @@ This is the current syntax for `./breeze <./breeze>`_:
DockerHub. You need to be logged in to the registry in order to be
able to pull/push from
and you need to be committer to push to Apache Airflow' GitHub
registry.
- --github-registry GITHUB_REGISTRY
- GitHub registry used. GitHub has legacy Packages registry and Public
Beta Container
- registry.
-
- Default: ghcr.io.
-
- If you use this flag, automatically --use-github-registry flag is
enabled.
-
- ghcr.io docker.pkg.github.com
-
-g, --github-repository GITHUB_REPOSITORY
GitHub repository used to pull, push images when cache is used.
Default: apache/airflow.
@@ -2693,16 +2673,6 @@ This is the current syntax for `./breeze <./breeze>`_:
DockerHub. You need to be logged in to the registry in order to be
able to pull/push from
and you need to be committer to push to Apache Airflow' GitHub
registry.
- --github-registry GITHUB_REGISTRY
- GitHub registry used. GitHub has legacy Packages registry and Public
Beta Container
- registry.
-
- Default: ghcr.io.
-
- If you use this flag, automatically --use-github-registry flag is
enabled.
-
- ghcr.io docker.pkg.github.com
-
-g, --github-repository GITHUB_REPOSITORY
GitHub repository used to pull, push images when cache is used.
Default: apache/airflow.
diff --git a/CI.rst b/CI.rst
index 56957a3..29abae5 100644
--- a/CI.rst
+++ b/CI.rst
@@ -54,7 +54,7 @@ it can be ~6-7 minutes and in case base image of Python
releases new patch-level
Container Registry used as cache
--------------------------------
-For the CI builds of our we are using Container Registry to store results of
the "Build Image" workflow
+For the CI builds of our we are using GitHub Container Registry to store
results of the "Build Image" workflow
and pass it to the "CI Build" workflow.
Currently in main version of Airflow we run tests in 4 different versions of
Python (3.6, 3.7, 3.8, 3.9)
@@ -70,40 +70,23 @@ This is especially important in our case where Pull
Requests to Airflow might co
and it would be a huge security issue if anyone from outside could
utilise the WRITE access to Apache Airflow repository via an external Pull
Request.
-Thanks to the WRITE access and fact that the 'pull_request_target' by default
uses the 'main' version of the
+Thanks to the WRITE access and fact that the ``pull_request_target`` by
default uses the ``main`` version of the
sources, we can safely run some logic there will checkout the incoming Pull
Request, build the container
image from the sources from the incoming PR and push such image to an GitHub
Docker Registry - so that
this image can be built only once and used by all the jobs running tests. The
image is tagged with unique
``COMMIT_SHA`` of the incoming Pull Request and the tests run in the Pull
Request can simply pull such image
rather than build it from the scratch. Pulling such image takes ~ 1 minute,
thanks to that we are saving
a lot of precious time for jobs.
+4
+We use `GitHub Container Registry
<https://docs.github.com/en/packages/guides/about-github-container-registry>`_
+GitHub Package Registry ``GITHUB_TOKEN`` is needed to push to the registry.
You also have to manually manage
+permissions of the images, after creating image for the first time (pushing it
using your personal token)
+you need to set their visibility to ``Public`` and enable
+`Inheriting access from repository
<https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility#inheriting-access-for-a-container-image-from-a-repository>`_
+Those images have specific naming schema. See `Images documentation
<IMAGES.rst>`_ for details.
-We can use either of the two available GitHub Container registries as cache:
-
-* Legacy `GitHub Package Registry <https://github.com/features/packages>`_
which is not very
- stable, uses old infrastructure of GitHub and it lacks certain features -
notably it does not allow
- us to delete the old image. The benefit of using GitHub Package Registry is
that it works
- out-of-the-box (write authentication is done using ``GITHUB_TOKEN`` and
users do not have to do any
- action to make it work in case they want to run build using their own forks.
Also those images
- do not provide public access, so you need to login to
``docker.pkg.github.com`` docker registry
- using your username and personal token to be able to pull those images.
-
-* The new `GitHub Container Registry
<https://docs.github.com/en/packages/guides/about-github-container-registry>`_
- which is in Public Beta, has many more features (including permission
management, public access and
- image retention possibility). Similarly as in case of GitHub Package
Registry ``GITHUB_TOKEN`` is needed
- to push to the repositories. You also have to manually manage permissions of
the images,
- i.e. after creating images for the first time, you need to set their
visibility to ``Public`` and
- add ``Admin`` permissions to group of people managing the images (in our
case ``airflow-committers`` group).
- This makes it not very suitable to use GitHub container registry if you want
to run builds of Airflow
- in your own forks (note - it does not affect pull requests from forks to
Airflow).
-
-Those two images have different naming schemas. See `Images documentation
<IMAGES.rst>`_ for details.
-
-You can interact with the GitHub Registry images (pull/push) via `Breeze
<BREEZE.rst>`_ - you can
-pass ``--github-registry`` flag with either ``docker.pkg.github.com`` for
GitHub Package Registry or
-``ghcr.io`` for GitHub Container Registry and pull/push operations will be
performed using the chosen
-registry, using appropriate naming convention. This allows building and
pushing the images locally by
-committers who have access to push/pull those images.
+You can interact with the GitHub Registry images (pull/push) via `Breeze
<BREEZE.rst>`_ - by passing
+``--use-github-registry`` flag.
Locally replicating CI failures
-------------------------------
@@ -787,7 +770,7 @@ cd27124534b46c9688a1d89e75fcd137ab5137e3, in python 3.8
environment you can run:
.. code-block:: bash
- ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3
--github-registry ghcr.io --python 3.8
+ ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3
--use=github-registry --python 3.8
You will be dropped into a shell with the exact version that was used during
the CI run and you will
be able to run pytest tests manually, easily reproducing the environment that
was used in CI. Note that in
@@ -848,8 +831,7 @@ In order to add a new version the following operations
should be done (example u
.. code-block:: bash
./breeze push-image --python 3.9
- ./breeze push-image --python 3.9 --github-registry ghcr.io
- ./breeze push-image --python 3.9 --github-registry docker.pkg.github.com
+ ./breeze push-image --python 3.9 --use-github-registry
* Find the 3 new images (main, ci, build) created in
`GitHub Container registry
<https://github.com/orgs/apache/packages?tab=packages&ecosystem=container&q=airflow>`_
diff --git a/IMAGES.rst b/IMAGES.rst
index c3a1805..47ce39f 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -270,14 +270,11 @@ Choosing image registry
=======================
By default images are pulled and pushed from and to DockerHub registry when
you use Breeze's push-image
-or build commands. But as described in `CI Documentation <CI.rst>`_, you can
choose different image
-registry by setting ``GITHUB_REGISTRY`` to ``docker.pkg.github.com`` for
GitHub Package Registry or
-``ghcr.io`` for GitHub Container Registry.
+or build commands.
-Default is the GitHub Package Registry one. The Pull Request forks have no
access to the secret but they
-auto-detect the registry used when they wait for the images.
+But as described in `CI Documentation <CI.rst>`_, you can choose GitHub
Container Registry.
-Our images are named following conventions below.
+Naming convention for DockerHub images.
Images used during CI builds:
@@ -320,53 +317,16 @@ For example:
You can see those CI DockerHub images at
`<https://hub.docker.com/r/apache/airflow>`_
-Using GitHub registries as build cache
---------------------------------------
+Using GitHub Container Registry as build cache
+----------------------------------------------
-By default DockerHub registry is used when you push or pull such images.
-However for CI builds we keep the images in GitHub registry as well - this way
we can easily push
-the images automatically after merge requests and use such images for Pull
Requests
-as cache - which makes it much it much faster for CI builds (images are
available in cache
-right after merged request in main finishes it's build), The difference is
visible especially if
-significant changes are done in the Dockerfile.CI.
-
-The images are named differently (in Docker definition of image names -
registry URL is part of the
-image name if DockerHub is not used as registry). Also GitHub has its own
structure for registries
-each project has its own registry naming convention that should be followed.
The name of
-images for GitHub registry are different as they must follow limitation of the
registry used.
-
-We are still using GitHub Packages as registry, but we are in the process of
testing and switching
-to GitHub Container Registry, and the naming conventions are slightly
different (GitHub Packages
-required all packages to have "organization/repository/" URL prefix
("apache/airflow/",
-where in GitHub Container Registry, all images are in "organization" not in
"repository" and they are all
-in organization wide "apache/" namespace rather than in "apache/airflow/" one).
-We are adding "airflow-" as prefix for image names of all Airflow images
instead.
+We are using GitHub Container Registry as build cache.The images are all in
organization wide "apache/"
+namespace. We are adding "airflow-" as prefix for image names of all Airflow
images.
The images are linked to the repository via
``org.opencontainers.image.source`` label in the image.
-Naming convention for GitHub Packages
--------------------------------------
-
-Images with a commit SHA (built for pull requests and pushes)
-
-.. code-block:: bash
-
- docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-ci-v2:<COMMIT_SHA> -
for CI images
- docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-v2:<COMMIT_SHA>
- for production images
-
docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-build-v2:<COMMIT_SHA> -
for production build stage
- docker.pkg.github.com/apache-airflow/python-v2:X.Y-slim-buster-<COMMIT_SHA>
- for base Python images
-
-Latest images (pushed when main merge succeeds):
-
-.. code-block:: bash
-
- docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-ci-v2:latest -
for CI images
- docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-v2:latest -
for production images
- docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-build-v2:latest -
for production build stage
- docker.pkg.github.com/apache/airflow/python-v2:X.Y-slim-buster -
for base Python images
-
+See
https://docs.github.com/en/packages/learn-github-packages/connecting-a-repository-to-a-package
-Naming convention for GitHub Container Registry
------------------------------------------------
+Naming convention for the GitHub packages.
Images with a commit SHA (built for pull requests and pushes)
@@ -386,40 +346,28 @@ Latest images (pushed when main merge succeeds):
ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-build-v2:latest - for production
build stage
ghcr.io/apache/airflow-python-v2:X.Y-slim-buster - for base Python
images
-Note that we never push or pull "release" images to GitHub registry. It is
only used for CI builds
-
+Note that we never push or pull "release" images to GitHub registry. Those are
only pushed to DockerHub.
You can see all the current GitHub images at
`<https://github.com/apache/airflow/packages>`_
-
-In order to interact with the GitHub images you need to add
``--use-github-registry`` flag to the pull/push
-commands in Breeze. This way the images will be pulled/pushed from/to GitHub
rather than from/to
-DockerHub. Images are build locally as ``apache/airflow`` images but then they
are tagged with the right
-GitHub tags for you. You can also specify ``--github-registry`` option and
choose which of the
-GitHub registries are used (``docker.pkg.github.com`` chooses GitHub Packages
and ``ghcr.io`` chooses
-GitHub Container Registry).
+In order to interact with the GitHub Container Registry you need to add
``--use-github-registry``
+flag to the pull/push commands in Breeze. This way the images will be
pulled/pushed from/to GitHub
+rather than from/to DockerHub. Images are build locally as ``apache/airflow``
images but then they are
+tagged with the right GitHub tags for you automatically.
You can read more about the CI configuration and how CI builds are using
DockerHub/GitHub images
in `<CI.rst>`_.
Note that you need to be committer and have the right to push to DockerHub and
GitHub and you need to
-be logged in. Only committers can push images directly. You need to login with
your
-Personal Access Token with "packages" scope to be able to push to those
repositories or pull from them
+be logged in to the registry. Only committers can push images directly. You
need to login with your
+Personal Access Token with "packages" write scope to be able to push to those
repositories or pull from them
in case of GitHub Packages.
-GitHub Packages:
-
-.. code-block:: bash
-
- docker login docker.pkg.github.com
-
GitHub Container Registry
.. code-block:: bash
docker login ghcr.io
-Interacting with container registries
-=====================================
Since there are different naming conventions used for Airflow images and there
are multiple images used,
`Breeze <BREEZE.rst>`_ provides easy to use management interface for the
images. The
@@ -431,23 +379,12 @@ to refresh them.
This can be done with ``Breeze`` command line which has easy-to-use tool to
manage those images. For
example:
-
Force building Python 3.6 CI image using local cache and pushing it container
registry:
.. code-block:: bash
- ./breeze build-image --python 3.6 --force-build-images --build-cache-local
- ./breeze push-image --python 3.6 --github-registry ghcr.io
-
-
-Building Python 3.7 PROD images (both build and final image) using cache pulled
-from ``docker.pkg.github.com`` and pushing it back:
-
-.. code-block:: bash
-
- ./breeze build-image --production-image --python 3.7 --github-registry
docker.pkg.github.com
- ./breeze push-image --production-image --python 3.7 --github-registry
docker.pkg.github.com
-
+ ./breeze build-image --python 3.6 --force-build-images
--force-pull-base-python-image --build-cache-local
+ ./breeze push-image --python 3.6 --use-github-registry
Building Python 3.8 CI image using cache pulled from DockerHub and pushing it
back:
@@ -461,13 +398,13 @@ tool that allows to reproduce CI failures locally, enter
the images and fix them
to pass ``--github-image-id`` and the registry and Breeze will download and
execute commands using
the same image that was used during the CI build.
-For example this command will run the same Python 3.8 image as was used in
210056909
-run with enabled Kerberos integration (assuming docker.pkg.github.com was used
as build cache).
+For example this command will run the same Python 3.8 image as was used in
build identified with
+9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e commit SHA with enabled Kerberos
integration.
.. code-block:: bash
- ./breeze --github-image-id 210056909 \
- --github-registry docker.pkg.github.com \
+ ./breeze --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e \
+ --use-github-registry \
--python 3.8 --integration kerberos
You can see more details and examples in `Breeze <BREEZE.rst>`_
@@ -774,11 +711,9 @@ GitHub Registies in order to be able to do that.
./breeze build-image --python ${python_version}
--build-cache-local \
--production-image --verbose
./breeze push-image
- ./breeze push-image --github-registry ghcr.io
- ./breeze push-image --github-registry docker.pkg.github.com
+ ./breeze push-image --use-github-registry
./breeze push-image --production-image
- ./breeze push-image --github-registry ghcr.io --production-image
- ./breeze push-image --github-registry docker.pkg.github.com
--production-image
+ ./breeze push-image --production-image --use-github-registry
done
Running the CI image
diff --git a/breeze b/breeze
index de1fe1a..e239960 100755
--- a/breeze
+++ b/breeze
@@ -1159,15 +1159,6 @@ function breeze::parse_arguments() {
export USE_GITHUB_REGISTRY="true"
shift
;;
- --github-registry)
- echo
- echo "Using GitHub registry."
- echo "GitHub registry used: ${2}"
- echo
- export GITHUB_REGISTRY="${2}"
- export USE_GITHUB_REGISTRY="true"
- shift 2
- ;;
-g | --github-repository)
echo
echo "Using GitHub registry."
@@ -1624,10 +1615,6 @@ function breeze::prepare_formatted_versions() {
tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed
"s/^/${list_prefix}/")
readonly FORMATTED_GENERATE_CONSTRAINTS_MODE
- FORMATTED_GITHUB_REGISTRY=$(echo "${_breeze_allowed_github_registrys=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed
"s/^/${list_prefix}/")
- readonly FORMATTED_GITHUB_REGISTRY
-
FORMATTED_POSTGRES_VERSIONS=$(echo
"${_breeze_allowed_postgres_versions=""}" |
tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed
"s/^/${list_prefix}/")
readonly FORMATTED_POSTGRES_VERSIONS
@@ -2846,16 +2833,6 @@ function breeze::flag_pull_push_docker_images() {
DockerHub. You need to be logged in to the registry in order to be
able to pull/push from
and you need to be committer to push to Apache Airflow' GitHub
registry.
---github-registry GITHUB_REGISTRY
- GitHub registry used. GitHub has legacy Packages registry and Public
Beta Container
- registry.
-
- Default: ${_breeze_default_github_registry:=}.
-
- If you use this flag, automatically --use-github-registry flag is
enabled.
-
-${FORMATTED_GITHUB_REGISTRY}
-
-g, --github-repository GITHUB_REPOSITORY
GitHub repository used to pull, push images when cache is used.
Default: ${_breeze_default_github_repository:=}.
@@ -3186,7 +3163,6 @@ function breeze::check_and_save_all_params() {
parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres
version" "--postgres-version"
parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version"
"--mysql-version"
parameters::check_and_save_allowed_param "MSSQL_VERSION" "MSSql version"
"--mssql-version"
- parameters::check_and_save_allowed_param "GITHUB_REGISTRY" "GitHub
Registry" "--github-registry"
parameters::check_allowed_param TEST_TYPE "Type of tests" "--test-type"
parameters::check_allowed_param PACKAGE_FORMAT "Format of packages to
build" "--package-format"
diff --git a/breeze-complete b/breeze-complete
index 6b67d53..ec81c6e 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -27,8 +27,6 @@ _breeze_allowed_python_major_minor_versions="3.6 3.7 3.8 3.9"
_breeze_allowed_backends="sqlite mysql postgres mssql"
_breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq
redis statsd trino all"
_breeze_allowed_generate_constraints_modes="source-providers pypi-providers
no-providers"
-# registrys is good here even if it is not correct english. We are adding s
automatically to all variables
-_breeze_allowed_github_registrys="ghcr.io docker.pkg.github.com"
_breeze_allowed_kubernetes_modes="image"
_breeze_allowed_kubernetes_versions="v1.20.2 v1.19.7 v1.18.15"
_breeze_allowed_helm_versions="v3.2.4"
@@ -46,7 +44,6 @@ _breeze_allowed_installation_methods=". apache-airflow"
{
# Default values for the commands & flags used
_breeze_default_backend=$(echo "${_breeze_allowed_backends}" | awk '{print
$1}')
- _breeze_default_github_registry=$(echo
"${_breeze_allowed_github_registrys}" | awk '{print $1}')
_breeze_default_generate_providers_mode=$(echo
"${_breeze_allowed_generate_constraints_modes}" | awk '{print $1}')
_breeze_default_kubernetes_mode=$(echo
"${_breeze_allowed_kubernetes_modes}" | awk '{print $1}')
_breeze_default_kubernetes_version=$(echo
"${_breeze_allowed_kubernetes_versions}" | awk '{print $1}')
@@ -323,9 +320,6 @@ function breeze_complete::get_known_values_breeze() {
--installation-method)
_breeze_known_values="${_breeze_allowed_installation_methods}"
;;
- --github-registry)
- _breeze_known_values="${_breeze_allowed_github_registrys}"
- ;;
--generate-constraints-mode)
_breeze_known_values="${_breeze_allowed_generate_constraints_modes}"
;;
diff --git a/scripts/ci/libraries/_initialization.sh
b/scripts/ci/libraries/_initialization.sh
index 92c8aaa..34e700d 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -566,9 +566,9 @@ function initialization::initialize_git_variables() {
function initialization::initialize_github_variables() {
# Defaults for interacting with GitHub
+ export GITHUB_REGISTRY="ghcr.io"
export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
export GITHUB_REGISTRY_IMAGE_SUFFIX=${GITHUB_REGISTRY_IMAGE_SUFFIX:="-v2"}
- export GITHUB_REGISTRY=${GITHUB_REGISTRY:="ghcr.io"}
export
GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
export
GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
export
GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
@@ -730,7 +730,6 @@ Production image build variables:
Detected GitHub environment:
USE_GITHUB_REGISTRY: '${USE_GITHUB_REGISTRY}'
- GITHUB_REGISTRY: '${GITHUB_REGISTRY}'
GITHUB_REPOSITORY: '${GITHUB_REPOSITORY}'
GITHUB_USERNAME: '${GITHUB_USERNAME}'
GITHUB_TOKEN: '${GITHUB_TOKEN}'
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh
b/scripts/ci/libraries/_push_pull_remove_images.sh
index a6a1c5c..f8a1d3e 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -291,35 +291,6 @@ function push_pull_remove_images::push_prod_images() {
fi
}
-# waits for an image to be available in GitHub Packages. Should be run with
`set +e`
-# the build automatically determines which registry to use based one the
images available
-function push_pull_remove_images::check_for_image_in_github_packages() {
- local github_repository_lowercase
- github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]'
'[:lower:]')"
- local github_api_endpoint
-
github_api_endpoint="https://docker.pkg.github.com/v2/${github_repository_lowercase}";
- local image_name_in_github_registry="${1}"
- local image_tag_in_github_registry=${2}
- local
image_to_wait_for=${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry}
- local github_api_call
-
github_api_call="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
- echo "GitHub Packages: checking for ${image_to_wait_for} via
${github_api_call}!"
- http_status=$(curl --silent --output "${OUTPUT_LOG}" --write-out
"%{http_code}" \
- --connect-timeout 60 --max-time 60 \
- -X GET "${github_api_call}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}")
- if [[ ${http_status} == "200" ]]; then
- echo "Image: ${image_to_wait_for} found in GitHub Packages:
${COLOR_GREEN}OK. ${COLOR_RESET}"
- echo "::set-output name=githubRegistry::docker.pkg.github.com"
- echo
- echo "Setting githubRegistry output to docker.pkg.github.com"
- echo
- return 0
- else
- cat "${OUTPUT_LOG}"
- echo "${COLOR_YELLOW}Still waiting. Status code
${http_status}!${COLOR_RESET}"
- return 1
- fi
-}
# waits for an image to be available in GitHub Container Registry. Should be
run with `set +e`
function
push_pull_remove_images::check_for_image_in_github_container_registry() {
@@ -332,10 +303,6 @@ function
push_pull_remove_images::check_for_image_in_github_container_registry()
local res=$?
if [[ ${res} == "0" ]]; then
echo "Image: ${image_to_wait_for} found in Container Registry:
${COLOR_GREEN}OK.${COLOR_RESET}"
- echo
- echo "Setting githubRegistry output to ghcr.io"
- echo
- echo "::set-output name=githubRegistry::ghcr.io"
return 0
else
echo "${COLOR_YELLOW}Still waiting. Not found!${COLOR_RESET}"
@@ -352,9 +319,6 @@ function
push_pull_remove_images::wait_for_github_registry_image() {
if
push_pull_remove_images::check_for_image_in_github_container_registry "${@}";
then
break
fi
- if push_pull_remove_images::check_for_image_in_github_packages "${@}";
then
- break
- fi
sleep 30
done
set -e
