This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch test-cache-refreshing in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 53f8aa6ccd2ce3ad38d79eccc11bffba822ea68e Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sat Mar 23 13:34:41 2024 +0100 Test cache refreshing in CI --- .github/workflows/ci.yml | 597 +++-------------------------------------------- 1 file changed, 27 insertions(+), 570 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37b6dd3c7d..f4567c8dae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,593 +27,50 @@ on: # yamllint disable-line rule:truthy permissions: # All other permissions are set to none contents: read - # Technically read access while waiting for images should be more than enough. However, - # there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission - # denied error when attempting to just pull private image, changing the token permission to write solves the - # issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml - # changes. If it is for a fork, then the token is read-only anyway. packages: write -env: - DB_RESET: "true" - VERBOSE: "true" - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.actor }} - # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the - # Airflow one is going to be used - CONSTRAINTS_GITHUB_REPOSITORY: >- - ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && - secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} - # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" - USE_SUDO: "true" - INCLUDE_NOT_READY_PROVIDERS: "true" - AIRFLOW_ENABLE_AIP_44: "true" - MOUNT_SOURCES: "skip" concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - - build-info: - name: "Build info" - # The runs-on cannot refer to env. or secrets. context, so we have no - # option but to specify a hard-coded list here. This is "safe", the - # runner checks if the user is an owner or collaborator of the repo - # before running the workflow. - runs-on: ["ubuntu-22.04"] - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - outputs: - image-tag: ${{ github.event.pull_request.head.sha || github.sha }} - cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} - affected-providers-list-as-string: >- - ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} - upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - python-versions: ${{ steps.selective-checks.outputs.python-versions }} - python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} - all-python-versions-list-as-string: >- - ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} - default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} - kubernetes-versions-list-as-string: >- - ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} - kubernetes-combos-list-as-string: >- - ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} - default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} - postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} - default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} - mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} - default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} - default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} - full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} - parallel-test-types-list-as-string: >- - ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} - postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} - mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} - sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} - skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }} - run-tests: ${{ steps.selective-checks.outputs.run-tests }} - run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} - run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} - run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} - ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} - prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} - docs-build: ${{ steps.selective-checks.outputs.docs-build }} - mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }} - needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} - needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} - needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} - needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} - default-branch: ${{ steps.selective-checks.outputs.default-branch }} - default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} - docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} - skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} - providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }} - helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} - debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} - runs-on: ${{steps.selective-checks.outputs.runs-on}} - is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} - is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} - is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} - is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} - is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} - is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} - latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} - chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} - has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} - source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} - pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} - in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} - build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} - canary-run: ${{ steps.source-run-info.outputs.canary-run }} - run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v4 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Get information about the Workflow" - id: source-run-info - run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} - - name: Selective checks - id: selective-checks - env: - PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" - COMMIT_REF: "${{ github.sha }}" - VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - - name: env - run: printenv - env: - PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} - GITHUB_CONTEXT: ${{ toJson(github) }} - - basic-tests: - name: "Basic tests" - needs: [build-info] - uses: ./.github/workflows/basic-tests.yml - with: - run-www-tests: ${{needs.build-info.outputs.run-www-tests}} - needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} - default-python-version: ${{needs.build-info.outputs.default-python-version}} - basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} - canary-run: ${{needs.build-info.outputs.canary-run}} - latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} - - build-ci-images: - name: > - ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} - CI images in-workflow - needs: [build-info] - uses: ./.github/workflows/ci-image-build.yml + push-cache: + name: Push Image Cache + uses: ./.github/workflows/push-image-cache.yml permissions: contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the - # "in-workflow-build" condition packages: write secrets: inherit with: - do-build: ${{ needs.build-info.outputs.in-workflow-build }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - upload-constraints: "true" - branch: ${{ needs.build-info.outputs.default-branch }} + runs-on: "[\"ubuntu-22.04\"]" + # runs-on: "[\"self-hosted\", \"Linux\", \"X64\"]" + include-prod-images: "false" + push-latest-images: "false" + image-tag: "test-tag1234" + python-versions: "[ \"3.8\" ]" + branch: "main" use-uv: "true" - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - docker-cache: ${{ needs.build-info.outputs.cache-directive }} - - wait-for-ci-images: - timeout-minutes: 120 - name: "Wait for CI images" - runs-on: ["ubuntu-22.04"] - needs: [build-info, build-ci-images] - if: needs.build-info.outputs.ci-image-build == 'true' - env: - BACKEND: sqlite - # Force more parallelism for pull even on public images - PARALLELISM: 6 - INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: "Install Breeze" - uses: ./.github/actions/breeze - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Login to ghcr.io - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} - id: wait-for-images - run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest - env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - if: needs.build-info.outputs.in-workflow-build == 'false' - - additional-ci-image-checks: - name: "Additional CI image checks" - needs: [build-info, wait-for-ci-images] - uses: ./.github/workflows/additional-ci-image-checks.yml - with: - runs-on: ${{needs.build-info.outputs.runs-on}} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} - docker-cache: ${{ needs.build-info.outputs.cache-directive }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - mypy-folders: ${{ needs.build-info.outputs.mypy-folders }} - needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} - latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - - static-checks-mypy-and-constraints-generation: - name: "Static checks, mypy and constraints generation" - needs: [build-info, wait-for-ci-images] - uses: ./.github/workflows/static-checks-mypy-and-constraints-generation.yml - with: - runs-on: ${{needs.build-info.outputs.runs-on}} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - branch: ${{ needs.build-info.outputs.default-branch }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} - basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - - docs: - name: "Docs" - uses: ./.github/workflows/docs.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - if: > - needs.build-info.outputs.docs-build == 'true' && - needs.build-info.outputs.latest-versions-only != 'true' - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - - providers: - name: "Provider checks" - uses: ./.github/workflows/check-providers.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - if: > - needs.build-info.outputs.skip-providers-tests != 'true' && - needs.build-info.outputs.latest-versions-only != 'true' - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }} - providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }} - skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - - tests-helm: - name: "Helm tests" - uses: ./.github/workflows/helm-tests.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - if: > - needs.build-info.outputs.needs-helm-tests == 'true' && - needs.build-info.outputs.default-branch == 'main' && - needs.build-info.outputs.latest-versions-only != 'true' - - tests-postgres: - name: "Postgres tests" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - backend: "postgres" - test-name: "Postgres" - test-scope: "DB" - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - backend-versions: ${{ needs.build-info.outputs.postgres-versions }} - excludes: ${{ needs.build-info.outputs.postgres-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-migration-tests: "true" - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.run-tests == 'true' - - tests-mysql: - name: "MySQL tests" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - backend: "mysql" - test-name: "MySQL" - test-scope: "DB" - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - backend-versions: ${{ needs.build-info.outputs.mysql-versions }} - excludes: ${{ needs.build-info.outputs.mysql-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - run-migration-tests: "true" - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.run-tests == 'true' + include-success-outputs: "true" + constraints-branch: "constraints-main" + docker-cache: "registry" - tests-sqlite: - name: "Sqlite tests" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - backend: "sqlite" - test-name: "Sqlite" - test-name-separator: "" - test-scope: "DB" - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - # No versions for sqlite - backend-versions: "['']" - excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - run-migration-tests: "true" - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.run-tests == 'true' - - tests-non-db: - name: "Non-DB tests" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - backend: "sqlite" - test-name: "" - test-name-separator: "" - test-scope: "Non-DB" - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - # No versions for non-db - backend-versions: "['']" - excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.run-tests == 'true' - - tests-special: - name: "Special tests" - uses: ./.github/workflows/special-tests.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.run-tests == 'true' - - tests-integration: - name: Integration Tests - needs: [build-info, wait-for-ci-images] - uses: ./.github/workflows/integration-tests.yml - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} - default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }} - skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }} - is-airflow-runner: ${{ needs.build-info.outputs.is-airflow-runner }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.run-tests == 'true' - - build-prod-images: - name: > - ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} - PROD images in-workflow - needs: [build-info, build-ci-images] - uses: ./.github/workflows/prod-image-build.yml + build-ci-images: + name: Build CI images + needs: [push-cache] + uses: ./.github/workflows/ci-image-build.yml permissions: contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the - # "in-workflow-build" condition packages: write secrets: inherit with: - build-type: "Regular" - do-build: ${{ needs.build-info.outputs.in-workflow-build }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - branch: ${{ needs.build-info.outputs.default-branch }} - push-image: "true" + runs-on: "[\"ubuntu-22.04\"]" + # runs-on: "[\"self-hosted\", \"Linux\", \"X64\"]" + do-build: "true" + image-tag: "test-tag1234" + python-versions: "[ \"3.8\" ]" + upload-constraints: "true" + branch: "main" use-uv: "true" - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - docker-cache: ${{ needs.build-info.outputs.cache-directive }} - - wait-for-prod-images: - timeout-minutes: 80 - name: "Wait for PROD images" - runs-on: ["ubuntu-22.04"] - needs: [build-info, wait-for-ci-images, build-prod-images] - if: needs.build-info.outputs.prod-image-build == 'true' - env: - BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" - # Force more parallelism for pull on public images - PARALLELISM: 6 - INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" - IMAGE_TAG: ${{ needs.build-info.outputs.image-tag }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: "Install Breeze" - uses: ./.github/actions/breeze - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Login to ghcr.io - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} - # We wait for the images to be available either from "build-images.yml' run as pull_request_target - # or from build-prod-images (or build-prod-images-release-branch) above. - # We are utilising single job to wait for all images because this job merely waits - # For the images to be available. - run: breeze prod-image pull --wait-for-image --run-in-parallel - env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.in-workflow-build == 'false' - - additional-prod-image-tests: - name: "Additional PROD image tests" - needs: [build-info, wait-for-prod-images, static-checks-mypy-and-constraints-generation] - uses: ./.github/workflows/additional-prod-image-tests.yml - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - docker-cache: ${{ needs.build-info.outputs.cache-directive }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - if: needs.build-info.outputs.prod-image-build == 'true' - - tests-kubernetes: - name: "Kubernetes tests" - uses: ./.github/workflows/k8s-tests.yml - needs: [build-info, wait-for-prod-images] - permissions: - contents: read - packages: read - secrets: inherit - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }} - kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - if: > - ( needs.build-info.outputs.run-kubernetes-tests == 'true' || - needs.build-info.outputs.needs-helm-tests == 'true') - - finalize-tests: - name: Finalize tests - permissions: - contents: write - packages: write - secrets: inherit - needs: - - build-info - - docs - - wait-for-ci-images - - wait-for-prod-images - - tests-sqlite - - tests-mysql - - tests-postgres - - tests-non-db - - tests-special - - tests-integration - - static-checks-mypy-and-constraints-generation - uses: ./.github/workflows/finalize-tests.yml - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - docker-cache: ${{ needs.build-info.outputs.cache-directive }} - canary-run: ${{ needs.build-info.outputs.canary-run }} + upgrade-to-newer-dependencies: "false" + constraints-branch: "constraints-main" + docker-cache: "registry"