This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch improve-live-staging-approach-for-docs
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 79038c3873dd9b536c2da3efaffba2a99feece0a
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed May 14 22:17:06 2025 -0400
Improve flow for live/staging doc publishing
There are a couple of improvements for live/staging doc publishing.
1) we use auto/live/staging rather than buckets in the drop-downs
2) auto selection of live/staging depending on the tag used to
produce the documentation
3) we use env variables to pass inputs/outputs to scripts - which
is good security practice
4) watermark changes are applied to documentation published to
staging site
---
.github/workflows/publish-docs-to-s3.yml | 65 +++++++++++++++++++++++---------
1 file changed, 48 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index 748a466cf8c..b1db642b3cb 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -24,6 +24,15 @@ on: # yamllint disable-line rule:truthy
description: "The branch or tag to checkout for the docs publishing"
required: true
type: string
+ destination:
+ description: "The destination location in S3"
+ required: false
+ default: auto
+ type: choice
+ options:
+ - auto
+ - live
+ - staging
include-docs:
description: |
Space separated list of docs to build.
@@ -38,14 +47,6 @@ on: # yamllint disable-line rule:truthy
required: false
default: "no-docs-excluded"
type: string
- destination-location:
- description: "The destination location in S3, default is live site"
- required: false
- default: "s3://live-docs-airflow-apache-org/docs"
- type: choice
- options:
- - s3://live-docs-airflow-apache-org/docs
- - s3://staging-docs-airflow-apache-org/docs
env:
AIRFLOW_ROOT_PATH: "/home/runner/work/temp-airflow-repo-reference" #
checkout dir for referenced tag
@@ -62,9 +63,11 @@ jobs:
REF: ${{ inputs.ref }}
INCLUDE_DOCS: ${{ inputs.include-docs }}
EXCLUDE_DOCS: ${{ inputs.exclude-docs }}
- DESTINATION_LOCATION: ${{ inputs.destination-location }}
+ DESTINATION: ${{ inputs.destination }}
outputs:
include-docs: ${{ inputs.include-docs == 'all' && '' ||
inputs.include-docs }}
+ destination-location: ${{ steps.parameters.outputs.destination-location
}}
+ destination: ${{ steps.parameters.outputs.destination }}
if: contains(fromJSON('[
"ashb",
"eladkal",
@@ -78,13 +81,29 @@ jobs:
steps:
- name: "Input parameters summary"
shell: bash
+ id: parameters
run: |
echo "Input parameters summary"
echo "========================="
echo "Ref: '${REF}'"
echo "Included docs : '${INCLUDE_DOCS}'"
echo "Exclude docs: '${EXCLUDE_DOCS}'"
- echo "Destination location: '${DESTINATION_LOCATION}'"
+ echo "Destination: '${DESTINATION}'"
+ if [[ "${DESTINATION}" == "auto" ]]; then
+ if [[ "${REF}" =~ ^.*[0-9]*\.[0-9]*\.[0-9]*$ ]]; then
+ echo "${REF} looks like final release, using live destination"
+ DESTINATION="live"
+ else
+ echo "${REF} does not looks like final release, using staging
destination"
+ DESTINATION="staging"
+ fi
+ fi
+ echo "destination=${DESTINATION}" >> ${GITHUB_OUTPUT}
+ if [[ "${DESTINATION}" == "live" ]]; then
+ echo
"destination-location=s3://live-docs-airflow-apache-org/docs/" >>
${GITHUB_OUTPUT}
+ else
+ echo
"destination-location=s3://staging-docs-airflow-apache-org/docs/" >>
${GITHUB_OUTPUT}
+ fi
build-ci-images:
name: Build CI images
@@ -179,18 +198,18 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Prepare breeze & CI image: 3.9"
- uses: ./.github/actions/prepare_breeze_and_image
- with:
- platform: "linux/amd64"
- python: 3.9
- use-uv: true
- name: "Checkout ${{ inputs.ref }}"
run: |
git clone https://github.com/apache/airflow.git
"${AIRFLOW_ROOT_PATH}"
cd "${AIRFLOW_ROOT_PATH}" && git checkout ${REF}
env:
REF: ${{ inputs.ref }}
+ - name: "Prepare breeze & CI image: 3.9"
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ platform: "linux/amd64"
+ python: 3.9
+ use-uv: true
- name: "Download docs prepared as artifacts"
uses: actions/download-artifact@v4
with:
@@ -220,6 +239,18 @@ jobs:
run: breeze release-management add-back-references docker-stack
- name: "Generate back references for helm-chart"
run: breeze release-management add-back-references helm-chart
+ - name: "Update watermarks"
+ env:
+ SOURCE_DIR_PATH: "/mnt/airflow-site/docs-archive/"
+ # yamllint enable rule:line-length
+ run: |
+ curl -sSf -o add_watermark.py
https://raw.githubusercontent.com/apache/airflow-site/refs/heads/main/.github/scripts/add_watermark.py
+ chmod a+x add_watermark.py
+ mkdir -p images
+ curl -sSf -o images/staging.png
https://raw.githubusercontent.com/apache/airflow-site/refs/heads/main/.github/scripts/images/staging.png
+ uv run add_watermark.py --pattern 'main.min*css' --folder
${SOURCE_DIR_PATH}/generated/_build/docs \
+ --image-directory images --url-prefix /docs/images
+ if: needs.build-info.outputs.destination == 'staging'
- name: Install AWS CLI v2
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"; -o
/tmp/awscliv2.zip
@@ -235,7 +266,7 @@ jobs:
aws-region: us-east-2
- name: "Syncing docs to S3"
env:
- DESTINATION_LOCATION: "${{ inputs.destination-location }}"
+ DESTINATION_LOCATION: "${{
needs.build-info.outputs.destination-location }}"
SOURCE_DIR_PATH: "/mnt/airflow-site/docs-archive/"
EXCLUDE_DOCS: "${{ inputs.exclude-docs }}"
run: |
