This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 695fef9a3ca Automatically invalidate cloudfront cache when publishing
docs (#51507)
695fef9a3ca is described below
commit 695fef9a3cafb8d62c9d0063926fd9e5c3aa9b1e
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Jun 9 12:45:51 2025 +0200
Automatically invalidate cloudfront cache when publishing docs (#51507)
When we publish new documentation (especially overriding old
documentation) to S3, we should invalidate cloudfront cache so that
it takes effect immediately
---
.../commands/release_management_commands.py | 11 ++++-
.../src/airflow_breeze/utils/publish_docs_to_s3.py | 50 +++++++++++++++++++---
2 files changed, 54 insertions(+), 7 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 0c2747feda4..f8fef973447 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -127,7 +127,6 @@ from airflow_breeze.utils.docker_command_utils import (
fix_ownership_using_docker,
perform_environment_checks,
)
-from airflow_breeze.utils.docs_publisher import DocsPublisher
from airflow_breeze.utils.github import download_constraints_file,
get_active_airflow_versions
from airflow_breeze.utils.packages import (
PackageSuspendedException,
@@ -1691,6 +1690,8 @@ def run_docs_publishing(
verbose: bool,
output: Output | None,
) -> tuple[int, str]:
+ from airflow_breeze.utils.docs_publisher import DocsPublisher
+
builder = DocsPublisher(package_name=package_name, output=output,
verbose=verbose)
return builder.publish(override_versioned=override_versioned,
airflow_site_dir=airflow_site_directory)
@@ -3951,3 +3952,11 @@ def publish_docs_to_s3(
docs_to_s3.publish_all_docs()
if stable_versions:
docs_to_s3.publish_stable_version_docs()
+ from airflow_breeze.utils.publish_docs_to_s3 import version_error
+
+ if version_error:
+ get_console().print(
+ "[error]There was an error with the version of the docs. "
+ "Please check the version in the docs and try again.[/]"
+ )
+ sys.exit(1)
diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
index 55f5426e826..eb64174e8df 100644
--- a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
+++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
@@ -24,7 +24,6 @@ from functools import cached_property
import awswrangler as wr
import boto3
-import semver
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.parallel import check_async_run_results,
run_with_pool
@@ -36,6 +35,15 @@ NON_SHORT_NAME_PACKAGES = ["docker-stack", "helm-chart",
"apache-airflow"]
PACKAGES_METADATA_EXCLUDE_NAMES = ["docker-stack", "apache-airflow-providers"]
s3_client = boto3.client("s3")
+cloudfront_client = boto3.client("cloudfront")
+
+version_error = False
+
+
+def get_cloudfront_distribution(destination_location):
+ if "live-docs" in destination_location:
+ return "E26P75MP9PMULE"
+ return "E197MS0XRJC5F3"
class S3DocsPublish:
@@ -260,19 +268,38 @@ class S3DocsPublish:
# We keep metadata in the same location with constant file name so that
# its easy to reference in airflow-site with url
# ex:
https://staging-docs-airflow-apache-org.s3.us-east-2.amazonaws.com/manifest/packages-metadata.json
-
+ get_console().print("[info]Uploading packages-metadata.json to S3\n")
s3_client.put_object(
Bucket=bucket,
Key="manifest/packages-metadata.json",
Body=json.dumps(all_packages_infos, indent=2),
ContentType="application/json",
)
+ get_console().print("[success]packages-metadata.json file generated
successfully\n")
+ distribution_id =
get_cloudfront_distribution(self.destination_location)
+ get_console().print(
+ f"[info]Invalidating CloudFront cache for the uploaded files:
distribution id {distribution_id}\n"
+ )
+ # We invalidate all CloudFront caches so that all uploaded files are
available immediately
+ cloudfront_client.create_invalidation(
+ DistributionId=distribution_id,
+ InvalidationBatch={
+ "Paths": {
+ "Quantity": 1,
+ "Items": ["/*"],
+ },
+ "CallerReference": str(int(os.environ.get("GITHUB_RUN_ID",
0))),
+ },
+ )
+ get_console().print(
+ f"[success]CloudFront cache request invalidated successfully:
{distribution_id}\n"
+ )
def dump_docs_package_metadata(self, package_versions: dict[str,
list[str]]):
all_packages_infos = [
{
"package-name": package_name,
- "all-versions": (all_versions :=
self.get_all_versions(versions)),
+ "all-versions": (all_versions :=
self.get_all_versions(package_name, versions)),
"stable-version": all_versions[-1],
}
for package_name, versions in package_versions.items()
@@ -281,10 +308,21 @@ class S3DocsPublish:
return all_packages_infos
@staticmethod
- def get_all_versions(versions: list[str]) -> list[str]:
+ def get_all_versions(package_name: str, versions: list[str]) -> list[str]:
+ from packaging.version import Version
+
+ good_versions = []
+ for version in versions:
+ try:
+ Version(version)
+ good_versions.append(version)
+ except ValueError as e:
+ get_console().print(f"[error]Invalid version {version}: {e}\n")
+ global version_error
+ version_error = True
return sorted(
- versions,
- key=lambda d: semver.VersionInfo.parse(d),
+ good_versions,
+ key=lambda d: Version(d),
)
@staticmethod
