This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a9ad129058a Skip allowlisted non-PEP 440 docs versions when publishing
docs to S3 (#76020)
a9ad129058a is described below
commit a9ad129058a104d9bd70bb86eca0c92f9d89d0cf
Author: Jason(Zhe-You) Liu <[email protected]>
AuthorDate: Tue Nov 3 12:09:23 2026 +0800
Skip allowlisted non-PEP 440 docs versions when publishing docs to S3
(#76020)
* Use an explicit allowlist for skipped docs versions
* Cover an excluded version listed under another package
---------
---
dev/breeze/src/airflow_breeze/global_constants.py | 6 +++
.../src/airflow_breeze/utils/publish_docs_to_s3.py | 15 ++++---
dev/breeze/tests/test_publish_docs_to_s3.py | 46 ++++++++++++++++------
3 files changed, 48 insertions(+), 19 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index e129805779d..2b259570604 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -366,6 +366,12 @@ SCHEMA_DESTINATION_LOCATIONS = [
PACKAGES_METADATA_EXCLUDE_NAMES = ["docker-stack", "apache-airflow-providers"]
+# Published docs folders that are not PEP 440 versions, such as Maven
snapshots staged for testing.
+# Any other invalid version folder still fails the docs publishing.
+PACKAGES_METADATA_EXCLUDE_VERSIONS: dict[str, list[str]] = {
+ "java-sdk": ["1.0.0-SNAPSHOT"],
+}
+
@clearable_cache
def all_selective_core_test_types() -> tuple[str, ...]:
diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
index 7c8fb990596..e6722be51ef 100644
--- a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
+++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
@@ -25,7 +25,10 @@ from functools import cached_property
import boto3
-from airflow_breeze.global_constants import PACKAGES_METADATA_EXCLUDE_NAMES
+from airflow_breeze.global_constants import (
+ PACKAGES_METADATA_EXCLUDE_NAMES,
+ PACKAGES_METADATA_EXCLUDE_VERSIONS,
+)
from airflow_breeze.utils.console import console_print
from airflow_breeze.utils.parallel import check_async_run_results,
run_with_pool
@@ -47,9 +50,6 @@ s3_client = boto3.client("s3")
cloudfront_client = boto3.client("cloudfront")
-SNAPSHOT_SUFFIX = "-SNAPSHOT"
-
-
class VersionError:
"""Class to track version errors during processing."""
@@ -347,11 +347,10 @@ class S3DocsPublish:
console_print(f"[info]Getting package versions for {package_name}
from:\n")
console_print(versions)
all_versions: list[Version] = []
+ excluded_versions =
PACKAGES_METADATA_EXCLUDE_VERSIONS.get(package_name, [])
for v in versions:
- # A Maven development build such as the Java SDK's
`1.0.0-SNAPSHOT` is not a release and
- # is not a valid Python version either; one that reached S3 is
left out, not an error.
- if v.upper().endswith(SNAPSHOT_SUFFIX):
- console_print(f"[warning]Skipping development build {v} of
{package_name}\n")
+ if v in excluded_versions:
+ console_print(f"[info]Skipping excluded version
{package_name}/{v}\n")
continue
try:
all_versions.append(Version(v))
diff --git a/dev/breeze/tests/test_publish_docs_to_s3.py
b/dev/breeze/tests/test_publish_docs_to_s3.py
index 531b1ff0a2a..08fe58b7f53 100644
--- a/dev/breeze/tests/test_publish_docs_to_s3.py
+++ b/dev/breeze/tests/test_publish_docs_to_s3.py
@@ -299,19 +299,43 @@ class TestPublishDocsToS3:
assert mock_generate_packages_metadata.call_count == expected_calls
+ @patch.dict(
+
"airflow_breeze.utils.publish_docs_to_s3.PACKAGES_METADATA_EXCLUDE_VERSIONS",
+ {"java-sdk": ["1.0.0-SNAPSHOT"], "snapshot-only-sdk":
["1.0.0-SNAPSHOT"]},
+ clear=True,
+ )
+ @patch.object(VersionError, "version_error", False)
+ def test_dump_docs_package_metadata_skips_excluded_versions(self):
+ metadata = self.publish_docs_to_s3.dump_docs_package_metadata(
+ {
+ "java-sdk": ["1.0.0-SNAPSHOT", "1.0.0-beta1"],
+ "snapshot-only-sdk": ["1.0.0-SNAPSHOT"],
+ }
+ )
[email protected](
- ("versions", "expected"),
- [
- pytest.param(["1.0.0-SNAPSHOT", "1.0.0-beta1"], ["1.0.0b1"],
id="maven-snapshot-skipped"),
- pytest.param(["2.1.0-snapshot", "2.0.0"], ["2.0.0"],
id="lowercase-snapshot-skipped"),
- ],
-)
-def test_get_latest_minor_versions_skips_development_builds(versions,
expected):
- VersionError.set_version_error(False)
+ assert metadata == [
+ {"package-name": "java-sdk", "all-versions": ["1.0.0b1"],
"stable-version": "1.0.0b1"},
+ ]
+ assert not VersionError.has_any_error()
+
+ @patch.dict(
+
"airflow_breeze.utils.publish_docs_to_s3.PACKAGES_METADATA_EXCLUDE_VERSIONS",
+ {"java-sdk": ["1.0.0-SNAPSHOT"]},
+ clear=True,
+ )
+ @pytest.mark.parametrize(
+ ("package_name", "invalid_version"),
+ [
+ pytest.param("java-sdk", "1.1.0-SNAPSHOT",
id="version_not_excluded"),
+ pytest.param("ts-sdk", "1.0.0-SNAPSHOT",
id="version_excluded_for_another_package"),
+ ],
+ )
+ @patch.object(VersionError, "version_error", False)
+ def test_get_latest_minor_versions_flags_versions_not_excluded(self,
package_name, invalid_version):
+ versions = S3DocsPublish.get_latest_minor_versions(package_name,
[invalid_version, "1.0.0-beta1"])
- assert S3DocsPublish.get_latest_minor_versions("java-sdk", versions) ==
expected
- assert VersionError.has_any_error() is False
+ assert versions == ["1.0.0b1"]
+ assert VersionError.has_any_error()
def test_get_latest_minor_versions_still_fails_on_other_invalid_versions():
