This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new e637f08188 Fix errors in common.io release documentation generation 
with prevention (#35236)
e637f08188 is described below

commit e637f08188bd7c9348531dc3f013e95aecb3572c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Oct 28 18:11:08 2023 +0200

    Fix errors in common.io release documentation generation with prevention 
(#35236)
    
    The common.io provider release caused a few problems when preparing the
    new wave of the providers:
    
    * it missed link to security.rst (which is generated automatically
      anticipating SBOM generation in the near future)
    * it did not have proper min-version set in
      MIN_AIRFLOW_VERSION_EXCEPTIONS
    * it had single-backtick description that resulted in generated commit
      and changelog code - since it failed documentation building
    
    All those errors have not been caught at PR time. This PR fixes them
    and introduces prevention from similar mistakes - future providers with
    similar errors will get nice errors generated in CI at "docs publishing
    step".
    
    * if some generated file is missing in the index user is asked to add
      the files in the index
    * if there are `backtick` errors they will be reported and pointed at
    * if there are liniting problems with generated .rst files they will
      be reported
    * the "min airflow version" is now taken directly from provider.yaml
      not from exception list - but only when it is bigger than current
      min version. This way, we will also not forget to remove the provider
      from the list of exceptions when we bump min version above the
      exception version
---
 airflow/providers/common/io/__init__.py            | 27 ++++++
 airflow/providers/common/io/provider.yaml          |  2 +-
 dev/provider_packages/prepare_provider_packages.py | 98 +++++++++++++++++-----
 .../apache-airflow-providers-common-io/commits.rst | 16 +---
 docs/apache-airflow-providers-common-io/index.rst  | 34 ++++++++
 .../security.rst                                   | 38 +++++++++
 setup.py                                           |  1 +
 7 files changed, 183 insertions(+), 33 deletions(-)

diff --git a/airflow/providers/common/io/__init__.py 
b/airflow/providers/common/io/__init__.py
index 13a83393a9..f5bcab9b8b 100644
--- a/airflow/providers/common/io/__init__.py
+++ b/airflow/providers/common/io/__init__.py
@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,3 +15,29 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+#
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
+#
+# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
+# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` 
DIRECTORY
+#
+from __future__ import annotations
+
+import packaging.version
+
+__all__ = ["__version__"]
+
+__version__ = "1.0.0"
+
+try:
+    from airflow import __version__ as airflow_version
+except ImportError:
+    from airflow.version import version as airflow_version
+
+if 
packaging.version.parse(packaging.version.parse(airflow_version).base_version) 
< packaging.version.parse(
+    "2.8.0"
+):
+    raise RuntimeError(
+        f"The package `apache-airflow-providers-common-io:{__version__}` 
requires Apache Airflow 2.8.0+"
+    )
diff --git a/airflow/providers/common/io/provider.yaml 
b/airflow/providers/common/io/provider.yaml
index de5ae0935a..78723c4fa1 100644
--- a/airflow/providers/common/io/provider.yaml
+++ b/airflow/providers/common/io/provider.yaml
@@ -19,7 +19,7 @@
 package-name: apache-airflow-providers-common-io
 name: Common IO
 description: |
-  `Common IO Provider`
+  ``Common IO Provider``
 
 suspended: false
 versions:
diff --git a/dev/provider_packages/prepare_provider_packages.py 
b/dev/provider_packages/prepare_provider_packages.py
index 6d5a82327c..546125acf9 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -55,10 +55,6 @@ from yaml import safe_load
 ALL_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
 
 MIN_AIRFLOW_VERSION = "2.5.0"
-# In case you have some providers that you want to have different min-airflow 
version for,
-# Add them as exceptions here. Make sure to remove it once the min-airflow 
version is bumped
-# to the same version that is required by the exceptional provider
-MIN_AIRFLOW_VERSION_EXCEPTIONS = {"openlineage": "2.7.0"}
 
 INITIAL_CHANGELOG_CONTENT = """
  .. Licensed to the Apache Software Foundation (ASF) under one
@@ -1122,8 +1118,11 @@ def get_provider_jinja_context(
     for p in provider_details.excluded_python_versions:
         python_requires += f", !={p}"
     min_airflow_version = MIN_AIRFLOW_VERSION
-    if 
MIN_AIRFLOW_VERSION_EXCEPTIONS.get(provider_details.provider_package_id):
-        min_airflow_version = 
MIN_AIRFLOW_VERSION_EXCEPTIONS[provider_details.provider_package_id]
+    for dependency in provider_info["dependencies"]:
+        if dependency.startswith("apache-airflow>="):
+            current_min_airflow_version = dependency.split(">=")[1]
+            if Version(current_min_airflow_version) > 
Version(min_airflow_version):
+                min_airflow_version = current_min_airflow_version
     context: dict[str, Any] = {
         "ENTITY_TYPES": list(EntityType),
         "README_FILE": "README.rst",
@@ -1335,25 +1334,32 @@ def update_release_notes(
     )
     jinja_context["DETAILED_CHANGES_RST"] = changes
     jinja_context["DETAILED_CHANGES_PRESENT"] = bool(changes)
-    update_changelog_rst(
+    errors = False
+    if not update_changelog_rst(
         jinja_context,
         provider_package_id,
         provider_details.documentation_provider_package_path,
         regenerate_missing_docs,
-    )
-    update_security_rst(
+    ):
+        errors = True
+    if not update_security_rst(
         jinja_context,
         provider_package_id,
         provider_details.documentation_provider_package_path,
         regenerate_missing_docs,
-    )
+    ):
+        errors = True
     if not force:
-        update_commits_rst(
+        if not update_commits_rst(
             jinja_context,
             provider_package_id,
             provider_details.documentation_provider_package_path,
             regenerate_missing_docs,
-        )
+        ):
+            errors = True
+    if errors:
+        console.print("[red]There were errors when generating 
documentation[/]")
+        sys.exit(1)
     return True
 
 
@@ -1453,6 +1459,11 @@ def update_index_rst(
     replace_content(index_file_path, old_text, new_text, provider_package_id)
 
 
+# Taken from pygrep hooks we are using in pre-commit
+# https://github.com/pre-commit/pygrep-hooks/blob/main/.pre-commit-hooks.yaml
+BACKTICKS_CHECK = re.compile(r"^(?!    ).*(^| )`[^`]+`([^_]|$)", re.MULTILINE)
+
+
 def _update_file(
     context: dict[str, Any],
     template_name: str,
@@ -1461,10 +1472,10 @@ def _update_file(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
+) -> bool:
     file_path = target_path / file_name
     if regenerate_missing_docs and file_path.exists():
-        return
+        return True
     new_text = render_template(
         template_name=template_name, context=context, extension=extension, 
keep_trailing_newline=True
     )
@@ -1474,6 +1485,53 @@ def _update_file(
         with open(file_path) as readme_file_read:
             old_text = readme_file_read.read()
     replace_content(file_path, old_text, new_text, provider_package_id)
+    index_path = target_path / "index.rst"
+    if not index_path.exists():
+        console.print(f"[red]ERROR! The index must exist for the provider 
docs: {index_path}")
+        sys.exit(1)
+
+    expected_link_in_index = f"<{file_name.split('.')[0]}>"
+    if expected_link_in_index not in index_path.read_text():
+        console.print(
+            f"\n[red]ERROR! The {index_path} must contain "
+            f"link to the generated documentation:[/]\n\n"
+            f"[yellow]{expected_link_in_index}[/]\n\n"
+            f"[bright_blue]Please make sure to add it to {index_path}.\n"
+        )
+
+    console.print(f"Checking for backticks correctly generated in: 
{file_path}")
+    match = BACKTICKS_CHECK.search(file_path.read_text())
+    if match:
+        console.print(
+            f"\n[red]ERROR: Single backticks (`) found in {file_path}:[/]\n\n"
+            f"[yellow]{match.group(0)}[/]\n\n"
+            f"[bright_blue]Please fix them by replacing with double backticks 
(``).[/]\n"
+        )
+        return False
+
+    # TODO: uncomment me. Linting revealed that our already generated provider 
docs have duplicate links
+    #       in the generated files, we should fix those and uncomment linting 
as separate step - so that
+    #       we do not hold current release for fixing the docs.
+    # console.print(f"Linting: {file_path}")
+    # errors = restructuredtext_lint.lint_file(file_path)
+    # real_errors = False
+    # if errors:
+    #     for error in errors:
+    #         # Skip known issue: linter with doc role similar to 
https://github.com/OCA/pylint-odoo/issues/38
+    #         if (
+    #             'No role entry for "doc"' in error.message
+    #             or 'Unknown interpreted text role "doc"' in error.message
+    #         ):
+    #             continue
+    #         real_errors = True
+    #         console.print(f"* [red] {error.message}")
+    #     if real_errors:
+    #         console.print(f"\n[red] Errors found in {file_path}")
+    #         return False
+
+    console.print(f"[green]Generated {file_path} for {provider_package_id} is 
OK[/]")
+
+    return True
 
 
 def update_changelog_rst(
@@ -1481,8 +1539,8 @@ def update_changelog_rst(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
-    _update_file(
+) -> bool:
+    return _update_file(
         context=context,
         template_name="PROVIDER_CHANGELOG",
         extension=".rst",
@@ -1498,8 +1556,8 @@ def update_security_rst(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
-    _update_file(
+) -> bool:
+    return _update_file(
         context=context,
         template_name="PROVIDER_SECURITY",
         extension=".rst",
@@ -1515,8 +1573,8 @@ def update_commits_rst(
     provider_package_id: str,
     target_path: Path,
     regenerate_missing_docs: bool,
-):
-    _update_file(
+) -> bool:
+    return _update_file(
         context=context,
         template_name="PROVIDER_COMMITS",
         extension=".rst",
diff --git a/docs/apache-airflow-providers-common-io/commits.rst 
b/docs/apache-airflow-providers-common-io/commits.rst
index a5ae8822a2..a9592f90c1 100644
--- a/docs/apache-airflow-providers-common-io/commits.rst
+++ b/docs/apache-airflow-providers-common-io/commits.rst
@@ -16,21 +16,13 @@
     specific language governing permissions and limitations
     under the License.
 
+.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE 
OVERWRITTEN AT RELEASE TIME!
 
 Package apache-airflow-providers-common-io
-------------------------------------------
+------------------------------------------------------
+
+``Common IO Provider``
 
-Common IO Provider
 
 This is detailed commit list of changes for versions provider package: 
``common.io``.
 For high-level changelog, see :doc:`package information including changelog 
<index>`.
-
-1.0.0
-.....
-
-Latest change: 2023-10-19
-
-=================================================================================================
  ===========  ========================================================
-Commit                                                                         
                    Committed    Subject
-=================================================================================================
  ===========  ========================================================
-=================================================================================================
  ===========  ========================================================
diff --git a/docs/apache-airflow-providers-common-io/index.rst 
b/docs/apache-airflow-providers-common-io/index.rst
index 5c6c7853f8..2822abf88c 100644
--- a/docs/apache-airflow-providers-common-io/index.rst
+++ b/docs/apache-airflow-providers-common-io/index.rst
@@ -27,6 +27,7 @@
 
     Home <self>
     Changelog <changelog>
+    Security <security>
 
 .. toctree::
     :hidden:
@@ -68,3 +69,36 @@
     :caption: Commits
 
     Detailed list of commits <commits>
+
+
+Package apache-airflow-providers-common-io
+------------------------------------------------------
+
+``Common IO Provider``
+
+
+Release: 1.0.0
+
+Provider package
+----------------
+
+This is a provider package for ``common.io`` provider. All classes for this 
provider package
+are in ``airflow.providers.common.io`` python package.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see 
``Requirements`` below)
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-common-io``
+
+Requirements
+------------
+
+The minimum Apache Airflow version supported by this provider package is 
``2.8.0``.
+
+==================  ==================
+PIP package         Version required
+==================  ==================
+``apache-airflow``  ``>=2.8.0``
+==================  ==================
diff --git a/docs/apache-airflow-providers-common-io/security.rst 
b/docs/apache-airflow-providers-common-io/security.rst
new file mode 100644
index 0000000000..66c6f79a4e
--- /dev/null
+++ b/docs/apache-airflow-providers-common-io/security.rst
@@ -0,0 +1,38 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Releasing security patches
+--------------------------
+
+Airflow providers are released independently from Airflow itself and the 
information about vulnerabilities
+is published separately. You can upgrade providers independently from Airflow 
itself, following the
+instructions found in :doc:`apache-airflow:installation/installing-from-pypi`.
+
+When we release Provider version, the development is always done from the 
``main`` branch where we prepare
+the next version. The provider uses strict `SemVer <https://semver.org>`_ 
versioning policy. Depending on
+the scope of the change, Provider will get ''MAJOR'' version upgrade when 
there are
+breaking changes, ``MINOR`` version upgrade when there are new features or 
``PATCHLEVEL`` version upgrade
+when there are only bug fixes (including security bugfixes) - and this is the 
only version that receives
+security fixes by default, so you should upgrade to latest version of the 
provider if you want to receive
+all released security fixes.
+
+The only exception to that rule is when we have a critical security fix and 
good reason to provide an
+out-of-band release for the provider, in which case stakeholders in the 
provider might decide to cherry-pick
+and prepare a branch for an older version of the provider following the
+`mixed governance model 
<https://github.com/apache/airflow/blob/main/PROVIDERS.rst#mixed-governance-model>`_
+and requires interested parties to cherry-pick and test the fixes.
diff --git a/setup.py b/setup.py
index 2a0846164a..e0d1a8da78 100644
--- a/setup.py
+++ b/setup.py
@@ -436,6 +436,7 @@ _devel_only_devscripts = [
     "pipdeptree",
     "pygithub",
     "rich-click>=1.7.0",
+    "restructuredtext-lint",
     "semver",
     "towncrier",
     "twine",

Reply via email to