This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e637f08188 Fix errors in common.io release documentation generation
with prevention (#35236)
e637f08188 is described below
commit e637f08188bd7c9348531dc3f013e95aecb3572c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Oct 28 18:11:08 2023 +0200
Fix errors in common.io release documentation generation with prevention
(#35236)
The common.io provider release caused a few problems when preparing the
new wave of the providers:
* it missed link to security.rst (which is generated automatically
anticipating SBOM generation in the near future)
* it did not have proper min-version set in
MIN_AIRFLOW_VERSION_EXCEPTIONS
* it had single-backtick description that resulted in generated commit
and changelog code - since it failed documentation building
All those errors have not been caught at PR time. This PR fixes them
and introduces prevention from similar mistakes - future providers with
similar errors will get nice errors generated in CI at "docs publishing
step".
* if some generated file is missing in the index user is asked to add
the files in the index
* if there are `backtick` errors they will be reported and pointed at
* if there are liniting problems with generated .rst files they will
be reported
* the "min airflow version" is now taken directly from provider.yaml
not from exception list - but only when it is bigger than current
min version. This way, we will also not forget to remove the provider
from the list of exceptions when we bump min version above the
exception version
---
airflow/providers/common/io/__init__.py | 27 ++++++
airflow/providers/common/io/provider.yaml | 2 +-
dev/provider_packages/prepare_provider_packages.py | 98 +++++++++++++++++-----
.../apache-airflow-providers-common-io/commits.rst | 16 +---
docs/apache-airflow-providers-common-io/index.rst | 34 ++++++++
.../security.rst | 38 +++++++++
setup.py | 1 +
7 files changed, 183 insertions(+), 33 deletions(-)
diff --git a/airflow/providers/common/io/__init__.py
b/airflow/providers/common/io/__init__.py
index 13a83393a9..f5bcab9b8b 100644
--- a/airflow/providers/common/io/__init__.py
+++ b/airflow/providers/common/io/__init__.py
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,3 +15,29 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
+#
+# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
+# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages`
DIRECTORY
+#
+from __future__ import annotations
+
+import packaging.version
+
+__all__ = ["__version__"]
+
+__version__ = "1.0.0"
+
+try:
+ from airflow import __version__ as airflow_version
+except ImportError:
+ from airflow.version import version as airflow_version
+
+if
packaging.version.parse(packaging.version.parse(airflow_version).base_version)
< packaging.version.parse(
+ "2.8.0"
+):
+ raise RuntimeError(
+ f"The package `apache-airflow-providers-common-io:{__version__}`
requires Apache Airflow 2.8.0+"
+ )
diff --git a/airflow/providers/common/io/provider.yaml
b/airflow/providers/common/io/provider.yaml
index de5ae0935a..78723c4fa1 100644
--- a/airflow/providers/common/io/provider.yaml
+++ b/airflow/providers/common/io/provider.yaml
@@ -19,7 +19,7 @@
package-name: apache-airflow-providers-common-io
name: Common IO
description: |
- `Common IO Provider`
+ ``Common IO Provider``
suspended: false
versions:
diff --git a/dev/provider_packages/prepare_provider_packages.py
b/dev/provider_packages/prepare_provider_packages.py
index 6d5a82327c..546125acf9 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -55,10 +55,6 @@ from yaml import safe_load
ALL_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
MIN_AIRFLOW_VERSION = "2.5.0"
-# In case you have some providers that you want to have different min-airflow
version for,
-# Add them as exceptions here. Make sure to remove it once the min-airflow
version is bumped
-# to the same version that is required by the exceptional provider
-MIN_AIRFLOW_VERSION_EXCEPTIONS = {"openlineage": "2.7.0"}
INITIAL_CHANGELOG_CONTENT = """
.. Licensed to the Apache Software Foundation (ASF) under one
@@ -1122,8 +1118,11 @@ def get_provider_jinja_context(
for p in provider_details.excluded_python_versions:
python_requires += f", !={p}"
min_airflow_version = MIN_AIRFLOW_VERSION
- if
MIN_AIRFLOW_VERSION_EXCEPTIONS.get(provider_details.provider_package_id):
- min_airflow_version =
MIN_AIRFLOW_VERSION_EXCEPTIONS[provider_details.provider_package_id]
+ for dependency in provider_info["dependencies"]:
+ if dependency.startswith("apache-airflow>="):
+ current_min_airflow_version = dependency.split(">=")[1]
+ if Version(current_min_airflow_version) >
Version(min_airflow_version):
+ min_airflow_version = current_min_airflow_version
context: dict[str, Any] = {
"ENTITY_TYPES": list(EntityType),
"README_FILE": "README.rst",
@@ -1335,25 +1334,32 @@ def update_release_notes(
)
jinja_context["DETAILED_CHANGES_RST"] = changes
jinja_context["DETAILED_CHANGES_PRESENT"] = bool(changes)
- update_changelog_rst(
+ errors = False
+ if not update_changelog_rst(
jinja_context,
provider_package_id,
provider_details.documentation_provider_package_path,
regenerate_missing_docs,
- )
- update_security_rst(
+ ):
+ errors = True
+ if not update_security_rst(
jinja_context,
provider_package_id,
provider_details.documentation_provider_package_path,
regenerate_missing_docs,
- )
+ ):
+ errors = True
if not force:
- update_commits_rst(
+ if not update_commits_rst(
jinja_context,
provider_package_id,
provider_details.documentation_provider_package_path,
regenerate_missing_docs,
- )
+ ):
+ errors = True
+ if errors:
+ console.print("[red]There were errors when generating
documentation[/]")
+ sys.exit(1)
return True
@@ -1453,6 +1459,11 @@ def update_index_rst(
replace_content(index_file_path, old_text, new_text, provider_package_id)
+# Taken from pygrep hooks we are using in pre-commit
+# https://github.com/pre-commit/pygrep-hooks/blob/main/.pre-commit-hooks.yaml
+BACKTICKS_CHECK = re.compile(r"^(?! ).*(^| )`[^`]+`([^_]|$)", re.MULTILINE)
+
+
def _update_file(
context: dict[str, Any],
template_name: str,
@@ -1461,10 +1472,10 @@ def _update_file(
provider_package_id: str,
target_path: Path,
regenerate_missing_docs: bool,
-):
+) -> bool:
file_path = target_path / file_name
if regenerate_missing_docs and file_path.exists():
- return
+ return True
new_text = render_template(
template_name=template_name, context=context, extension=extension,
keep_trailing_newline=True
)
@@ -1474,6 +1485,53 @@ def _update_file(
with open(file_path) as readme_file_read:
old_text = readme_file_read.read()
replace_content(file_path, old_text, new_text, provider_package_id)
+ index_path = target_path / "index.rst"
+ if not index_path.exists():
+ console.print(f"[red]ERROR! The index must exist for the provider
docs: {index_path}")
+ sys.exit(1)
+
+ expected_link_in_index = f"<{file_name.split('.')[0]}>"
+ if expected_link_in_index not in index_path.read_text():
+ console.print(
+ f"\n[red]ERROR! The {index_path} must contain "
+ f"link to the generated documentation:[/]\n\n"
+ f"[yellow]{expected_link_in_index}[/]\n\n"
+ f"[bright_blue]Please make sure to add it to {index_path}.\n"
+ )
+
+ console.print(f"Checking for backticks correctly generated in:
{file_path}")
+ match = BACKTICKS_CHECK.search(file_path.read_text())
+ if match:
+ console.print(
+ f"\n[red]ERROR: Single backticks (`) found in {file_path}:[/]\n\n"
+ f"[yellow]{match.group(0)}[/]\n\n"
+ f"[bright_blue]Please fix them by replacing with double backticks
(``).[/]\n"
+ )
+ return False
+
+ # TODO: uncomment me. Linting revealed that our already generated provider
docs have duplicate links
+ # in the generated files, we should fix those and uncomment linting
as separate step - so that
+ # we do not hold current release for fixing the docs.
+ # console.print(f"Linting: {file_path}")
+ # errors = restructuredtext_lint.lint_file(file_path)
+ # real_errors = False
+ # if errors:
+ # for error in errors:
+ # # Skip known issue: linter with doc role similar to
https://github.com/OCA/pylint-odoo/issues/38
+ # if (
+ # 'No role entry for "doc"' in error.message
+ # or 'Unknown interpreted text role "doc"' in error.message
+ # ):
+ # continue
+ # real_errors = True
+ # console.print(f"* [red] {error.message}")
+ # if real_errors:
+ # console.print(f"\n[red] Errors found in {file_path}")
+ # return False
+
+ console.print(f"[green]Generated {file_path} for {provider_package_id} is
OK[/]")
+
+ return True
def update_changelog_rst(
@@ -1481,8 +1539,8 @@ def update_changelog_rst(
provider_package_id: str,
target_path: Path,
regenerate_missing_docs: bool,
-):
- _update_file(
+) -> bool:
+ return _update_file(
context=context,
template_name="PROVIDER_CHANGELOG",
extension=".rst",
@@ -1498,8 +1556,8 @@ def update_security_rst(
provider_package_id: str,
target_path: Path,
regenerate_missing_docs: bool,
-):
- _update_file(
+) -> bool:
+ return _update_file(
context=context,
template_name="PROVIDER_SECURITY",
extension=".rst",
@@ -1515,8 +1573,8 @@ def update_commits_rst(
provider_package_id: str,
target_path: Path,
regenerate_missing_docs: bool,
-):
- _update_file(
+) -> bool:
+ return _update_file(
context=context,
template_name="PROVIDER_COMMITS",
extension=".rst",
diff --git a/docs/apache-airflow-providers-common-io/commits.rst
b/docs/apache-airflow-providers-common-io/commits.rst
index a5ae8822a2..a9592f90c1 100644
--- a/docs/apache-airflow-providers-common-io/commits.rst
+++ b/docs/apache-airflow-providers-common-io/commits.rst
@@ -16,21 +16,13 @@
specific language governing permissions and limitations
under the License.
+.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE
OVERWRITTEN AT RELEASE TIME!
Package apache-airflow-providers-common-io
-------------------------------------------
+------------------------------------------------------
+
+``Common IO Provider``
-Common IO Provider
This is detailed commit list of changes for versions provider package:
``common.io``.
For high-level changelog, see :doc:`package information including changelog
<index>`.
-
-1.0.0
-.....
-
-Latest change: 2023-10-19
-
-=================================================================================================
=========== ========================================================
-Commit
Committed Subject
-=================================================================================================
=========== ========================================================
-=================================================================================================
=========== ========================================================
diff --git a/docs/apache-airflow-providers-common-io/index.rst
b/docs/apache-airflow-providers-common-io/index.rst
index 5c6c7853f8..2822abf88c 100644
--- a/docs/apache-airflow-providers-common-io/index.rst
+++ b/docs/apache-airflow-providers-common-io/index.rst
@@ -27,6 +27,7 @@
Home <self>
Changelog <changelog>
+ Security <security>
.. toctree::
:hidden:
@@ -68,3 +69,36 @@
:caption: Commits
Detailed list of commits <commits>
+
+
+Package apache-airflow-providers-common-io
+------------------------------------------------------
+
+``Common IO Provider``
+
+
+Release: 1.0.0
+
+Provider package
+----------------
+
+This is a provider package for ``common.io`` provider. All classes for this
provider package
+are in ``airflow.providers.common.io`` python package.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see
``Requirements`` below)
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-common-io``
+
+Requirements
+------------
+
+The minimum Apache Airflow version supported by this provider package is
``2.8.0``.
+
+================== ==================
+PIP package Version required
+================== ==================
+``apache-airflow`` ``>=2.8.0``
+================== ==================
diff --git a/docs/apache-airflow-providers-common-io/security.rst
b/docs/apache-airflow-providers-common-io/security.rst
new file mode 100644
index 0000000000..66c6f79a4e
--- /dev/null
+++ b/docs/apache-airflow-providers-common-io/security.rst
@@ -0,0 +1,38 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Releasing security patches
+--------------------------
+
+Airflow providers are released independently from Airflow itself and the
information about vulnerabilities
+is published separately. You can upgrade providers independently from Airflow
itself, following the
+instructions found in :doc:`apache-airflow:installation/installing-from-pypi`.
+
+When we release Provider version, the development is always done from the
``main`` branch where we prepare
+the next version. The provider uses strict `SemVer <https://semver.org>`_
versioning policy. Depending on
+the scope of the change, Provider will get ''MAJOR'' version upgrade when
there are
+breaking changes, ``MINOR`` version upgrade when there are new features or
``PATCHLEVEL`` version upgrade
+when there are only bug fixes (including security bugfixes) - and this is the
only version that receives
+security fixes by default, so you should upgrade to latest version of the
provider if you want to receive
+all released security fixes.
+
+The only exception to that rule is when we have a critical security fix and
good reason to provide an
+out-of-band release for the provider, in which case stakeholders in the
provider might decide to cherry-pick
+and prepare a branch for an older version of the provider following the
+`mixed governance model
<https://github.com/apache/airflow/blob/main/PROVIDERS.rst#mixed-governance-model>`_
+and requires interested parties to cherry-pick and test the fixes.
diff --git a/setup.py b/setup.py
index 2a0846164a..e0d1a8da78 100644
--- a/setup.py
+++ b/setup.py
@@ -436,6 +436,7 @@ _devel_only_devscripts = [
"pipdeptree",
"pygithub",
"rich-click>=1.7.0",
+ "restructuredtext-lint",
"semver",
"towncrier",
"twine",