This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 23b87534e6 Better handling build errors when building docs with --spellcheck-only (#36336) 23b87534e6 is described below commit 23b87534e636e407f3f68107d442f55b771216f3 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Wed Dec 20 20:49:18 2023 +0100 Better handling build errors when building docs with --spellcheck-only (#36336) When running docs building with --spellcheck-only and building docs fails, the errors which were not spelling errors were not handled well - first of all, the build errors were not printted separately as errors (you only saw generaic spellchecking failed), but more importantly, when building docs failed it has not been repeated again. This caused problems if there were changes that required to rebuild another packag - which happens when you add related changes to several packages. This PR fixes it by: * allowing spellchecking to contribute build errors when the --spellcheck-only flag is used * when build is retried, spellcheck-only flag is ignored - so that the documentation for related packages can be actually rebuilt, not only spell-checked * when --spellcheck-only is used to build packages, all original packages are rebuilt not only the failed one - this allows to rebuild dependent packages --- docs/build_docs.py | 33 ++++++++++++++++++++++++--------- docs/exts/docs_build/docs_builder.py | 18 ++++++++++++------ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/docs/build_docs.py b/docs/build_docs.py index aaba029b60..6eac628b96 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -183,7 +183,8 @@ class SpellCheckResult(NamedTuple): package_name: str log_file_name: str - errors: list[SpellingError] + spelling_errors: list[SpellingError] + build_errors: list[DocBuildError] def perform_docs_build_for_single_package(build_specification: BuildSpecification) -> BuildDocsResult: @@ -204,11 +205,13 @@ def perform_spell_check_for_single_package(build_specification: BuildSpecificati """Performs single package spell check.""" builder = AirflowDocsBuilder(package_name=build_specification.package_name) console.print(f"[info]{build_specification.package_name:60}:[/] Checking spelling started") + spelling_errors, build_errors = builder.check_spelling( + verbose=build_specification.verbose, + ) result = SpellCheckResult( package_name=build_specification.package_name, - errors=builder.check_spelling( - verbose=build_specification.verbose, - ), + spelling_errors=spelling_errors, + build_errors=build_errors, log_file_name=builder.log_spelling_filename, ) console.print(f"[info]{build_specification.package_name:60}:[/] Checking spelling completed") @@ -280,8 +283,10 @@ def run_sequentially( verbose=verbose, ) ) - if spellcheck_result.errors: - all_spelling_errors[package_name].extend(spellcheck_result.errors) + if spellcheck_result.spelling_errors: + all_spelling_errors[package_name].extend(spellcheck_result.spelling_errors) + if spellcheck_only: + all_build_errors[package_name].extend(spellcheck_result.build_errors) print_spelling_output(spellcheck_result) @@ -306,6 +311,7 @@ def run_in_parallel( if not docs_only: run_spell_check_in_parallel( all_spelling_errors=all_spelling_errors, + all_build_errors=all_build_errors, current_packages=current_packages, verbose=verbose, pool=pool, @@ -363,6 +369,7 @@ def print_spelling_output(result: SpellCheckResult): def run_spell_check_in_parallel( all_spelling_errors: dict[str, list[SpellingError]], + all_build_errors: dict[str, list[DocBuildError]], current_packages: list[str], verbose: bool, pool, @@ -377,8 +384,9 @@ def run_spell_check_in_parallel( console.print() result_list = pool.map(perform_spell_check_for_single_package, spell_check_specifications) for result in result_list: - if result.errors: - all_spelling_errors[result.package_name].extend(result.errors) + if result.spelling_errors: + all_spelling_errors[result.package_name].extend(result.spelling_errors) + all_build_errors[result.package_name].extend(result.build_errors) print_spelling_output(result) @@ -495,6 +503,7 @@ def main(): jobs, package_build_errors, package_spelling_errors, + current_packages, spellcheck_only, ) @@ -507,6 +516,7 @@ def main(): jobs, package_build_errors, package_spelling_errors, + current_packages, spellcheck_only, ) @@ -537,6 +547,7 @@ def retry_building_docs_if_needed( jobs, package_build_errors, package_spelling_errors, + current_packages, spellcheck_only, ): to_retry_packages = [ @@ -544,6 +555,10 @@ def retry_building_docs_if_needed( for package_name, errors in package_build_errors.items() if any(any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD) for e in errors) ] + if to_retry_packages and spellcheck_only: + # in case spellchecking fails with retry all packages should be rebuilt without spell-checking + # in case the failed package refers to another package + to_retry_packages = current_packages if to_retry_packages: for package_name in to_retry_packages: if package_name in all_build_errors: @@ -554,7 +569,7 @@ def retry_building_docs_if_needed( package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=to_retry_packages, docs_only=docs_only, - spellcheck_only=spellcheck_only, + spellcheck_only=False, jobs=jobs, verbose=args.verbose, ) diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index d72c8941f6..e44b5544ca 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -125,7 +125,7 @@ class AirflowDocsBuilder: os.makedirs(api_dir, exist_ok=True) os.makedirs(self._build_dir, exist_ok=True) - def check_spelling(self, verbose: bool) -> list[SpellingError]: + def check_spelling(self, verbose: bool) -> tuple[list[SpellingError], list[DocBuildError]]: """ Checks spelling @@ -133,6 +133,7 @@ class AirflowDocsBuilder: :return: list of errors """ spelling_errors = [] + build_errors = [] os.makedirs(self._build_dir, exist_ok=True) shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True) os.makedirs(self.log_spelling_output_dir, exist_ok=True) @@ -182,12 +183,17 @@ class AirflowDocsBuilder: ), ) ) - warning_text = "" + spelling_warning_text = "" for filepath in glob(f"{self.log_spelling_output_dir}/**/*.spelling", recursive=True): with open(filepath) as spelling_file: - warning_text += spelling_file.read() - - spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir)) + spelling_warning_text += spelling_file.read() + spelling_errors.extend(parse_spelling_warnings(spelling_warning_text, self._src_dir)) + if os.path.isfile(self.log_spelling_filename): + with open(self.log_spelling_filename) as warning_file: + warning_text = warning_file.read() + # Remove 7-bit C1 ANSI escape sequences + warning_text = re.sub(r"\x1B[@-_][0-?]*[ -/]*[@-~]", "", warning_text) + build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir)) console.print(f"[info]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]") else: if spelling_errors: @@ -198,7 +204,7 @@ class AirflowDocsBuilder: console.print( f"[info]{self.package_name:60}:[/] [green]Finished spell-checking successfully[/]" ) - return spelling_errors + return spelling_errors, build_errors def build_sphinx_docs(self, verbose: bool) -> list[DocBuildError]: """