kaxil commented on code in PR #71477: URL: https://github.com/apache/airflow/pull/71477#discussion_r4109113796
########## dev/registry/registry_tools/docs_guides.py: ########## @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Map a provider's classes to the how-to guide sections that document them. + +A module's ``docs_url`` points at generated API reference, which tells a reader +what the arguments are but not how the thing is meant to be used. The prose +guides carry that, and they already mark it: a how-to guide documents one class +(or a class and its task-flow decorator) per section, titled with the name(s) +(``HookToolset``, ``SQLToolset``, ``AgentOperator`` & ``@task.agent``). + +So the mapping is read back out of the guides rather than curated anywhere: a +hand-maintained name-to-guide table would rot silently every time a guide is +split, renamed, or a class is dropped, and a rotten link is worse than none. +Callers supply the reST they can see (a git tag, or the working tree) and get +back only the anchors those sources actually contain. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from pathlib import PurePosixPath +from typing import Any + +# reST underlines an (optionally overlined) section title with a run of one +# punctuation character, at least as long as the title itself. +_ADORNMENT_CHARS = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" + +_SKIPPED_PAGE_NAMES = frozenset({"changelog.rst", "commits.rst"}) + + +def is_guide_page(relative_path: str) -> bool: + """Whether a path relative to a provider's docs directory is a how-to guide page. + + Callers hand every ``.rst`` they can see to this before it ever reaches + ``collect_guide_anchors``. Two kinds of real, built pages must not go + further: + + - Anything under a ``_``-prefixed path segment, at any depth + (``_api/hook/index.rst``, ``operators/_partials/foo.rst``, top-level + ``_partials/foo.rst``): Sphinx/autoapi output and partials are directive + markup, not the hand-written, reST-underlined titles this module's + leading-inline-literal convention parses. + - ``changelog.rst`` and ``commits.rst``: real release-note pages, not + how-to guides, that can carry inline-literal-formatted headings by + coincidence. + """ + path = PurePosixPath(relative_path) + if any(part.startswith("_") for part in path.parts): + return False + return path.name not in _SKIPPED_PAGE_NAMES + + +# A single inline-literal name: a class (``HookToolset``) or a task-flow +# decorator (``@task.llm_file_analysis``) -- narrow enough that it still can't +# match arbitrary prose wrapped in backticks. +_INLINE_LITERAL_NAME = r"``(@?[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*)``" +_INLINE_LITERAL_NAME_RE = re.compile(_INLINE_LITERAL_NAME) + +# Only titles opening with a run of inline-literal names are treated as +# documenting them, so prose headings ("Bounded query results") never produce a +# link. A run is one or more names joined by "&", "," or "/" -- how guides write +# a section that covers both an operator and its decorator +# (``AgentOperator`` & ``@task.agent``). The run stops at the first thing that +# is neither a name nor a separator, so it never reaches into prose. +_LEADING_LITERAL_NAME_RUN = re.compile(rf"^{_INLINE_LITERAL_NAME}(?:\s*[&,/]\s*{_INLINE_LITERAL_NAME})*") Review Comment: Since the rebase onto main, this no longer fits common/ai's guides. The docs reorg retitled the dedicated pages to a `Prose: ``Class``` shape (`Airflow hooks as tools: ``HookToolset```, `Agents with tools: ``AgentOperator`` and ``@task.agent```), so none of them match, and `collect_guide_anchors` over the working tree now yields 6 names instead of the 19 from last round. Two of those 6 land on the wrong section: `HookToolset` resolves to `agent_security.html#hooktoolset-guidelines` and `LLMBatchOperator` to `operators/llm_batch.html#llmbatchoperator-or-the-vendor-batch-operators`, because a subsection that happens to open with the name wins the sorted-page tie-break over the page title. Accepting a trailing `: ``X`` [and ``Y``]` run and preferring a page's top-level title over its subsections would restore it. An explicit label per guide section, resolved through the `objects.inv` already fetched for `docs_url`, would also survive the next retitle. Either way the fixtures need the new title shape (they all encode the old one), and the AGENTS.md line saying common/ai follows the convention most thoroughly needs another look. ########## dev/registry/extract_versions.py: ########## @@ -129,6 +130,70 @@ def git_show(tag: str, path: str) -> str | None: return None +def git_ls_tree(tag: str, prefix: str) -> list[str]: + """List the file paths under a prefix at a specific git tag. + + Decodes via the process locale (`text=True`), unlike `git_cat_file_batch`'s + UTF-8 decode -- and `git_cat_file_batch` re-encodes these same strings + (passed straight through by `read_guide_docs`) into its stdin, so a + non-UTF-8 locale could break byte-for-byte round-trip. No such path exists + under `providers/` today. + """ + try: + result = subprocess.run( + ["git", "-c", "core.quotePath=false", "ls-tree", "-r", "--name-only", tag, "--", prefix], + capture_output=True, + text=True, Review Comment: Dropping `text=True` here and decoding `result.stdout` as UTF-8 would make this agree with `git_cat_file_batch`, and the docstring paragraph explaining the mismatch could go. ########## dev/registry/tests/test_extract_versions.py: ########## @@ -210,3 +213,187 @@ def test_external_services_propagates_to_version_metadata( assert result is not None assert result["connection_types"][0]["external_services"] == ["openai", "anthropic"] + + +class TestExtractModulesGuideUrls: + """A class the provider's guides document in a section of its own must get a + ``guide_url`` for every version, not just the latest. A superseded version's + page is rendered only from the per-version file this module writes, so a link + resolved on the latest path alone disappears the moment a new version lands. + """ + + PROVIDER_YAML = { + "toolsets": [ + { + "integration-name": "Test", + "python-modules": ["airflow.providers.test.toolsets.hook"], + } + ] + } + SOURCE = 'class HookToolset:\n """A toolset."""\n' + GUIDE = "``HookToolset``\n---------------\n\nProse.\n" + + def _extract(self, layout="new", docs_paths=("providers/test/docs/toolsets.rst",)): + def fake_git_show(_tag, path): + return self.SOURCE if path.endswith(".py") else None + + def fake_git_cat_file_batch(_tag, paths): + return {p: self.GUIDE for p in paths} + + with ( + patch("extract_versions.git_ls_tree", autospec=True, return_value=list(docs_paths)), + patch("extract_versions.git_show", autospec=True, side_effect=fake_git_show), + patch("extract_versions.git_cat_file_batch", autospec=True, side_effect=fake_git_cat_file_batch), + ): + return extract_modules_from_yaml( + self.PROVIDER_YAML, "providers-test/1.0.0", layout, "test", "test", "1.0.0" + ) + + def test_documented_class_gets_a_versioned_guide_url(self): + modules = self._extract() + + assert [m["name"] for m in modules] == ["HookToolset"] + assert modules[0]["guide_url"] == ( + "https://airflow.apache.org/docs/apache-airflow-providers-test/1.0.0/toolsets.html#hooktoolset"; + ) + + def test_undocumented_class_gets_no_guide_url(self): + modules = self._extract(docs_paths=()) + + assert [m["name"] for m in modules] == ["HookToolset"] + assert "guide_url" not in modules[0] + + def test_old_layout_gets_no_guide_url(self): + # Pre-per-provider tags kept docs in a top-level tree, so there is no + # provider-relative page path to build a link from. + modules = self._extract(layout="old") + + assert "guide_url" not in modules[0] + + +class TestReadGuideDocs: + def test_skips_generated_and_release_note_pages_before_calling_git_show(self): + docs_prefix = "providers/test/docs/" + paths = [ + docs_prefix + "_api/x/index.rst", + docs_prefix + "changelog.rst", + docs_prefix + "toolsets.rst", + ] + + with ( + patch("extract_versions.git_ls_tree", autospec=True, return_value=paths), + patch( + "extract_versions.git_cat_file_batch", + autospec=True, + side_effect=lambda tag, paths: {p: "Prose.\n" for p in paths}, + ) as mock_git_cat_file_batch, + ): + result = read_guide_docs("providers-test/1.0.0", "new", "test") + + # Mutation canary: if the `is_guide_page` filter in read_guide_docs is + # removed, this dict grows two more keys and this assertion goes red. + assert set(result) == {"toolsets.rst"} + # Mutation canary: without the filter, the two skipped paths would also + # be included in the batch call's path list -- this assertion goes red + # too, proving the filtering runs before the batch call, not just + # before the dict write. + assert mock_git_cat_file_batch.call_args_list == [ + call("providers-test/1.0.0", [docs_prefix + "toolsets.rst"]) + ] + + def test_skips_a_page_whose_content_is_an_empty_string(self): + # Mutation canary: replacing `batch_result.get(full_path)` with + # `full_path in batch_result` in read_guide_docs's comprehension makes + # this page reappear as `{"empty.rst": ""}`, turning this assertion red. + docs_prefix = "providers/test/docs/" + paths = [docs_prefix + "empty.rst"] + + with ( + patch("extract_versions.git_ls_tree", autospec=True, return_value=paths), + patch( + "extract_versions.git_cat_file_batch", + autospec=True, + return_value={docs_prefix + "empty.rst": ""}, + ), + ): + result = read_guide_docs("providers-test/1.0.0", "new", "test") + + assert result == {} + + +class TestGitLsTree: + def test_passes_quote_path_false_to_git(self): + mock_result = MagicMock() Review Comment: The `subprocess.run` patches in `TestGitLsTree` and `TestGitCatFileBatch` are the only ones in this file without `autospec`, and the results are bare `MagicMock()`. `patch(..., autospec=True)` with `MagicMock(spec=subprocess.CompletedProcess)` would keep them in line with the rest. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
