https://github.com/Neil-N4 created
https://github.com/llvm/llvm-project/pull/208003
Adds a line-based Markdown parser producing the AST landed in #205609. Handles
paragraphs, ATX headings, fenced code blocks, thematic breaks, and unordered
lists.
The parser lives in its own MarkdownParser.{h,cpp} so the node definitions stay
decoupled. Blocks are parsed through a LineCursor with one helper per block
type rather than one loop with manual index tracking. ASTContext owns the
bump-pointer arena backing the nodes.
Follow-ups: ordered lists, inline emphasis/code, and clang-doc comment-pipeline
integration.
>From a1e681a20e6f86a15ab64fd332fbcbf91eb6e1ea Mon Sep 17 00:00:00 2001
From: Neil-N4 <[email protected]>
Date: Tue, 7 Jul 2026 03:58:12 -0400
Subject: [PATCH] [clang-doc] Add Markdown parser
Parses paragraphs, ATX headings, fenced code blocks, thematic breaks, and
unordered lists into the AST from Markdown.h. Each block type has its own
helper driven by a LineCursor; ASTContext owns the node arena.
---
.../clang-doc/markdown/CMakeLists.txt | 1 +
.../clang-doc/markdown/MarkdownParser.cpp | 182 ++++++++++++++++++
.../clang-doc/markdown/MarkdownParser.h | 62 ++++++
.../clang-doc/MarkdownParserTest.cpp | 94 +++++++++
4 files changed, 339 insertions(+)
create mode 100644 clang-tools-extra/clang-doc/markdown/MarkdownParser.cpp
create mode 100644 clang-tools-extra/clang-doc/markdown/MarkdownParser.h
diff --git a/clang-tools-extra/clang-doc/markdown/CMakeLists.txt
b/clang-tools-extra/clang-doc/markdown/CMakeLists.txt
index 11e6958e2e828..cee442955eac8 100644
--- a/clang-tools-extra/clang-doc/markdown/CMakeLists.txt
+++ b/clang-tools-extra/clang-doc/markdown/CMakeLists.txt
@@ -4,4 +4,5 @@ set(LLVM_LINK_COMPONENTS
add_clang_library(clangDocMarkdown STATIC
Markdown.cpp
+ MarkdownParser.cpp
)
diff --git a/clang-tools-extra/clang-doc/markdown/MarkdownParser.cpp
b/clang-tools-extra/clang-doc/markdown/MarkdownParser.cpp
new file mode 100644
index 0000000000000..aa57daef21222
--- /dev/null
+++ b/clang-tools-extra/clang-doc/markdown/MarkdownParser.cpp
@@ -0,0 +1,182 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MarkdownParser.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace clang::doc::markdown {
+
+namespace {
+
+/// A forward cursor over the lines of the input. Owns no storage; it points
+/// into a caller-provided array of lines and tracks the current position.
+/// Block parsers consume lines through this cursor instead of juggling a raw
+/// index, which keeps the parsing helpers free of manual bookkeeping.
+class LineCursor {
+public:
+ explicit LineCursor(llvm::ArrayRef<llvm::StringRef> Lines) : Lines(Lines) {}
+
+ bool atEnd() const { return Pos >= Lines.size(); }
+
+ /// The current line, trimmed of surrounding whitespace.
+ llvm::StringRef peek() const { return Lines[Pos].trim(); }
+
+ /// The current line as it appears in the source, without trimming.
+ llvm::StringRef peekRaw() const { return Lines[Pos]; }
+
+ void advance() { ++Pos; }
+
+private:
+ llvm::ArrayRef<llvm::StringRef> Lines;
+ size_t Pos = 0;
+};
+
+} // namespace
+
+static bool isListMarker(llvm::StringRef Line) {
+ return Line.starts_with("- ") || Line.starts_with("* ") ||
+ Line.starts_with("+ ");
+}
+
+static bool isThematicBreak(llvm::StringRef Line) {
+ if (Line.empty())
+ return false;
+ char Marker = Line.front();
+ if (Marker != '-' && Marker != '*' && Marker != '_')
+ return false;
+ llvm::SmallString<8> Allowed;
+ Allowed += Marker;
+ Allowed += ' ';
+ if (Line.find_first_not_of(Allowed) != llvm::StringRef::npos)
+ return false;
+ return Line.count(Marker) >= 3;
+}
+
+static bool isFence(llvm::StringRef Line) {
+ return Line.starts_with("```") || Line.starts_with("~~~");
+}
+
+/// Returns the heading level (1-6) if Line is an ATX heading, or 0 otherwise.
+static unsigned getHeadingLevel(llvm::StringRef Line) {
+ unsigned Level = Line.take_while([](char C) { return C == '#'; }).size();
+ if (Level == 0 || Level > 6 || Level >= Line.size() || Line[Level] != ' ')
+ return 0;
+ return Level;
+}
+
+/// True if Line begins a new block, meaning paragraph accumulation must stop.
+static bool startsNewBlock(llvm::StringRef Line) {
+ return Line.empty() || isThematicBreak(Line) || isFence(Line) ||
+ isListMarker(Line) || getHeadingLevel(Line) != 0;
+}
+
+static TextNode *makeText(llvm::StringRef Text, ASTContext &Ctx) {
+ return Ctx.allocate<TextNode>(Ctx.intern(Text));
+}
+
+static BlockNode *parseThematicBreak(LineCursor &Cursor, ASTContext &Ctx) {
+ Cursor.advance();
+ return Ctx.allocate<ThematicBreakNode>();
+}
+
+static BlockNode *parseFencedCode(LineCursor &Cursor, ASTContext &Ctx) {
+ llvm::StringRef Opening = Cursor.peek();
+ char Fence = Opening.front();
+ llvm::StringRef Lang = Opening.drop_front(3).trim();
+ Cursor.advance();
+
+ llvm::SmallString<256> Code;
+ while (!Cursor.atEnd()) {
+ llvm::StringRef Line = Cursor.peek();
+ bool IsClosingFence = Line.size() >= 3 && Line[0] == Fence &&
+ Line[1] == Fence && Line[2] == Fence;
+ if (IsClosingFence) {
+ Cursor.advance();
+ break;
+ }
+ if (!Code.empty())
+ Code += '\n';
+ Code += Cursor.peekRaw();
+ Cursor.advance();
+ }
+ return Ctx.allocate<FencedCodeNode>(Lang, Ctx.intern(Code));
+}
+
+static BlockNode *parseHeading(LineCursor &Cursor, ASTContext &Ctx) {
+ llvm::StringRef Line = Cursor.peek();
+ unsigned Level = getHeadingLevel(Line);
+ llvm::StringRef Content = Line.drop_front(Level + 1).trim();
+ auto *Heading = Ctx.allocate<HeadingNode>(Level);
+ Heading->addChild(*makeText(Content, Ctx));
+ Cursor.advance();
+ return Heading;
+}
+
+static BlockNode *parseUnorderedList(LineCursor &Cursor, ASTContext &Ctx) {
+ auto *List = Ctx.allocate<UnorderedListNode>();
+ while (!Cursor.atEnd() && isListMarker(Cursor.peek())) {
+ llvm::StringRef ItemText = Cursor.peek().drop_front(2).trim();
+ auto *Item = Ctx.allocate<ListItemNode>();
+ Item->addChild(*makeText(ItemText, Ctx));
+ List->addItem(*Item);
+ Cursor.advance();
+ }
+ return List;
+}
+
+static BlockNode *parseParagraph(LineCursor &Cursor, ASTContext &Ctx) {
+ llvm::SmallString<256> Text;
+ while (!Cursor.atEnd() && !startsNewBlock(Cursor.peek())) {
+ if (!Text.empty())
+ Text += ' ';
+ Text += Cursor.peek();
+ Cursor.advance();
+ }
+ auto *Para = Ctx.allocate<ParagraphNode>();
+ Para->addChild(*makeText(Text, Ctx));
+ return Para;
+}
+
+/// Parses the block starting at the cursor's current position. The cursor is
+/// guaranteed to be on a non-empty line that begins a block.
+static BlockNode *parseBlock(LineCursor &Cursor, ASTContext &Ctx) {
+ llvm::StringRef Line = Cursor.peek();
+
+ // Thematic breaks are checked first: "---" and "- - -" would otherwise be
+ // misread as a fence or a list marker.
+ if (isThematicBreak(Line))
+ return parseThematicBreak(Cursor, Ctx);
+ if (isFence(Line))
+ return parseFencedCode(Cursor, Ctx);
+ if (getHeadingLevel(Line) != 0)
+ return parseHeading(Cursor, Ctx);
+ if (isListMarker(Line))
+ return parseUnorderedList(Cursor, Ctx);
+ return parseParagraph(Cursor, Ctx);
+}
+
+DocumentNode *parseMarkdown(llvm::StringRef Text, ASTContext &Ctx) {
+ auto *Doc = Ctx.allocate<DocumentNode>();
+
+ llvm::SmallVector<llvm::StringRef> Lines;
+ Text.split(Lines, '\n');
+
+ LineCursor Cursor(Lines);
+ while (!Cursor.atEnd()) {
+ if (Cursor.peek().empty()) {
+ Cursor.advance();
+ continue;
+ }
+ Doc->addChild(*parseBlock(Cursor, Ctx));
+ }
+ return Doc;
+}
+
+} // namespace clang::doc::markdown
diff --git a/clang-tools-extra/clang-doc/markdown/MarkdownParser.h
b/clang-tools-extra/clang-doc/markdown/MarkdownParser.h
new file mode 100644
index 0000000000000..d89c1e2116d3e
--- /dev/null
+++ b/clang-tools-extra/clang-doc/markdown/MarkdownParser.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Declares the Markdown parser entry point and the ASTContext that owns the
+/// lifetime of the nodes it produces.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_MARKDOWN_MARKDOWNPARSER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_MARKDOWN_MARKDOWNPARSER_H
+
+#include "Markdown.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Allocator.h"
+#include <type_traits>
+#include <utility>
+
+namespace clang::doc::markdown {
+
+/// Restricts ASTContext::allocate to Markdown node types.
+template <typename T>
+using IsMarkdownNode = std::enable_if_t<std::is_base_of_v<InlineNode, T> ||
+ std::is_base_of_v<BlockNode, T> ||
+ std::is_same_v<T, ListItemNode>>;
+
+/// Owns the bump-pointer arena backing every node produced by the parser.
+/// Nodes allocated through allocate() live as long as the ASTContext.
+class ASTContext {
+public:
+ ASTContext() = default;
+
+ template <typename T, typename... Args, typename = IsMarkdownNode<T>>
+ T *allocate(Args &&...Params) {
+ return new (Arena.Allocate<T>()) T(std::forward<Args>(Params)...);
+ }
+
+ /// Copies a string into the arena so its lifetime matches the ASTContext.
+ /// Used when the parser needs to synthesize text that is not a contiguous
+ /// slice of the original buffer (e.g. joined paragraph lines).
+ llvm::StringRef intern(llvm::StringRef S) {
+ char *Buf = Arena.Allocate<char>(S.size());
+ llvm::copy(S, Buf);
+ return llvm::StringRef(Buf, S.size());
+ }
+
+private:
+ llvm::BumpPtrAllocator Arena;
+};
+
+/// Parses Markdown text into a DocumentNode owned by Ctx.
+DocumentNode *parseMarkdown(llvm::StringRef Text, ASTContext &Ctx);
+
+} // namespace clang::doc::markdown
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_MARKDOWN_MARKDOWNPARSER_H
diff --git a/clang-tools-extra/unittests/clang-doc/MarkdownParserTest.cpp
b/clang-tools-extra/unittests/clang-doc/MarkdownParserTest.cpp
index 3bc3dfd852f3e..d0757eac9e14c 100644
--- a/clang-tools-extra/unittests/clang-doc/MarkdownParserTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/MarkdownParserTest.cpp
@@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
+#include "markdown/MarkdownParser.h"
#include "markdown/Markdown.h"
#include "gtest/gtest.h"
@@ -115,4 +116,97 @@ TEST(MarkdownNodeTest, UnorderedListWithItems) {
EXPECT_EQ(firstChildText(List.items().front().children()), "item text");
}
+/// Returns the single block child of Doc, asserting there is exactly one.
+static const BlockNode &onlyBlock(const DocumentNode &Doc) {
+ EXPECT_EQ(std::distance(Doc.children().begin(), Doc.children().end()), 1);
+ return Doc.children().front();
+}
+
+TEST(MarkdownParserTest, EmptyInput) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("", Ctx);
+ EXPECT_TRUE(Doc->children().empty());
+}
+
+TEST(MarkdownParserTest, BlankLinesOnly) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("\n\n \n", Ctx);
+ EXPECT_TRUE(Doc->children().empty());
+}
+
+TEST(MarkdownParserTest, PlainParagraph) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("hello world", Ctx);
+ const auto &Para = llvm::cast<ParagraphNode>(onlyBlock(*Doc));
+ EXPECT_EQ(firstChildText(Para.children()), "hello world");
+}
+
+TEST(MarkdownParserTest, ParagraphJoinsLines) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("line one\nline two", Ctx);
+ const auto &Para = llvm::cast<ParagraphNode>(onlyBlock(*Doc));
+ EXPECT_EQ(firstChildText(Para.children()), "line one line two");
+}
+
+TEST(MarkdownParserTest, Heading) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("### Title", Ctx);
+ const auto &Heading = llvm::cast<HeadingNode>(onlyBlock(*Doc));
+ EXPECT_EQ(Heading.getLevel(), 3u);
+ EXPECT_EQ(firstChildText(Heading.children()), "Title");
+}
+
+TEST(MarkdownParserTest, HeadingTooManyHashesIsParagraph) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("####### not a heading", Ctx);
+ EXPECT_TRUE(llvm::isa<ParagraphNode>(onlyBlock(*Doc)));
+}
+
+TEST(MarkdownParserTest, FencedCode) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("```cpp\nint x = 0;\n```", Ctx);
+ const auto &Code = llvm::cast<FencedCodeNode>(onlyBlock(*Doc));
+ EXPECT_EQ(Code.getLang(), "cpp");
+ EXPECT_EQ(Code.getCode(), "int x = 0;");
+}
+
+TEST(MarkdownParserTest, FencedCodePreservesInteriorBlankLines) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("```\na\n\nb\n```", Ctx);
+ const auto &Code = llvm::cast<FencedCodeNode>(onlyBlock(*Doc));
+ EXPECT_EQ(Code.getCode(), "a\n\nb");
+}
+
+TEST(MarkdownParserTest, ThematicBreak) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("---", Ctx);
+ EXPECT_TRUE(llvm::isa<ThematicBreakNode>(onlyBlock(*Doc)));
+}
+
+TEST(MarkdownParserTest, SpacedThematicBreak) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("- - -", Ctx);
+ EXPECT_TRUE(llvm::isa<ThematicBreakNode>(onlyBlock(*Doc)));
+}
+
+TEST(MarkdownParserTest, UnorderedList) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("- one\n- two\n- three", Ctx);
+ const auto &List = llvm::cast<UnorderedListNode>(onlyBlock(*Doc));
+ EXPECT_EQ(std::distance(List.items().begin(), List.items().end()), 3);
+ EXPECT_EQ(firstChildText(List.items().front().children()), "one");
+}
+
+TEST(MarkdownParserTest, MultipleBlocks) {
+ ASTContext Ctx;
+ DocumentNode *Doc = parseMarkdown("# Heading\n\nA paragraph.\n\n- item",
Ctx);
+ llvm::SmallVector<NodeKind> Kinds;
+ for (const BlockNode &B : Doc->children())
+ Kinds.push_back(B.getKind());
+ ASSERT_EQ(Kinds.size(), 3u);
+ EXPECT_EQ(Kinds[0], NodeKind::NK_Heading);
+ EXPECT_EQ(Kinds[1], NodeKind::NK_Paragraph);
+ EXPECT_EQ(Kinds[2], NodeKind::NK_UnorderedList);
+}
+
} // namespace
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
