https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From 49fd778d21adb49080d1f9b360aa5cda2b7359a6 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 01/10] Comment parsing: add argument parsing for @throw
@throws @exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h | 3 +
clang/lib/AST/CommentParser.cpp | 133 ++++++++++++
clang/unittests/AST/CommentParser.cpp | 235 ++++++++++++++++++++-
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd..06b2fa9b5531c 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws : BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws : BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a..5884a25d00785 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef<Comment::Argument>
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef<Comment::Argument>
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c..c70fa1b05cb24 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+ assert(!isEnd());
+ assert(Pos.BufferPtr != Pos.BufferEnd);
+ if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+ } else {
+ return '\0';
+ }
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+ unsigned offset = 1;
+ char C = peekNext(offset++);
+ while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+ }
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+ unsigned IncrementCounter = 0;
+ while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+ break;
+ case '<': {
+ IncrementCounter++;
+ } break;
+ case '>': {
+ IncrementCounter--;
+ if (!IncrementCounter)
+ return true;
+ } break;
+ }
+ }
+ return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+ if (isEnd())
+ return false;
+ Position SavedPos = Pos;
+ consumeWhitespace();
+ SmallString<32> NextToken;
+ SmallString<32> WordText;
+ const char *WordBegin = Pos.BufferPtr;
+ SourceLocation Loc = getSourceLocation();
+ StringRef ConstVal = StringRef("const");
+ bool ConstPointer = false;
+
+ while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+ if (C == '<') {
+ if (!lexTemplateType(WordText))
+ return false;
+ } else {
+ WordText.push_back(C);
+ consumeChar();
+ }
+ } else {
+ if (WordText.equals(ConstVal)) {
+ WordText.push_back(C);
+ consumeChar();
+ } else if (WordText.ends_with(StringRef("*")) ||
+ WordText.ends_with(StringRef("&"))) {
+ NextToken.clear();
+ peekNextToken(NextToken);
+ if (NextToken.equals(ConstVal)) {
+ ConstPointer = true;
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ consumeChar();
+ break;
+ }
+ } else {
+ NextToken.clear();
+ peekNextToken(NextToken);
+ if ((NextToken.ends_with(StringRef("*")) ||
+ NextToken.ends_with(StringRef("&"))) &&
+ !ConstPointer) {
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ consumeChar();
+ break;
+ }
+ }
+ }
+ }
+
+ const unsigned Length = WordText.size();
+ if (Length == 0) {
+ Pos = SavedPos;
+ return false;
+ }
+
+ char *TextPtr = Allocator.Allocate<char>(Length + 1);
+
+ memcpy(TextPtr, WordText.c_str(), Length + 1);
+ StringRef Text = StringRef(TextPtr, Length);
+
+ formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
+ return true;
+ }
+
/// Extract a word -- sequence of non-whitespace characters.
bool lexWord(Token &Tok) {
if (isEnd())
@@ -295,6 +407,7 @@ Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer,
unsigned NumArgs) {
Comment::Argument[NumArgs];
unsigned ParsedArgs = 0;
Token Arg;
+
while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
Args[ParsedArgs] = Comment::Argument{
SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
@@ -304,6 +417,23 @@ Parser::parseCommandArgs(TextTokenRetokenizer
&Retokenizer, unsigned NumArgs) {
return llvm::ArrayRef(Args, ParsedArgs);
}
+ArrayRef<Comment::Argument>
+Parser::parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer,
+ unsigned NumArgs) {
+ auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
+ Comment::Argument[NumArgs];
+ unsigned ParsedArgs = 0;
+ Token Arg;
+
+ while (ParsedArgs < NumArgs && Retokenizer.lexDataType(Arg)) {
+ Args[ParsedArgs] = Comment::Argument{
+ SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
+ ParsedArgs++;
+ }
+
+ return llvm::ArrayRef(Args, ParsedArgs);
+}
+
BlockCommandComment *Parser::parseBlockCommand() {
assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
@@ -356,6 +486,9 @@ BlockCommandComment *Parser::parseBlockCommand() {
parseParamCommandArgs(PC, Retokenizer);
else if (TPC)
parseTParamCommandArgs(TPC, Retokenizer);
+ else if (Info->IsThrowsCommand)
+ S.actOnBlockCommandArgs(
+ BC, parseThrowCommandArgs(Retokenizer, Info->NumArgs));
else
S.actOnBlockCommandArgs(BC, parseCommandArgs(Retokenizer,
Info->NumArgs));
diff --git a/clang/unittests/AST/CommentParser.cpp
b/clang/unittests/AST/CommentParser.cpp
index c3479672ae2a3..e01d654aa1cea 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -1427,8 +1427,241 @@ TEST_F(CommentParserTest, Deprecated) {
}
}
+TEST_F(CommentParserTest, ThrowsCommandHasArg1) {
+ const char *Sources[] = {
+ "/// @throws int This function throws an integer",
+ ("/// @throws\n"
+ "/// int This function throws an integer"),
+ ("/// @throws \n"
+ "/// int This function throws an integer"),
+ ("/// @throws\n"
+ "/// int\n"
+ "/// This function throws an integer"),
+ ("/// @throws \n"
+ "/// int \n"
+ "/// This function throws an integer"),
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "int");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg2) {
+ const char *Sources[] = {
+ "/// @throws const int This function throws a const integer",
+ ("/// @throws\n"
+ "/// const int This function throws a const integer"),
+ ("/// @throws \n"
+ "/// const int This function throws a const integer"),
+ ("/// @throws\n"
+ "/// const int\n"
+ "/// This function throws a const integer"),
+ ("/// @throws \n"
+ "/// const int \n"
+ "/// This function throws a const integer"),
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "const int");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg3) {
+ const char *Sources[] = {
+ "/// @throws const int * This function throws a pointer to a const "
+ "integer\n",
+ ("/// @throws\n"
+ "/// const int * This function throws a pointer to a const integer"),
+ ("/// @throws \n"
+ "/// const int * This function throws a pointer to a const integer"),
+ ("/// @throws\n"
+ "/// const int *\n"
+ "/// This function throws a pointer to a const integer"),
+ ("/// @throws \n"
+ "/// const int *\n"
+ "/// This function throws a pointer to a const integer"),
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "const int *");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg4) {
+ const char *Sources[] = {
+ "/// @throws const int * const This function throws a const pointer to a
"
+ "const integer",
+ ("/// @throws\n"
+ "/// const int * const This function throws a const pointer to a const "
+ "integer"),
+ ("/// @throws \n"
+ "/// const int * const This function throws a const pointer to a const "
+ "integer"),
+ ("/// @throws\n"
+ "/// const int * const\n"
+ "/// This function throws a const pointer to a const integer"),
+ ("/// @throws \n"
+ "/// const int * const\n"
+ "/// This function throws a const pointer to a const integer"),
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "const int * const");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg5) {
+ const char *Sources[] = {
+ "/// @throws int** This function throws a double pointer to an integer",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "int**");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg6) {
+ const char *Sources[] = {
+ "/// @throws const char ** double pointer to a constant char pointer",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "const char **");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg7) {
+ const char *Sources[] = {
+ "/// @throws Error<T> error of type Error<T>",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 3)); // Extra children because <T> is
parsed
+ // as a series of TextComments
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "Error<T>");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg8) {
+ const char *Sources[] = {
+ "/// @throws Error<Container<T>> nested templates",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "Error<Container<T>>");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg9) {
+ const char *Sources[] = {
+ "/// @throws Error<Ts...> variadic templates",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "Error<Ts...>");
+ }
+ }
+}
+
} // unnamed namespace
} // end namespace comments
} // end namespace clang
-
>From 7da378bacb80c24f1986d70a6e3d8d1e5f4a3a37 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Mon, 11 Mar 2024 16:13:33 -0700
Subject: [PATCH 02/10] Fix comment to XML tests
---
.../Index/comment-to-html-xml-conversion.cpp | 53 ++++++++-----------
1 file changed, 21 insertions(+), 32 deletions(-)
diff --git a/clang/test/Index/comment-to-html-xml-conversion.cpp
b/clang/test/Index/comment-to-html-xml-conversion.cpp
index d9eefb909653c..291aecf44d112 100644
--- a/clang/test/Index/comment-to-html-xml-conversion.cpp
+++ b/clang/test/Index/comment-to-html-xml-conversion.cpp
@@ -1046,82 +1046,71 @@ void comment_to_xml_conversion_todo_4();
/// Aaa.
/// \throws Bbb.
void comment_to_xml_conversion_exceptions_1();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_1:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_1</Name><USR>c:@F@comment_to_xml_conversion_exceptions_1#</USR><Declaration>void
comment_to_xml_conversion_exceptions_1()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para> Bbb.</Para></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_1:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_1</Name><USR>c:@F@comment_to_xml_conversion_exceptions_1#</USR><Declaration>void
comment_to_xml_conversion_exceptions_1()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Bbb.]))))]
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Bbb.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace)))]
/// Aaa.
/// \throw Bbb.
void comment_to_xml_conversion_exceptions_2();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_2:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_2</Name><USR>c:@F@comment_to_xml_conversion_exceptions_2#</USR><Declaration>void
comment_to_xml_conversion_exceptions_2()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para> Bbb.</Para></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_2:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_2</Name><USR>c:@F@comment_to_xml_conversion_exceptions_2#</USR><Declaration>void
comment_to_xml_conversion_exceptions_2()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throw]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Bbb.]))))]
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throw] Arg[0]=Bbb.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace)))]
/// Aaa.
/// \exception Bbb.
void comment_to_xml_conversion_exceptions_3();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_3:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_3</Name><USR>c:@F@comment_to_xml_conversion_exceptions_3#</USR><Declaration>void
comment_to_xml_conversion_exceptions_3()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para> Bbb.</Para></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_3:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_3</Name><USR>c:@F@comment_to_xml_conversion_exceptions_3#</USR><Declaration>void
comment_to_xml_conversion_exceptions_3()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[exception]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Bbb.]))))]
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[exception]
Arg[0]=Bbb.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace)))]
/// Aaa.
/// \throws Bbb.
/// \throws Ccc.
/// \throws Ddd.
void comment_to_xml_conversion_exceptions_4();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_4:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_4</Name><USR>c:@F@comment_to_xml_conversion_exceptions_4#</USR><Declaration>void
comment_to_xml_conversion_exceptions_4()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para> Bbb. </Para><Para> Ccc. </Para><Para>
Ddd.</Para></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_4:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_4</Name><USR>c:@F@comment_to_xml_conversion_exceptions_4#</USR><Declaration>void
comment_to_xml_conversion_exceptions_4()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Bbb.] HasTrailingNewline)
-// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace)))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Ccc.] HasTrailingNewline)
-// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace)))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Ddd.]))))]
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Bbb.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace))
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Ccc.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace))
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Ddd.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace)))]
/// Aaa.
/// \throws Bbb.
/// \throw Ccc.
void comment_to_xml_conversion_exceptions_5();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_5:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_5</Name><USR>c:@F@comment_to_xml_conversion_exceptions_5#</USR><Declaration>void
comment_to_xml_conversion_exceptions_5()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para> Bbb. </Para><Para>
Ccc.</Para></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_5:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_5</Name><USR>c:@F@comment_to_xml_conversion_exceptions_5#</USR><Declaration>void
comment_to_xml_conversion_exceptions_5()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Bbb.] HasTrailingNewline)
-// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace)))
-// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throw]
-// CHECK-NEXT: (CXComment_Paragraph
-// CHECK-NEXT: (CXComment_Text Text=[ Ccc.]))))]
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Bbb.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace))
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throw] Arg[0]=Ccc.
+// CHECK-NEXT: (CXComment_Paragraph IsWhitespace)))]
#endif
>From ea7b93354adecf668f012d66a8b34af13a7e6276 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Tue, 12 Mar 2024 20:15:10 -0700
Subject: [PATCH 03/10] Update XML output routine for Throws commands to fix
XML validation
XML validation was failing due to the Exception XML being empty, since
the actual exception type was being parsed as an argument instead of
as a ParagraphComment.
This was a result of the change we made to argument parsing.
As a result, I updated the XML output to still output the argument
text in the Para XML, as it was emitted before.
---
clang/lib/Index/CommentToXML.cpp | 34 ++++++++++-----
.../Index/comment-to-html-xml-conversion.cpp | 42 ++++++++++++++++---
2 files changed, 59 insertions(+), 17 deletions(-)
diff --git a/clang/lib/Index/CommentToXML.cpp b/clang/lib/Index/CommentToXML.cpp
index 3372fbba43831..cd7226e71171c 100644
--- a/clang/lib/Index/CommentToXML.cpp
+++ b/clang/lib/Index/CommentToXML.cpp
@@ -546,7 +546,8 @@ class CommentASTToXMLConverter :
void visitParagraphComment(const ParagraphComment *C);
void appendParagraphCommentWithKind(const ParagraphComment *C,
- StringRef Kind);
+ StringRef ParagraphKind,
+ StringRef PrependBodyText);
void visitBlockCommandComment(const BlockCommandComment *C);
void visitParamCommandComment(const ParamCommandComment *C);
@@ -680,15 +681,15 @@ CommentASTToXMLConverter::visitHTMLEndTagComment(const
HTMLEndTagComment *C) {
Result << "></" << C->getTagName() << "></rawHTML>";
}
-void
-CommentASTToXMLConverter::visitParagraphComment(const ParagraphComment *C) {
- appendParagraphCommentWithKind(C, StringRef());
+void CommentASTToXMLConverter::visitParagraphComment(
+ const ParagraphComment *C) {
+ appendParagraphCommentWithKind(C, StringRef(), StringRef());
}
void CommentASTToXMLConverter::appendParagraphCommentWithKind(
- const ParagraphComment *C,
- StringRef ParagraphKind) {
- if (C->isWhitespace())
+ const ParagraphComment *C, StringRef ParagraphKind,
+ StringRef PrependBodyText) {
+ if (C->isWhitespace() && PrependBodyText.empty())
return;
if (ParagraphKind.empty())
@@ -696,8 +697,11 @@ void
CommentASTToXMLConverter::appendParagraphCommentWithKind(
else
Result << "<Para kind=\"" << ParagraphKind << "\">";
- for (Comment::child_iterator I = C->child_begin(), E = C->child_end();
- I != E; ++I) {
+ if (!PrependBodyText.empty())
+ Result << PrependBodyText << " ";
+
+ for (Comment::child_iterator I = C->child_begin(), E = C->child_end(); I !=
E;
+ ++I) {
visit(*I);
}
Result << "</Para>";
@@ -706,8 +710,15 @@ void
CommentASTToXMLConverter::appendParagraphCommentWithKind(
void CommentASTToXMLConverter::visitBlockCommandComment(
const BlockCommandComment *C) {
StringRef ParagraphKind;
+ StringRef ExceptionType;
- switch (C->getCommandID()) {
+ const unsigned CommandID = C->getCommandID();
+ const CommandInfo *Info = Traits.getCommandInfo(CommandID);
+ if (Info->IsThrowsCommand && C->getNumArgs() > 0) {
+ ExceptionType = C->getArgText(0);
+ }
+
+ switch (CommandID) {
case CommandTraits::KCI_attention:
case CommandTraits::KCI_author:
case CommandTraits::KCI_authors:
@@ -732,7 +743,8 @@ void CommentASTToXMLConverter::visitBlockCommandComment(
break;
}
- appendParagraphCommentWithKind(C->getParagraph(), ParagraphKind);
+ appendParagraphCommentWithKind(C->getParagraph(), ParagraphKind,
+ ExceptionType);
}
void CommentASTToXMLConverter::visitParamCommandComment(
diff --git a/clang/test/Index/comment-to-html-xml-conversion.cpp
b/clang/test/Index/comment-to-html-xml-conversion.cpp
index 291aecf44d112..e0a7cff5a9a3d 100644
--- a/clang/test/Index/comment-to-html-xml-conversion.cpp
+++ b/clang/test/Index/comment-to-html-xml-conversion.cpp
@@ -1046,7 +1046,7 @@ void comment_to_xml_conversion_todo_4();
/// Aaa.
/// \throws Bbb.
void comment_to_xml_conversion_exceptions_1();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_1:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_1</Name><USR>c:@F@comment_to_xml_conversion_exceptions_1#</USR><Declaration>void
comment_to_xml_conversion_exceptions_1()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_1:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_1</Name><USR>c:@F@comment_to_xml_conversion_exceptions_1#</USR><Declaration>void
comment_to_xml_conversion_exceptions_1()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb. </Para></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
@@ -1058,7 +1058,7 @@ void comment_to_xml_conversion_exceptions_1();
/// Aaa.
/// \throw Bbb.
void comment_to_xml_conversion_exceptions_2();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_2:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_2</Name><USR>c:@F@comment_to_xml_conversion_exceptions_2#</USR><Declaration>void
comment_to_xml_conversion_exceptions_2()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_2:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_2</Name><USR>c:@F@comment_to_xml_conversion_exceptions_2#</USR><Declaration>void
comment_to_xml_conversion_exceptions_2()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb. </Para></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
@@ -1070,7 +1070,7 @@ void comment_to_xml_conversion_exceptions_2();
/// Aaa.
/// \exception Bbb.
void comment_to_xml_conversion_exceptions_3();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_3:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_3</Name><USR>c:@F@comment_to_xml_conversion_exceptions_3#</USR><Declaration>void
comment_to_xml_conversion_exceptions_3()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_3:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_3</Name><USR>c:@F@comment_to_xml_conversion_exceptions_3#</USR><Declaration>void
comment_to_xml_conversion_exceptions_3()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb. </Para></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
@@ -1084,7 +1084,7 @@ void comment_to_xml_conversion_exceptions_3();
/// \throws Ccc.
/// \throws Ddd.
void comment_to_xml_conversion_exceptions_4();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_4:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_4</Name><USR>c:@F@comment_to_xml_conversion_exceptions_4#</USR><Declaration>void
comment_to_xml_conversion_exceptions_4()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_4:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_4</Name><USR>c:@F@comment_to_xml_conversion_exceptions_4#</USR><Declaration>void
comment_to_xml_conversion_exceptions_4()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb. </Para><Para>Ccc. </Para><Para>Ddd.
</Para></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
@@ -1101,7 +1101,7 @@ void comment_to_xml_conversion_exceptions_4();
/// \throws Bbb.
/// \throw Ccc.
void comment_to_xml_conversion_exceptions_5();
-// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_5:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_5</Name><USR>c:@F@comment_to_xml_conversion_exceptions_5#</USR><Declaration>void
comment_to_xml_conversion_exceptions_5()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions></Exceptions></Function>]
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_5:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_5</Name><USR>c:@F@comment_to_xml_conversion_exceptions_5#</USR><Declaration>void
comment_to_xml_conversion_exceptions_5()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb. </Para><Para>Ccc.
</Para></Exceptions></Function>]
// CHECK-NEXT: CommentAST=[
// CHECK-NEXT: (CXComment_FullComment
// CHECK-NEXT: (CXComment_Paragraph
@@ -1112,5 +1112,35 @@ void comment_to_xml_conversion_exceptions_5();
// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throw] Arg[0]=Ccc.
// CHECK-NEXT: (CXComment_Paragraph IsWhitespace)))]
-#endif
+/// Aaa.
+/// \throws Bbb subsequent arg text
+void comment_to_xml_conversion_exceptions_6();
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_6:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_6</Name><USR>c:@F@comment_to_xml_conversion_exceptions_6#</USR><Declaration>void
comment_to_xml_conversion_exceptions_6()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb subsequent arg
text</Para></Exceptions></Function>]
+// CHECK-NEXT: CommentAST=[
+// CHECK-NEXT: (CXComment_FullComment
+// CHECK-NEXT: (CXComment_Paragraph
+// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
+// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Bbb
+// CHECK-NEXT: (CXComment_Paragraph
+// CHECK-NEXT: (CXComment_Text Text=[subsequent arg text]))))]
+/// Aaa.
+/// \throws Bbb subsequent arg text
+/// \throw Ccc subsequent arg text
+void comment_to_xml_conversion_exceptions_7();
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6:
FunctionDecl=comment_to_xml_conversion_exceptions_7:{{.*}}
FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp"
line="[[@LINE-1]]"
column="6"><Name>comment_to_xml_conversion_exceptions_7</Name><USR>c:@F@comment_to_xml_conversion_exceptions_7#</USR><Declaration>void
comment_to_xml_conversion_exceptions_7()</Declaration><Abstract><Para> Aaa.
</Para></Abstract><Exceptions><Para>Bbb subsequent arg text </Para><Para>Ccc
subsequent arg text</Para></Exceptions></Function>]
+// CHECK-NEXT: CommentAST=[
+// CHECK-NEXT: (CXComment_FullComment
+// CHECK-NEXT: (CXComment_Paragraph
+// CHECK-NEXT: (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
+// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throws] Arg[0]=Bbb
+// CHECK-NEXT: (CXComment_Paragraph
+// CHECK-NEXT: (CXComment_Text Text=[subsequent arg text]
HasTrailingNewline)
+// CHECK-NEXT: (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT: (CXComment_BlockCommand CommandName=[throw] Arg[0]=Ccc
+// CHECK-NEXT: (CXComment_Paragraph
+// CHECK-NEXT: (CXComment_Text Text=[subsequent arg text]))))]
+
+#endif
>From b1c73a10ec57820479ec55ce6205723441817017 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Fri, 15 Mar 2024 11:42:50 -0700
Subject: [PATCH 04/10] Support more qualifiers, such as volatile
---
clang/lib/AST/CommentParser.cpp | 99 ++++++++++++++++++++-------
clang/unittests/AST/CommentParser.cpp | 86 +++++++++++++++++++++++
2 files changed, 162 insertions(+), 23 deletions(-)
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index c70fa1b05cb24..a2aa3cfc6d5b5 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -131,6 +131,48 @@ class TextTokenRetokenizer {
return false;
}
+ bool isDataTypeQualifier(SmallString<32> &WordText) {
+ if (WordText.ends_with(StringRef("const")))
+ return true;
+ if (WordText.ends_with(StringRef("volatile")))
+ return true;
+ if (WordText.ends_with(StringRef("unsigned")))
+ return true;
+ if (WordText.ends_with(StringRef("signed")))
+ return true;
+ if (WordText.ends_with(StringRef("long")))
+ return true;
+ if (WordText.ends_with(StringRef("short")))
+ return true;
+ if (WordText.ends_with(StringRef("restrict")))
+ return true;
+ if (WordText.ends_with(StringRef("auto")))
+ return true;
+ if (WordText.ends_with(StringRef("register")))
+ return true;
+ if (WordText.ends_with(StringRef("static")))
+ return true;
+ if (WordText.ends_with(StringRef("extern")))
+ return true;
+ if (WordText.ends_with(StringRef("struct")))
+ return true;
+ if (WordText.ends_with(StringRef("typedef")))
+ return true;
+ if (WordText.ends_with(StringRef("union")))
+ return true;
+ if (WordText.ends_with(StringRef("void")))
+ return true;
+ return false;
+ }
+
+ bool isScopeResolutionOperator(SmallString<32> &WordText) {
+ return WordText.ends_with(StringRef("::"));
+ }
+
+ bool continueParsing(SmallString<32> &WordText) {
+ return isDataTypeQualifier(WordText) ||
isScopeResolutionOperator(WordText);
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -192,7 +234,7 @@ class TextTokenRetokenizer {
}
/// Extract a type argument
- bool lexDataType(Token &Tok) {
+ bool lexType(Token &Tok) {
if (isEnd())
return false;
Position SavedPos = Pos;
@@ -202,6 +244,8 @@ class TextTokenRetokenizer {
const char *WordBegin = Pos.BufferPtr;
SourceLocation Loc = getSourceLocation();
StringRef ConstVal = StringRef("const");
+ StringRef PointerVal = StringRef("*");
+ StringRef ReferenceVal = StringRef("&");
bool ConstPointer = false;
while (!isEnd()) {
@@ -215,32 +259,41 @@ class TextTokenRetokenizer {
consumeChar();
}
} else {
- if (WordText.equals(ConstVal)) {
- WordText.push_back(C);
+ if (ConstPointer) {
consumeChar();
- } else if (WordText.ends_with(StringRef("*")) ||
- WordText.ends_with(StringRef("&"))) {
- NextToken.clear();
- peekNextToken(NextToken);
- if (NextToken.equals(ConstVal)) {
- ConstPointer = true;
- WordText.push_back(C);
- consumeChar();
- } else {
- consumeChar();
- break;
- }
+ break;
} else {
- NextToken.clear();
- peekNextToken(NextToken);
- if ((NextToken.ends_with(StringRef("*")) ||
- NextToken.ends_with(StringRef("&"))) &&
- !ConstPointer) {
+ if (continueParsing(WordText)) {
WordText.push_back(C);
consumeChar();
} else {
- consumeChar();
- break;
+ NextToken.clear();
+ peekNextToken(NextToken);
+ if (WordText.ends_with(PointerVal) ||
+ WordText.ends_with(ReferenceVal)) {
+ if (NextToken.equals(ConstVal)) {
+ ConstPointer = true;
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ consumeChar();
+ break;
+ }
+ } else {
+ if ((NextToken.ends_with(PointerVal) ||
+ NextToken.ends_with(ReferenceVal))) {
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ if (continueParsing(NextToken)) {
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ consumeChar();
+ break;
+ }
+ }
+ }
}
}
}
@@ -425,7 +478,7 @@ Parser::parseThrowCommandArgs(TextTokenRetokenizer
&Retokenizer,
unsigned ParsedArgs = 0;
Token Arg;
- while (ParsedArgs < NumArgs && Retokenizer.lexDataType(Arg)) {
+ while (ParsedArgs < NumArgs && Retokenizer.lexType(Arg)) {
Args[ParsedArgs] = Comment::Argument{
SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
ParsedArgs++;
diff --git a/clang/unittests/AST/CommentParser.cpp
b/clang/unittests/AST/CommentParser.cpp
index e01d654aa1cea..e2e77dd95a9a5 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -1661,6 +1661,92 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg9) {
}
}
+TEST_F(CommentParserTest, ThrowsCommandHasArg10) {
+ const char *Sources[] = {
+ "/// @throws const std::map<int, std::string> * pointer to a const map",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "const std::map<int, std::string> *");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg11) {
+ const char *Sources[] = {
+ "/// @throws const std :: map<int, std :: string> * pointer to a "
+ "const map with spaces",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) ==
+ "const std :: map<int, std :: string> *");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg12) {
+ const char *Sources[] = {
+ "/// @throws volatile int a volatile integer",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "volatile int");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg13) {
+ const char *Sources[] = {
+ "/// @throws volatile double * volatile pointer to a double",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "volatile double *");
+ }
+ }
+}
+
} // unnamed namespace
} // end namespace comments
>From 4432d3282d5f3b7d97dc58a62c9f2ad81cbb734a Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Sun, 24 Mar 2024 00:39:56 -0700
Subject: [PATCH 05/10] Refactor integer argument parsing and add corresponding
test cases
---
clang/lib/AST/CommentParser.cpp | 127 ++++++++++++++++++--------
clang/unittests/AST/CommentParser.cpp | 42 +++++++++
2 files changed, 130 insertions(+), 39 deletions(-)
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index a2aa3cfc6d5b5..63baebe59cdfd 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -108,69 +108,112 @@ class TextTokenRetokenizer {
}
}
+ bool continueInt(SmallString<32> &NextToken) {
+ return NextToken.ends_with(StringRef("char")) ||
+ NextToken.ends_with(StringRef("int")) ||
+ NextToken.ends_with(StringRef("char*")) ||
+ NextToken.ends_with(StringRef("int*")) ||
+ NextToken.ends_with(StringRef("char&")) ||
+ NextToken.ends_with(StringRef("int&"));
+ }
+
+ bool lexInt(SmallString<32> &WordText, SmallString<32> &NextToken) {
+ unsigned LongCounter = (WordText.ends_with(StringRef("long"))) ? 1 : 0;
+ bool complete = false;
+
+ while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+
+ NextToken.clear();
+ peekNextToken(NextToken);
+
+ if (WordText.ends_with(StringRef("long"))) {
+ LongCounter++;
+ if (continueInt(NextToken)) {
+ WordText.push_back(C);
+ consumeChar();
+ complete = true;
+ continue;
+ } else {
+ if (LongCounter == 2) {
+ return true;
+ }
+ }
+ } else {
+
+ if (complete || continueInt(WordText)) {
+ return true;
+ }
+ }
+
+ if (NextToken.ends_with(StringRef("long"))) {
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
/// Extract a template type
- bool lexTemplateType(SmallString<32> &WordText) {
+ bool lexTemplate(SmallString<32> &WordText) {
unsigned IncrementCounter = 0;
while (!isEnd()) {
const char C = peek();
WordText.push_back(C);
consumeChar();
switch (C) {
- default:
- break;
case '<': {
IncrementCounter++;
- } break;
+ break;
+ }
case '>': {
IncrementCounter--;
if (!IncrementCounter)
return true;
- } break;
+ break;
+ }
+ default:
+ break;
}
}
return false;
}
- bool isDataTypeQualifier(SmallString<32> &WordText) {
- if (WordText.ends_with(StringRef("const")))
- return true;
- if (WordText.ends_with(StringRef("volatile")))
- return true;
- if (WordText.ends_with(StringRef("unsigned")))
- return true;
- if (WordText.ends_with(StringRef("signed")))
- return true;
- if (WordText.ends_with(StringRef("long")))
- return true;
- if (WordText.ends_with(StringRef("short")))
- return true;
- if (WordText.ends_with(StringRef("restrict")))
- return true;
- if (WordText.ends_with(StringRef("auto")))
- return true;
- if (WordText.ends_with(StringRef("register")))
- return true;
- if (WordText.ends_with(StringRef("static")))
- return true;
- if (WordText.ends_with(StringRef("extern")))
- return true;
- if (WordText.ends_with(StringRef("struct")))
- return true;
- if (WordText.ends_with(StringRef("typedef")))
- return true;
- if (WordText.ends_with(StringRef("union")))
- return true;
- if (WordText.ends_with(StringRef("void")))
- return true;
- return false;
+ bool isTypeQualifier(SmallString<32> &WordText) {
+ return WordText.ends_with(StringRef("const")) ||
+ WordText.ends_with(StringRef("volatile")) ||
+ WordText.ends_with(StringRef("short")) ||
+ WordText.ends_with(StringRef("restrict")) ||
+ WordText.ends_with(StringRef("auto")) ||
+ WordText.ends_with(StringRef("register")) ||
+ WordText.ends_with(StringRef("static")) ||
+ WordText.ends_with(StringRef("extern")) ||
+ WordText.ends_with(StringRef("struct")) ||
+ WordText.ends_with(StringRef("typedef")) ||
+ WordText.ends_with(StringRef("union")) ||
+ WordText.ends_with(StringRef("void"));
}
bool isScopeResolutionOperator(SmallString<32> &WordText) {
return WordText.ends_with(StringRef("::"));
}
+ bool isInt(SmallString<32> &WordText) {
+ return WordText.ends_with(StringRef("unsigned")) ||
+ WordText.ends_with(StringRef("long")) ||
+ WordText.ends_with(StringRef("signed"));
+ }
+
bool continueParsing(SmallString<32> &WordText) {
- return isDataTypeQualifier(WordText) ||
isScopeResolutionOperator(WordText);
+ return isTypeQualifier(WordText) || isScopeResolutionOperator(WordText);
}
/// Add a token.
@@ -252,7 +295,7 @@ class TextTokenRetokenizer {
const char C = peek();
if (!isWhitespace(C)) {
if (C == '<') {
- if (!lexTemplateType(WordText))
+ if (!lexTemplate(WordText))
return false;
} else {
WordText.push_back(C);
@@ -263,6 +306,12 @@ class TextTokenRetokenizer {
consumeChar();
break;
} else {
+ if (isInt(WordText)) {
+ WordText.push_back(C);
+ consumeChar();
+ if (!lexInt(WordText, NextToken))
+ return false;
+ }
if (continueParsing(WordText)) {
WordText.push_back(C);
consumeChar();
diff --git a/clang/unittests/AST/CommentParser.cpp
b/clang/unittests/AST/CommentParser.cpp
index e2e77dd95a9a5..385a98e2a036b 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -1747,6 +1747,48 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg13) {
}
}
+TEST_F(CommentParserTest, ThrowsCommandHasArg14) {
+ const char *Sources[] = {
+ "/// @throws unsigned long at least a 32-bit integer",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "unsigned long");
+ }
+ }
+}
+
+TEST_F(CommentParserTest, ThrowsCommandHasArg15) {
+ const char *Sources[] = {
+ "/// @throws unsigned long long at least a 64-bit integer",
+ };
+
+ for (size_t i = 0, e = std::size(Sources); i != e; i++) {
+ FullComment *FC = parseString(Sources[i]);
+ ASSERT_TRUE(HasChildCount(FC, 2));
+
+ ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+ {
+ BlockCommandComment *BCC;
+ ParagraphComment *PC;
+ ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
+ ASSERT_TRUE(HasChildCount(PC, 1));
+ ASSERT_TRUE(BCC->getNumArgs() == 1);
+ ASSERT_TRUE(BCC->getArgText(0) == "unsigned long long");
+ }
+ }
+}
+
} // unnamed namespace
} // end namespace comments
>From 421f585c2954548e736ef6259097ac89faa991de Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Tue, 2 Apr 2024 21:37:53 -0700
Subject: [PATCH 06/10] Address review feedback
- Refactored some functionality with the help of clang-tidy
- Added comments to the complicated lexing functions
- Renamed variables to follow LLVM/Clang conventions
---
clang/lib/AST/CommentParser.cpp | 118 +++++++++++++++++++-------------
1 file changed, 71 insertions(+), 47 deletions(-)
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 63baebe59cdfd..f6d2ff55e3beb 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -78,7 +78,7 @@ class TextTokenRetokenizer {
char peekNext(unsigned offset) const {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
- if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ if (Pos.BufferPtr + offset < Pos.BufferEnd) {
return *(Pos.BufferPtr + offset);
} else {
return '\0';
@@ -108,7 +108,7 @@ class TextTokenRetokenizer {
}
}
- bool continueInt(SmallString<32> &NextToken) {
+ bool shouldContinueLexingIntegralType(SmallString<32> &NextToken) {
return NextToken.ends_with(StringRef("char")) ||
NextToken.ends_with(StringRef("int")) ||
NextToken.ends_with(StringRef("char*")) ||
@@ -117,9 +117,10 @@ class TextTokenRetokenizer {
NextToken.ends_with(StringRef("int&"));
}
- bool lexInt(SmallString<32> &WordText, SmallString<32> &NextToken) {
+ /// Lex an integral type, such as unsigned long long, etc.
+ bool lexIntegral(SmallString<32> &WordText, SmallString<32> &NextToken) {
unsigned LongCounter = (WordText.ends_with(StringRef("long"))) ? 1 : 0;
- bool complete = false;
+ bool IsLexingComplete = false;
while (!isEnd()) {
const char C = peek();
@@ -127,29 +128,33 @@ class TextTokenRetokenizer {
WordText.push_back(C);
consumeChar();
} else {
-
NextToken.clear();
peekNextToken(NextToken);
if (WordText.ends_with(StringRef("long"))) {
LongCounter++;
- if (continueInt(NextToken)) {
+ // Use the next token to determine if we should continue parsing
+ if (shouldContinueLexingIntegralType(NextToken)) {
WordText.push_back(C);
consumeChar();
- complete = true;
+ IsLexingComplete = true;
continue;
- } else {
- if (LongCounter == 2) {
- return true;
- }
}
- } else {
+ // Maximum number of consecutive "long" is 2, so we can return if
+ // we've hit that.
+ if (LongCounter == 2) {
+ return true;
+ }
+ }
- if (complete || continueInt(WordText)) {
+ // If current word doesn't end with long, check if we should exit early
+ else {
+ if (IsLexingComplete || shouldContinueLexingIntegralType(WordText)) {
return true;
}
}
+ // If next token ends with long then we consume it and continue parsing
if (NextToken.ends_with(StringRef("long"))) {
WordText.push_back(C);
consumeChar();
@@ -206,7 +211,7 @@ class TextTokenRetokenizer {
return WordText.ends_with(StringRef("::"));
}
- bool isInt(SmallString<32> &WordText) {
+ bool isIntegral(SmallString<32> &WordText) {
return WordText.ends_with(StringRef("unsigned")) ||
WordText.ends_with(StringRef("long")) ||
WordText.ends_with(StringRef("signed"));
@@ -280,7 +285,12 @@ class TextTokenRetokenizer {
bool lexType(Token &Tok) {
if (isEnd())
return false;
+
+ // Save current position in case we need to rollback because the type is
+ // empty.
Position SavedPos = Pos;
+
+ // Consume any leading whitespace.
consumeWhitespace();
SmallString<32> NextToken;
SmallString<32> WordText;
@@ -289,10 +299,12 @@ class TextTokenRetokenizer {
StringRef ConstVal = StringRef("const");
StringRef PointerVal = StringRef("*");
StringRef ReferenceVal = StringRef("&");
- bool ConstPointer = false;
+ bool IsTypeConstPointerOrRef = false;
while (!isEnd()) {
const char C = peek();
+ // For non-whitespace characters we check if it's a template or otherwise
+ // continue reading the text into a word.
if (!isWhitespace(C)) {
if (C == '<') {
if (!lexTemplate(WordText))
@@ -301,47 +313,59 @@ class TextTokenRetokenizer {
WordText.push_back(C);
consumeChar();
}
- } else {
- if (ConstPointer) {
+ }
+ // For whitespace, we start inspecting the constructed word
+ else {
+ // If we encounter a pointer/reference, we can stop parsing since we're
+ // only parsing expressions.
+ if (IsTypeConstPointerOrRef) {
consumeChar();
break;
- } else {
- if (isInt(WordText)) {
- WordText.push_back(C);
- consumeChar();
- if (!lexInt(WordText, NextToken))
- return false;
- }
- if (continueParsing(WordText)) {
- WordText.push_back(C);
- consumeChar();
+ }
+ // Parse out integral types
+ if (isIntegral(WordText)) {
+ WordText.push_back(C);
+ consumeChar();
+ if (!lexIntegral(WordText, NextToken))
+ return false;
+ }
+ // Certain types, like qualified names or types with CVR to name a few,
+ // may have whitespace inside of the typename, so we need to check and
+ // continue parsing if that's the case
+ if (continueParsing(WordText)) {
+ WordText.push_back(C);
+ consumeChar();
+ }
+ // Handles cases without qualified names or type qualifiers
+ else {
+ NextToken.clear();
+ peekNextToken(NextToken);
+ // Check for pointer/ref vals, and mark the type as a pointer/ref for
+ // the rest of the lex
+ if (WordText.ends_with(PointerVal) ||
+ WordText.ends_with(ReferenceVal)) {
+ if (NextToken.equals(ConstVal)) {
+ IsTypeConstPointerOrRef = true;
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ consumeChar();
+ break;
+ }
} else {
- NextToken.clear();
- peekNextToken(NextToken);
- if (WordText.ends_with(PointerVal) ||
- WordText.ends_with(ReferenceVal)) {
- if (NextToken.equals(ConstVal)) {
- ConstPointer = true;
+ // Check if the next token is a pointer/ref
+ if ((NextToken.ends_with(PointerVal) ||
+ NextToken.ends_with(ReferenceVal))) {
+ WordText.push_back(C);
+ consumeChar();
+ } else {
+ if (continueParsing(NextToken)) {
WordText.push_back(C);
consumeChar();
} else {
consumeChar();
break;
}
- } else {
- if ((NextToken.ends_with(PointerVal) ||
- NextToken.ends_with(ReferenceVal))) {
- WordText.push_back(C);
- consumeChar();
- } else {
- if (continueParsing(NextToken)) {
- WordText.push_back(C);
- consumeChar();
- } else {
- consumeChar();
- break;
- }
- }
}
}
}
>From 9eb681d45d2491ba07dd9ea0cdd0ac1d7e8d8438 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Wed, 29 May 2024 15:06:41 +0200
Subject: [PATCH 07/10] Parse only the space-delimited word after @throw
This is in response to a request by upstream to simplify the parser
previously developed.
---
clang/include/clang/AST/CommentParser.h | 2 +
clang/lib/AST/CommentParser.cpp | 174 +---------------
clang/unittests/AST/CommentParser.cpp | 252 +-----------------------
3 files changed, 9 insertions(+), 419 deletions(-)
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index 5884a25d00785..636bd40aeaa7f 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,8 @@ class Parser {
ArrayRef<Comment::Argument>
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ /// Parse arguments for \\throws command supported args are in form of class
+ /// or template.
ArrayRef<Comment::Argument>
parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index f6d2ff55e3beb..098866b358135 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,25 +75,6 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
- char peekNext(unsigned offset) const {
- assert(!isEnd());
- assert(Pos.BufferPtr != Pos.BufferEnd);
- if (Pos.BufferPtr + offset < Pos.BufferEnd) {
- return *(Pos.BufferPtr + offset);
- } else {
- return '\0';
- }
- }
-
- void peekNextToken(SmallString<32> &WordText) const {
- unsigned offset = 1;
- char C = peekNext(offset++);
- while (!isWhitespace(C) && C != '\0') {
- WordText.push_back(C);
- C = peekNext(offset++);
- }
- }
-
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -108,65 +89,6 @@ class TextTokenRetokenizer {
}
}
- bool shouldContinueLexingIntegralType(SmallString<32> &NextToken) {
- return NextToken.ends_with(StringRef("char")) ||
- NextToken.ends_with(StringRef("int")) ||
- NextToken.ends_with(StringRef("char*")) ||
- NextToken.ends_with(StringRef("int*")) ||
- NextToken.ends_with(StringRef("char&")) ||
- NextToken.ends_with(StringRef("int&"));
- }
-
- /// Lex an integral type, such as unsigned long long, etc.
- bool lexIntegral(SmallString<32> &WordText, SmallString<32> &NextToken) {
- unsigned LongCounter = (WordText.ends_with(StringRef("long"))) ? 1 : 0;
- bool IsLexingComplete = false;
-
- while (!isEnd()) {
- const char C = peek();
- if (!isWhitespace(C)) {
- WordText.push_back(C);
- consumeChar();
- } else {
- NextToken.clear();
- peekNextToken(NextToken);
-
- if (WordText.ends_with(StringRef("long"))) {
- LongCounter++;
- // Use the next token to determine if we should continue parsing
- if (shouldContinueLexingIntegralType(NextToken)) {
- WordText.push_back(C);
- consumeChar();
- IsLexingComplete = true;
- continue;
- }
- // Maximum number of consecutive "long" is 2, so we can return if
- // we've hit that.
- if (LongCounter == 2) {
- return true;
- }
- }
-
- // If current word doesn't end with long, check if we should exit early
- else {
- if (IsLexingComplete || shouldContinueLexingIntegralType(WordText)) {
- return true;
- }
- }
-
- // If next token ends with long then we consume it and continue parsing
- if (NextToken.ends_with(StringRef("long"))) {
- WordText.push_back(C);
- consumeChar();
- } else {
- return true;
- }
- }
- }
-
- return false;
- }
-
/// Extract a template type
bool lexTemplate(SmallString<32> &WordText) {
unsigned IncrementCounter = 0;
@@ -192,35 +114,6 @@ class TextTokenRetokenizer {
return false;
}
- bool isTypeQualifier(SmallString<32> &WordText) {
- return WordText.ends_with(StringRef("const")) ||
- WordText.ends_with(StringRef("volatile")) ||
- WordText.ends_with(StringRef("short")) ||
- WordText.ends_with(StringRef("restrict")) ||
- WordText.ends_with(StringRef("auto")) ||
- WordText.ends_with(StringRef("register")) ||
- WordText.ends_with(StringRef("static")) ||
- WordText.ends_with(StringRef("extern")) ||
- WordText.ends_with(StringRef("struct")) ||
- WordText.ends_with(StringRef("typedef")) ||
- WordText.ends_with(StringRef("union")) ||
- WordText.ends_with(StringRef("void"));
- }
-
- bool isScopeResolutionOperator(SmallString<32> &WordText) {
- return WordText.ends_with(StringRef("::"));
- }
-
- bool isIntegral(SmallString<32> &WordText) {
- return WordText.ends_with(StringRef("unsigned")) ||
- WordText.ends_with(StringRef("long")) ||
- WordText.ends_with(StringRef("signed"));
- }
-
- bool continueParsing(SmallString<32> &WordText) {
- return isTypeQualifier(WordText) || isScopeResolutionOperator(WordText);
- }
-
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -292,14 +185,9 @@ class TextTokenRetokenizer {
// Consume any leading whitespace.
consumeWhitespace();
- SmallString<32> NextToken;
SmallString<32> WordText;
const char *WordBegin = Pos.BufferPtr;
SourceLocation Loc = getSourceLocation();
- StringRef ConstVal = StringRef("const");
- StringRef PointerVal = StringRef("*");
- StringRef ReferenceVal = StringRef("&");
- bool IsTypeConstPointerOrRef = false;
while (!isEnd()) {
const char C = peek();
@@ -313,62 +201,9 @@ class TextTokenRetokenizer {
WordText.push_back(C);
consumeChar();
}
- }
- // For whitespace, we start inspecting the constructed word
- else {
- // If we encounter a pointer/reference, we can stop parsing since we're
- // only parsing expressions.
- if (IsTypeConstPointerOrRef) {
- consumeChar();
- break;
- }
- // Parse out integral types
- if (isIntegral(WordText)) {
- WordText.push_back(C);
- consumeChar();
- if (!lexIntegral(WordText, NextToken))
- return false;
- }
- // Certain types, like qualified names or types with CVR to name a few,
- // may have whitespace inside of the typename, so we need to check and
- // continue parsing if that's the case
- if (continueParsing(WordText)) {
- WordText.push_back(C);
- consumeChar();
- }
- // Handles cases without qualified names or type qualifiers
- else {
- NextToken.clear();
- peekNextToken(NextToken);
- // Check for pointer/ref vals, and mark the type as a pointer/ref for
- // the rest of the lex
- if (WordText.ends_with(PointerVal) ||
- WordText.ends_with(ReferenceVal)) {
- if (NextToken.equals(ConstVal)) {
- IsTypeConstPointerOrRef = true;
- WordText.push_back(C);
- consumeChar();
- } else {
- consumeChar();
- break;
- }
- } else {
- // Check if the next token is a pointer/ref
- if ((NextToken.ends_with(PointerVal) ||
- NextToken.ends_with(ReferenceVal))) {
- WordText.push_back(C);
- consumeChar();
- } else {
- if (continueParsing(NextToken)) {
- WordText.push_back(C);
- consumeChar();
- } else {
- consumeChar();
- break;
- }
- }
- }
- }
+ } else {
+ consumeChar();
+ break;
}
}
@@ -462,8 +297,7 @@ class TextTokenRetokenizer {
memcpy(TextPtr, WordText.c_str(), Length + 1);
StringRef Text = StringRef(TextPtr, Length);
- formTokenWithChars(Tok, Loc, WordBegin,
- Pos.BufferPtr - WordBegin, Text);
+ formTokenWithChars(Tok, Loc, WordBegin, Pos.BufferPtr - WordBegin, Text);
return true;
}
diff --git a/clang/unittests/AST/CommentParser.cpp
b/clang/unittests/AST/CommentParser.cpp
index 385a98e2a036b..8071bb69ab867 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -1459,103 +1459,6 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg1) {
}
TEST_F(CommentParserTest, ThrowsCommandHasArg2) {
- const char *Sources[] = {
- "/// @throws const int This function throws a const integer",
- ("/// @throws\n"
- "/// const int This function throws a const integer"),
- ("/// @throws \n"
- "/// const int This function throws a const integer"),
- ("/// @throws\n"
- "/// const int\n"
- "/// This function throws a const integer"),
- ("/// @throws \n"
- "/// const int \n"
- "/// This function throws a const integer"),
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "const int");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg3) {
- const char *Sources[] = {
- "/// @throws const int * This function throws a pointer to a const "
- "integer\n",
- ("/// @throws\n"
- "/// const int * This function throws a pointer to a const integer"),
- ("/// @throws \n"
- "/// const int * This function throws a pointer to a const integer"),
- ("/// @throws\n"
- "/// const int *\n"
- "/// This function throws a pointer to a const integer"),
- ("/// @throws \n"
- "/// const int *\n"
- "/// This function throws a pointer to a const integer"),
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "const int *");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg4) {
- const char *Sources[] = {
- "/// @throws const int * const This function throws a const pointer to a
"
- "const integer",
- ("/// @throws\n"
- "/// const int * const This function throws a const pointer to a const "
- "integer"),
- ("/// @throws \n"
- "/// const int * const This function throws a const pointer to a const "
- "integer"),
- ("/// @throws\n"
- "/// const int * const\n"
- "/// This function throws a const pointer to a const integer"),
- ("/// @throws \n"
- "/// const int * const\n"
- "/// This function throws a const pointer to a const integer"),
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "const int * const");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg5) {
const char *Sources[] = {
"/// @throws int** This function throws a double pointer to an integer",
};
@@ -1576,28 +1479,7 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg5) {
}
}
-TEST_F(CommentParserTest, ThrowsCommandHasArg6) {
- const char *Sources[] = {
- "/// @throws const char ** double pointer to a constant char pointer",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "const char **");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg7) {
+TEST_F(CommentParserTest, ThrowsCommandHasArg3) {
const char *Sources[] = {
"/// @throws Error<T> error of type Error<T>",
};
@@ -1619,7 +1501,7 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg7) {
}
}
-TEST_F(CommentParserTest, ThrowsCommandHasArg8) {
+TEST_F(CommentParserTest, ThrowsCommandHasArg4) {
const char *Sources[] = {
"/// @throws Error<Container<T>> nested templates",
};
@@ -1640,7 +1522,7 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg8) {
}
}
-TEST_F(CommentParserTest, ThrowsCommandHasArg9) {
+TEST_F(CommentParserTest, ThrowsCommandHasArg5) {
const char *Sources[] = {
"/// @throws Error<Ts...> variadic templates",
};
@@ -1661,134 +1543,6 @@ TEST_F(CommentParserTest, ThrowsCommandHasArg9) {
}
}
-TEST_F(CommentParserTest, ThrowsCommandHasArg10) {
- const char *Sources[] = {
- "/// @throws const std::map<int, std::string> * pointer to a const map",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "const std::map<int, std::string> *");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg11) {
- const char *Sources[] = {
- "/// @throws const std :: map<int, std :: string> * pointer to a "
- "const map with spaces",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) ==
- "const std :: map<int, std :: string> *");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg12) {
- const char *Sources[] = {
- "/// @throws volatile int a volatile integer",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "volatile int");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg13) {
- const char *Sources[] = {
- "/// @throws volatile double * volatile pointer to a double",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "volatile double *");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg14) {
- const char *Sources[] = {
- "/// @throws unsigned long at least a 32-bit integer",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "unsigned long");
- }
- }
-}
-
-TEST_F(CommentParserTest, ThrowsCommandHasArg15) {
- const char *Sources[] = {
- "/// @throws unsigned long long at least a 64-bit integer",
- };
-
- for (size_t i = 0, e = std::size(Sources); i != e; i++) {
- FullComment *FC = parseString(Sources[i]);
- ASSERT_TRUE(HasChildCount(FC, 2));
-
- ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
- {
- BlockCommandComment *BCC;
- ParagraphComment *PC;
- ASSERT_TRUE(HasBlockCommandAt(FC, Traits, 1, BCC, "throws", PC));
- ASSERT_TRUE(HasChildCount(PC, 1));
- ASSERT_TRUE(BCC->getNumArgs() == 1);
- ASSERT_TRUE(BCC->getArgText(0) == "unsigned long long");
- }
- }
-}
-
} // unnamed namespace
} // end namespace comments
>From 0684ecfe8e9793365f51b70eba96a5602c89eba2 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Mon, 3 Jun 2024 14:52:06 -0700
Subject: [PATCH 08/10] Rename IncrementCounter -> BracketCount
---
clang/lib/AST/CommentParser.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 098866b358135..e746aebaf8066 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -91,19 +91,19 @@ class TextTokenRetokenizer {
/// Extract a template type
bool lexTemplate(SmallString<32> &WordText) {
- unsigned IncrementCounter = 0;
+ unsigned BracketCount = 0;
while (!isEnd()) {
const char C = peek();
WordText.push_back(C);
consumeChar();
switch (C) {
case '<': {
- IncrementCounter++;
+ BracketCount++;
break;
}
case '>': {
- IncrementCounter--;
- if (!IncrementCounter)
+ BracketCount--;
+ if (!BracketCount)
return true;
break;
}
>From 0e56adeacaf9501acb098bcb136c1127ad9d738b Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Mon, 3 Jun 2024 14:52:22 -0700
Subject: [PATCH 09/10] Fix comment string
---
clang/include/clang/AST/CommentParser.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index 636bd40aeaa7f..a2d0e30835e2c 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,7 +100,7 @@ class Parser {
ArrayRef<Comment::Argument>
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
- /// Parse arguments for \\throws command supported args are in form of class
+ /// Parse arguments for \throws command supported args are in form of class
/// or template.
ArrayRef<Comment::Argument>
parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
>From 6b165dee7dad075e41ca31fa309313d6595f9407 Mon Sep 17 00:00:00 2001
From: hdoc <git...@hdoc.io>
Date: Mon, 3 Jun 2024 14:59:03 -0700
Subject: [PATCH 10/10] Revert clang-format changes that sneaked through
---
clang/lib/AST/CommentParser.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index e746aebaf8066..5baf81a509fb6 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -297,7 +297,8 @@ class TextTokenRetokenizer {
memcpy(TextPtr, WordText.c_str(), Length + 1);
StringRef Text = StringRef(TextPtr, Length);
- formTokenWithChars(Tok, Loc, WordBegin, Pos.BufferPtr - WordBegin, Text);
+ formTokenWithChars(Tok, Loc, WordBegin,
+ Pos.BufferPtr - WordBegin, Text);
return true;
}
@@ -367,7 +368,6 @@ Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer,
unsigned NumArgs) {
Comment::Argument[NumArgs];
unsigned ParsedArgs = 0;
Token Arg;
-
while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
Args[ParsedArgs] = Comment::Argument{
SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
