[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 1/5] Comment parsing: add argument parsing for @throw @throws
@exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd8..06b2fa9b5531c6 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a1..5884a25d007851 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c3..c70fa1b05cb241 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: Ping https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
sdkrystian wrote:
Shouldn't this be `if (Pos.BufferPtr + offset < Pos.BufferEnd)` (unless this
depends of the buffer being null terminated)? Otherwise this allows for an
`offset` which results in `Pos.BufferEnd` being read.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+StringRef PointerVal = StringRef("*");
+StringRef ReferenceVal = StringRef("&");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplate(WordText))
+return false;
+} else {
+ WordText.push_back(C);
sdkrystian wrote:
Does this treat any non-whitespace character (other than `<`) as being part of
a name?
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+StringRef PointerVal = StringRef("*");
+StringRef ReferenceVal = StringRef("&");
+bool ConstPointer = false;
+
+while (!isEnd()) {
sdkrystian wrote:
Some comments wouldn't hurt...
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +108,114 @@ class TextTokenRetokenizer {
}
}
+ bool continueInt(SmallString<32> &NextToken) {
+return NextToken.ends_with(StringRef("char")) ||
sdkrystian wrote:
Don't really understand what this is for...
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+StringRef PointerVal = StringRef("*");
sdkrystian wrote:
What about pointers to members? Arrays? Function? More complex declarators like
`void(*)(int, int)`?
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc edited https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc edited https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
sdkrystian wrote:
Shouldn't we support `volatile` as well?
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 1/2] Comment parsing: add argument parsing for @throw @throws
@exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd8..06b2fa9b5531c6 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a1..5884a25d007851 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c3..c70fa1b05cb241 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
hdoc wrote:
Yes, it should. We'll update the PR to support that. Thank you for noticing and
mentioning this.
Likewise in the future we'll probably want to add support for `restrict`. It's
not necessary for this PR since `throw/throws/exception` is only useful in C++
but when parsing Doxygen comments in C codebases we'll want to have the option
of supporting that.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 1/3] Comment parsing: add argument parsing for @throw @throws
@exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd8..06b2fa9b5531c6 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a1..5884a25d007851 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c3..c70fa1b05cb241 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 1/4] Comment parsing: add argument parsing for @throw @throws
@exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd8..06b2fa9b5531c6 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a1..5884a25d007851 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c3..c70fa1b05cb241 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: Ping https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
cor3ntin wrote: Before I do an in-depth review i think supporting arguments to `@throw` etc makes a lot of sense. However, I don't think we want to have a `type-id` parser in the comment parser. The grammar is complicated, and the approach seems brittle and hard to maintain. Given use cases, I think it's sufficient to consider anything before a space as part of a type. You can try to support things like `foo` just by balancing `<>`, but I really don't think we want to parse arbitrary types (including function types, arrays, lambdas, etc) as types that you can throw. If people want to throw "unsigned int" exceptions, well... it won't work. I think it's a reasonable tradeoff. 99% of exceptions are going to be class types. In the long term, if we really want to support arbitrary constructs there, I think it would take a more robust approach such as the one that was [proposed here (and subsequently abandoned) ](https://discourse.llvm.org/t/rfc-a-c-pseudo-parser-for-tooling/59217 ) https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From 49fd778d21adb49080d1f9b360aa5cda2b7359a6 Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 01/10] Comment parsing: add argument parsing for @throw
@throws @exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd..06b2fa9b5531c 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a..5884a25d00785 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c..c70fa1b05cb24 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return false;
+
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From 49fd778d21adb49080d1f9b360aa5cda2b7359a6 Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 01/11] Comment parsing: add argument parsing for @throw
@throws @exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd..06b2fa9b5531c 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a..5884a25d00785 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c..c70fa1b05cb24 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return false;
+
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplate(SmallString<32> &WordText) {
hdoc wrote:
Added some more test cases that check this edge case behavior, like unequal
numbers of opening and closing brackets as well as some other unusual edge
cases.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin approved this pull request. LGTM, Thanks Will you need us to merge that for you? https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: @cor3ntin yes, please merge this as we do not have commit access. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin closed https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
github-actions[bot] wrote: @hdoc Congratulations on having your first Pull Request (PR) merged into the LLVM Project! Your changes will be combined with recent changes from other authors, then tested by our [build bots](https://lab.llvm.org/buildbot/). If there is a problem with a build, you may receive a report in an email or a comment on this PR. Please check whether problems have been caused by your change specifically, as the builds can include changes from many authors. It is not uncommon for your change to be included in a build that fails due to someone else's changes, or infrastructure issues. How to do this, and the rest of the post-merge process, is covered in detail [here](https://llvm.org/docs/MyFirstTypoFix.html#myfirsttypofix-issues-after-landing-your-pr). If your change does cause a problem, it may be reverted, or you can revert it yourself. This is a normal part of [LLVM development](https://llvm.org/docs/DeveloperPolicy.html#patch-reversion-policy). You can fix your changes and open a new PR to merge them again. If you don't get any reports, no action is required from you. Your changes are working as expected, well done! https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 1/7] Comment parsing: add argument parsing for @throw @throws
@exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd..06b2fa9b5531c 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a..5884a25d00785 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c..c70fa1b05cb24 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return false;
+
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: @cor3ntin we appreciate the feedback. Likewise, we agree that the home-grown parser just for parsing `type-id`s in comments is less than ideal. We had some alternative approaches in mind, like using an instance of `clang::Lexer`, but went with our original approach at first. I've refactored the PR and reduced the functionality to the subset you requested. I think that makes more sense from a maintainability perspective, as you highlighted. Please review the PR when you get a chance. Thank you. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin commented: The general direction looks good now. I left a few nitpicky comments. Thanks! https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplate(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
cor3ntin wrote:
It would be nice to modify `consumeChar` to return the consumed char
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin edited https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +174,54 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexType(Token &Tok) {
+if (isEnd())
+ return false;
+
+// Save current position in case we need to rollback because the type is
+// empty.
+Position SavedPos = Pos;
+
+// Consume any leading whitespace.
+consumeWhitespace();
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+
+while (!isEnd()) {
+ const char C = peek();
+ // For non-whitespace characters we check if it's a template or otherwise
+ // continue reading the text into a word.
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplate(WordText))
+return false;
+} else {
+ WordText.push_back(C);
+ consumeChar();
+}
+ } else {
+consumeChar();
+break;
+ }
+}
+
+const unsigned Length = WordText.size();
+if (Length == 0) {
+ Pos = SavedPos;
+ return false;
+}
+
+char *TextPtr = Allocator.Allocate(Length + 1);
+
+memcpy(TextPtr, WordText.c_str(), Length + 1);
+StringRef Text = StringRef(TextPtr, Length);
+
+formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
cor3ntin wrote:
I'd like to see that extracted in a separate function (and replace the few
places where we do the exact same thing)
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplate(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
cor3ntin wrote:
`BracketCount` would be a better name
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -295,6 +367,7 @@ Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer,
unsigned NumArgs) {
Comment::Argument[NumArgs];
unsigned ParsedArgs = 0;
Token Arg;
+
cor3ntin wrote:
Can you revert whitespace only changes?
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -100,6 +100,11 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ /// Parse arguments for \\throws command supported args are in form of class
cor3ntin wrote:
```suggestion
/// Parse arguments for \throws command supported args are in form of class
```
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+StringRef PointerVal = StringRef("*");
+StringRef ReferenceVal = StringRef("&");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplate(WordText))
+return false;
+} else {
+ WordText.push_back(C);
cor3ntin wrote:
Which I think is fine, we are just trying to find a boundary
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726
>From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001
From: hdoc
Date: Mon, 11 Mar 2024 01:13:25 -0700
Subject: [PATCH 01/10] Comment parsing: add argument parsing for @throw
@throws @exception
Doxygen allows for the @throw, @throws, and @exception commands to
have an attached argument indicating the type being thrown. Currently,
Clang's AST parsing doesn't support parsing out this argument from doc
comments. The result is missing compatibility with Doxygen.
We would find it helpful if the AST exposed these thrown types as
BlockCommandComment arguments so that we could generate better
documentation.
This PR implements parsing of arguments for the @throw, @throws, and
@exception commands. Each command can only have one argument, matching
the semantics of Doxygen. We have also added unit tests to validate
the functionality.
---
clang/include/clang/AST/CommentCommands.td | 6 +-
clang/include/clang/AST/CommentParser.h| 3 +
clang/lib/AST/CommentParser.cpp| 133
clang/unittests/AST/CommentParser.cpp | 235 -
4 files changed, 373 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/CommentCommands.td
b/clang/include/clang/AST/CommentCommands.td
index e839031752cdd..06b2fa9b5531c 100644
--- a/clang/include/clang/AST/CommentCommands.td
+++ b/clang/include/clang/AST/CommentCommands.td
@@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand =
1; }
// HeaderDoc command for template parameter documentation.
def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
-def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; }
-def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; }
-def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; }
+def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs
= 1; }
+def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs =
1; }
+def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let
NumArgs = 1;}
def Deprecated : BlockCommand<"deprecated"> {
let IsEmptyParagraphAllowed = 1;
diff --git a/clang/include/clang/AST/CommentParser.h
b/clang/include/clang/AST/CommentParser.h
index e11e818b1af0a..5884a25d00785 100644
--- a/clang/include/clang/AST/CommentParser.h
+++ b/clang/include/clang/AST/CommentParser.h
@@ -100,6 +100,9 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ ArrayRef
+ parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+
BlockCommandComment *parseBlockCommand();
InlineCommandComment *parseInlineCommand();
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 8adfd85d0160c..c70fa1b05cb24 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -75,6 +75,25 @@ class TextTokenRetokenizer {
return *Pos.BufferPtr;
}
+ char peekNext(unsigned offset) const {
+assert(!isEnd());
+assert(Pos.BufferPtr != Pos.BufferEnd);
+if (Pos.BufferPtr + offset <= Pos.BufferEnd) {
+ return *(Pos.BufferPtr + offset);
+} else {
+ return '\0';
+}
+ }
+
+ void peekNextToken(SmallString<32> &WordText) const {
+unsigned offset = 1;
+char C = peekNext(offset++);
+while (!isWhitespace(C) && C != '\0') {
+ WordText.push_back(C);
+ C = peekNext(offset++);
+}
+ }
+
void consumeChar() {
assert(!isEnd());
assert(Pos.BufferPtr != Pos.BufferEnd);
@@ -89,6 +108,29 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplateType(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
+ switch (C) {
+ default:
+break;
+ case '<': {
+IncrementCounter++;
+ } break;
+ case '>': {
+IncrementCounter--;
+if (!IncrementCounter)
+ return true;
+ } break;
+ }
+}
+return false;
+ }
+
/// Add a token.
/// Returns true on success, false if there are no interesting tokens to
/// fetch from lexer.
@@ -149,6 +191,76 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexDataType(Token &Tok) {
+if (isEnd())
+ return false;
+Position SavedPos = Pos;
+consumeWhitespace();
+SmallString<32> NextToken;
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+StringRef ConstVal = StringRef("const");
+bool ConstPointer = false;
+
+while (!isEnd()) {
+ const char C = peek();
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplateType(WordText))
+return false;
+
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -295,6 +367,7 @@ Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer,
unsigned NumArgs) {
Comment::Argument[NumArgs];
unsigned ParsedArgs = 0;
Token Arg;
+
hdoc wrote:
Fixed, I try to keep the clang-format changes restricted to only the regions I
touch but it looks like some sneaked through
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -100,6 +100,11 @@ class Parser {
ArrayRef
parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs);
+ /// Parse arguments for \\throws command supported args are in form of class
hdoc wrote:
Fixed
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplate(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
+while (!isEnd()) {
+ const char C = peek();
+ WordText.push_back(C);
+ consumeChar();
hdoc wrote:
I would prefer to do that as a separate refactor PR, I think it's outside the
scope of this change.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplate(SmallString<32> &WordText) {
+unsigned IncrementCounter = 0;
hdoc wrote:
That's a much better name, fixed and thank you.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +174,54 @@ class TextTokenRetokenizer {
addToken();
}
+ /// Extract a type argument
+ bool lexType(Token &Tok) {
+if (isEnd())
+ return false;
+
+// Save current position in case we need to rollback because the type is
+// empty.
+Position SavedPos = Pos;
+
+// Consume any leading whitespace.
+consumeWhitespace();
+SmallString<32> WordText;
+const char *WordBegin = Pos.BufferPtr;
+SourceLocation Loc = getSourceLocation();
+
+while (!isEnd()) {
+ const char C = peek();
+ // For non-whitespace characters we check if it's a template or otherwise
+ // continue reading the text into a word.
+ if (!isWhitespace(C)) {
+if (C == '<') {
+ if (!lexTemplate(WordText))
+return false;
+} else {
+ WordText.push_back(C);
+ consumeChar();
+}
+ } else {
+consumeChar();
+break;
+ }
+}
+
+const unsigned Length = WordText.size();
+if (Length == 0) {
+ Pos = SavedPos;
+ return false;
+}
+
+char *TextPtr = Allocator.Allocate(Length + 1);
+
+memcpy(TextPtr, WordText.c_str(), Length + 1);
+StringRef Text = StringRef(TextPtr, Length);
+
+formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
hdoc wrote:
As above, I would prefer to do that as a separate refactor PR, I think it's
outside the scope of this change.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer {
}
}
+ /// Extract a template type
+ bool lexTemplate(SmallString<32> &WordText) {
AaronBallman wrote:
This should work okay for simple (most) cases, but we should have tests showing
the boundaries, e.g., `Foo<(1 > 0)>` or a typo like `Foo` etc.
https://github.com/llvm/llvm-project/pull/84726
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
