[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 1/5] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd8..06b2fa9b5531c6 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a1..5884a25d007851 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c3..c70fa1b05cb241 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: Ping https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { sdkrystian wrote: Shouldn't this be `if (Pos.BufferPtr + offset < Pos.BufferEnd)` (unless this depends of the buffer being null terminated)? Otherwise this allows for an `offset` which results in `Pos.BufferEnd` being read. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +StringRef PointerVal = StringRef("*"); +StringRef ReferenceVal = StringRef("&"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplate(WordText)) +return false; +} else { + WordText.push_back(C); sdkrystian wrote: Does this treat any non-whitespace character (other than `<`) as being part of a name? https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +StringRef PointerVal = StringRef("*"); +StringRef ReferenceVal = StringRef("&"); +bool ConstPointer = false; + +while (!isEnd()) { sdkrystian wrote: Some comments wouldn't hurt... https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +108,114 @@ class TextTokenRetokenizer { } } + bool continueInt(SmallString<32> &NextToken) { +return NextToken.ends_with(StringRef("char")) || sdkrystian wrote: Don't really understand what this is for... https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +StringRef PointerVal = StringRef("*"); sdkrystian wrote: What about pointers to members? Arrays? Function? More complex declarators like `void(*)(int, int)`? https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc edited https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc edited https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); sdkrystian wrote: Shouldn't we support `volatile` as well? https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 1/2] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd8..06b2fa9b5531c6 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a1..5884a25d007851 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c3..c70fa1b05cb241 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); hdoc wrote: Yes, it should. We'll update the PR to support that. Thank you for noticing and mentioning this. Likewise in the future we'll probably want to add support for `restrict`. It's not necessary for this PR since `throw/throws/exception` is only useful in C++ but when parsing Doxygen comments in C codebases we'll want to have the option of supporting that. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 1/3] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd8..06b2fa9b5531c6 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a1..5884a25d007851 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c3..c70fa1b05cb241 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 1/4] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd8..06b2fa9b5531c6 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a1..5884a25d007851 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c3..c70fa1b05cb241 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return fals
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: Ping https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
cor3ntin wrote: Before I do an in-depth review i think supporting arguments to `@throw` etc makes a lot of sense. However, I don't think we want to have a `type-id` parser in the comment parser. The grammar is complicated, and the approach seems brittle and hard to maintain. Given use cases, I think it's sufficient to consider anything before a space as part of a type. You can try to support things like `foo` just by balancing `<>`, but I really don't think we want to parse arbitrary types (including function types, arrays, lambdas, etc) as types that you can throw. If people want to throw "unsigned int" exceptions, well... it won't work. I think it's a reasonable tradeoff. 99% of exceptions are going to be class types. In the long term, if we really want to support arbitrary constructs there, I think it would take a more robust approach such as the one that was [proposed here (and subsequently abandoned) ](https://discourse.llvm.org/t/rfc-a-c-pseudo-parser-for-tooling/59217 ) https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From 49fd778d21adb49080d1f9b360aa5cda2b7359a6 Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 01/10] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd..06b2fa9b5531c 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a..5884a25d00785 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c..c70fa1b05cb24 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return false; +
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From 49fd778d21adb49080d1f9b360aa5cda2b7359a6 Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 01/11] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd..06b2fa9b5531c 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a..5884a25d00785 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c..c70fa1b05cb24 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return false; +
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplate(SmallString<32> &WordText) { hdoc wrote: Added some more test cases that check this edge case behavior, like unequal numbers of opening and closing brackets as well as some other unusual edge cases. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin approved this pull request. LGTM, Thanks Will you need us to merge that for you? https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: @cor3ntin yes, please merge this as we do not have commit access. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin closed https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
github-actions[bot] wrote: @hdoc Congratulations on having your first Pull Request (PR) merged into the LLVM Project! Your changes will be combined with recent changes from other authors, then tested by our [build bots](https://lab.llvm.org/buildbot/). If there is a problem with a build, you may receive a report in an email or a comment on this PR. Please check whether problems have been caused by your change specifically, as the builds can include changes from many authors. It is not uncommon for your change to be included in a build that fails due to someone else's changes, or infrastructure issues. How to do this, and the rest of the post-merge process, is covered in detail [here](https://llvm.org/docs/MyFirstTypoFix.html#myfirsttypofix-issues-after-landing-your-pr). If your change does cause a problem, it may be reverted, or you can revert it yourself. This is a normal part of [LLVM development](https://llvm.org/docs/DeveloperPolicy.html#patch-reversion-policy). You can fix your changes and open a new PR to merge them again. If you don't get any reports, no action is required from you. Your changes are working as expected, well done! https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 1/7] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd..06b2fa9b5531c 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a..5884a25d00785 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c..c70fa1b05cb24 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return false; +
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
hdoc wrote: @cor3ntin we appreciate the feedback. Likewise, we agree that the home-grown parser just for parsing `type-id`s in comments is less than ideal. We had some alternative approaches in mind, like using an instance of `clang::Lexer`, but went with our original approach at first. I've refactored the PR and reduced the functionality to the subset you requested. I think that makes more sense from a maintainability perspective, as you highlighted. Please review the PR when you get a chance. Thank you. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin commented: The general direction looks good now. I left a few nitpicky comments. Thanks! https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplate(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); cor3ntin wrote: It would be nice to modify `consumeChar` to return the consumed char https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/cor3ntin edited https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +174,54 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexType(Token &Tok) { +if (isEnd()) + return false; + +// Save current position in case we need to rollback because the type is +// empty. +Position SavedPos = Pos; + +// Consume any leading whitespace. +consumeWhitespace(); +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); + +while (!isEnd()) { + const char C = peek(); + // For non-whitespace characters we check if it's a template or otherwise + // continue reading the text into a word. + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplate(WordText)) +return false; +} else { + WordText.push_back(C); + consumeChar(); +} + } else { +consumeChar(); +break; + } +} + +const unsigned Length = WordText.size(); +if (Length == 0) { + Pos = SavedPos; + return false; +} + +char *TextPtr = Allocator.Allocate(Length + 1); + +memcpy(TextPtr, WordText.c_str(), Length + 1); +StringRef Text = StringRef(TextPtr, Length); + +formTokenWithChars(Tok, Loc, WordBegin, Length, Text); cor3ntin wrote: I'd like to see that extracted in a separate function (and replace the few places where we do the exact same thing) https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplate(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; cor3ntin wrote: `BracketCount` would be a better name https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -295,6 +367,7 @@ Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) { Comment::Argument[NumArgs]; unsigned ParsedArgs = 0; Token Arg; + cor3ntin wrote: Can you revert whitespace only changes? https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -100,6 +100,11 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + /// Parse arguments for \\throws command supported args are in form of class cor3ntin wrote: ```suggestion /// Parse arguments for \throws command supported args are in form of class ``` https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +276,93 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +StringRef PointerVal = StringRef("*"); +StringRef ReferenceVal = StringRef("&"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplate(WordText)) +return false; +} else { + WordText.push_back(C); cor3ntin wrote: Which I think is fine, we are just trying to find a boundary https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
https://github.com/hdoc updated https://github.com/llvm/llvm-project/pull/84726 >From ec3f444913d9162de4494cdb09b336b1b00380fa Mon Sep 17 00:00:00 2001 From: hdoc Date: Mon, 11 Mar 2024 01:13:25 -0700 Subject: [PATCH 01/10] Comment parsing: add argument parsing for @throw @throws @exception Doxygen allows for the @throw, @throws, and @exception commands to have an attached argument indicating the type being thrown. Currently, Clang's AST parsing doesn't support parsing out this argument from doc comments. The result is missing compatibility with Doxygen. We would find it helpful if the AST exposed these thrown types as BlockCommandComment arguments so that we could generate better documentation. This PR implements parsing of arguments for the @throw, @throws, and @exception commands. Each command can only have one argument, matching the semantics of Doxygen. We have also added unit tests to validate the functionality. --- clang/include/clang/AST/CommentCommands.td | 6 +- clang/include/clang/AST/CommentParser.h| 3 + clang/lib/AST/CommentParser.cpp| 133 clang/unittests/AST/CommentParser.cpp | 235 - 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/AST/CommentCommands.td b/clang/include/clang/AST/CommentCommands.td index e839031752cdd..06b2fa9b5531c 100644 --- a/clang/include/clang/AST/CommentCommands.td +++ b/clang/include/clang/AST/CommentCommands.td @@ -132,9 +132,9 @@ def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; } // HeaderDoc command for template parameter documentation. def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; } -def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; } -def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; } -def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; } +def Throws: BlockCommand<"throws"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Throw : BlockCommand<"throw"> { let IsThrowsCommand = 1; let NumArgs = 1; } +def Exception : BlockCommand<"exception"> { let IsThrowsCommand = 1; let NumArgs = 1;} def Deprecated : BlockCommand<"deprecated"> { let IsEmptyParagraphAllowed = 1; diff --git a/clang/include/clang/AST/CommentParser.h b/clang/include/clang/AST/CommentParser.h index e11e818b1af0a..5884a25d00785 100644 --- a/clang/include/clang/AST/CommentParser.h +++ b/clang/include/clang/AST/CommentParser.h @@ -100,6 +100,9 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + ArrayRef + parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + BlockCommandComment *parseBlockCommand(); InlineCommandComment *parseInlineCommand(); diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index 8adfd85d0160c..c70fa1b05cb24 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -75,6 +75,25 @@ class TextTokenRetokenizer { return *Pos.BufferPtr; } + char peekNext(unsigned offset) const { +assert(!isEnd()); +assert(Pos.BufferPtr != Pos.BufferEnd); +if (Pos.BufferPtr + offset <= Pos.BufferEnd) { + return *(Pos.BufferPtr + offset); +} else { + return '\0'; +} + } + + void peekNextToken(SmallString<32> &WordText) const { +unsigned offset = 1; +char C = peekNext(offset++); +while (!isWhitespace(C) && C != '\0') { + WordText.push_back(C); + C = peekNext(offset++); +} + } + void consumeChar() { assert(!isEnd()); assert(Pos.BufferPtr != Pos.BufferEnd); @@ -89,6 +108,29 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplateType(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); + switch (C) { + default: +break; + case '<': { +IncrementCounter++; + } break; + case '>': { +IncrementCounter--; +if (!IncrementCounter) + return true; + } break; + } +} +return false; + } + /// Add a token. /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. @@ -149,6 +191,76 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexDataType(Token &Tok) { +if (isEnd()) + return false; +Position SavedPos = Pos; +consumeWhitespace(); +SmallString<32> NextToken; +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); +StringRef ConstVal = StringRef("const"); +bool ConstPointer = false; + +while (!isEnd()) { + const char C = peek(); + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplateType(WordText)) +return false; +
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -295,6 +367,7 @@ Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) { Comment::Argument[NumArgs]; unsigned ParsedArgs = 0; Token Arg; + hdoc wrote: Fixed, I try to keep the clang-format changes restricted to only the regions I touch but it looks like some sneaked through https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -100,6 +100,11 @@ class Parser { ArrayRef parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs); + /// Parse arguments for \\throws command supported args are in form of class hdoc wrote: Fixed https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplate(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; +while (!isEnd()) { + const char C = peek(); + WordText.push_back(C); + consumeChar(); hdoc wrote: I would prefer to do that as a separate refactor PR, I think it's outside the scope of this change. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplate(SmallString<32> &WordText) { +unsigned IncrementCounter = 0; hdoc wrote: That's a much better name, fixed and thank you. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -149,6 +174,54 @@ class TextTokenRetokenizer { addToken(); } + /// Extract a type argument + bool lexType(Token &Tok) { +if (isEnd()) + return false; + +// Save current position in case we need to rollback because the type is +// empty. +Position SavedPos = Pos; + +// Consume any leading whitespace. +consumeWhitespace(); +SmallString<32> WordText; +const char *WordBegin = Pos.BufferPtr; +SourceLocation Loc = getSourceLocation(); + +while (!isEnd()) { + const char C = peek(); + // For non-whitespace characters we check if it's a template or otherwise + // continue reading the text into a word. + if (!isWhitespace(C)) { +if (C == '<') { + if (!lexTemplate(WordText)) +return false; +} else { + WordText.push_back(C); + consumeChar(); +} + } else { +consumeChar(); +break; + } +} + +const unsigned Length = WordText.size(); +if (Length == 0) { + Pos = SavedPos; + return false; +} + +char *TextPtr = Allocator.Allocate(Length + 1); + +memcpy(TextPtr, WordText.c_str(), Length + 1); +StringRef Text = StringRef(TextPtr, Length); + +formTokenWithChars(Tok, Loc, WordBegin, Length, Text); hdoc wrote: As above, I would prefer to do that as a separate refactor PR, I think it's outside the scope of this change. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [Clang][Comments] Add argument parsing for @throw @throws @exception (PR #84726)
@@ -89,6 +89,31 @@ class TextTokenRetokenizer { } } + /// Extract a template type + bool lexTemplate(SmallString<32> &WordText) { AaronBallman wrote: This should work okay for simple (most) cases, but we should have tests showing the boundaries, e.g., `Foo<(1 > 0)>` or a typo like `Foo` etc. https://github.com/llvm/llvm-project/pull/84726 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits