Author: jvikstrom Date: Tue Aug 20 06:34:01 2019 New Revision: 369385 URL: http://llvm.org/viewvc/llvm-project?rev=369385&view=rev Log: [Syntax] Added function to get macro expansion tokens to TokenBuffer.
Summary: Returns the first token in every mapping where the token is an identifier. This API is required to be able to highlight macro expansions in clangd. Reviewers: hokein, ilya-biryukov Subscribers: kadircet, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D66470 Modified: cfe/trunk/include/clang/Tooling/Syntax/Tokens.h cfe/trunk/lib/Tooling/Syntax/Tokens.cpp cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp Modified: cfe/trunk/include/clang/Tooling/Syntax/Tokens.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Syntax/Tokens.h?rev=369385&r1=369384&r2=369385&view=diff ============================================================================== --- cfe/trunk/include/clang/Tooling/Syntax/Tokens.h (original) +++ cfe/trunk/include/clang/Tooling/Syntax/Tokens.h Tue Aug 20 06:34:01 2019 @@ -236,6 +236,16 @@ public: /// #pragma, etc. llvm::ArrayRef<syntax::Token> spelledTokens(FileID FID) const; + /// Get all tokens that expand a macro in \p FID. For the following input + /// #define FOO B + /// #define FOO2(X) int X + /// FOO2(XY) + /// int B; + /// FOO; + /// macroExpansions() returns {"FOO2", "FOO"} (from line 3 and 5 + /// respecitvely). + std::vector<const syntax::Token *> macroExpansions(FileID FID) const; + const SourceManager &sourceManager() const { return *SourceMgr; } std::string dumpForTests() const; Modified: cfe/trunk/lib/Tooling/Syntax/Tokens.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Syntax/Tokens.cpp?rev=369385&r1=369384&r2=369385&view=diff ============================================================================== --- cfe/trunk/lib/Tooling/Syntax/Tokens.cpp (original) +++ cfe/trunk/lib/Tooling/Syntax/Tokens.cpp Tue Aug 20 06:34:01 2019 @@ -232,6 +232,21 @@ TokenBuffer::expansionStartingAt(const s return E; } +std::vector<const syntax::Token *> +TokenBuffer::macroExpansions(FileID FID) const { + auto FileIt = Files.find(FID); + assert(FileIt != Files.end() && "file not tracked by token buffer"); + auto &File = FileIt->second; + std::vector<const syntax::Token *> Expansions; + auto &Spelled = File.SpelledTokens; + for (auto Mapping : File.Mappings) { + const syntax::Token *Token = &Spelled[Mapping.BeginSpelled]; + if (Token->kind() == tok::TokenKind::identifier) + Expansions.push_back(Token); + } + return Expansions; +} + std::vector<syntax::Token> syntax::tokenize(FileID FID, const SourceManager &SM, const LangOptions &LO) { std::vector<syntax::Token> Tokens; Modified: cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp?rev=369385&r1=369384&r2=369385&view=diff ============================================================================== --- cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp (original) +++ cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp Tue Aug 20 06:34:01 2019 @@ -755,4 +755,27 @@ TEST_F(TokenBufferTest, TokensToFileRang // We don't test assertion failures because death tests are slow. } +TEST_F(TokenBufferTest, macroExpansions) { + llvm::Annotations Code(R"cpp( + #define FOO B + #define FOO2 BA + #define CALL(X) int X + #define G CALL(FOO2) + int B; + $macro[[FOO]]; + $macro[[CALL]](A); + $macro[[G]]; + )cpp"); + recordTokens(Code.code()); + auto &SM = *SourceMgr; + auto Expansions = Buffer.macroExpansions(SM.getMainFileID()); + std::vector<FileRange> ExpectedMacroRanges; + for (auto Range : Code.ranges("macro")) + ExpectedMacroRanges.push_back( + FileRange(SM.getMainFileID(), Range.Begin, Range.End)); + std::vector<FileRange> ActualMacroRanges; + for (auto Expansion : Expansions) + ActualMacroRanges.push_back(Expansion->range(SM)); + EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges); +} } // namespace _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits