sammccall created this revision.
sammccall added reviewers: ilya-biryukov, kbobyrev.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Useful when positions are used to target nodes, with before/after ambiguity.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D71356
Files:
clang/include/clang/Tooling/Syntax/Tokens.h
clang/lib/Tooling/Syntax/Tokens.cpp
clang/unittests/Tooling/Syntax/TokensTest.cpp
Index: clang/unittests/Tooling/Syntax/TokensTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -793,4 +793,45 @@
ActualMacroRanges.push_back(Expansion->range(SM));
EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);
}
+
+TEST_F(TokenBufferTest, Touching) {
+ llvm::Annotations Code("^i^nt^ ^a^b^=^1;^");
+ recordTokens(Code.code());
+
+ auto Touching = [&](int Index) {
+ SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
+ Code.points()[Index]);
+ return spelledTokensTouching(Loc, Buffer);
+ };
+ auto Identifier = [&](int Index) {
+ SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
+ Code.points()[Index]);
+ const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer);
+ return Tok ? Tok->text(*SourceMgr) : "";
+ };
+
+ EXPECT_THAT(Touching(0), SameRange(findSpelled("int")));
+ EXPECT_EQ(Identifier(0), "");
+ EXPECT_THAT(Touching(1), SameRange(findSpelled("int")));
+ EXPECT_EQ(Identifier(1), "");
+ EXPECT_THAT(Touching(2), SameRange(findSpelled("int")));
+ EXPECT_EQ(Identifier(2), "");
+
+ EXPECT_THAT(Touching(3), SameRange(findSpelled("ab")));
+ EXPECT_EQ(Identifier(3), "ab");
+ EXPECT_THAT(Touching(4), SameRange(findSpelled("ab")));
+ EXPECT_EQ(Identifier(4), "ab");
+
+ EXPECT_THAT(Touching(5), SameRange(findSpelled("ab =")));
+ EXPECT_EQ(Identifier(5), "ab");
+
+ EXPECT_THAT(Touching(6), SameRange(findSpelled("= 1")));
+ EXPECT_EQ(Identifier(6), "");
+
+ EXPECT_THAT(Touching(7), SameRange(findSpelled(";")));
+ EXPECT_EQ(Identifier(7), "");
+
+ ASSERT_EQ(Code.points().size(), 8u);
+}
+
} // namespace
Index: clang/lib/Tooling/Syntax/Tokens.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Tokens.cpp
+++ clang/lib/Tooling/Syntax/Tokens.cpp
@@ -248,6 +248,29 @@
return E;
}
+llvm::ArrayRef<syntax::Token>
+syntax::spelledTokensTouching(SourceLocation Loc,
+ const syntax::TokenBuffer &Tokens) {
+ assert(Loc.isFileID());
+ llvm::ArrayRef<syntax::Token> All =
+ Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc));
+ // SourceLocation < SourceLocation is OK for one file ID.
+ auto *Right = llvm::partition_point(
+ All, [&](const syntax::Token &Tok) { return Tok.location() < Loc; });
+ bool AcceptRight = Right != All.end() && !(Loc < Right->location());
+ bool AcceptLeft = Right != All.begin() && !((Right - 1)->endLocation() < Loc);
+ return llvm::makeArrayRef(Right - int(AcceptLeft), Right + int(AcceptRight));
+}
+
+const syntax::Token *
+syntax::spelledIdentifierTouching(SourceLocation Loc,
+ const syntax::TokenBuffer &Tokens) {
+ for (const syntax::Token &Tok : spelledTokensTouching(Loc, Tokens))
+ if (Tok.kind() == tok::identifier)
+ return &Tok;
+ return nullptr;
+}
+
std::vector<const syntax::Token *>
TokenBuffer::macroExpansions(FileID FID) const {
auto FileIt = Files.find(FID);
Index: clang/include/clang/Tooling/Syntax/Tokens.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Tokens.h
+++ clang/include/clang/Tooling/Syntax/Tokens.h
@@ -309,6 +309,17 @@
const SourceManager *SourceMgr;
};
+/// The spelled tokens that overlap or touch a spelling location Loc.
+/// This always returns 0-2 tokens.
+llvm::ArrayRef<syntax::Token>
+spelledTokensTouching(SourceLocation Loc, const syntax::TokenBuffer &Tokens);
+
+/// The identifier token that overlaps or touches a spelling location Loc.
+/// If there is none, returns nullptr.
+const syntax::Token *
+spelledIdentifierTouching(SourceLocation Loc,
+ const syntax::TokenBuffer &Tokens);
+
/// Lex the text buffer, corresponding to \p FID, in raw mode and record the
/// resulting spelled tokens. Does minimal post-processing on raw identifiers,
/// setting the appropriate token kind (instead of the raw_identifier reported
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
