Author: bruno Date: Sat Jan 30 18:47:51 2016 New Revision: 259311 URL: http://llvm.org/viewvc/llvm-project?rev=259311&view=rev Log: [Parser] Update CachedTokens while parsing ObjectiveC template argument list
Consider the following ObjC++ snippet: -- @protocol PA; @protocol PB; @class NSArray<ObjectType>; typedef int some_t; id<PA> FA(NSArray<id<PB>> *h, some_t group); -- This would hit an assertion in the parser after generating an annotation token while trying to update the token cache: Assertion failed: (CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc() && "The annotation should be until the most recent cached token") ... 7 clang::Preprocessor::AnnotatePreviousCachedTokens(clang::Token const&) + 494 8 clang::Parser::TryAnnotateTypeOrScopeTokenAfterScopeSpec(bool, bool, clang::CXXScopeSpec&, bool) + 1163 9 clang::Parser::TryAnnotateTypeOrScopeToken(bool, bool) + 361 10 clang::Parser::isCXXDeclarationSpecifier(clang::Parser::TPResult, bool*) + 598 ... The cached preprocessor token in this case is: greatergreater '>>' Loc=<testcase.mm:7:24> while the annotation ("NSArray<id<PB>>") ends at "testcase.mm:7:25", hence the assertion. Properly update the CachedTokens during template parsing to contain two greater tokens instead of a greatergreater. Differential Revision: http://reviews.llvm.org/D15173 rdar://problem/23494277 Modified: cfe/trunk/include/clang/Lex/Preprocessor.h cfe/trunk/lib/Lex/PPCaching.cpp cfe/trunk/lib/Parse/ParseTemplate.cpp cfe/trunk/test/Parser/objcxx11-protocol-in-template.mm Modified: cfe/trunk/include/clang/Lex/Preprocessor.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=259311&r1=259310&r2=259311&view=diff ============================================================================== --- cfe/trunk/include/clang/Lex/Preprocessor.h (original) +++ cfe/trunk/include/clang/Lex/Preprocessor.h Sat Jan 30 18:47:51 2016 @@ -1185,6 +1185,17 @@ public: return CachedTokens[CachedLexPos-1].getLastLoc(); } + /// \brief Whether \p Tok is the most recent token (`CachedLexPos - 1`) in + /// CachedTokens. + bool IsPreviousCachedToken(const Token &Tok) const; + + /// \brief Replace token in `CachedLexPos - 1` in CachedTokens by the tokens + /// in \p NewToks. + /// + /// Useful when a token needs to be split in smaller ones and CachedTokens + /// most recent token must to be updated to reflect that. + void ReplacePreviousCachedToken(ArrayRef<Token> NewToks); + /// \brief Replace the last token with an annotation token. /// /// Like AnnotateCachedTokens(), this routine replaces an Modified: cfe/trunk/lib/Lex/PPCaching.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPCaching.cpp?rev=259311&r1=259310&r2=259311&view=diff ============================================================================== --- cfe/trunk/lib/Lex/PPCaching.cpp (original) +++ cfe/trunk/lib/Lex/PPCaching.cpp Sat Jan 30 18:47:51 2016 @@ -116,3 +116,29 @@ void Preprocessor::AnnotatePreviousCache } } } + +bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const { + // There's currently no cached token... + if (!CachedLexPos) + return false; + + const Token LastCachedTok = CachedTokens[CachedLexPos - 1]; + if (LastCachedTok.getKind() != Tok.getKind()) + return false; + + int RelOffset = 0; + if ((!getSourceManager().isInSameSLocAddrSpace( + Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) || + RelOffset) + return false; + + return true; +} + +void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) { + assert(CachedLexPos != 0 && "Expected to have some cached tokens"); + CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(), + NewToks.end()); + CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size()); + CachedLexPos += NewToks.size() - 1; +} Modified: cfe/trunk/lib/Parse/ParseTemplate.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseTemplate.cpp?rev=259311&r1=259310&r2=259311&view=diff ============================================================================== --- cfe/trunk/lib/Parse/ParseTemplate.cpp (original) +++ cfe/trunk/lib/Parse/ParseTemplate.cpp Sat Jan 30 18:47:51 2016 @@ -831,6 +831,7 @@ bool Parser::ParseGreaterThanInTemplateL } // Strip the initial '>' from the token. + Token PrevTok = Tok; if (RemainingToken == tok::equal && Next.is(tok::equal) && areTokensAdjacent(Tok, Next)) { // Join two adjacent '=' tokens into one, for cases like: @@ -847,6 +848,17 @@ bool Parser::ParseGreaterThanInTemplateL PP.getSourceManager(), getLangOpts())); + // The advance from '>>' to '>' in a ObjectiveC template argument list needs + // to be properly reflected in the token cache to allow correct interaction + // between annotation and backtracking. + if (ObjCGenericList && PrevTok.getKind() == tok::greatergreater && + RemainingToken == tok::greater && PP.IsPreviousCachedToken(PrevTok)) { + PrevTok.setKind(RemainingToken); + PrevTok.setLength(1); + Token NewToks[] = {PrevTok, Tok}; + PP.ReplacePreviousCachedToken(NewToks); + } + if (!ConsumeLastToken) { // Since we're not supposed to consume the '>' token, we need to push // this token and revert the current token back to the '>'. Modified: cfe/trunk/test/Parser/objcxx11-protocol-in-template.mm URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Parser/objcxx11-protocol-in-template.mm?rev=259311&r1=259310&r2=259311&view=diff ============================================================================== --- cfe/trunk/test/Parser/objcxx11-protocol-in-template.mm (original) +++ cfe/trunk/test/Parser/objcxx11-protocol-in-template.mm Sat Jan 30 18:47:51 2016 @@ -8,3 +8,11 @@ template<class T> class vector {}; vector<id<P>> v; vector<vector<id<P>>> v2; + +@protocol PA; +@protocol PB; + +@class NSArray<ObjectType>; +typedef int some_t; + +id<PA> FA(NSArray<id<PB>> *h, some_t group); _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits