llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang Author: Chen Miao (ChenMiaoi) <details> <summary>Changes</summary> When returning whitespace tokens, the lexer does not propagate leading-space information to the following token. Preserve this information in whitespace retention mode, accounting for whether the whitespace ends with a newline. Avoid inserting an extra space in traditional preprocessor output when the previous token already supplies whitespace. --- Full diff: https://github.com/llvm/llvm-project/pull/236408.diff 4 Files Affected: - (modified) clang/lib/Frontend/PrintPreprocessedOutput.cpp (+13-2) - (modified) clang/lib/Lex/Lexer.cpp (+2-2) - (modified) clang/test/Preprocessor/traditional-cpp.c (+8) - (modified) clang/unittests/Lex/LexerTest.cpp (+46) ``````````diff diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index ab5ac8a4a69c6d..00c122ed7a56c9 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -792,6 +792,17 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, !Tok.is(tok::annot_embed) && !Tok.is(tok::annot_module_name))) return; + bool HasLeadingSpace = Tok.hasLeadingSpace(); + if (HasLeadingSpace && PP.getLangOpts().TraditionalCPP && + PrevTok.is(tok::unknown) && PrevTok.getLength()) { + // Traditional preprocessing emits whitespace as tokens. Do not add another + // space if the previous token already supplied it. + SmallString<128> SpellingBuffer; + StringRef Spelling = PP.getSpelling(PrevTok, SpellingBuffer); + if (!Spelling.empty() && isWhitespace(Spelling.back())) + HasLeadingSpace = false; + } + // EmittedDirectiveOnThisLine takes priority over RequireSameLine. if ((!RequireSameLine || EmittedDirectiveOnThisLine) && MoveToLine(Tok, /*RequireStartOfLine=*/EmittedDirectiveOnThisLine)) { @@ -808,7 +819,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, // expect leading white space, if a macro expansion in column 1 starts // with an empty macro argument, or an empty nested macro expansion. In // this case, move the token to column 2. - if (ColNo == 1 && Tok.hasLeadingSpace()) + if (ColNo == 1 && HasLeadingSpace) ColNo = 2; // This hack prevents stuff like: @@ -831,7 +842,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, // whitespace-minimization mode // - The whitespace is necessary to keep the tokens apart and there is not // already a newline between them - if (RequireSpace || (!MinimizeWhitespace && Tok.hasLeadingSpace()) || + if (RequireSpace || (!MinimizeWhitespace && HasLeadingSpace) || ((EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) && AvoidConcat(PrevPrevTok, PrevTok, Tok))) *OS << ' '; diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index b659780bac593a..8a3676d94503f7 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -2663,7 +2663,7 @@ bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { IsAtStartOfLine = true; IsAtPhysicalStartOfLine = true; } - // FIXME: The next token will not have LeadingSpace set. + HasLeadingSpace = !isVerticalWhitespace(CurPtr[-1]); return true; } @@ -3875,7 +3875,7 @@ bool Lexer::LexTokenInternal(Token &Result) { // whitespace. if (isKeepWhitespaceMode()) { FormTokenWithChars(Result, CurPtr, tok::unknown); - // FIXME: The next token will not have LeadingSpace set. + HasLeadingSpace = true; return true; } diff --git a/clang/test/Preprocessor/traditional-cpp.c b/clang/test/Preprocessor/traditional-cpp.c index f311be0bb59360..23fddb1cdc4147 100644 --- a/clang/test/Preprocessor/traditional-cpp.c +++ b/clang/test/Preprocessor/traditional-cpp.c @@ -29,6 +29,14 @@ foo // bar indented! tab separated values +/* CHECK: {{^}}a b{{$}} + * CHECK: {{^}}a b{{$}} + * CHECK: {{^}} indented_with_spaces{{$}} + */ +a b +a b + indented_with_spaces + #define bracket(x) >>>x<<< bracket(| spaces |) /* CHECK: {{^}}>>>| spaces |<<<{{$}} diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp index 8f632ccc5b9315..8ca1abbe826c90 100644 --- a/clang/unittests/Lex/LexerTest.cpp +++ b/clang/unittests/Lex/LexerTest.cpp @@ -719,6 +719,52 @@ TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) { 1U); } +TEST_F(LexerTest, LeadingSpaceWithWhitespaceRetention) { + const struct { + const char *Whitespace; + bool LeadingSpace; + bool StartOfLine; + } Cases[] = { + {"", false, false}, {" ", true, false}, + {"\t", true, false}, {" \t\f\v", true, false}, + {"\n", false, true}, {"\r", false, true}, + {"\r\n", false, true}, {"\n ", true, true}, + {"\r\n\t", true, true}, {" \n", false, true}, + {" \n \n", false, true}, {" \n \n\t", true, true}, + }; + + for (const auto &Case : Cases) { + for (bool KeepWhitespace : {false, true}) { + SCOPED_TRACE(Case.Whitespace); + SCOPED_TRACE(KeepWhitespace); + auto Buffer = llvm::MemoryBuffer::getMemBufferCopy( + std::string("a+") + Case.Whitespace + "b+c"); + Lexer L(SourceLocation(), LangOpts, Buffer->getBufferStart(), + Buffer->getBufferStart(), Buffer->getBufferEnd()); + L.SetKeepWhitespaceMode(KeepWhitespace); + + for (unsigned I = 0; I != 5; ++I) { + Token T; + do { + L.LexFromRawLexer(T); + if (T.is(tok::unknown)) { + ASSERT_TRUE(KeepWhitespace); + ASSERT_GT(T.getLength(), 0u); + } + } while (T.is(tok::unknown)); + + ASSERT_EQ(T.getKind(), + I == 1 || I == 3 ? tok::plus : tok::raw_identifier); + EXPECT_EQ(T.hasLeadingSpace(), I == 2 && Case.LeadingSpace); + EXPECT_EQ(T.isAtStartOfLine(), I == 0 || (I == 2 && Case.StartOfLine)); + } + Token T; + L.LexFromRawLexer(T); + EXPECT_TRUE(T.is(tok::eof)); + } + } +} + TEST_F(LexerTest, RawAndNormalLexSameForLineComments) { const llvm::StringLiteral Source = R"cpp( // First line comment. `````````` </details> https://github.com/llvm/llvm-project/pull/236408 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
