https://github.com/anutosh491 updated https://github.com/llvm/llvm-project/pull/127569
>From 7f0c62d7a5febc7a402f1887d984182de2b6054f Mon Sep 17 00:00:00 2001 From: anutosh491 <[email protected]> Date: Mon, 19 Oct 2026 10:24:22 +0530 Subject: [PATCH] [clang-repl] Use EOF for incremental input --- clang/include/clang/Basic/TokenKinds.def | 3 -- clang/include/clang/Parse/Parser.h | 3 +- .../lib/Frontend/PrintPreprocessedOutput.cpp | 7 +-- clang/lib/Interpreter/IncrementalParser.cpp | 6 +-- clang/lib/Lex/PPLexerChange.cpp | 8 +-- clang/lib/Lex/Preprocessor.cpp | 15 ++++-- clang/lib/Parse/ParseCXXInlineMethods.cpp | 1 - clang/lib/Parse/ParseDecl.cpp | 6 +-- clang/lib/Parse/ParseStmt.cpp | 6 ++- clang/lib/Parse/Parser.cpp | 2 - clang/lib/Serialization/ASTReader.cpp | 1 - clang/lib/Serialization/ASTWriter.cpp | 1 - .../Interpreter/repl-input-end-recovery.cpp | 50 +++++++++++++++++++ 13 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 clang/test/Interpreter/repl-input-end-recovery.cpp diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 528825ad68b8dc..802947e31cbc18 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -917,9 +917,6 @@ ANNOTATION(module_name) // into the name of a header unit. ANNOTATION(header_unit) -// Annotation for end of input in clang-repl. -ANNOTATION(repl_input_end) - // Annotation for #embed ANNOTATION(embed) diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index 9a1abd4f3872de..7f91c7d4670ff1 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -829,8 +829,7 @@ class Parser : public CodeCompletionHandler { bool isEofOrEom() { tok::TokenKind Kind = Tok.getKind(); return Kind == tok::eof || Kind == tok::annot_module_begin || - Kind == tok::annot_module_end || Kind == tok::annot_module_include || - Kind == tok::annot_repl_input_end; + Kind == tok::annot_module_end || Kind == tok::annot_module_include; } static void setTypeAnnotation(Token &Tok, TypeResult T) { diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index 23da82769e2aeb..ab5ac8a4a69c6d 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -789,8 +789,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, if (Tok.is(tok::eof) || (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) && !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) && - !Tok.is(tok::annot_repl_input_end) && !Tok.is(tok::annot_embed) && - !Tok.is(tok::annot_module_name))) + !Tok.is(tok::annot_embed) && !Tok.is(tok::annot_module_name))) return; // EmittedDirectiveOnThisLine takes priority over RequireSameLine. @@ -946,8 +945,6 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, // -traditional-cpp the lexer keeps /all/ whitespace, including comments. PP.Lex(Tok); continue; - } else if (Tok.is(tok::annot_repl_input_end)) { - // Fall through to exit the loop. } else if (Tok.is(tok::eod)) { // Don't print end of directive tokens, since they are typically newlines // that mess up our line tracking. These come from unknown pre-processor @@ -1084,7 +1081,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, Callbacks->setEmittedTokensOnThisLine(); IsStartOfLine = false; - if (Tok.is(tok::eof) || Tok.is(tok::annot_repl_input_end)) + if (Tok.is(tok::eof)) break; PP.Lex(Tok); diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp index 3cfabb5e5daa18..4d1ba3473d48ef 100644 --- a/clang/lib/Interpreter/IncrementalParser.cpp +++ b/clang/lib/Interpreter/IncrementalParser.cpp @@ -65,7 +65,7 @@ IncrementalParser::ParseOrWrapTopLevelDecl() { C.addTranslationUnitDecl(); // Skip previous eof due to last incremental input. - if (P->getCurToken().is(tok::annot_repl_input_end)) { + if (P->getCurToken().is(tok::eof)) { P->ConsumeAnyToken(); // FIXME: Clang does not call ExitScope on finalizing the regular TU, we // might want to do that around HandleEndOfTranslationUnit. @@ -165,11 +165,11 @@ IncrementalParser::Parse(llvm::StringRef input) { Token Tok; do { PP.Lex(Tok); - } while (Tok.isNot(tok::annot_repl_input_end)); + } while (Tok.isNot(tok::eof)); } else { Token AssertTok; PP.Lex(AssertTok); - assert(AssertTok.is(tok::annot_repl_input_end) && + assert(AssertTok.is(tok::eof) && "Lexer must be EOF when starting incremental parse!"); } diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp index 552902e70f094b..995a44dd2904a1 100644 --- a/clang/lib/Lex/PPLexerChange.cpp +++ b/clang/lib/Lex/PPLexerChange.cpp @@ -532,13 +532,7 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { Result.startToken(); CurLexer->BufferPtr = EndPos; - if (getLangOpts().IncrementalExtensions) { - CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end); - Result.setAnnotationEndLoc(Result.getLocation()); - Result.setAnnotationValue(nullptr); - } else { - CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); - } + CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); if (isCodeCompletionEnabled()) { // Inserting the code-completion point increases the source buffer by 1, diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 84907adc5d744a..a8717424f1d155 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1020,12 +1020,22 @@ void Preprocessor::Lex(Token &Result) { // This token is injected to represent the translation of '#include "a.h"' // into "import a.h;". Mimic the notional ';'. case tok::annot_module_include: - case tok::annot_repl_input_end: case tok::semi: TrackGMFState.handleSemi(); StdCXXImportSeqState.handleSemi(); ModuleDeclState.handleSemi(); break; + case tok::eof: + if (getLangOpts().IncrementalExtensions) { + TrackGMFState.handleSemi(); + StdCXXImportSeqState.handleSemi(); + ModuleDeclState.handleSemi(); + } else { + TrackGMFState.handleMisc(); + StdCXXImportSeqState.handleMisc(); + ModuleDeclState.handleMisc(); + } + break; case tok::header_name: case tok::annot_header_unit: StdCXXImportSeqState.handleHeaderName(); @@ -1098,8 +1108,7 @@ void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) { while (1) { Token Tok; Lex(Tok); - if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod, - tok::annot_repl_input_end)) + if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod)) break; if (Tokens != nullptr) Tokens->push_back(Tok); diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp index 6b478ec2e36f6d..22e15cbe5b260e 100644 --- a/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -902,7 +902,6 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2, case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: - case tok::annot_repl_input_end: // Ran out of tokens. return false; diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 40249287184446..5b76b5d2ef274b 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -2155,7 +2155,6 @@ void Parser::SkipMalformedDecl() { case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: - case tok::annot_repl_input_end: return; default: @@ -5899,9 +5898,8 @@ Parser::DeclGroupPtrTy Parser::ParseTopLevelStmtDecl() { TLSD->setStmt(R.get()); } - if (Tok.is(tok::annot_repl_input_end) && - Tok.getAnnotationValue() != nullptr) { - ConsumeAnnotationToken(); + if (Tok.is(tok::eof) && Tok.getEofData()) { + ConsumeToken(); TLSD->setSemiMissing(); } diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp index 14bea6a1a79487..de82003f109328 100644 --- a/clang/lib/Parse/ParseStmt.cpp +++ b/clang/lib/Parse/ParseStmt.cpp @@ -584,7 +584,9 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) { Token *CurTok = nullptr; // If the semicolon is missing at the end of REPL input, we want to print // the result. Note we shouldn't eat the token since the callback needs it. - if (Tok.is(tok::annot_repl_input_end)) + // Preserve the data on EOF tokens that delimit cached declarations. + if (Tok.is(tok::eof) && getLangOpts().IncrementalExtensions && + !Tok.getEofData()) CurTok = &Tok; else // Otherwise, eat the semicolon. @@ -592,7 +594,7 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) { StmtResult R = handleExprStmt(Expr, StmtCtx); if (CurTok && !R.isInvalid()) - CurTok->setAnnotationValue(R.get()); + CurTok->setEofData(R.get()); return R; } diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index c5d6a17fbfcd34..f0f4c7195381e5 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -354,7 +354,6 @@ bool Parser::SkipUntil(ArrayRef<tok::TokenKind> Toks, SkipUntilFlags Flags) { case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: - case tok::annot_repl_input_end: // Stop before we change submodules. They generally indicate a "good" // place to pick up parsing again (except in the special case where // we're trying to skip to EOF). @@ -690,7 +689,6 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, return false; case tok::eof: - case tok::annot_repl_input_end: // Check whether -fmax-tokens= was reached. if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) { PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total) diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index e6f6be496e5e58..56f1aa8a7f932a 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -2194,7 +2194,6 @@ Token ASTReader::ReadToken(ModuleFile &M, const RecordDataImpl &Record, case tok::annot_pragma_unused: case tok::annot_pragma_openacc: case tok::annot_pragma_openacc_end: - case tok::annot_repl_input_end: break; default: llvm_unreachable("missing deserialization code for annotation token"); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 0afba1c84fd627..48735e32eb5f66 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -5432,7 +5432,6 @@ void ASTWriter::AddToken(const Token &Tok, RecordDataImpl &Record) { case tok::annot_pragma_unused: case tok::annot_pragma_openacc: case tok::annot_pragma_openacc_end: - case tok::annot_repl_input_end: break; default: llvm_unreachable("missing serialization code for annotation token"); diff --git a/clang/test/Interpreter/repl-input-end-recovery.cpp b/clang/test/Interpreter/repl-input-end-recovery.cpp new file mode 100644 index 00000000000000..0b9d94117d4edd --- /dev/null +++ b/clang/test/Interpreter/repl-input-end-recovery.cpp @@ -0,0 +1,50 @@ +// REQUIRES: host-supports-jit +// RUN: cat %s | clang-repl > %t.out 2> %t.err +// RUN: FileCheck %s --check-prefix=OUT < %t.out +// RUN: FileCheck %s --check-prefix=ERR --implicit-check-not='fatal error:' < %t.err +// RUN: not clang-repl -Xcc -fms-extensions '__pragma(section("broken", )' \ +// RUN: 'extern "C" int puts(const char *); ::puts("recovered");' 2>&1 \ +// RUN: | FileCheck %s --check-prefix=MS + +// MS: error: missing terminating ')' character +// MS: error: Parsing failed. +// MS: {{^recovered$}} + +extern "C" int printf(const char *, ...); + +void foo() { int x = 5; +// ERR: error: expected '}' +// ERR: error: Parsing failed. + +int g1 = 0; void foo() { g1 = 5; } foo(); printf("g1 = %d\n", ::g1); +// OUT: g1 = 5 + +void (*test)() = [](){ if } +// ERR: error: expected '(' after 'if' +// ERR: error: Parsing failed. + +int g2 = 0; void (*test)() = [](){ if (1) g2 = 7; }; test(); printf("g2 = %d\n", ::g2); +// OUT: g2 = 7 + +namespace myspace { +// ERR: error: expected '}' +// ERR: error: Parsing failed. + +namespace myspace { int v = 11; } printf("v = %d\n", ::myspace::v); +// OUT: v = 11 + +extern "C" { int cvalue; +// ERR: error: expected '}' +// ERR: error: Parsing failed. + +extern "C" { int cvalue = 13; } printf("cvalue = %d\n", ::cvalue); +// OUT: cvalue = 13 + +struct X { using type = int }; +// ERR: error: expected ';' after alias declaration +// ERR: error: Parsing failed. + +struct X { using type = int; }; X::type t = 3; printf("t = %d\n", ::t); +// OUT: t = 3 + +%quit _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
