https://github.com/cor3ntin updated https://github.com/llvm/llvm-project/pull/208757
>From ee0debec1f54e761bdb23c57be1ff7a42a29cc54 Mon Sep 17 00:00:00 2001 From: Corentin Jabot <[email protected]> Date: Fri, 10 Jul 2026 17:25:19 +0200 Subject: [PATCH 1/2] [Clang] Detects invalid unicode characters in token formed by `##` When parsing identifiers with invalid (non-xid_continue), we would emit a diagnostics and continue, in an effort to improve recovery. However, in raw mode we would not emit a diagnostic, but still continue. So identifiers formed in raw mode could be gibberish. We fixed that by not-continuing on error in raw mode. Fixes #101342 --- clang/docs/ReleaseNotes.md | 1 + clang/lib/Lex/Lexer.cpp | 55 +++++++++++++-------- clang/test/Preprocessor/ucn-pp-identifier.c | 15 ++++++ 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 96317a3eece97..0792ea012b02f 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -800,6 +800,7 @@ latest release, please see the [Clang Web Site](https://clang.llvm.org) or the - Fixed crash when checking for overflow for unary operator that can't overflow (#GH170072) - Clang no longer handles a `" q-char-sequence "` header name as a string literal (#GH132643). - Under `-fdollars-in-identifiers`, the `$` can now appear in user-defined-literals. (#GH173985) +- Clang now detects invalid unicode characters in identifiers formed by token concatenation. (#GH101342) - Fixed an assertion where we improperly handled implicit conversions to integral types from an atomic-type with a conversion function. (#GH201770) - Fixed assertion failures involving code completion with delayed default arguments and exception specifications. (#GH200879) - Fixed a regression where calling a function that takes a class-type parameter by value inside `decltype` of a concept could be incorrectly rejected when used as a non-type template argument. (#GH175831) diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 32ad310ca2672..d5085ca6d4c8a 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -1809,11 +1809,13 @@ static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C, } } -static void diagnoseInvalidUnicodeCodepointInIdentifier( - DiagnosticsEngine &Diags, const LangOptions &LangOpts, uint32_t CodePoint, - CharSourceRange Range, bool IsFirst) { +static bool CheckCodepointValidInIdentifier(const Preprocessor *PP, + const LangOptions &LangOpts, + uint32_t CodePoint, + CharSourceRange Range, bool IsFirst, + bool Diagnose) { if (isASCII(CodePoint)) - return; + return true; bool IsExtension; bool IsIDStart = isAllowedInitiallyIDChar(CodePoint, LangOpts, IsExtension); @@ -1821,19 +1823,23 @@ static void diagnoseInvalidUnicodeCodepointInIdentifier( IsIDStart || isAllowedIDChar(CodePoint, LangOpts, IsExtension); if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue)) - return; + return true; + + if (!Diagnose) + return false; bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue; if (!IsFirst || InvalidOnlyAtStart) { - Diags.Report(Range.getBegin(), diag::err_character_not_allowed_identifier) + PP->Diag(Range.getBegin(), diag::err_character_not_allowed_identifier) << Range << codepointAsHexString(CodePoint) << int(InvalidOnlyAtStart) << FixItHint::CreateRemoval(Range); } else { - Diags.Report(Range.getBegin(), diag::err_character_not_allowed) + PP->Diag(Range.getBegin(), diag::err_character_not_allowed) << Range << codepointAsHexString(CodePoint) << FixItHint::CreateRemoval(Range); } + return false; } bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, @@ -1847,13 +1853,15 @@ bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, if (!isAllowedIDChar(CodePoint, LangOpts, IsExtension)) { if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint)) return false; - if (!isLexingRawMode() && !ParsingPreprocessorDirective && - !PP->isPreprocessedOutput()) - diagnoseInvalidUnicodeCodepointInIdentifier( - PP->getDiagnostics(), LangOpts, CodePoint, - makeCharRange(*this, CurPtr, UCNPtr), - /*IsFirst=*/false); + bool DiagnoseAndContinue = !isLexingRawMode() && + !ParsingPreprocessorDirective && + !PP->isPreprocessedOutput(); + if (!CheckCodepointValidInIdentifier( + PP, LangOpts, CodePoint, makeCharRange(*this, CurPtr, UCNPtr), + /*IsFirst=*/false, DiagnoseAndContinue) && + !DiagnoseAndContinue) + return false; // We got a unicode codepoint that is neither a space nor a // a valid identifier part. // Carry on as if the codepoint was valid for recovery purposes. @@ -1900,11 +1908,16 @@ bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) { if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint)) return false; - if (!isLexingRawMode() && !ParsingPreprocessorDirective && - !PP->isPreprocessedOutput()) - diagnoseInvalidUnicodeCodepointInIdentifier( - PP->getDiagnostics(), LangOpts, CodePoint, - makeCharRange(*this, CharStart, UnicodePtr), /*IsFirst=*/false); + bool DiagnoseAndContinue = !isLexingRawMode() && + !ParsingPreprocessorDirective && + !PP->isPreprocessedOutput(); + + if (!CheckCodepointValidInIdentifier( + PP, LangOpts, CodePoint, + makeCharRange(*this, CharStart, UnicodePtr), /*IsFirst=*/false, + DiagnoseAndContinue) && + !DiagnoseAndContinue) + return false; // We got a unicode codepoint that is neither a space nor a // a valid identifier part. Carry on as if the codepoint was // valid for recovery purposes. @@ -1960,9 +1973,9 @@ bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C, // loophole in the mapping of Unicode characters to basic character set // characters that allows us to map these particular characters to, say, // whitespace. - diagnoseInvalidUnicodeCodepointInIdentifier( - PP->getDiagnostics(), LangOpts, C, - makeCharRange(*this, BufferPtr, CurPtr), /*IsStart*/ true); + CheckCodepointValidInIdentifier(PP, LangOpts, C, + makeCharRange(*this, BufferPtr, CurPtr), + /*IsStart=*/true, /*Diagnose=*/true); BufferPtr = CurPtr; return false; } diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c index ee008a73eb882..0cb9965822984 100644 --- a/clang/test/Preprocessor/ucn-pp-identifier.c +++ b/clang/test/Preprocessor/ucn-pp-identifier.c @@ -162,3 +162,18 @@ int a\N{LATIN CAPITAL LETTER A WITH GRAVE??>; // GH64161 int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character <U+202D> not allowed in an identifier}} + + +// GH101342 +// We should not form identifiers by concatenating invalid UCNs. + +#define DOT • +#define CONCAT_IMPL(Left, Separator, Right) Left##Separator##Right +#define CONCAT2(Left, Separator, Right) CONCAT_IMPL(Left, Separator, Right) +#define MAKE_CLASS_NAME(A, B) CONCAT2(A, DOT, B) + +// struct foo•bar {} x; +struct MAKE_CLASS_NAME(foo, bar); +// expected-error@-1 {{pasting formed 'foo•', an invalid preprocessing token}} \ +// expected-error@-1 {{pasting formed '•bar', an invalid preprocessing token}} \ +// expected-error@-1 {{expected unqualified-id}} >From 4bba8f5b7b68d9d9c95633f7d59ea0168ea21467 Mon Sep 17 00:00:00 2001 From: Corentin Jabot <[email protected]> Date: Fri, 10 Jul 2026 18:31:42 +0200 Subject: [PATCH 2/2] tests --- clang/test/Preprocessor/GH101342.c | 20 ++++++++++++++++++++ clang/test/Preprocessor/ucn-pp-identifier.c | 14 +------------- 2 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 clang/test/Preprocessor/GH101342.c diff --git a/clang/test/Preprocessor/GH101342.c b/clang/test/Preprocessor/GH101342.c new file mode 100644 index 0000000000000..5ff401bde9c8e --- /dev/null +++ b/clang/test/Preprocessor/GH101342.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 %s -fsyntax-only -pedantic -verify +// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify + +// GH101342 +// We should not form identifiers by concatenating invalid UCNs. + +#define DOT • +#define CONCAT_IMPL(Left, Separator, Right) Left##Separator##Right +#define CONCAT(Left, Separator, Right) CONCAT_IMPL(Left, Separator, Right) +#define MAKE_CLASS_NAME(A, B) CONCAT(A, DOT, B) + +// struct foo•bar {} x; +struct MAKE_CLASS_NAME(foo, bar); +// expected-error@-1 {{pasting formed 'foo•', an invalid preprocessing token}} \ +// expected-error@-1 {{pasting formed '•bar', an invalid preprocessing token}} +#ifdef __cplusplus +// expected-error@-4 {{expected unqualified-id}} +#else +// expected-error@-6 {{expected identifier or '('}} +#endif diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c index 0cb9965822984..e702743d47a3c 100644 --- a/clang/test/Preprocessor/ucn-pp-identifier.c +++ b/clang/test/Preprocessor/ucn-pp-identifier.c @@ -164,16 +164,4 @@ int a\N{LATIN CAPITAL LETTER A WITH GRAVE??>; int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character <U+202D> not allowed in an identifier}} -// GH101342 -// We should not form identifiers by concatenating invalid UCNs. - -#define DOT • -#define CONCAT_IMPL(Left, Separator, Right) Left##Separator##Right -#define CONCAT2(Left, Separator, Right) CONCAT_IMPL(Left, Separator, Right) -#define MAKE_CLASS_NAME(A, B) CONCAT2(A, DOT, B) - -// struct foo•bar {} x; -struct MAKE_CLASS_NAME(foo, bar); -// expected-error@-1 {{pasting formed 'foo•', an invalid preprocessing token}} \ -// expected-error@-1 {{pasting formed '•bar', an invalid preprocessing token}} \ -// expected-error@-1 {{expected unqualified-id}} + _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
