https://github.com/lenary updated https://github.com/llvm/llvm-project/pull/213765
>From 4dfee23575066814a8d82fe15987a2b7700781b7 Mon Sep 17 00:00:00 2001 From: Sam Elliott <[email protected]> Date: Mon, 3 Aug 2026 14:18:18 -0700 Subject: [PATCH 1/2] [clang] Support Trailing Comments in Config Files Before this change, if users put trailing comments into a config file, these would be added verbatim as processed arguments. This change ensures that trailing comments after flags are not added to the processed arguments. This means that comments in config files work more closely to how comments in unix shell scripts work. This change is only applied to config files (i.e. from `--config=file`), and not GNU response files (i.e. from `@file`). --- clang/docs/ReleaseNotes.md | 2 + clang/docs/UsersManual.md | 15 +++-- llvm/lib/Support/CommandLine.cpp | 45 +++++++++++-- llvm/unittests/Support/CommandLineTest.cpp | 73 ++++++++++++++++++++++ 4 files changed, 126 insertions(+), 9 deletions(-) diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 7108392abbaa14..522ff0d517747f 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -173,6 +173,8 @@ features cannot lower the translation-unit ABI level; - All options of the `-fzero-call-used-regs` compiler flag are now allowed on RISC-V. +- `--config` files now support trailing `#` comments after an option, in addition to whole-line comments. + ### Removed Compiler Flags ### Attribute Changes in Clang diff --git a/clang/docs/UsersManual.md b/clang/docs/UsersManual.md index bce811eed283b7..1418d7efdd4150 100644 --- a/clang/docs/UsersManual.md +++ b/clang/docs/UsersManual.md @@ -1099,11 +1099,14 @@ x86_64-pc-linux-gnu.cfg It is not an error if either of these files is not found. -The configuration file consists of command-line options specified on one or -more lines. Lines composed of whitespace characters only are ignored as well as -lines in which the first non-blank character is `#`. Long options may be split -between several lines by a trailing backslash. Here is an example of a -configuration file: +The configuration file consists of command-line options specified on one or more +lines. Lines composed of whitespace characters only are ignored as well as lines +in which the first non-blank character is `#`. A `#` elsewhere on a line also +starts a comment that runs to the end of the line, as long as it begins a new +word (i.e. is preceded by whitespace) and is not inside a quoted string. Long +options may be split between several lines by a trailing backslash. + +Here is an example of a configuration file: ``` # Several options on line @@ -1115,6 +1118,8 @@ include/c++/5.4.0 # other config files may be included @linux.options + +-Wall # trailing comments are supported ``` Files included by `@file` directives in configuration files are resolved diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp index a9102fe30efe33..eced4ac6f10fd1 100644 --- a/llvm/lib/Support/CommandLine.cpp +++ b/llvm/lib/Support/CommandLine.cpp @@ -1116,10 +1116,21 @@ void cl::tokenizeConfigFile(StringRef Source, StringSaver &Saver, ++Cur; continue; } - // Find end of the current line. + // Find end of the current line. As well as splicing backslash-newline + // continuations, this recognizes a trailing "# comment" (a '#' that + // begins a new, unquoted token) and, like the whole-line comment case + // above, extends it to the next literal newline without honoring + // escapes or continuations inside the comment text. const char *Start = Cur; + const char *LineEnd = nullptr; + bool InQuote = false; + char QuoteChar = 0; + // Start is known not to be whitespace or '#' (checked above), so we are + // logically at the start of the first token on this line. + bool AtTokenStart = true; for (const char *End = Source.end(); Cur != End; ++Cur) { - if (*Cur == '\\') { + char C = *Cur; + if (C == '\\') { if (Cur + 1 != End) { ++Cur; if (*Cur == '\n' || @@ -1128,13 +1139,39 @@ void cl::tokenizeConfigFile(StringRef Source, StringSaver &Saver, if (*Cur == '\r') ++Cur; Start = Cur + 1; + // AtTokenStart carries over unchanged: the spliced text abuts + // directly, introducing no new token boundary. + continue; } } - } else if (*Cur == '\n') + // An escaped literal character always extends the current token. + AtTokenStart = false; + continue; + } + if (InQuote) { + if (C == QuoteChar) + InQuote = false; + AtTokenStart = false; + continue; + } + if (isQuote(C)) { + InQuote = true; + QuoteChar = C; + AtTokenStart = false; + continue; + } + if (C == '\n') + break; + if (C == '#' && AtTokenStart) { + LineEnd = Cur; + while (Cur != End && *Cur != '\n') + ++Cur; break; + } + AtTokenStart = isWhitespace(C); } // Tokenize line. - Line.append(Start, Cur); + Line.append(Start, LineEnd ? LineEnd : Cur); cl::TokenizeGNUCommandLine(Line, Saver, NewArgv, MarkEOLs); } } diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp index 956d5b97c2703e..4339e1668146d8 100644 --- a/llvm/unittests/Support/CommandLineTest.cpp +++ b/llvm/unittests/Support/CommandLineTest.cpp @@ -438,6 +438,79 @@ TEST(CommandLineTest, TokenizeConfigFile11) { testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } +TEST(CommandLineTest, TokenizeConfigFileTrailingComment) { + const char *Input = "-c # comment\n"; + const char *const Output[] = {"-c"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileHashNotAtTokenStart) { + // A '#' that does not begin a new token (i.e. is not preceded by + // whitespace) is not a comment. + const char *Input = "-DFOO=1#2\n"; + const char *const Output[] = {"-DFOO=1#2"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileHashInQuotes) { + // A '#' inside a quoted string is not a comment, but a '#' that begins a + // new token after the closing quote still is. + const char *Input = "-DFOO=\"a # b\" # comment\n"; + const char *const Output[] = {"-DFOO=a # b"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileHashAfterClosingQuote) { + // A '#' immediately after a closing quote, with no intervening + // whitespace, does not begin a new token and so is not a comment. + const char *Input = "-DFOO=\"a\"#b\n"; + const char *const Output[] = {"-DFOO=a#b"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileEscapedHash) { + // A backslash-escaped '#' is not a comment start. + const char *Input = "-c \\#comment\n"; + const char *const Output[] = {"-c", "#comment"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileCommentAfterContinuationWithSpace) { + // The backslash-newline splice does not introduce a token boundary, but it + // also does not remove one that was already there: since there was a + // space before the backslash, the '#' immediately after the splice is + // still whitespace-preceded and starts a comment. + const char *Input = "-c \\\n# comment\n"; + const char *const Output[] = {"-c"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileHashAfterContinuationNoSpace) { + // With no space before the backslash, the spliced text abuts directly, so + // the '#' right after the splice does not begin a new token and is not a + // comment; it merges into a single token like other continuations (see + // TokenizeConfigFile4). + const char *Input = "abc\\\n#comment\n"; + const char *const Output[] = {"abc#comment"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileCommentAfterContinuationWithOption) { + const char *Input = "-c \\\noption # comment\n"; + const char *const Output[] = {"-c", "option"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileHashesInFlagNotComment) { + // clang's "-###" flag is made up entirely of '#' characters that never + // begin a new token (each is preceded by non-whitespace), so none of them + // should be treated as a comment start. Only the whitespace-preceded '#' + // that starts the real trailing comment should truncate the line. + const char *Input = "-### # comment\n"; + const char *const Output[] = {"-###"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + TEST(CommandLineTest, AliasesWithArguments) { static const size_t ARGC = 3; const char *const Inputs[][ARGC] = { >From 874c5e9d765e0b2497b341fb3124b4aaf8f22575 Mon Sep 17 00:00:00 2001 From: Fangrui Song <[email protected]> Date: Tue, 4 Aug 2026 23:31:04 -0700 Subject: [PATCH 2/2] [Support] Simplify config-file comment scanning Merge InQuote/QuoteChar into one variable and fold quote tracking into the '#' check chain. Check '\n' before quote handling so an unescaped newline always ends the line: quote state no longer leaks across lines, which merged the rest of the file into one argument after an unterminated quote. Trim comments and add tests. --- clang/docs/ReleaseNotes.md | 3 +- llvm/lib/Support/CommandLine.cpp | 37 ++++++---------- llvm/unittests/Support/CommandLineTest.cpp | 49 ++++++++++++++-------- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 522ff0d517747f..f45ed97b5b63b1 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -173,7 +173,8 @@ features cannot lower the translation-unit ABI level; - All options of the `-fzero-call-used-regs` compiler flag are now allowed on RISC-V. -- `--config` files now support trailing `#` comments after an option, in addition to whole-line comments. +- `--config` files now support trailing `#` comments after an option, in + addition to whole-line comments. ### Removed Compiler Flags diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp index eced4ac6f10fd1..298e79554c7c93 100644 --- a/llvm/lib/Support/CommandLine.cpp +++ b/llvm/lib/Support/CommandLine.cpp @@ -1116,17 +1116,13 @@ void cl::tokenizeConfigFile(StringRef Source, StringSaver &Saver, ++Cur; continue; } - // Find end of the current line. As well as splicing backslash-newline - // continuations, this recognizes a trailing "# comment" (a '#' that - // begins a new, unquoted token) and, like the whole-line comment case - // above, extends it to the next literal newline without honoring - // escapes or continuations inside the comment text. + // Find end of the current line, splicing backslash-newline continuations. + // A '#' that begins a new unquoted token starts a comment running to the + // next literal newline; escapes and continuations are not honored inside + // the comment. const char *Start = Cur; const char *LineEnd = nullptr; - bool InQuote = false; - char QuoteChar = 0; - // Start is known not to be whitespace or '#' (checked above), so we are - // logically at the start of the first token on this line. + char Quote = 0; bool AtTokenStart = true; for (const char *End = Source.end(); Cur != End; ++Cur) { char C = *Cur; @@ -1139,30 +1135,21 @@ void cl::tokenizeConfigFile(StringRef Source, StringSaver &Saver, if (*Cur == '\r') ++Cur; Start = Cur + 1; - // AtTokenStart carries over unchanged: the spliced text abuts - // directly, introducing no new token boundary. + // Splicing does not introduce a token boundary. continue; } } - // An escaped literal character always extends the current token. - AtTokenStart = false; - continue; - } - if (InQuote) { - if (C == QuoteChar) - InQuote = false; - AtTokenStart = false; - continue; - } - if (isQuote(C)) { - InQuote = true; - QuoteChar = C; AtTokenStart = false; continue; } if (C == '\n') break; - if (C == '#' && AtTokenStart) { + if (Quote) { + if (C == Quote) + Quote = 0; + } else if (isQuote(C)) { + Quote = C; + } else if (C == '#' && AtTokenStart) { LineEnd = Cur; while (Cur != End && *Cur != '\n') ++Cur; diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp index 4339e1668146d8..0a5ba1ebf967ee 100644 --- a/llvm/unittests/Support/CommandLineTest.cpp +++ b/llvm/unittests/Support/CommandLineTest.cpp @@ -444,51 +444,67 @@ TEST(CommandLineTest, TokenizeConfigFileTrailingComment) { testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } +TEST(CommandLineTest, TokenizeConfigFileTrailingCommentNoNewline) { + const char *Input = "-c # comment"; + const char *const Output[] = {"-c"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileLineAfterComment) { + const char *Input = "-a # comment\n-b\n"; + const char *const Output[] = {"-a", "-b"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + +TEST(CommandLineTest, TokenizeConfigFileNoContinuationInComment) { + // A backslash-newline inside a comment is not a continuation. + const char *Input = "-c # comment \\\n-d\n"; + const char *const Output[] = {"-c", "-d"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + TEST(CommandLineTest, TokenizeConfigFileHashNotAtTokenStart) { - // A '#' that does not begin a new token (i.e. is not preceded by - // whitespace) is not a comment. + // '#' not preceded by whitespace is not a comment. const char *Input = "-DFOO=1#2\n"; const char *const Output[] = {"-DFOO=1#2"}; testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } TEST(CommandLineTest, TokenizeConfigFileHashInQuotes) { - // A '#' inside a quoted string is not a comment, but a '#' that begins a - // new token after the closing quote still is. const char *Input = "-DFOO=\"a # b\" # comment\n"; const char *const Output[] = {"-DFOO=a # b"}; testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } TEST(CommandLineTest, TokenizeConfigFileHashAfterClosingQuote) { - // A '#' immediately after a closing quote, with no intervening - // whitespace, does not begin a new token and so is not a comment. const char *Input = "-DFOO=\"a\"#b\n"; const char *const Output[] = {"-DFOO=a#b"}; testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } +TEST(CommandLineTest, TokenizeConfigFileNewlineEndsQuote) { + // A literal newline ends the line even inside a quoted string. + const char *Input = "-DA=\"x\n-DB=y\n"; + const char *const Output[] = {"-DA=x", "-DB=y"}; + testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); +} + TEST(CommandLineTest, TokenizeConfigFileEscapedHash) { - // A backslash-escaped '#' is not a comment start. const char *Input = "-c \\#comment\n"; const char *const Output[] = {"-c", "#comment"}; testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } TEST(CommandLineTest, TokenizeConfigFileCommentAfterContinuationWithSpace) { - // The backslash-newline splice does not introduce a token boundary, but it - // also does not remove one that was already there: since there was a - // space before the backslash, the '#' immediately after the splice is - // still whitespace-preceded and starts a comment. + // The splice keeps the preceding whitespace in effect, so the '#' begins a + // token and starts a comment. const char *Input = "-c \\\n# comment\n"; const char *const Output[] = {"-c"}; testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); } TEST(CommandLineTest, TokenizeConfigFileHashAfterContinuationNoSpace) { - // With no space before the backslash, the spliced text abuts directly, so - // the '#' right after the splice does not begin a new token and is not a - // comment; it merges into a single token like other continuations (see + // No whitespace before the splice: '#' continues the token (cf. // TokenizeConfigFile4). const char *Input = "abc\\\n#comment\n"; const char *const Output[] = {"abc#comment"}; @@ -502,10 +518,7 @@ TEST(CommandLineTest, TokenizeConfigFileCommentAfterContinuationWithOption) { } TEST(CommandLineTest, TokenizeConfigFileHashesInFlagNotComment) { - // clang's "-###" flag is made up entirely of '#' characters that never - // begin a new token (each is preceded by non-whitespace), so none of them - // should be treated as a comment start. Only the whitespace-preceded '#' - // that starts the real trailing comment should truncate the line. + // None of the '#'s in -### begins a token. const char *Input = "-### # comment\n"; const char *const Output[] = {"-###"}; testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output); _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
