https://github.com/Rinn updated https://github.com/llvm/llvm-project/pull/205084
>From 25db0f0051754ebc88a1b42438d4d593b486f934 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 22 Jun 2026 12:29:50 +0100 Subject: [PATCH 01/10] Fix preprocessed block comment newline on windows When preprocessing a source file on windows with \r\n newlines while preserving comments, any newlines in a block comment are written directly which is then later normalized to \r\r\n. Fix this by only writing out \n in PrintPreprocessedTokens for those comments by copying the logic from PrintPPOutputPPCallbacks::HandleNewlinesInToken --- .../lib/Frontend/PrintPreprocessedOutput.cpp | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index 02266882c4c4a..16c09d1263e30 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -889,6 +889,26 @@ struct UnknownPragmaHandler : public PragmaHandler { }; } // end anonymous namespace +static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr, + unsigned Len) { + for (; Len; --Len, ++TokStr) { + if (*TokStr != '\n' && + *TokStr != '\r') { + *OS << *TokStr; + continue; + } + + *OS << '\n'; + + // If we have \n\r or \r\n, skip both and emit one newline. + if (Len != 1 && + (TokStr[1] == '\n' || TokStr[1] == '\r') && + TokStr[0] != TokStr[1]) { + ++TokStr; + --Len; + } + } +} static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, PrintPPOutputPPCallbacks *Callbacks) { @@ -1022,7 +1042,10 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, } else if (Tok.getLength() < std::size(Buffer)) { const char *TokPtr = Buffer; unsigned Len = PP.getSpelling(Tok, TokPtr); - Callbacks->OS->write(TokPtr, Len); + if (Tok.is(tok::comment)) + PrintPreprocessedComment(Callbacks->OS, TokPtr, Len); + else + Callbacks->OS->write(TokPtr, Len); // Tokens that can contain embedded newlines need to adjust our current // line number. @@ -1039,7 +1062,10 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, } } else { std::string S = PP.getSpelling(Tok); - Callbacks->OS->write(S.data(), S.size()); + if (Tok.is(tok::comment)) + PrintPreprocessedComment(Callbacks->OS, S.data(), S.size()); + else + Callbacks->OS->write(S.data(), S.size()); // Tokens that can contain embedded newlines need to adjust our current // line number. >From 515cd83647efe645fe4ece3c3e2e306cd57c09ec Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 22 Jun 2026 12:48:29 +0100 Subject: [PATCH 02/10] clang-format --- clang/lib/Frontend/PrintPreprocessedOutput.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index 16c09d1263e30..40c1b89366a7c 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -892,8 +892,7 @@ struct UnknownPragmaHandler : public PragmaHandler { static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr, unsigned Len) { for (; Len; --Len, ++TokStr) { - if (*TokStr != '\n' && - *TokStr != '\r') { + if (*TokStr != '\n' && *TokStr != '\r') { *OS << *TokStr; continue; } @@ -901,8 +900,7 @@ static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr, *OS << '\n'; // If we have \n\r or \r\n, skip both and emit one newline. - if (Len != 1 && - (TokStr[1] == '\n' || TokStr[1] == '\r') && + if (Len != 1 && (TokStr[1] == '\n' || TokStr[1] == '\r') && TokStr[0] != TokStr[1]) { ++TokStr; --Len; >From dec4c02c62716739b9969321f191970f02a6e046 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Fri, 26 Jun 2026 09:13:35 +0100 Subject: [PATCH 03/10] Apply suggestions from code review Co-authored-by: Aaron Ballman <[email protected]> --- clang/lib/Frontend/PrintPreprocessedOutput.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index 40c1b89366a7c..38a802dd7614b 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -1040,7 +1040,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, } else if (Tok.getLength() < std::size(Buffer)) { const char *TokPtr = Buffer; unsigned Len = PP.getSpelling(Tok, TokPtr); - if (Tok.is(tok::comment)) + if (Tok.is(tok::comment, tok::unknown)) PrintPreprocessedComment(Callbacks->OS, TokPtr, Len); else Callbacks->OS->write(TokPtr, Len); @@ -1060,7 +1060,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, } } else { std::string S = PP.getSpelling(Tok); - if (Tok.is(tok::comment)) + if (Tok.is(tok::comment, tok::unknown)) PrintPreprocessedComment(Callbacks->OS, S.data(), S.size()); else Callbacks->OS->write(S.data(), S.size()); >From b9afb477f1e82741f15275046d45176428cffb1e Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Fri, 26 Jun 2026 10:12:25 +0100 Subject: [PATCH 04/10] Fix Token.is only accepting one paramter, rename PrintPreprocessedComment --- clang/lib/Frontend/PrintPreprocessedOutput.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index 38a802dd7614b..57de5126bcd71 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -889,8 +889,8 @@ struct UnknownPragmaHandler : public PragmaHandler { }; } // end anonymous namespace -static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr, - unsigned Len) { +static void WriteMultilineToken(raw_ostream *OS, const char *TokStr, + unsigned Len) { for (; Len; --Len, ++TokStr) { if (*TokStr != '\n' && *TokStr != '\r') { *OS << *TokStr; @@ -1040,8 +1040,8 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, } else if (Tok.getLength() < std::size(Buffer)) { const char *TokPtr = Buffer; unsigned Len = PP.getSpelling(Tok, TokPtr); - if (Tok.is(tok::comment, tok::unknown)) - PrintPreprocessedComment(Callbacks->OS, TokPtr, Len); + if (Tok.is(tok::comment) || Tok.is(tok::unknown)) + WriteMultilineToken(Callbacks->OS, TokPtr, Len); else Callbacks->OS->write(TokPtr, Len); @@ -1060,8 +1060,8 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, } } else { std::string S = PP.getSpelling(Tok); - if (Tok.is(tok::comment, tok::unknown)) - PrintPreprocessedComment(Callbacks->OS, S.data(), S.size()); + if (Tok.is(tok::comment) || Tok.is(tok::unknown)) + WriteMultilineToken(Callbacks->OS, S.data(), S.size()); else Callbacks->OS->write(S.data(), S.size()); >From 18546015bba9055ea3d90a00b45da6e9ce8a0e98 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Wed, 1 Jul 2026 10:58:41 +0100 Subject: [PATCH 05/10] Add test --- clang/test/.gitattributes | 1 + clang/test/Preprocessor/Inputs/comment_save_crlf.h | 5 +++++ clang/test/Preprocessor/comment_save_crlf.c | 10 ++++++++++ 3 files changed, 16 insertions(+) create mode 100644 clang/test/.gitattributes create mode 100644 clang/test/Preprocessor/Inputs/comment_save_crlf.h create mode 100644 clang/test/Preprocessor/comment_save_crlf.c diff --git a/clang/test/.gitattributes b/clang/test/.gitattributes new file mode 100644 index 0000000000000..0e7628b9abfda --- /dev/null +++ b/clang/test/.gitattributes @@ -0,0 +1 @@ +Preprocessor/Inputs/comment_save_crlf.h -text diff --git a/clang/test/Preprocessor/Inputs/comment_save_crlf.h b/clang/test/Preprocessor/Inputs/comment_save_crlf.h new file mode 100644 index 0000000000000..6cd956eff14ff --- /dev/null +++ b/clang/test/Preprocessor/Inputs/comment_save_crlf.h @@ -0,0 +1,5 @@ +int a; +/* block comment + spanning multiple + CRLF-terminated lines */ +int b; diff --git a/clang/test/Preprocessor/comment_save_crlf.c b/clang/test/Preprocessor/comment_save_crlf.c new file mode 100644 index 0000000000000..7e8a01e3d1b02 --- /dev/null +++ b/clang/test/Preprocessor/comment_save_crlf.c @@ -0,0 +1,10 @@ +// REQUIRES: system-windows + +// RUN: %clang_cc1 -E -C -o %t %S/Inputs/comment_save_crlf.h +// RUN: FileCheck --strict-whitespace --match-full-lines --input-file=%t %s + +// CHECK:int a; +// CHECK-NEXT:/* block comment +// CHECK-NEXT: spanning multiple +// CHECK-NEXT: CRLF-terminated lines */ +// CHECK-NEXT:int b; >From b92e042469b25b01d7618b9af7526b579902f33e Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 13 Jul 2026 11:08:51 +0100 Subject: [PATCH 06/10] Revert "Add test" This reverts commit 18546015bba9055ea3d90a00b45da6e9ce8a0e98. --- clang/test/.gitattributes | 1 - clang/test/Preprocessor/Inputs/comment_save_crlf.h | 5 ----- clang/test/Preprocessor/comment_save_crlf.c | 10 ---------- 3 files changed, 16 deletions(-) delete mode 100644 clang/test/.gitattributes delete mode 100644 clang/test/Preprocessor/Inputs/comment_save_crlf.h delete mode 100644 clang/test/Preprocessor/comment_save_crlf.c diff --git a/clang/test/.gitattributes b/clang/test/.gitattributes deleted file mode 100644 index 0e7628b9abfda..0000000000000 --- a/clang/test/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -Preprocessor/Inputs/comment_save_crlf.h -text diff --git a/clang/test/Preprocessor/Inputs/comment_save_crlf.h b/clang/test/Preprocessor/Inputs/comment_save_crlf.h deleted file mode 100644 index 6cd956eff14ff..0000000000000 --- a/clang/test/Preprocessor/Inputs/comment_save_crlf.h +++ /dev/null @@ -1,5 +0,0 @@ -int a; -/* block comment - spanning multiple - CRLF-terminated lines */ -int b; diff --git a/clang/test/Preprocessor/comment_save_crlf.c b/clang/test/Preprocessor/comment_save_crlf.c deleted file mode 100644 index 7e8a01e3d1b02..0000000000000 --- a/clang/test/Preprocessor/comment_save_crlf.c +++ /dev/null @@ -1,10 +0,0 @@ -// REQUIRES: system-windows - -// RUN: %clang_cc1 -E -C -o %t %S/Inputs/comment_save_crlf.h -// RUN: FileCheck --strict-whitespace --match-full-lines --input-file=%t %s - -// CHECK:int a; -// CHECK-NEXT:/* block comment -// CHECK-NEXT: spanning multiple -// CHECK-NEXT: CRLF-terminated lines */ -// CHECK-NEXT:int b; >From 819433176693f59fe4ebcc923ad57e958d639a7e Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 13 Jul 2026 11:43:50 +0100 Subject: [PATCH 07/10] Add test --- clang/test/Preprocessor/Inputs/comment_save_crlf.h | 5 +++++ clang/test/Preprocessor/comment_save_crlf.c | 6 ++++++ clang/test/lit.cfg.py | 7 +++++++ 3 files changed, 18 insertions(+) create mode 100644 clang/test/Preprocessor/Inputs/comment_save_crlf.h create mode 100644 clang/test/Preprocessor/comment_save_crlf.c diff --git a/clang/test/Preprocessor/Inputs/comment_save_crlf.h b/clang/test/Preprocessor/Inputs/comment_save_crlf.h new file mode 100644 index 0000000000000..d942ddf0b1c64 --- /dev/null +++ b/clang/test/Preprocessor/Inputs/comment_save_crlf.h @@ -0,0 +1,5 @@ +int a; +/* multi-line + block + comment */ +int b; diff --git a/clang/test/Preprocessor/comment_save_crlf.c b/clang/test/Preprocessor/comment_save_crlf.c new file mode 100644 index 0000000000000..1aa4d967b2ff3 --- /dev/null +++ b/clang/test/Preprocessor/comment_save_crlf.c @@ -0,0 +1,6 @@ +// REQUIRES: system-windows + +// RUN: %{to-crlf} %S/Inputs/comment_save_crlf.h > %t.h +// RUN: %clang_cc1 -E -C -o %t.i %t.h +// RUN: %{reveal-cr} %t.i | FileCheck %s +// CHECK-NOT: <CR> diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index f7b3a77266cb8..42d5ccaee3daa 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -112,6 +112,13 @@ ), ) ) + +# Rewrite LF to CRLF +config.substitutions.append(("%{to-crlf}", f"{sed_cmd} -e 's/\r//g' -e 's/$/\r/'")) + +# Filtering command for testing for carriage return +config.substitutions.append(("%{reveal-cr}", f"{sed_cmd} -e 's/\r/<CR>/g'")) + # For each occurrence of a clang tool name, replace it with the full path to # the build directory holding that tool. We explicitly specify the directories # to search to ensure that we get the tools just built and not some random >From 558fa0296eb9ef5c1baed21718b47a3c3936cae2 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 13 Jul 2026 12:34:36 +0100 Subject: [PATCH 08/10] Update lit.cfg.py --- clang/test/lit.cfg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index 11dae7877b855..88bbd50e0ced8 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -114,10 +114,10 @@ ) # Rewrite LF to CRLF -config.substitutions.append(("%{to-crlf}", f"{sed_cmd} -e 's/\r//g' -e 's/$/\r/'")) +config.substitutions.append(("%{to-crlf}", f"{sed_cmd} -e 's|\r||g' -e 's|$|\r|'")) # Filtering command for testing for carriage return -config.substitutions.append(("%{reveal-cr}", f"{sed_cmd} -e 's/\r/<CR>/g'")) +config.substitutions.append(("%{reveal-cr}", f"{sed_cmd} -e 's|\r|<CR>|g'")) # For each occurrence of a clang tool name, replace it with the full path to # the build directory holding that tool. We explicitly specify the directories >From 26e322660afb0f1e021d95275af40769cd94a64e Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 13 Jul 2026 13:01:06 +0100 Subject: [PATCH 09/10] Update lit.cfg.py --- clang/test/lit.cfg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index 88bbd50e0ced8..c01eef449a82e 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -114,10 +114,10 @@ ) # Rewrite LF to CRLF -config.substitutions.append(("%{to-crlf}", f"{sed_cmd} -e 's|\r||g' -e 's|$|\r|'")) +config.substitutions.append(("%{to-crlf}", rf"{sed_cmd} -e 's|\r||g' -e 's|$|\r|'")) # Filtering command for testing for carriage return -config.substitutions.append(("%{reveal-cr}", f"{sed_cmd} -e 's|\r|<CR>|g'")) +config.substitutions.append(("%{reveal-cr}", rf"{sed_cmd} -e 's|\r|<CR>|g'")) # For each occurrence of a clang tool name, replace it with the full path to # the build directory holding that tool. We explicitly specify the directories >From 0e6c1e1b6b350565d61620f123989575460ef3b2 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff <[email protected]> Date: Mon, 13 Jul 2026 13:22:42 +0100 Subject: [PATCH 10/10] windows only test substitutions --- clang/test/lit.cfg.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index c01eef449a82e..11c8da86cbf7a 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -113,11 +113,11 @@ ) ) -# Rewrite LF to CRLF -config.substitutions.append(("%{to-crlf}", rf"{sed_cmd} -e 's|\r||g' -e 's|$|\r|'")) - -# Filtering command for testing for carriage return -config.substitutions.append(("%{reveal-cr}", rf"{sed_cmd} -e 's|\r|<CR>|g'")) +if platform.system() in ["Windows"]: + # Rewrite LF to CRLF + config.substitutions.append(("%{to-crlf}", rf"{sed_cmd} -e 's|\r||g' -e 's|$|\r|'")) + # Filtering command for testing for carriage return + config.substitutions.append(("%{reveal-cr}", rf"{sed_cmd} -e 's|\r|<CR>|g'")) # For each occurrence of a clang tool name, replace it with the full path to # the build directory holding that tool. We explicitly specify the directories _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
