This revision was automatically updated to reflect the committed changes.
Closed by commit rG739b410f1ff5: Add a warning, flags and pragmas to limit the
number of pre-processor tokens in… (authored by hans).
Herald added a project: clang.
Changed prior to commit:
https://reviews.llvm.org/D72703?vs=239944&id=240568#toc
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D72703/new/
https://reviews.llvm.org/D72703
Files:
clang/include/clang/Basic/DiagnosticGroups.td
clang/include/clang/Basic/DiagnosticParseKinds.td
clang/include/clang/Basic/LangOptions.def
clang/include/clang/Driver/Options.td
clang/include/clang/Lex/Preprocessor.h
clang/include/clang/Parse/Parser.h
clang/lib/Driver/ToolChains/Clang.cpp
clang/lib/Frontend/CompilerInvocation.cpp
clang/lib/Lex/Preprocessor.cpp
clang/lib/Parse/ParsePragma.cpp
clang/lib/Parse/Parser.cpp
clang/test/Driver/autocomplete.c
clang/test/Parser/max-tokens.cpp
Index: clang/test/Parser/max-tokens.cpp
===================================================================
--- /dev/null
+++ clang/test/Parser/max-tokens.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify %s -DMAX_TOKENS -fmax-tokens 2
+// RUN: %clang_cc1 -fsyntax-only -verify %s -DMAX_TOKENS_OVERRIDE -fmax-tokens 9
+
+int x, y, z;
+
+#pragma clang max_tokens_here // expected-error {{missing argument to '#pragma clang max_tokens_here'; expected integer}}
+#pragma clang max_tokens_here foo // expected-error {{expected an integer argument in '#pragma clang max_tokens_here'}}
+#pragma clang max_tokens_here 123 456 // expected-warning{{extra tokens at end of '#pragma clang max_tokens_here' - ignored}}
+
+#pragma clang max_tokens_here 1 // expected-warning{{the number of preprocessor source tokens (7) exceeds this token limit (1)}}
+
+
+#pragma clang max_tokens_total // expected-error{{missing argument to '#pragma clang max_tokens_total'; expected integer}}
+#pragma clang max_tokens_total foo // expected-error{{expected an integer argument in '#pragma clang max_tokens_total'}}
+#pragma clang max_tokens_total 123 456 // expected-warning{{extra tokens at end of '#pragma clang max_tokens_total' - ignored}}
+
+#ifdef MAX_TOKENS_OVERRIDE
+#pragma clang max_tokens_total 3 // expected-warning@+4{{the total number of preprocessor source tokens (8) exceeds the token limit (3)}}
+ // expected-note@-1{{total token limit set here}}
+#elif MAX_TOKENS
+// expected-warning@+1{{the total number of preprocessor source tokens (8) exceeds the token limit (2)}}
+#endif
Index: clang/test/Driver/autocomplete.c
===================================================================
--- clang/test/Driver/autocomplete.c
+++ clang/test/Driver/autocomplete.c
@@ -99,6 +99,7 @@
// WARNING-NEXT: -Wmain-return-type
// WARNING-NEXT: -Wmalformed-warning-check
// WARNING-NEXT: -Wmany-braces-around-scalar-init
+// WARNING-NEXT: -Wmax-tokens
// WARNING-NEXT: -Wmax-unsigned-zero
// RUN: %clang --autocomplete=-Wno-invalid-pp- | FileCheck %s -check-prefix=NOWARNING
// NOWARNING: -Wno-invalid-pp-token
Index: clang/lib/Parse/Parser.cpp
===================================================================
--- clang/lib/Parse/Parser.cpp
+++ clang/lib/Parse/Parser.cpp
@@ -650,6 +650,16 @@
return false;
case tok::eof:
+ // Check whether -fmax-tokens was reached.
+ if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) {
+ PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total)
+ << PP.getTokenCount() << PP.getMaxTokens();
+ SourceLocation OverrideLoc = PP.getMaxTokensOverrideLoc();
+ if (OverrideLoc.isValid()) {
+ PP.Diag(OverrideLoc, diag::note_max_tokens_total_override);
+ }
+ }
+
// Late template parsing can begin.
if (getLangOpts().DelayedTemplateParsing)
Actions.SetLateTemplateParser(LateTemplateParserCallback,
Index: clang/lib/Parse/ParsePragma.cpp
===================================================================
--- clang/lib/Parse/ParsePragma.cpp
+++ clang/lib/Parse/ParsePragma.cpp
@@ -262,6 +262,18 @@
ParsedAttributes AttributesForPragmaAttribute;
};
+struct PragmaMaxTokensHereHandler : public PragmaHandler {
+ PragmaMaxTokensHereHandler() : PragmaHandler("max_tokens_here") {}
+ void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
+ Token &FirstToken) override;
+};
+
+struct PragmaMaxTokensTotalHandler : public PragmaHandler {
+ PragmaMaxTokensTotalHandler() : PragmaHandler("max_tokens_total") {}
+ void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
+ Token &FirstToken) override;
+};
+
} // end namespace
void Parser::initializePragmaHandlers() {
@@ -382,6 +394,12 @@
AttributePragmaHandler =
std::make_unique<PragmaAttributeHandler>(AttrFactory);
PP.AddPragmaHandler("clang", AttributePragmaHandler.get());
+
+ MaxTokensHerePragmaHandler = std::make_unique<PragmaMaxTokensHereHandler>();
+ PP.AddPragmaHandler("clang", MaxTokensHerePragmaHandler.get());
+
+ MaxTokensTotalPragmaHandler = std::make_unique<PragmaMaxTokensTotalHandler>();
+ PP.AddPragmaHandler("clang", MaxTokensTotalPragmaHandler.get());
}
void Parser::resetPragmaHandlers() {
@@ -487,6 +505,12 @@
PP.RemovePragmaHandler("clang", AttributePragmaHandler.get());
AttributePragmaHandler.reset();
+
+ PP.RemovePragmaHandler("clang", MaxTokensHerePragmaHandler.get());
+ MaxTokensHerePragmaHandler.reset();
+
+ PP.RemovePragmaHandler("clang", MaxTokensTotalPragmaHandler.get());
+ MaxTokensTotalPragmaHandler.reset();
}
/// Handle the annotation token produced for #pragma unused(...)
@@ -3279,3 +3303,64 @@
PP.EnterTokenStream(std::move(TokenArray), 1,
/*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
+
+// Handle '#pragma clang max_tokens 12345'.
+void PragmaMaxTokensHereHandler::HandlePragma(Preprocessor &PP,
+ PragmaIntroducer Introducer,
+ Token &Tok) {
+ PP.Lex(Tok);
+ if (Tok.is(tok::eod)) {
+ PP.Diag(Tok.getLocation(), diag::err_pragma_missing_argument)
+ << "clang max_tokens_here" << /*Expected=*/true << "integer";
+ return;
+ }
+
+ SourceLocation Loc = Tok.getLocation();
+ uint64_t MaxTokens;
+ if (Tok.isNot(tok::numeric_constant) ||
+ !PP.parseSimpleIntegerLiteral(Tok, MaxTokens)) {
+ PP.Diag(Tok.getLocation(), diag::err_pragma_expected_integer)
+ << "clang max_tokens_here";
+ return;
+ }
+
+ if (Tok.isNot(tok::eod)) {
+ PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
+ << "clang max_tokens_here";
+ return;
+ }
+
+ if (PP.getTokenCount() > MaxTokens) {
+ PP.Diag(Loc, diag::warn_max_tokens)
+ << PP.getTokenCount() << (unsigned)MaxTokens;
+ }
+}
+
+// Handle '#pragma clang max_file_tokens 12345'.
+void PragmaMaxTokensTotalHandler::HandlePragma(Preprocessor &PP,
+ PragmaIntroducer Introducer,
+ Token &Tok) {
+ PP.Lex(Tok);
+ if (Tok.is(tok::eod)) {
+ PP.Diag(Tok.getLocation(), diag::err_pragma_missing_argument)
+ << "clang max_tokens_total" << /*Expected=*/true << "integer";
+ return;
+ }
+
+ SourceLocation Loc = Tok.getLocation();
+ uint64_t MaxTokens;
+ if (Tok.isNot(tok::numeric_constant) ||
+ !PP.parseSimpleIntegerLiteral(Tok, MaxTokens)) {
+ PP.Diag(Tok.getLocation(), diag::err_pragma_expected_integer)
+ << "clang max_tokens_total";
+ return;
+ }
+
+ if (Tok.isNot(tok::eod)) {
+ PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
+ << "clang max_tokens_total";
+ return;
+ }
+
+ PP.overrideMaxTokens(MaxTokens, Loc);
+}
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -166,6 +166,8 @@
this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
if (ExcludedConditionalDirectiveSkipMappings)
ExcludedConditionalDirectiveSkipMappings->clear();
+
+ MaxTokens = LangOpts.MaxTokens;
}
Preprocessor::~Preprocessor() {
@@ -962,8 +964,12 @@
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
- if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected))
- OnToken(Result);
+
+ if (LexLevel == 0 && !Result.getFlag(Token::IsReinjected)) {
+ ++TokenCount;
+ if (OnToken)
+ OnToken(Result);
+ }
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/lib/Frontend/CompilerInvocation.cpp
===================================================================
--- clang/lib/Frontend/CompilerInvocation.cpp
+++ clang/lib/Frontend/CompilerInvocation.cpp
@@ -3292,6 +3292,8 @@
Opts.CompleteMemberPointers = Args.hasArg(OPT_fcomplete_member_pointers);
Opts.BuildingPCHWithObjectFile = Args.hasArg(OPT_building_pch_with_obj);
+
+ Opts.MaxTokens = getLastArgIntValue(Args, OPT_fmax_tokens, 0, Diags);
}
static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) {
Index: clang/lib/Driver/ToolChains/Clang.cpp
===================================================================
--- clang/lib/Driver/ToolChains/Clang.cpp
+++ clang/lib/Driver/ToolChains/Clang.cpp
@@ -5717,6 +5717,8 @@
Args.AddLastArg(CmdArgs, options::OPT_dM);
Args.AddLastArg(CmdArgs, options::OPT_dD);
+ Args.AddLastArg(CmdArgs, options::OPT_fmax_tokens);
+
// Handle serialized diagnostics.
if (Arg *A = Args.getLastArg(options::OPT__serialize_diags)) {
CmdArgs.push_back("-serialize-diagnostic-file");
Index: clang/include/clang/Parse/Parser.h
===================================================================
--- clang/include/clang/Parse/Parser.h
+++ clang/include/clang/Parse/Parser.h
@@ -201,6 +201,8 @@
std::unique_ptr<PragmaHandler> STDCCXLIMITHandler;
std::unique_ptr<PragmaHandler> STDCUnknownHandler;
std::unique_ptr<PragmaHandler> AttributePragmaHandler;
+ std::unique_ptr<PragmaHandler> MaxTokensHerePragmaHandler;
+ std::unique_ptr<PragmaHandler> MaxTokensTotalPragmaHandler;
std::unique_ptr<CommentHandler> CommentSemaHandler;
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -416,6 +416,14 @@
/// of phase 4 of translation or for some other situation.
unsigned LexLevel = 0;
+ /// The number of (LexLevel 0) preprocessor tokens.
+ unsigned TokenCount = 0;
+
+ /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
+ /// warning, or zero for unlimited.
+ unsigned MaxTokens = 0;
+ SourceLocation MaxTokensOverrideLoc;
+
public:
struct PreambleSkipInfo {
SourceLocation HashTokenLoc;
@@ -1010,6 +1018,19 @@
}
/// \}
+ /// Get the number of tokens processed so far.
+ unsigned getTokenCount() const { return TokenCount; }
+
+ /// Get the max number of tokens before issuing a -Wmax-tokens warning.
+ unsigned getMaxTokens() const { return MaxTokens; }
+
+ void overrideMaxTokens(unsigned Value, SourceLocation Loc) {
+ MaxTokens = Value;
+ MaxTokensOverrideLoc = Loc;
+ };
+
+ SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
+
/// Register a function that would be called on each token in the final
/// expanded token stream.
/// This also reports annotation tokens produced by the parser.
Index: clang/include/clang/Driver/Options.td
===================================================================
--- clang/include/clang/Driver/Options.td
+++ clang/include/clang/Driver/Options.td
@@ -646,6 +646,9 @@
def interface_stub_version_EQ : JoinedOrSeparate<["-"], "interface-stub-version=">, Flags<[CC1Option]>;
def exported__symbols__list : Separate<["-"], "exported_symbols_list">;
def e : JoinedOrSeparate<["-"], "e">, Group<Link_Group>;
+def fmax_tokens : Separate<["-"], "fmax-tokens">,
+ HelpText<"Max total number of preprocessed tokens for -Wmax-tokens.">,
+ Group<f_Group>, Flags<[CC1Option]>;
def fPIC : Flag<["-"], "fPIC">, Group<f_Group>;
def fno_PIC : Flag<["-"], "fno-PIC">, Group<f_Group>;
def fPIE : Flag<["-"], "fPIE">, Group<f_Group>;
Index: clang/include/clang/Basic/LangOptions.def
===================================================================
--- clang/include/clang/Basic/LangOptions.def
+++ clang/include/clang/Basic/LangOptions.def
@@ -344,6 +344,8 @@
LANGOPT(RegisterStaticDestructors, 1, 1, "Register C++ static destructors")
+COMPATIBLE_VALUE_LANGOPT(MaxTokens, 32, 0, "Max number of tokens per TU or 0")
+
#undef LANGOPT
#undef COMPATIBLE_LANGOPT
#undef BENIGN_LANGOPT
Index: clang/include/clang/Basic/DiagnosticParseKinds.td
===================================================================
--- clang/include/clang/Basic/DiagnosticParseKinds.td
+++ clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1046,6 +1046,8 @@
def warn_pragma_expected_init_seg : Warning<
"expected 'compiler', 'lib', 'user', or a string literal for the section name in '#pragma %0' - ignored">,
InGroup<IgnoredPragmas>;
+
+def err_pragma_expected_integer : Error<"expected an integer argument in '#pragma %0'">;
def warn_pragma_expected_integer : Warning<
"expected integer between %0 and %1 inclusive in '#pragma %2' - ignored">,
InGroup<IgnoredPragmas>;
@@ -1375,4 +1377,14 @@
"expected 'auto' or 'decltype(auto)' after concept name">;
}
+def warn_max_tokens : Warning<
+ "the number of preprocessor source tokens (%0) exceeds this token limit (%1)">,
+ InGroup<MaxTokens>;
+
+def warn_max_tokens_total : Warning<
+ "the total number of preprocessor source tokens (%0) exceeds the token limit (%1)">,
+ InGroup<MaxTokens>;
+
+def note_max_tokens_total_override : Note<"total token limit set here">;
+
} // end of Parser diagnostics
Index: clang/include/clang/Basic/DiagnosticGroups.td
===================================================================
--- clang/include/clang/Basic/DiagnosticGroups.td
+++ clang/include/clang/Basic/DiagnosticGroups.td
@@ -1149,3 +1149,32 @@
def CTADMaybeUnsupported : DiagGroup<"ctad-maybe-unsupported">;
def FortifySource : DiagGroup<"fortify-source">;
+
+def MaxTokens : DiagGroup<"max-tokens"> {
+ code Documentation = [{
+The warning is issued if the number of pre-processor tokens exceeds
+the token limit, which can be set in three ways:
+
+1. As a limit at a specific point in a file, using the ``clang max_tokens_here``
+ pragma:
+
+ .. code-block: c++
+ #pragma clang max_tokens_here 1234
+
+2. As a per-translation unit limit, using the ``-fmax-tokens`` command-line
+ flag:
+
+ .. code-block: console
+ clang -c a.cpp -fmax-tokens 1234
+
+3. As a per-translation unit limit using the ``clang max_tokens_total`` pragma,
+ which works like and overrides the ``-fmax-tokens`` flag:
+
+ .. code-block: c++
+ #pragma clang max_file_tokens 1234
+
+These limits can be helpful in limiting code growth through included files.
+
+Setting a token limit of zero means no limit.
+}];
+}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
