This revision was automatically updated to reflect the committed changes. Closed by commit rG739b410f1ff5: Add a warning, flags and pragmas to limit the number of pre-processor tokens in… (authored by hans). Herald added a project: clang.
Changed prior to commit: https://reviews.llvm.org/D72703?vs=239944&id=240568#toc Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D72703/new/ https://reviews.llvm.org/D72703 Files: clang/include/clang/Basic/DiagnosticGroups.td clang/include/clang/Basic/DiagnosticParseKinds.td clang/include/clang/Basic/LangOptions.def clang/include/clang/Driver/Options.td clang/include/clang/Lex/Preprocessor.h clang/include/clang/Parse/Parser.h clang/lib/Driver/ToolChains/Clang.cpp clang/lib/Frontend/CompilerInvocation.cpp clang/lib/Lex/Preprocessor.cpp clang/lib/Parse/ParsePragma.cpp clang/lib/Parse/Parser.cpp clang/test/Driver/autocomplete.c clang/test/Parser/max-tokens.cpp
Index: clang/test/Parser/max-tokens.cpp =================================================================== --- /dev/null +++ clang/test/Parser/max-tokens.cpp @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s +// RUN: %clang_cc1 -fsyntax-only -verify %s -DMAX_TOKENS -fmax-tokens 2 +// RUN: %clang_cc1 -fsyntax-only -verify %s -DMAX_TOKENS_OVERRIDE -fmax-tokens 9 + +int x, y, z; + +#pragma clang max_tokens_here // expected-error {{missing argument to '#pragma clang max_tokens_here'; expected integer}} +#pragma clang max_tokens_here foo // expected-error {{expected an integer argument in '#pragma clang max_tokens_here'}} +#pragma clang max_tokens_here 123 456 // expected-warning{{extra tokens at end of '#pragma clang max_tokens_here' - ignored}} + +#pragma clang max_tokens_here 1 // expected-warning{{the number of preprocessor source tokens (7) exceeds this token limit (1)}} + + +#pragma clang max_tokens_total // expected-error{{missing argument to '#pragma clang max_tokens_total'; expected integer}} +#pragma clang max_tokens_total foo // expected-error{{expected an integer argument in '#pragma clang max_tokens_total'}} +#pragma clang max_tokens_total 123 456 // expected-warning{{extra tokens at end of '#pragma clang max_tokens_total' - ignored}} + +#ifdef MAX_TOKENS_OVERRIDE +#pragma clang max_tokens_total 3 // expected-warning@+4{{the total number of preprocessor source tokens (8) exceeds the token limit (3)}} + // expected-note@-1{{total token limit set here}} +#elif MAX_TOKENS +// expected-warning@+1{{the total number of preprocessor source tokens (8) exceeds the token limit (2)}} +#endif Index: clang/test/Driver/autocomplete.c =================================================================== --- clang/test/Driver/autocomplete.c +++ clang/test/Driver/autocomplete.c @@ -99,6 +99,7 @@ // WARNING-NEXT: -Wmain-return-type // WARNING-NEXT: -Wmalformed-warning-check // WARNING-NEXT: -Wmany-braces-around-scalar-init +// WARNING-NEXT: -Wmax-tokens // WARNING-NEXT: -Wmax-unsigned-zero // RUN: %clang --autocomplete=-Wno-invalid-pp- | FileCheck %s -check-prefix=NOWARNING // NOWARNING: -Wno-invalid-pp-token Index: clang/lib/Parse/Parser.cpp =================================================================== --- clang/lib/Parse/Parser.cpp +++ clang/lib/Parse/Parser.cpp @@ -650,6 +650,16 @@ return false; case tok::eof: + // Check whether -fmax-tokens was reached. + if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) { + PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total) + << PP.getTokenCount() << PP.getMaxTokens(); + SourceLocation OverrideLoc = PP.getMaxTokensOverrideLoc(); + if (OverrideLoc.isValid()) { + PP.Diag(OverrideLoc, diag::note_max_tokens_total_override); + } + } + // Late template parsing can begin. if (getLangOpts().DelayedTemplateParsing) Actions.SetLateTemplateParser(LateTemplateParserCallback, Index: clang/lib/Parse/ParsePragma.cpp =================================================================== --- clang/lib/Parse/ParsePragma.cpp +++ clang/lib/Parse/ParsePragma.cpp @@ -262,6 +262,18 @@ ParsedAttributes AttributesForPragmaAttribute; }; +struct PragmaMaxTokensHereHandler : public PragmaHandler { + PragmaMaxTokensHereHandler() : PragmaHandler("max_tokens_here") {} + void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, + Token &FirstToken) override; +}; + +struct PragmaMaxTokensTotalHandler : public PragmaHandler { + PragmaMaxTokensTotalHandler() : PragmaHandler("max_tokens_total") {} + void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, + Token &FirstToken) override; +}; + } // end namespace void Parser::initializePragmaHandlers() { @@ -382,6 +394,12 @@ AttributePragmaHandler = std::make_unique<PragmaAttributeHandler>(AttrFactory); PP.AddPragmaHandler("clang", AttributePragmaHandler.get()); + + MaxTokensHerePragmaHandler = std::make_unique<PragmaMaxTokensHereHandler>(); + PP.AddPragmaHandler("clang", MaxTokensHerePragmaHandler.get()); + + MaxTokensTotalPragmaHandler = std::make_unique<PragmaMaxTokensTotalHandler>(); + PP.AddPragmaHandler("clang", MaxTokensTotalPragmaHandler.get()); } void Parser::resetPragmaHandlers() { @@ -487,6 +505,12 @@ PP.RemovePragmaHandler("clang", AttributePragmaHandler.get()); AttributePragmaHandler.reset(); + + PP.RemovePragmaHandler("clang", MaxTokensHerePragmaHandler.get()); + MaxTokensHerePragmaHandler.reset(); + + PP.RemovePragmaHandler("clang", MaxTokensTotalPragmaHandler.get()); + MaxTokensTotalPragmaHandler.reset(); } /// Handle the annotation token produced for #pragma unused(...) @@ -3279,3 +3303,64 @@ PP.EnterTokenStream(std::move(TokenArray), 1, /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } + +// Handle '#pragma clang max_tokens 12345'. +void PragmaMaxTokensHereHandler::HandlePragma(Preprocessor &PP, + PragmaIntroducer Introducer, + Token &Tok) { + PP.Lex(Tok); + if (Tok.is(tok::eod)) { + PP.Diag(Tok.getLocation(), diag::err_pragma_missing_argument) + << "clang max_tokens_here" << /*Expected=*/true << "integer"; + return; + } + + SourceLocation Loc = Tok.getLocation(); + uint64_t MaxTokens; + if (Tok.isNot(tok::numeric_constant) || + !PP.parseSimpleIntegerLiteral(Tok, MaxTokens)) { + PP.Diag(Tok.getLocation(), diag::err_pragma_expected_integer) + << "clang max_tokens_here"; + return; + } + + if (Tok.isNot(tok::eod)) { + PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) + << "clang max_tokens_here"; + return; + } + + if (PP.getTokenCount() > MaxTokens) { + PP.Diag(Loc, diag::warn_max_tokens) + << PP.getTokenCount() << (unsigned)MaxTokens; + } +} + +// Handle '#pragma clang max_file_tokens 12345'. +void PragmaMaxTokensTotalHandler::HandlePragma(Preprocessor &PP, + PragmaIntroducer Introducer, + Token &Tok) { + PP.Lex(Tok); + if (Tok.is(tok::eod)) { + PP.Diag(Tok.getLocation(), diag::err_pragma_missing_argument) + << "clang max_tokens_total" << /*Expected=*/true << "integer"; + return; + } + + SourceLocation Loc = Tok.getLocation(); + uint64_t MaxTokens; + if (Tok.isNot(tok::numeric_constant) || + !PP.parseSimpleIntegerLiteral(Tok, MaxTokens)) { + PP.Diag(Tok.getLocation(), diag::err_pragma_expected_integer) + << "clang max_tokens_total"; + return; + } + + if (Tok.isNot(tok::eod)) { + PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) + << "clang max_tokens_total"; + return; + } + + PP.overrideMaxTokens(MaxTokens, Loc); +} Index: clang/lib/Lex/Preprocessor.cpp =================================================================== --- clang/lib/Lex/Preprocessor.cpp +++ clang/lib/Lex/Preprocessor.cpp @@ -166,6 +166,8 @@ this->PPOpts->ExcludedConditionalDirectiveSkipMappings; if (ExcludedConditionalDirectiveSkipMappings) ExcludedConditionalDirectiveSkipMappings->clear(); + + MaxTokens = LangOpts.MaxTokens; } Preprocessor::~Preprocessor() { @@ -962,8 +964,12 @@ LastTokenWasAt = Result.is(tok::at); --LexLevel; - if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected)) - OnToken(Result); + + if (LexLevel == 0 && !Result.getFlag(Token::IsReinjected)) { + ++TokenCount; + if (OnToken) + OnToken(Result); + } } /// Lex a header-name token (including one formed from header-name-tokens if Index: clang/lib/Frontend/CompilerInvocation.cpp =================================================================== --- clang/lib/Frontend/CompilerInvocation.cpp +++ clang/lib/Frontend/CompilerInvocation.cpp @@ -3292,6 +3292,8 @@ Opts.CompleteMemberPointers = Args.hasArg(OPT_fcomplete_member_pointers); Opts.BuildingPCHWithObjectFile = Args.hasArg(OPT_building_pch_with_obj); + + Opts.MaxTokens = getLastArgIntValue(Args, OPT_fmax_tokens, 0, Diags); } static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) { Index: clang/lib/Driver/ToolChains/Clang.cpp =================================================================== --- clang/lib/Driver/ToolChains/Clang.cpp +++ clang/lib/Driver/ToolChains/Clang.cpp @@ -5717,6 +5717,8 @@ Args.AddLastArg(CmdArgs, options::OPT_dM); Args.AddLastArg(CmdArgs, options::OPT_dD); + Args.AddLastArg(CmdArgs, options::OPT_fmax_tokens); + // Handle serialized diagnostics. if (Arg *A = Args.getLastArg(options::OPT__serialize_diags)) { CmdArgs.push_back("-serialize-diagnostic-file"); Index: clang/include/clang/Parse/Parser.h =================================================================== --- clang/include/clang/Parse/Parser.h +++ clang/include/clang/Parse/Parser.h @@ -201,6 +201,8 @@ std::unique_ptr<PragmaHandler> STDCCXLIMITHandler; std::unique_ptr<PragmaHandler> STDCUnknownHandler; std::unique_ptr<PragmaHandler> AttributePragmaHandler; + std::unique_ptr<PragmaHandler> MaxTokensHerePragmaHandler; + std::unique_ptr<PragmaHandler> MaxTokensTotalPragmaHandler; std::unique_ptr<CommentHandler> CommentSemaHandler; Index: clang/include/clang/Lex/Preprocessor.h =================================================================== --- clang/include/clang/Lex/Preprocessor.h +++ clang/include/clang/Lex/Preprocessor.h @@ -416,6 +416,14 @@ /// of phase 4 of translation or for some other situation. unsigned LexLevel = 0; + /// The number of (LexLevel 0) preprocessor tokens. + unsigned TokenCount = 0; + + /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens + /// warning, or zero for unlimited. + unsigned MaxTokens = 0; + SourceLocation MaxTokensOverrideLoc; + public: struct PreambleSkipInfo { SourceLocation HashTokenLoc; @@ -1010,6 +1018,19 @@ } /// \} + /// Get the number of tokens processed so far. + unsigned getTokenCount() const { return TokenCount; } + + /// Get the max number of tokens before issuing a -Wmax-tokens warning. + unsigned getMaxTokens() const { return MaxTokens; } + + void overrideMaxTokens(unsigned Value, SourceLocation Loc) { + MaxTokens = Value; + MaxTokensOverrideLoc = Loc; + }; + + SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; } + /// Register a function that would be called on each token in the final /// expanded token stream. /// This also reports annotation tokens produced by the parser. Index: clang/include/clang/Driver/Options.td =================================================================== --- clang/include/clang/Driver/Options.td +++ clang/include/clang/Driver/Options.td @@ -646,6 +646,9 @@ def interface_stub_version_EQ : JoinedOrSeparate<["-"], "interface-stub-version=">, Flags<[CC1Option]>; def exported__symbols__list : Separate<["-"], "exported_symbols_list">; def e : JoinedOrSeparate<["-"], "e">, Group<Link_Group>; +def fmax_tokens : Separate<["-"], "fmax-tokens">, + HelpText<"Max total number of preprocessed tokens for -Wmax-tokens.">, + Group<f_Group>, Flags<[CC1Option]>; def fPIC : Flag<["-"], "fPIC">, Group<f_Group>; def fno_PIC : Flag<["-"], "fno-PIC">, Group<f_Group>; def fPIE : Flag<["-"], "fPIE">, Group<f_Group>; Index: clang/include/clang/Basic/LangOptions.def =================================================================== --- clang/include/clang/Basic/LangOptions.def +++ clang/include/clang/Basic/LangOptions.def @@ -344,6 +344,8 @@ LANGOPT(RegisterStaticDestructors, 1, 1, "Register C++ static destructors") +COMPATIBLE_VALUE_LANGOPT(MaxTokens, 32, 0, "Max number of tokens per TU or 0") + #undef LANGOPT #undef COMPATIBLE_LANGOPT #undef BENIGN_LANGOPT Index: clang/include/clang/Basic/DiagnosticParseKinds.td =================================================================== --- clang/include/clang/Basic/DiagnosticParseKinds.td +++ clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1046,6 +1046,8 @@ def warn_pragma_expected_init_seg : Warning< "expected 'compiler', 'lib', 'user', or a string literal for the section name in '#pragma %0' - ignored">, InGroup<IgnoredPragmas>; + +def err_pragma_expected_integer : Error<"expected an integer argument in '#pragma %0'">; def warn_pragma_expected_integer : Warning< "expected integer between %0 and %1 inclusive in '#pragma %2' - ignored">, InGroup<IgnoredPragmas>; @@ -1375,4 +1377,14 @@ "expected 'auto' or 'decltype(auto)' after concept name">; } +def warn_max_tokens : Warning< + "the number of preprocessor source tokens (%0) exceeds this token limit (%1)">, + InGroup<MaxTokens>; + +def warn_max_tokens_total : Warning< + "the total number of preprocessor source tokens (%0) exceeds the token limit (%1)">, + InGroup<MaxTokens>; + +def note_max_tokens_total_override : Note<"total token limit set here">; + } // end of Parser diagnostics Index: clang/include/clang/Basic/DiagnosticGroups.td =================================================================== --- clang/include/clang/Basic/DiagnosticGroups.td +++ clang/include/clang/Basic/DiagnosticGroups.td @@ -1149,3 +1149,32 @@ def CTADMaybeUnsupported : DiagGroup<"ctad-maybe-unsupported">; def FortifySource : DiagGroup<"fortify-source">; + +def MaxTokens : DiagGroup<"max-tokens"> { + code Documentation = [{ +The warning is issued if the number of pre-processor tokens exceeds +the token limit, which can be set in three ways: + +1. As a limit at a specific point in a file, using the ``clang max_tokens_here`` + pragma: + + .. code-block: c++ + #pragma clang max_tokens_here 1234 + +2. As a per-translation unit limit, using the ``-fmax-tokens`` command-line + flag: + + .. code-block: console + clang -c a.cpp -fmax-tokens 1234 + +3. As a per-translation unit limit using the ``clang max_tokens_total`` pragma, + which works like and overrides the ``-fmax-tokens`` flag: + + .. code-block: c++ + #pragma clang max_file_tokens 1234 + +These limits can be helpful in limiting code growth through included files. + +Setting a token limit of zero means no limit. +}]; +}
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits