https://github.com/hnakamura5 updated https://github.com/llvm/llvm-project/pull/76059
>From b0080a41c1802517e4a02976058231cf37a82adb Mon Sep 17 00:00:00 2001 From: hnakamura5 <hnakamu...@outlook.com> Date: Fri, 3 Nov 2023 20:58:17 +0900 Subject: [PATCH] [clang-format] Support of TableGen formatting. Currently, TableGen has its language style but the it does not works well. This patch adds total support of TableGen formatting including the support for the code (multi line string), DAG args, bang operators, the cond operator, and the paste operators. --- clang/include/clang/Format/Format.h | 47 ++ clang/lib/Format/ContinuationIndenter.cpp | 18 +- clang/lib/Format/Format.cpp | 29 ++ clang/lib/Format/FormatToken.h | 98 ++++ clang/lib/Format/FormatTokenLexer.cpp | 142 ++++++ clang/lib/Format/FormatTokenLexer.h | 6 + clang/lib/Format/TokenAnnotator.cpp | 426 +++++++++++++++++- clang/lib/Format/UnwrappedLineParser.cpp | 54 ++- clang/lib/Format/WhitespaceManager.cpp | 31 +- clang/lib/Format/WhitespaceManager.h | 14 + clang/unittests/Format/FormatTestTableGen.cpp | 307 +++++++++++++ clang/unittests/Format/TokenAnnotatorTest.cpp | 50 ++ 12 files changed, 1197 insertions(+), 25 deletions(-) diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index 8604dea689f937..30a38aed99866e 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -396,6 +396,36 @@ struct FormatStyle { /// \version 17 ShortCaseStatementsAlignmentStyle AlignConsecutiveShortCaseStatements; + /// Style of aligning consecutive TableGen cond operator colons. + /// \code + /// !cond(!eq(size, 1) : 1, + /// !eq(size, 16): 1, + /// true : 0) + /// \endcode + /// \version 18 + AlignConsecutiveStyle AlignConsecutiveTableGenCondOperatorColons; + + /// Style of aligning consecutive TableGen DAGArg operator colons. + /// Intended to be used with TableGenBreakInsideDAGArgList + /// \code + /// let dagarg = (ins + /// a :$src1, + /// aa :$src2, + /// aaa:$src3 + /// ) + /// \endcode + /// \version 18 + AlignConsecutiveStyle AlignConsecutiveTableGenBreakingDAGArgColons; + + /// Style of aligning consecutive TableGen def colons. + /// \code + /// def Def : Parent {} + /// def DefDef : Parent {} + /// def DefDefDef : Parent {} + /// \endcode + /// \version 18 + AlignConsecutiveStyle AlignConsecutiveTableGenDefinitions; + /// Different styles for aligning escaped newlines. enum EscapedNewlineAlignmentStyle : int8_t { /// Don't align escaped newlines. @@ -3037,6 +3067,7 @@ struct FormatStyle { bool isProto() const { return Language == LK_Proto || Language == LK_TextProto; } + bool isTableGen() const { return Language == LK_TableGen; } /// Language, this format style is targeted at. /// \version 3.5 @@ -4656,6 +4687,15 @@ struct FormatStyle { /// \version 8 std::vector<std::string> StatementMacros; + /// Tablegen + bool TableGenAllowBreakBeforeInheritColon; + bool TableGenAllowBreakAfterInheritColon; + bool TableGenBreakInsideCondOperator; + bool TableGenBreakInsideDAGArgList; + bool TableGenPreferBreakInsideSquareBracket; + bool TableGenSpaceAroundDAGArgColon; + std::vector<std::string> TableGenBreakingDAGArgOperators; + /// The number of columns used for tab stops. /// \version 3.7 unsigned TabWidth; @@ -4753,6 +4793,13 @@ struct FormatStyle { AlignConsecutiveMacros == R.AlignConsecutiveMacros && AlignConsecutiveShortCaseStatements == R.AlignConsecutiveShortCaseStatements && + AlignConsecutiveTableGenCondOperatorColons == + R.AlignConsecutiveTableGenCondOperatorColons && + AlignConsecutiveTableGenBreakingDAGArgColons == + R.AlignConsecutiveTableGenBreakingDAGArgColons && + AlignConsecutiveTableGenDefinitions == + R.AlignConsecutiveTableGenDefinitions && + AlignConsecutiveMacros == R.AlignConsecutiveMacros && AlignEscapedNewlines == R.AlignEscapedNewlines && AlignOperands == R.AlignOperands && AlignTrailingComments == R.AlignTrailingComments && diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp index bd319f21b05f86..176dc7744a5576 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp @@ -800,6 +800,7 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun, if (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign && !CurrentState.IsCSharpGenericTypeConstraint && Previous.opensScope() && Previous.isNot(TT_ObjCMethodExpr) && Previous.isNot(TT_RequiresClause) && + Previous.isNot(TT_TableGenDAGArgOpener) && !(Current.MacroParent && Previous.MacroParent) && (Current.isNot(TT_LineComment) || Previous.isOneOf(BK_BracedInit, TT_VerilogMultiLineListLParen))) { @@ -1229,7 +1230,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { return CurrentState.Indent; } if ((Current.isOneOf(tok::r_brace, tok::r_square) || - (Current.is(tok::greater) && Style.isProto())) && + (Current.is(tok::greater) && (Style.isProto() || Style.isTableGen()))) && State.Stack.size() > 1) { if (Current.closesBlockOrBlockTypeList(Style)) return State.Stack[State.Stack.size() - 2].NestedBlockIndent; @@ -1257,6 +1258,12 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { Current.Next->isOneOf(tok::semi, tok::kw_const, tok::l_brace))) { return State.Stack[State.Stack.size() - 2].LastSpace; } + // When DAGArg closer exists top of line, it must be aligned in the similar + // way as function call above. + if (Current.is(tok::r_paren) && State.Stack.size() > 1 && + (Current.isOneOf(TT_TableGenDAGArgCloser, TT_TableGenParamAngleCloser))) { + return State.Stack[State.Stack.size() - 2].LastSpace; + } if (Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent && (Current.is(tok::r_paren) || (Current.is(tok::r_brace) && Current.MatchingParen && @@ -1593,6 +1600,9 @@ unsigned ContinuationIndenter::moveStateToNextToken(LineState &State, State.StartOfStringLiteral = State.Column + 1; if (Current.is(TT_CSharpStringLiteral) && State.StartOfStringLiteral == 0) { State.StartOfStringLiteral = State.Column + 1; + } else if (Current.is(TT_TableGenMultiLineString) && + State.StartOfStringLiteral == 0) { + State.StartOfStringLiteral = State.Column + 1; } else if (Current.isStringLiteral() && State.StartOfStringLiteral == 0) { State.StartOfStringLiteral = State.Column; } else if (!Current.isOneOf(tok::comment, tok::identifier, tok::hash) && @@ -1672,7 +1682,9 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, (!Previous || Previous->isNot(tok::kw_return) || (Style.Language != FormatStyle::LK_Java && PrecedenceLevel > 0)) && (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign || - PrecedenceLevel != prec::Comma || Current.NestingLevel == 0)) { + PrecedenceLevel != prec::Comma || Current.NestingLevel == 0) && + (!Style.isTableGen() || !Previous || + !Previous->isNot(TT_TableGenDAGArgListComma))) { NewParenState.Indent = std::max( std::max(State.Column, NewParenState.Indent), CurrentState.LastSpace); } @@ -1915,6 +1927,8 @@ void ContinuationIndenter::moveStatePastScopeCloser(LineState &State) { (Current.isOneOf(tok::r_paren, tok::r_square, TT_TemplateString) || (Current.is(tok::r_brace) && State.NextToken != State.Line->First) || State.NextToken->is(TT_TemplateCloser) || + State.NextToken->is(TT_TableGenParamAngleCloser) || + State.NextToken->is(TT_TableGenListCloser) || (Current.is(tok::greater) && Current.is(TT_DictLiteral)))) { State.Stack.pop_back(); } diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 28271181e07d0c..9ba3d5dbd530e7 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -926,6 +926,12 @@ template <> struct MappingTraits<FormatStyle> { IO.mapOptional("AlignConsecutiveMacros", Style.AlignConsecutiveMacros); IO.mapOptional("AlignConsecutiveShortCaseStatements", Style.AlignConsecutiveShortCaseStatements); + IO.mapOptional("AlignConsecutiveTableGenCondOperatorColons", + Style.AlignConsecutiveTableGenCondOperatorColons); + IO.mapOptional("AlignConsecutiveTableGenBreakingDAGArgColons", + Style.AlignConsecutiveTableGenBreakingDAGArgColons); + IO.mapOptional("AlignConsecutiveTableGenDefinitions", + Style.AlignConsecutiveTableGenDefinitions); IO.mapOptional("AlignEscapedNewlines", Style.AlignEscapedNewlines); IO.mapOptional("AlignOperands", Style.AlignOperands); IO.mapOptional("AlignTrailingComments", Style.AlignTrailingComments); @@ -1124,6 +1130,20 @@ template <> struct MappingTraits<FormatStyle> { IO.mapOptional("StatementAttributeLikeMacros", Style.StatementAttributeLikeMacros); IO.mapOptional("StatementMacros", Style.StatementMacros); + IO.mapOptional("TableGenAllowBreakAfterInheritColon", + Style.TableGenAllowBreakAfterInheritColon); + IO.mapOptional("TableGenAllowBreakBeforeInheritColon", + Style.TableGenAllowBreakBeforeInheritColon); + IO.mapOptional("TableGenBreakInsideCondOperator", + Style.TableGenBreakInsideCondOperator); + IO.mapOptional("TableGenBreakInsideDAGArgList", + Style.TableGenBreakInsideDAGArgList); + IO.mapOptional("TableGenPreferBreakInsideSquareBracket", + Style.TableGenPreferBreakInsideSquareBracket); + IO.mapOptional("TableGenSpaceAroundDAGArgColon", + Style.TableGenSpaceAroundDAGArgColon); + IO.mapOptional("TableGenBreakingDAGArgOperators", + Style.TableGenBreakingDAGArgOperators); IO.mapOptional("TabWidth", Style.TabWidth); IO.mapOptional("TypeNames", Style.TypeNames); IO.mapOptional("TypenameMacros", Style.TypenameMacros); @@ -1437,6 +1457,9 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) { LLVMStyle.AlignConsecutiveDeclarations = {}; LLVMStyle.AlignConsecutiveMacros = {}; LLVMStyle.AlignConsecutiveShortCaseStatements = {}; + LLVMStyle.AlignConsecutiveTableGenBreakingDAGArgColons = {}; + LLVMStyle.AlignConsecutiveTableGenCondOperatorColons = {}; + LLVMStyle.AlignConsecutiveTableGenDefinitions = {}; LLVMStyle.AlignTrailingComments = {}; LLVMStyle.AlignTrailingComments.Kind = FormatStyle::TCAS_Always; LLVMStyle.AlignTrailingComments.OverEmptyLines = 0; @@ -1585,6 +1608,12 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) { LLVMStyle.StatementAttributeLikeMacros.push_back("Q_EMIT"); LLVMStyle.StatementMacros.push_back("Q_UNUSED"); LLVMStyle.StatementMacros.push_back("QT_REQUIRE_VERSION"); + LLVMStyle.TableGenAllowBreakAfterInheritColon = true; + LLVMStyle.TableGenAllowBreakBeforeInheritColon = false; + LLVMStyle.TableGenBreakInsideCondOperator = true; + LLVMStyle.TableGenBreakInsideDAGArgList = false; + LLVMStyle.TableGenPreferBreakInsideSquareBracket = false; + LLVMStyle.TableGenSpaceAroundDAGArgColon = false; LLVMStyle.TabWidth = 8; LLVMStyle.UseTab = FormatStyle::UT_Never; LLVMStyle.VerilogBreakBetweenInstancePorts = true; diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 3f9664f8f78a3e..bd16567233bb22 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -148,6 +148,23 @@ namespace format { TYPE(StructLBrace) \ TYPE(StructRBrace) \ TYPE(StructuredBindingLSquare) \ + TYPE(TableGenBangOperator) \ + TYPE(TableGenCondOperator) \ + TYPE(TableGenCondOperatorColon) \ + TYPE(TableGenCondOperatorComma) \ + TYPE(TableGenDAGArgCloser) \ + TYPE(TableGenDAGArgListColon) \ + TYPE(TableGenDAGArgListColonToAlign) \ + TYPE(TableGenDAGArgListComma) \ + TYPE(TableGenDAGArgOpener) \ + TYPE(TableGenDAGArgOperatorID) \ + TYPE(TableGenListCloser) \ + TYPE(TableGenListOpener) \ + TYPE(TableGenMultiLineString) \ + TYPE(TableGenParamAngleOpener) \ + TYPE(TableGenParamAngleCloser) \ + TYPE(TableGenTrailingPasteOperator) \ + TYPE(TableGenValueSuffix) \ TYPE(TemplateCloser) \ TYPE(TemplateOpener) \ TYPE(TemplateString) \ @@ -671,6 +688,8 @@ struct FormatToken { return true; if (is(TT_DictLiteral) && is(tok::less)) return true; + if (is(TT_TableGenParamAngleOpener)) + return true; return isOneOf(tok::l_paren, tok::l_brace, tok::l_square, TT_TemplateOpener); } @@ -681,6 +700,10 @@ struct FormatToken { return true; if (is(TT_DictLiteral) && is(tok::greater)) return true; + if (is(TT_TableGenParamAngleCloser)) + return true; + if (is(TT_TableGenListCloser)) + return true; return isOneOf(tok::r_paren, tok::r_brace, tok::r_square, TT_TemplateCloser); } @@ -1202,6 +1225,21 @@ struct AdditionalKeywords { kw_verilogHashHash = &IdentTable.get("##"); kw_apostrophe = &IdentTable.get("\'"); + // TableGen keywords + kw_bit = &IdentTable.get("bit"); + kw_bits = &IdentTable.get("bits"); + kw_code = &IdentTable.get("code"); + kw_dag = &IdentTable.get("dag"); + kw_def = &IdentTable.get("def"); + kw_defm = &IdentTable.get("defm"); + kw_defset = &IdentTable.get("defset"); + kw_defvar = &IdentTable.get("defvar"); + kw_dump = &IdentTable.get("dump"); + kw_include = &IdentTable.get("include"); + kw_list = &IdentTable.get("list"); + kw_multiclass = &IdentTable.get("multiclass"); + kw_then = &IdentTable.get("then"); + // Keep this at the end of the constructor to make sure everything here // is // already initialized. @@ -1294,6 +1332,27 @@ struct AdditionalKeywords { kw_wildcard, kw_wire, kw_with, kw_wor, kw_verilogHash, kw_verilogHashHash}); + + TableGenExtraKeywords = std::unordered_set<IdentifierInfo *>({ + kw_assert, + kw_bit, + kw_bits, + kw_code, + kw_dag, + kw_def, + kw_defm, + kw_defset, + kw_defvar, + kw_dump, + kw_foreach, + kw_in, + kw_include, + kw_let, + kw_list, + kw_multiclass, + kw_string, + kw_then, + }); } // Context sensitive keywords. @@ -1539,6 +1598,21 @@ struct AdditionalKeywords { // Symbols in Verilog that don't exist in C++. IdentifierInfo *kw_apostrophe; + // TableGen keywords + IdentifierInfo *kw_bit; + IdentifierInfo *kw_bits; + IdentifierInfo *kw_code; + IdentifierInfo *kw_dag; + IdentifierInfo *kw_def; + IdentifierInfo *kw_defm; + IdentifierInfo *kw_defset; + IdentifierInfo *kw_defvar; + IdentifierInfo *kw_dump; + IdentifierInfo *kw_include; + IdentifierInfo *kw_list; + IdentifierInfo *kw_multiclass; + IdentifierInfo *kw_then; + /// Returns \c true if \p Tok is a keyword or an identifier. bool isWordLike(const FormatToken &Tok) const { // getIdentifierinfo returns non-null for keywords as well as identifiers. @@ -1811,6 +1885,27 @@ struct AdditionalKeywords { } } + bool isTableGenDefinition(const FormatToken &Tok) const { + return Tok.isOneOf(kw_def, kw_defm, kw_defset, kw_defvar, kw_multiclass, + kw_let, tok::kw_class); + } + + bool isTableGenKeyword(const FormatToken &Tok) const { + switch (Tok.Tok.getKind()) { + case tok::kw_class: + case tok::kw_else: + case tok::kw_false: + case tok::kw_if: + case tok::kw_int: + case tok::kw_true: + return true; + default: + return Tok.is(tok::identifier) && + TableGenExtraKeywords.find(Tok.Tok.getIdentifierInfo()) != + TableGenExtraKeywords.end(); + } + } + private: /// The JavaScript keywords beyond the C++ keyword set. std::unordered_set<IdentifierInfo *> JsExtraKeywords; @@ -1820,6 +1915,9 @@ struct AdditionalKeywords { /// The Verilog keywords beyond the C++ keyword set. std::unordered_set<IdentifierInfo *> VerilogExtraKeywords; + + /// The TableGen keywords beyond the C++ keyword set. + std::unordered_set<IdentifierInfo *> TableGenExtraKeywords; }; inline bool isLineComment(const FormatToken &FormatTok) { diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp index 61430282c6f88c..085dc9facc6423 100644 --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -93,6 +93,10 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() { // string literals are correctly identified. handleCSharpVerbatimAndInterpolatedStrings(); } + if (Style.isTableGen()) { + handleTableGenMultilineString(); + handleTableGenNumericLikeIdentifier(); + } if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline) FirstInLineIndex = Tokens.size() - 1; } while (Tokens.back()->isNot(tok::eof)); @@ -272,6 +276,38 @@ void FormatTokenLexer::tryMergePreviousTokens() { return; } } + if (Style.isTableGen()) { + if (tryMergeTokens({tok::l_square, tok::l_brace}, + TT_TableGenMultiLineString)) { + Tokens.back()->Tok.setKind(tok::string_literal); + return; + } + if (Tokens.size() > 1 && Tokens.end()[-2]->is(tok::exclaim)) { + if (Tokens.back()->is(tok::identifier) || + (Tokens.back()->is(tok::kw_if) && Tokens.back())) { + } + } + if (tryMergeTokens({tok::exclaim, tok::identifier}, + TT_TableGenBangOperator)) { + Tokens.back()->Tok.setKind(tok::identifier); + if (Tokens.back()->TokenText == "!cond") + Tokens.back()->setType(TT_TableGenCondOperator); + return; + } + if (tryMergeTokens({tok::exclaim, tok::kw_if}, TT_TableGenBangOperator)) { + // Here, "! if" becomes "!if". That is, ! captures if even when the space + // exists. That is only one possibility in TableGen's syntax. + return; + } + if (tryMergeTokens({tok::plus, tok::numeric_constant}, TT_Unknown)) { + Tokens.back()->Tok.setKind(tok::numeric_constant); + return; + } + if (tryMergeTokens({tok::minus, tok::numeric_constant}, TT_Unknown)) { + Tokens.back()->Tok.setKind(tok::numeric_constant); + return; + } + } } bool FormatTokenLexer::tryMergeNSStringLiteral() { @@ -763,6 +799,109 @@ void FormatTokenLexer::handleCSharpVerbatimAndInterpolatedStrings() { resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset + 1))); } +void FormatTokenLexer::handleTableGenMultilineString() { + FormatToken *MultiLineString = Tokens.back(); + if (MultiLineString->isNot(TT_TableGenMultiLineString)) { + // Multi line string starts with [{ + return; + } + bool PrevIsRBrace = false; + const char *FirstBreak = nullptr; + const char *LastBreak = nullptr; + const char *Begin = MultiLineString->TokenText.begin(); + for (const char *Current = Begin, *End = Lex->getBuffer().end(); + Current != End; ++Current) { + if (*Current == ']' && PrevIsRBrace) { + // }] is the end of multi line string. + if (!FirstBreak) + FirstBreak = Current; + MultiLineString->TokenText = StringRef(Begin, Current - Begin + 1); + MultiLineString->ColumnWidth = encoding::columnWidthWithTabs( + StringRef(Begin, FirstBreak - Begin + 1), + MultiLineString->OriginalColumn, Style.TabWidth, Encoding); + if (LastBreak) { + MultiLineString->LastLineColumnWidth = encoding::columnWidthWithTabs( + StringRef(LastBreak + 1, Current - LastBreak), + MultiLineString->OriginalColumn, Style.TabWidth, Encoding); + } + resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Current + 1))); + return; + } + PrevIsRBrace = false; + if (*Current == '\n') { + MultiLineString->IsMultiline = true; + // Assure LastBreak is not equal to FirstBreak. + if (!FirstBreak) + FirstBreak = Current; + LastBreak = Current; + continue; + } + if (*Current == '}') { + // Memorize '}'. If next character is ']', we find the end. + PrevIsRBrace = true; + continue; + } + } +} + +void FormatTokenLexer::handleTableGenNumericLikeIdentifier() { + FormatToken *Tok = Tokens.back(); + if (Tok->isNot(tok::numeric_constant)) + return; + StringRef Text = Tok->TokenText; + if (Text.size() < 1 || Text[0] == '+' || Text[0] == '-') + return; + size_t I = 0; + // Identifiers in TalbleGen may begin from digits. + while (I < Text.size() && isdigit(Text[I])) + ++I; + if (I >= Text.size()) + return; + char Second = Text[I]; + if (I > 1 || Second != 'x' || Second != 'b') { + // If non-digit appears in integer that does not begin from + or -, + // it must be in the second character. + Tok->Tok.setKind(tok::identifier); + return; + } + while (I < Text.size()) { + switch (Text[I]) { + default: // Invalid char for identifier is also allowed here. + Tok->Tok.setKind(tok::identifier); + return; + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + if (Second != 'b') { + Tok->Tok.setKind(tok::identifier); + return; + } + [[fallthrough]]; + case '0': + case '1': + break; + } + ++I; + } +} + void FormatTokenLexer::handleTemplateStrings() { FormatToken *BacktickToken = Tokens.back(); @@ -1182,6 +1321,9 @@ FormatToken *FormatTokenLexer::getNextToken() { tok::kw_operator)) { FormatTok->Tok.setKind(tok::identifier); FormatTok->Tok.setIdentifierInfo(nullptr); + } else if (Style.isTableGen() && !Keywords.isTableGenKeyword(*FormatTok)) { + FormatTok->Tok.setKind(tok::identifier); + FormatTok->Tok.setIdentifierInfo(nullptr); } } else if (FormatTok->is(tok::greatergreater)) { FormatTok->Tok.setKind(tok::greater); diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h index bb6a8ab69c1be1..062c6afa35defa 100644 --- a/clang/lib/Format/FormatTokenLexer.h +++ b/clang/lib/Format/FormatTokenLexer.h @@ -95,6 +95,12 @@ class FormatTokenLexer { void handleCSharpVerbatimAndInterpolatedStrings(); + // Handles TableGen multiline strings. It has the form [{ ... }]. + void handleTableGenMultilineString(); + // Handles TableGen numeric like identifiers. + // It has forms of [0-9]*[_a-zA-Z]([_a-zA-Z0-9]*) where it is not an integer. + void handleTableGenNumericLikeIdentifier(); + void tryParsePythonComment(); bool tryMerge_TMacro(); diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index f3551af3424396..c98aa9a701e2b8 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -215,6 +215,9 @@ class AnnotatingParser { (Style.Language == FormatStyle::LK_Proto && Left->Previous && Left->Previous->isOneOf(TT_SelectorName, TT_DictLiteral))) { CurrentToken->setType(TT_DictLiteral); + } else if (Style.isTableGen()) { + CurrentToken->setType(TT_TableGenParamAngleCloser); + CurrentToken->Tok.setLength(1); } else { CurrentToken->setType(TT_TemplateCloser); CurrentToken->Tok.setLength(1); @@ -256,6 +259,20 @@ class AnnotatingParser { } } } + if (Style.isTableGen() && (!Line.startsWith(tok::kw_class) && + !Line.startsWith(Keywords.kw_multiclass))) { + if (CurrentToken->isOneOf(tok::comma, tok::equal)) { + // They appears as a separator. Unless it is not in class definition. + next(); + continue; + } + // In angle, there must be Value like tokens but for the case of class + // definition. Types are also able to be parsed in the same way with + // Values. + if (!parseTableGenValue()) + return false; + continue; + } if (!consumeToken()) return false; } @@ -387,6 +404,27 @@ class AnnotatingParser { OpeningParen.Previous->isOneOf(tok::kw_for, tok::kw_catch); Contexts.back().IsExpression = !IsForOrCatch; } + if (Style.isTableGen()) { + if (FormatToken *Prev = OpeningParen.Previous) { + if (Prev->is(TT_TableGenCondOperator)) { + Contexts.back().IsTableGenCondOpe = true; + Contexts.back().IsExpression = true; + } else if (Contexts.size() > 1 && + Contexts[Contexts.size() - 2].IsTableGenBangOpe) { + // FIXME: Hack to handle bang operators. The parent context's flag + // was set by parseTableGenSimpleValue(). + // We have to specify the context outside because the prev of "(" may + // be ">", not the bang operator in this case. + Contexts.back().IsTableGenBangOpe = true; + Contexts.back().IsExpression = true; + } else { + // Otherwise, this paren seems DAGArg. + if (!parseTableGenDAGArg()) + return false; + return parseTableGenDAGArgAndList(&OpeningParen); + } + } + } // Infer the role of the l_paren based on the previous token if we haven't // detected one yet. @@ -549,6 +587,22 @@ class AnnotatingParser { if (CurrentToken->is(tok::comma)) Contexts.back().CanBeExpression = true; + if (Style.isTableGen()) { + if (CurrentToken->is(tok::comma)) { + if (Contexts.back().IsTableGenCondOpe) + CurrentToken->setType(TT_TableGenCondOperatorComma); + next(); + } else if (CurrentToken->is(tok::colon)) { + if (Contexts.back().IsTableGenCondOpe) + CurrentToken->setType(TT_TableGenCondOperatorColon); + next(); + } + // In TableGen there must be Values in parens. + if (!parseTableGenValue()) + return false; + continue; + } + FormatToken *Tok = CurrentToken; if (!consumeToken()) return false; @@ -803,6 +857,8 @@ class AnnotatingParser { if (Left->BlockParameterCount > 1) Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName = 0; } + if (Style.isTableGen() && Left->is(TT_TableGenListOpener)) + CurrentToken->setType(TT_TableGenListCloser); next(); return true; } @@ -833,6 +889,19 @@ class AnnotatingParser { Left->setType(TT_ArrayInitializerLSquare); } FormatToken *Tok = CurrentToken; + if (Style.isTableGen()) { + if (CurrentToken->isOneOf(tok::comma, tok::minus, tok::ellipsis)) { + // '-' and '...' appears as a separator in slice. + next(); + } else { + // In TableGen there must be a list of Values in square brackets. + // It must be ValueList or SliceElements. + if (!parseTableGenValue()) + return false; + } + updateParameterCount(Left, Tok); + continue; + } if (!consumeToken()) return false; updateParameterCount(Left, Tok); @@ -840,6 +909,213 @@ class AnnotatingParser { return false; } + void nextTableGenNonComment() { + next(); + while (CurrentToken && CurrentToken->is(tok::comment)) + next(); + } + + bool isTableGenDAGArgBreakingOperator(const FormatToken &Tok) { + auto &Opes = Style.TableGenBreakingDAGArgOperators; + if (Opes.size() == 0 || std::find(Opes.begin(), Opes.end(), + Tok.TokenText.str()) != Opes.end()) { + return true; + } + return false; + } + + bool parseTableGenValue(bool ParseNameMode = false) { + if (!CurrentToken) + return false; + while (CurrentToken->is(tok::comment)) + next(); + if (!parseTableGenSimpleValue()) + return false; + if (!CurrentToken) + return true; + // Value "#" [Value] + if (CurrentToken->is(tok::hash)) { + if (CurrentToken->Next && + CurrentToken->Next->isOneOf(tok::colon, tok::semi, tok::l_brace)) { + // Trailing paste operator. + // These are only the allowed cases in TGParser::ParseValue(). + CurrentToken->setType(TT_TableGenTrailingPasteOperator); + next(); + return true; + } + FormatToken *HashTok = CurrentToken; + nextTableGenNonComment(); + HashTok->setType(TT_Unknown); + if (!parseTableGenValue(ParseNameMode)) + return false; + } + if (ParseNameMode && CurrentToken->is(tok::l_brace)) + return true; + if (CurrentToken->isOneOf(tok::l_brace, tok::l_square, tok::period)) { + // Delegate ValueSuffix to normal consumeToken + CurrentToken->setType(TT_TableGenValueSuffix); + FormatToken *Suffix = CurrentToken; + nextTableGenNonComment(); + if (Suffix->is(tok::l_square)) { + return parseSquare(); + } else if (Suffix->is(tok::l_brace)) { + Scopes.push_back(getScopeType(*Suffix)); + return parseBrace(); + } + return true; + } + return true; + } + + // TokVarName ::= "$" ualpha (ualpha | "0"..."9")* + bool tryToParseTableGenTokVar() { + if (!CurrentToken) + return false; + if (CurrentToken->is(tok::identifier) && + CurrentToken->TokenText.front() == '$') { + nextTableGenNonComment(); + return true; + } + return false; + } + + // DagArg ::= Value [":" TokVarName] | TokVarName + bool parseTableGenDAGArg(bool AlignColon = false) { + if (tryToParseTableGenTokVar()) + return true; + if (parseTableGenValue()) { + if (CurrentToken && CurrentToken->is(tok::colon)) { + if (AlignColon) + CurrentToken->setType(TT_TableGenDAGArgListColonToAlign); + else + CurrentToken->setType(TT_TableGenDAGArgListColon); + nextTableGenNonComment(); + return tryToParseTableGenTokVar(); + } + return true; + } + return false; + } + + // SimpleValue6 ::= "(" DagArg [DagArgList] ")" + // This parses SimpleValue 6's inside part of "(" ")" + bool parseTableGenDAGArgAndList(FormatToken *Opener) { + FormatToken *FirstTok = CurrentToken; + bool BreakInside = false; + if (!parseTableGenDAGArg()) + return false; + // Specialization for DAGArgOperatorID, single identifier DAGArg operator. + if (!FirstTok->isOneOf(TT_TableGenBangOperator, TT_TableGenCondOperator)) { + if (FirstTok->is(tok::identifier) && FirstTok->Next && + FirstTok->Next->isNot(tok::colon)) { + // Special case for identifier DAGArg operator + if (isTableGenDAGArgBreakingOperator(*FirstTok)) { + BreakInside = true; + FirstTok->setType(TT_TableGenDAGArgOperatorID); + } + } + } + // Parse the [DagArgList] part + bool FirstDAGArgListElm = true; + while (CurrentToken) { + if (!FirstDAGArgListElm && CurrentToken->is(tok::comma)) { + if (BreakInside) + CurrentToken->setType(TT_TableGenDAGArgListComma); + nextTableGenNonComment(); + } + if (CurrentToken && CurrentToken->is(tok::r_paren)) { + CurrentToken->setType(TT_TableGenDAGArgCloser); + Opener->MatchingParen = CurrentToken; + CurrentToken->MatchingParen = Opener; + nextTableGenNonComment(); + return true; + } + if (!parseTableGenDAGArg( + BreakInside && + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled)) { + return false; + } + FirstDAGArgListElm = false; + } + return false; + } + + bool parseTableGenSimpleValue() { + assert(Style.isTableGen()); + if (!CurrentToken) + return false; + FormatToken *Tok = CurrentToken; + nextTableGenNonComment(); + // SimpleValue 1, 2, 3: Literals + if (Tok->isOneOf(tok::numeric_constant, tok::string_literal, + TT_TableGenMultiLineString, tok::kw_true, tok::kw_false, + tok::question, tok::kw_int)) { + return true; + } + // SimpleValue 4: ValueList, Type + if (Tok->is(tok::l_brace)) { + Scopes.push_back(getScopeType(*Tok)); + return parseBrace(); + } + // SimpleValue 5: List initializer + if (Tok->is(tok::l_square)) { + Tok->setType(TT_TableGenListOpener); + if (!parseSquare()) + return false; + if (Tok->is(tok::less)) { + CurrentToken->setType(TT_TableGenParamAngleOpener); + return parseAngle(); + } + return true; + } + // SimpleValue 6: DAGArg [DAGArgList] + // SimpleValue6 ::= "(" DagArg [DagArgList] ")" + if (Tok->is(tok::l_paren)) { + Tok->setType(TT_TableGenDAGArgOpener); + return parseTableGenDAGArgAndList(Tok); + } + // SimpleValue 9: Bang operator + if (Tok->is(TT_TableGenBangOperator)) { + if (CurrentToken && CurrentToken->is(tok::less)) { + CurrentToken->setType(TT_TableGenParamAngleOpener); + nextTableGenNonComment(); + if (!parseAngle()) + return false; + } + if (!CurrentToken || CurrentToken->isNot(tok::l_paren)) + return false; + nextTableGenNonComment(); + // FIXME: Hack using inheritance to child context + Contexts.back().IsTableGenBangOpe = true; + bool Result = parseParens(); + Contexts.back().IsTableGenBangOpe = false; + return Result; + } + // SimpleValue 9: Cond operator + if (Tok->is(TT_TableGenCondOperator)) { + Tok = CurrentToken; + nextTableGenNonComment(); + if (!Tok || !Tok->is(tok::l_paren)) + return false; + bool Result = parseParens(); + return Result; + } + // We have to check identifier at the last because it mistakes + // bang/cond operators. + // SimpleValue 7: Identifiers + if (Tok->is(tok::identifier)) { + // SimpleValue 8: Anonymous record + if (CurrentToken && CurrentToken->is(tok::less)) { + CurrentToken->setType(TT_TableGenParamAngleOpener); + nextTableGenNonComment(); + return parseAngle(); + } + return true; + } + + return false; + } + bool couldBeInStructArrayInitializer() const { if (Contexts.size() < 2) return false; @@ -880,6 +1156,8 @@ class AnnotatingParser { OpeningBrace.getPreviousNonComment()->isNot(Keywords.kw_apostrophe))) { Contexts.back().VerilogMayBeConcatenation = true; } + if (Style.isTableGen()) + Contexts.back().ColonIsDictLiteral = false; unsigned CommaCount = 0; while (CurrentToken) { @@ -906,7 +1184,7 @@ class AnnotatingParser { FormatToken *Previous = CurrentToken->getPreviousNonComment(); if (Previous->is(TT_JsTypeOptionalQuestion)) Previous = Previous->getPreviousNonComment(); - if ((CurrentToken->is(tok::colon) && + if ((CurrentToken->is(tok::colon) && !Style.isTableGen() && (!Contexts.back().ColonIsDictLiteral || !Style.isCpp())) || Style.isProto()) { OpeningBrace.setType(TT_DictLiteral); @@ -915,10 +1193,12 @@ class AnnotatingParser { Previous->setType(TT_SelectorName); } } - if (CurrentToken->is(tok::colon) && OpeningBrace.is(TT_Unknown)) + if (CurrentToken->is(tok::colon) && OpeningBrace.is(TT_Unknown) && + !Style.isTableGen()) { OpeningBrace.setType(TT_DictLiteral); - else if (Style.isJavaScript()) + } else if (Style.isJavaScript()) { OpeningBrace.overwriteFixedType(TT_DictLiteral); + } } if (CurrentToken->is(tok::comma)) { if (Style.isJavaScript()) @@ -989,6 +1269,8 @@ class AnnotatingParser { // operators. if (Tok->is(TT_VerilogTableItem)) return true; + if (Tok->is(TT_TableGenMultiLineString)) + return true; switch (Tok->Tok.getKind()) { case tok::plus: case tok::minus: @@ -1119,6 +1401,10 @@ class AnnotatingParser { Tok->setType(TT_ObjCMethodExpr); } else if (Contexts.back().ContextKind == tok::l_paren && !Line.InPragmaDirective) { + if (Style.isTableGen() && Contexts.back().IsTableGenDAGArg) { + Tok->setType(TT_TableGenDAGArgListColon); + break; + } Tok->setType(TT_InlineASMColon); } break; @@ -1130,6 +1416,13 @@ class AnnotatingParser { Tok->setType(TT_JsTypeOperator); break; case tok::kw_if: + if (Style.isTableGen()) { + if (!parseTableGenValue()) + return false; + if (CurrentToken && CurrentToken->is(Keywords.kw_then)) + next(); // skip then + break; + } if (CurrentToken && CurrentToken->isOneOf(tok::kw_constexpr, tok::identifier)) { next(); @@ -1235,6 +1528,8 @@ class AnnotatingParser { } break; case tok::l_square: + if (Style.isTableGen()) + Tok->setType(TT_TableGenListOpener); if (!parseSquare()) return false; break; @@ -1264,6 +1559,8 @@ class AnnotatingParser { if (Previous && Previous->getType() != TT_DictLiteral) Previous->setType(TT_SelectorName); } + if (Style.isTableGen()) + Tok->setType(TT_TableGenParamAngleOpener); } else { Tok->setType(TT_BinaryOperator); NonTemplateLess.insert(Tok); @@ -1423,11 +1720,30 @@ class AnnotatingParser { if (!Tok->getPreviousNonComment()) Line.IsContinuation = true; } + if (Style.isTableGen()) { + if (Tok->is(Keywords.kw_assert)) { + if (!parseTableGenValue()) + return false; + } else if (Tok->isOneOf(Keywords.kw_def, Keywords.kw_defm) && + (!Tok->Next || + !Tok->Next->isOneOf(tok::colon, tok::l_brace))) { + // The case NameValue appears. + if (!parseTableGenValue(true)) + return false; + } + } break; case tok::arrow: if (Tok->Previous && Tok->Previous->is(tok::kw_noexcept)) Tok->setType(TT_TrailingReturnArrow); break; + case tok::equal: + // In TableGen, there must be a value after "="; + if (Style.isTableGen()) { + if (!parseTableGenValue()) + return false; + } + break; default: break; } @@ -1710,7 +2026,9 @@ class AnnotatingParser { TT_UnionLBrace, TT_RequiresClause, TT_RequiresClauseInARequiresExpression, TT_RequiresExpression, TT_RequiresExpressionLParen, TT_RequiresExpressionLBrace, - TT_BracedListLBrace)) { + TT_BracedListLBrace, TT_TableGenDAGArgOperatorID, + TT_TableGenMultiLineString, TT_TableGenBangOperator, + TT_TableGenCondOperator)) { CurrentToken->setType(TT_Unknown); } CurrentToken->Role.reset(); @@ -1757,6 +2075,9 @@ class AnnotatingParser { // Whether the braces may mean concatenation instead of structure or array // literal. bool VerilogMayBeConcatenation = false; + bool IsTableGenDAGArg = false; + bool IsTableGenBangOpe = false; + bool IsTableGenCondOpe = false; enum { Unknown, // Like the part after `:` in a constructor. @@ -2061,6 +2382,9 @@ class AnnotatingParser { // In JavaScript, `interface X { foo?(): bar; }` is an optional method // on the interface, not a ternary expression. Current.setType(TT_JsTypeOptionalQuestion); + } else if (Style.isTableGen()) { + // In TableGen, it is only a identifier like token. + Current.setType(TT_Unknown); } else { Current.setType(TT_ConditionalExpr); } @@ -2234,6 +2558,15 @@ class AnnotatingParser { if (PreviousNotConst->ClosesRequiresClause) return false; + if (Style.isTableGen()) { + // keywords such as let and def* defines names. + if (Keywords.isTableGenDefinition(*PreviousNotConst)) + return true; + // Otherwise C++ style declarations is available only inside the brace. + if (Contexts.back().ContextKind != tok::l_brace) + return false; + } + bool IsPPKeyword = PreviousNotConst->is(tok::identifier) && PreviousNotConst->Previous && PreviousNotConst->Previous->is(tok::hash); @@ -3740,6 +4073,22 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line, return 1; if (Right.is(tok::period)) return 500; + } else if (Style.isTableGen()) { + // Not prefer to break inside <>. + if (Left.is(TT_TableGenParamAngleOpener) || + Right.is(TT_TableGenParamAngleCloser)) { + return 100; + } + if (Keywords.isTableGenDefinition(Left)) + return 500; + // Allow in default but not prefer after ":" in definition + if (Left.is(tok::colon) && Right.is(tok::identifier)) + return 500; + if (!Style.TableGenPreferBreakInsideSquareBracket && + (Left.is(TT_TableGenListOpener) || Right.is(TT_TableGenListCloser))) { + // Whether to to break after [. + return 500; + } } if (Right.is(tok::identifier) && Right.Next && Right.Next->is(TT_DictLiteral)) @@ -4761,7 +5110,42 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, Left.endsSequence(tok::greatergreater, tok::l_brace))) { return false; } + } else if (Style.isTableGen()) { + if (Right.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign) || + Left.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign)) { + return Style.TableGenSpaceAroundDAGArgColon; + } + if (Right.is(TT_TableGenCondOperatorColon)) + return false; + if (Left.is(TT_TableGenDAGArgOperatorID) && + Right.isNot(TT_TableGenDAGArgCloser)) { + return true; + } + // Trailing paste requires space before '{' or ':'. Not before ';'. + if (Left.is(TT_TableGenTrailingPasteOperator) && + Right.isOneOf(tok::l_brace, tok::colon)) { + return true; + } + // Otherwise paste operator does not prefer space around. + if (Left.is(tok::hash) || Right.is(tok::hash)) + return false; + // Bang operator does not require any space before paren. + if (Right.is(tok::l_paren) && + Left.isOneOf(TT_TableGenBangOperator, TT_TableGenCondOperator)) { + return false; + } + if (Keywords.isTableGenDefinition(Left)) + return true; + + // Avoid to connect [ and {. [{ is start token of multiline string. + if (Left.is(tok::l_square) && Right.is(tok::l_brace)) + return true; + if (Left.is(tok::r_brace) && Right.is(tok::r_square)) + return true; } + if (Left.is(TT_ImplicitStringLiteral)) return Right.hasWhitespaceBefore(); if (Line.Type == LT_ObjCMethodDecl) { @@ -5393,6 +5777,23 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, return true; } + if (Style.isTableGen()) { + if (Left.is(TT_TableGenDAGArgOperatorID) && + Right.isNot(TT_TableGenDAGArgCloser)) { + return Style.TableGenBreakInsideDAGArgList; + } + if (Left.is(TT_TableGenDAGArgListComma)) + return Style.TableGenBreakInsideDAGArgList; + if (Left.is(TT_TableGenCondOperatorComma)) + return Style.TableGenBreakInsideCondOperator; + if (Right.is(TT_TableGenDAGArgCloser) && Right.MatchingParen && + Right.MatchingParen->Next->is(TT_TableGenDAGArgOperatorID) && + &Left != Right.MatchingParen->Next) { + // Avoid empty DAGArg such as (ins). + return Style.TableGenBreakInsideDAGArgList; + } + } + return false; } @@ -5501,6 +5902,23 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, return false; if (Left.is(TT_TemplateString) && Left.opensScope()) return true; + } else if (Style.isTableGen()) { + if (Right.is(tok::l_paren)) { + return !Left.isOneOf(TT_TableGenBangOperator, TT_TableGenCondOperator, + TT_TableGenParamAngleCloser); + } + if (Right.is(tok::colon)) + return Style.TableGenAllowBreakBeforeInheritColon; + if (Right.isOneOf(tok::semi, tok::l_brace)) { + // Avoid to break before "{"" or ";" in TableGen. + return false; + } + if (Left.is(TT_TableGenValueSuffix)) + return false; + if (Left.is(tok::colon) && Right.is(tok::identifier)) + return Style.TableGenAllowBreakAfterInheritColon; + if (Left.is(tok::hash) || Right.is(tok::hash)) + return false; } if (Left.is(tok::at)) diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index 684609747a5513..74d5c0464d624a 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -495,7 +495,8 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) { do { NextTok = Tokens->getNextToken(); } while (NextTok->is(tok::comment)); - while (NextTok->is(tok::hash) && !Line->InMacroBody) { + while (NextTok->is(tok::hash) && !Line->InMacroBody && + !Style.isTableGen()) { NextTok = Tokens->getNextToken(); do { NextTok = Tokens->getNextToken(); @@ -1652,7 +1653,7 @@ void UnwrappedLineParser::parseStructuralElement( // In Verilog labels can be any expression, so we don't do them here. // JS doesn't have macros, and within classes colons indicate fields, not // labels. - if (!Style.isJavaScript() && !Style.isVerilog() && + if (!Style.isJavaScript() && !Style.isVerilog() && !Style.isTableGen() && Tokens->peekNextToken()->is(tok::colon) && !Line->MustBeDeclaration) { nextToken(); Line->Tokens.begin()->Tok->MustBreakBefore = true; @@ -1781,6 +1782,12 @@ void UnwrappedLineParser::parseStructuralElement( addUnwrappedLine(); return; } + if (Style.isTableGen()) { + // Do nothing special. In this case the l_brace becomes FunctionLBrace. + // This is same as def and so on. + nextToken(); + break; + } [[fallthrough]]; case tok::kw_struct: case tok::kw_union: @@ -2014,6 +2021,15 @@ void UnwrappedLineParser::parseStructuralElement( nextToken(); if (FormatTok->is(tok::l_brace)) { + if (Style.isTableGen() && + Line->Tokens.begin()->Tok->is(Keywords.kw_defset)) { + // TableGen's defset statement has strange syntax of the form, + // defset type id = { ... } + parseBlock(/*MustBeDeclaration=*/false, /*AddLevels=*/1u, + /*MunchSemi=*/false); + addUnwrappedLine(); + break; + } // Block kind should probably be set to BK_BracedInit for any language. // C# needs this change to ensure that array initialisers and object // initialisers are indented the same way. @@ -2739,23 +2755,31 @@ FormatToken *UnwrappedLineParser::parseIfThenElse(IfStmtKind *IfKind, nextToken(); bool KeepIfBraces = true; - if (FormatTok->is(tok::kw_consteval)) { + if (Style.isTableGen()) { + while (!eof() && !(FormatTok->is(Keywords.kw_then))) { + // Simply skip until then. This range only contains a value. + nextToken(); + } nextToken(); } else { - KeepIfBraces = !Style.RemoveBracesLLVM || KeepBraces; - if (FormatTok->isOneOf(tok::kw_constexpr, tok::identifier)) + if (FormatTok->is(tok::kw_consteval)) { nextToken(); - if (FormatTok->is(tok::l_paren)) { - FormatTok->setFinalizedType(TT_ConditionLParen); - parseParens(); + } else { + KeepIfBraces = !Style.RemoveBracesLLVM || KeepBraces; + if (FormatTok->isOneOf(tok::kw_constexpr, tok::identifier)) + nextToken(); + if (FormatTok->is(tok::l_paren)) { + FormatTok->setFinalizedType(TT_ConditionLParen); + parseParens(); + } + } + handleAttributes(); + // The then action is optional in Verilog assert statements. + if (IsVerilogAssert && FormatTok->is(tok::semi)) { + nextToken(); + addUnwrappedLine(); + return nullptr; } - } - handleAttributes(); - // The then action is optional in Verilog assert statements. - if (IsVerilogAssert && FormatTok->is(tok::semi)) { - nextToken(); - addUnwrappedLine(); - return nullptr; } bool NeedsUnwrappedLine = false; diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index 3bc6915b8df0a7..693822f90820f8 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -111,6 +111,9 @@ const tooling::Replacements &WhitespaceManager::generateReplacements() { alignConsecutiveDeclarations(); alignConsecutiveBitFields(); alignConsecutiveAssignments(); + alignConsecutiveTableGenCondOperatorColons(); + AlignConsecutiveTableGenBreakingDAGArgColons(); + alignConsecutiveTableGenDefinition(); alignChainedConditionals(); alignTrailingComments(); alignEscapedNewlines(); @@ -848,8 +851,9 @@ void WhitespaceManager::alignConsecutiveAssignments() { /*RightJustify=*/true); } -void WhitespaceManager::alignConsecutiveBitFields() { - if (!Style.AlignConsecutiveBitFields.Enabled) +void WhitespaceManager::alignConsecutiveColons( + const FormatStyle::AlignConsecutiveStyle &AlignStyle, TokenType Type) { + if (!AlignStyle.Enabled) return; AlignTokens( @@ -863,9 +867,28 @@ void WhitespaceManager::alignConsecutiveBitFields() { if (&C != &Changes.back() && (&C + 1)->NewlinesBefore > 0) return false; - return C.Tok->is(TT_BitFieldColon); + return C.Tok->is(Type); }, - Changes, /*StartAt=*/0, Style.AlignConsecutiveBitFields); + Changes, /*StartAt=*/0, AlignStyle); +} + +void WhitespaceManager::alignConsecutiveBitFields() { + alignConsecutiveColons(Style.AlignConsecutiveBitFields, TT_BitFieldColon); +} + +void WhitespaceManager::alignConsecutiveTableGenCondOperatorColons() { + alignConsecutiveColons(Style.AlignConsecutiveTableGenCondOperatorColons, + TT_TableGenCondOperatorColon); +} + +void WhitespaceManager::AlignConsecutiveTableGenBreakingDAGArgColons() { + alignConsecutiveColons(Style.AlignConsecutiveTableGenBreakingDAGArgColons, + TT_TableGenDAGArgListColonToAlign); +} + +void WhitespaceManager::alignConsecutiveTableGenDefinition() { + alignConsecutiveColons(Style.AlignConsecutiveTableGenDefinitions, + TT_InheritanceColon); } void WhitespaceManager::alignConsecutiveShortCaseStatements() { diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h index 24fe492dcb0269..d5b59a1d5a6453 100644 --- a/clang/lib/Format/WhitespaceManager.h +++ b/clang/lib/Format/WhitespaceManager.h @@ -223,12 +223,26 @@ class WhitespaceManager { /// Align consecutive assignments over all \c Changes. void alignConsecutiveAssignments(); + /// Align consecutive colon. For bitfields, TableGen DAGArgs and defintions. + void + alignConsecutiveColons(const FormatStyle::AlignConsecutiveStyle &AlignStyle, + TokenType Type); + /// Align consecutive bitfields over all \c Changes. void alignConsecutiveBitFields(); /// Align consecutive declarations over all \c Changes. void alignConsecutiveDeclarations(); + /// Align consecutive TableGen cond operator colon over all \c Changes. + void alignConsecutiveTableGenCondOperatorColons(); + + /// Align consecutive TableGen DAGArg colon over all \c Changes. + void AlignConsecutiveTableGenBreakingDAGArgColons(); + + /// Align consecutive TableGen definition over all \c Changes. + void alignConsecutiveTableGenDefinition(); + /// Align consecutive declarations over all \c Changes. void alignChainedConditionals(); diff --git a/clang/unittests/Format/FormatTestTableGen.cpp b/clang/unittests/Format/FormatTestTableGen.cpp index 123f47d6d7f884..8301dbf7b37189 100644 --- a/clang/unittests/Format/FormatTestTableGen.cpp +++ b/clang/unittests/Format/FormatTestTableGen.cpp @@ -40,6 +40,13 @@ class FormatTestTableGen : public ::testing::Test { EXPECT_EQ(Code.str(), format(Code)) << "Expected code is not stable"; EXPECT_EQ(Code.str(), format(test::messUp(Code))); } + + static void verifyFormat(llvm::StringRef Code, const FormatStyle &Style) { + EXPECT_EQ(Code.str(), format(Code, 0, Code.size(), Style)) + << "Expected code is not stable"; + auto MessUp = test::messUp(Code); + EXPECT_EQ(Code.str(), format(MessUp, 0, MessUp.size(), Style)); + } }; TEST_F(FormatTestTableGen, FormatStringBreak) { @@ -55,5 +62,305 @@ TEST_F(FormatTestTableGen, NoSpacesInSquareBracketLists) { verifyFormat("def flag : Flag<[\"-\", \"--\"], \"foo\">;"); } +TEST_F(FormatTestTableGen, LiteralsAndIdentifiers) { + verifyFormat("def LiteralAndIdentifiers {\n" + " let someInteger = -42;\n" + " let 0startID = $TokVarName;\n" + " let 0xstartInteger = 0x42;\n" + " let someIdentifier = $TokVarName;\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, BangOperators) { + verifyFormat("def BangOperators {\n" + " let IfOpe = !if(\n" + " !not(!and(!gt(!add(1, 2), !sub(3, 4)), !isa<Ty>($x))),\n" + " !foldl(0, !listconcat(!range(5, 6), !range(7, 8)),\n" + " total, rec, !add(total, rec.Number)),\n" + " !tail(!range(9, 10)));\n" + " let ForeachOpe = !foreach(\n" + " arg, arglist,\n" + " !if(!isa<SomeType>(arg.Type),\n" + " !add(!cast<SomeOtherType>(arg).Number, x), arg));\n" + " let CondOpe1 = !cond(!eq(size, 1): 1,\n" + " !eq(size, 2): 1,\n" + " !eq(size, 4): 1,\n" + " !eq(size, 8): 1,\n" + " !eq(size, 16): 1,\n" + " true: 0);\n" + " let CondOpe2 = !cond(!lt(x, 0): \"negativenegative\",\n" + " !eq(x, 0): \"zerozero\",\n" + " true: \"positivepositive\");\n" + " let CondOpe2WithComment = !cond(!lt(x, 0): // negative\n" + " \"negativenegative\",\n" + " !eq(x, 0): // zero\n" + " \"zerozero\",\n" + " true: // default\n" + " \"positivepositive\");\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, Include) { + verifyFormat("include \"test/IncludeFile.h\"\n"); +} + +TEST_F(FormatTestTableGen, Types) { + verifyFormat("def Types : list<int>, bits<3>, list<list<string>> {}\n"); +} + +TEST_F(FormatTestTableGen, Values) { + verifyFormat("def SimpleValue {\n" + " let Integer = 42;\n" + " let String = \"some string\";\n" + "}\n"); + + // verifyFormat does not understand multiline TableGen code-literals + std::string DefWithCode = + "def SimpleValueCode {\n" + " let Code =\n" + " [{ A TokCode is nothing more than a multi-line string literal " + "delimited by \\[{ and }\\]. It can break across lines and the line " + "breaks are retained in the string. " + "(https://llvm.org/docs/TableGen/ProgRef.html#grammar-token-TokCode)}];\n" + "}\n"; + std::string DefWithCodeMessingUp = + "def SimpleValueCode {\n" + " let Code= " + "[{ A TokCode is nothing more than a multi-line string literal " + "delimited by \\[{ and }\\]. It can break across lines and the line " + "breaks are retained in the string. " + "(https://llvm.org/docs/TableGen/ProgRef.html#grammar-token-TokCode)}];\n" + " } \n"; + EXPECT_EQ(DefWithCode, format(DefWithCodeMessingUp)); + + verifyFormat("def SimpleValue2 {\n" + " let True = true;\n" + " let False = false;\n" + "}\n"); + + verifyFormat("class SimpleValue3<int x> { int Question = ?; }\n"); + + verifyFormat("def SimpleValue4 { let ValueList = {1, 2, 3}; }\n"); + + verifyFormat("def SimpleValue5 {\n" + " let SquareList = [1, 4, 9];\n" + " let SquareListWithType = [\"a\", \"b\", \"c\"]<string>;\n" + " let SquareListListWithType = [[1, 2], [3, 4, 5], [7]]<\n" + " list<int>>;\n" + " let SquareBitsListWithType = [ {1, 2}, {3, 4} ]<\n" + " list<bits<8>>>;\n" + "}\n"); + + verifyFormat("def SimpleValue6 {\n" + " let DAGArgIns = (ins i32:$src1, i32:$src2);\n" + " let DAGArgOuts = (outs i32:$dst1, i32:$dst2, i32:$dst3,\n" + " i32:$dst4, i32:$dst5, i32:$dst6, i32:$dst7);\n" + " let DAGArgOutsWithComment = (outs i32:$dst1, // dst1\n" + " i32:$dst2, // dst2\n" + " i32:$dst3, // dst3\n" + " i32:$dst4, // dst4\n" + " i32:$dst5, // dst5\n" + " i32:$dst6, // dst6\n" + " i32:$dst7 // dst7\n" + " );\n" + " let DAGArgBang = (!cast<SomeType>(\"Some\") i32:$src1,\n" + " i32:$src2);\n" + "}\n"); + + verifyFormat("def SimpleValue7 { let Identifier = SimpleValue; }\n"); + + verifyFormat("def SimpleValue8 { let Class = SimpleValue3<3>; }\n"); + + verifyFormat("def SuffixedValues {\n" + " let Bit = value{17};\n" + " let Bits = value{8...15};\n" + " let List = value[1];\n" + " let Slice1 = value[1, ];\n" + " let Slice2 = value[4...7, 17, 2...3, 4];\n" + " let Field = value.field;\n" + "}\n"); + + verifyFormat( + "def Paste#\"Operator\" { string Paste = \"Paste\"#operator; }\n"); + + verifyFormat("def [\"Traring\", \"Paste\"]# {\n" + " string X = Traring#;\n" + " string Y = List<\"Operator\">#;\n" + " string Z = [\"Traring\", \"Paste\", \"Traring\", \"Paste\",\n" + " \"Traring\", \"Paste\"]#;\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, ClassDefinition) { + verifyFormat("class Class<int x, int y = 1, string z = \"z\",\n" + " int w = -1> : Parent1,\n" + " Parent2<x, y> {\n" + " int Item1 = 1;\n" + " int Item2;\n" + " code Item3 = [{ Item3 }];\n" + " let Item4 = 4;\n" + " let Item5{1, 2} = 5;\n" + " defvar Item6 = 6;\n" + " let Item7 = ?;\n" + " assert !ge(x, 0), \"Assert7\";\n" + "}\n"); + + verifyFormat("class FPFormat<bits<3> val> { bits<3> Value = val; }\n"); +} + +TEST_F(FormatTestTableGen, Def) { + verifyFormat("def Def : Parent1<Def>, Parent2(defs Def) {\n" + " code Item1 = [{ Item1 }];\n" + " let Item2{1, 3...4} = {1, 2};\n" + " defvar Item3 = (ops nodty:$node1, nodty:$node2);\n" + " assert !le(Item2, 0), \"Assert4\";\n" + "}\n"); + + verifyFormat("class FPFormat<bits<3> val> { bits<3> Value = val; }\n"); + + verifyFormat("def NotFP : FPFormat<0>;\n"); +} + +TEST_F(FormatTestTableGen, Let) { + verifyFormat("let x = 1, y = value<type>,\n" + " z = !and(!gt(!add(1, 2), !sub(3, 4)), !isa<Ty>($x)) in {\n" + " class Class1 : Parent<x, y> { let Item1 = z; }\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, MultiClass) { + verifyFormat("multiclass Multiclass<int x> {\n" + " def : Def1<(item type:$src1),\n" + " (!if(!ge(x, 0), !mul(!add(x, 1), !sub(x, 2)),\n" + " !sub(x, 2)))>;\n" + " def Def2 : value<type>;\n" + " def Def3 : type { let value = 1; }\n" + " defm : SomeMultiClass<Def1, Def2>;\n" + " defvar DefVar = 6;\n" + " foreach i = [1, 2, 3] in {\n" + " def : Foreach#i<(item type:$src1),\n" + " (!if(!gt(x, i),\n" + " !mul(!add(x, i), !sub(x, i)),\n" + " !sub(x, !add(i, 1))))>;\n" + " }\n" + " if !gt(x, 0) then {\n" + " def : IfThen<x>;\n" + " } else {\n" + " def : IfElse<x>;\n" + " }\n" + " if (dagid x, 0) then {\n" + " def : If2<1>;\n" + " }\n" + " let y = 1, z = 2 in {\n" + " multiclass Multiclass2<int x> {\n" + " foreach i = [1, 2, 3] in {\n" + " def : Foreach#i<(item type:$src1),\n" + " (!if(!gt(z, i),\n" + " !mul(!add(y, i), !sub(x, i)),\n" + " !sub(z, !add(i, 1))))>;\n" + " }\n" + " }\n" + " }\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, Defm) { + verifyFormat("defm : Multiclass<0>;\n"); + + verifyFormat("defm Defm1 : Multiclass<1>;\n"); +} + +TEST_F(FormatTestTableGen, Defset) { + verifyFormat("defset list<Class> DefSet1 = {\n" + " def Def1 : Class<1>;\n" + " def Def2 : Class<2>;\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, Defvar) { + verifyFormat("defvar DefVar1 = !cond(!ge(!size(PaseOperator.Paste), 1): 1,\n" + " true: 0);\n"); +} + +TEST_F(FormatTestTableGen, ForEach) { + verifyFormat( + "foreach i = [1, 2, 3] in {\n" + " def : Foreach#i<(item type:$src1),\n" + " (!if(!lt(x, i),\n" + " !shl(!mul(x, i), !size(\"string\")),\n" + " !size(!strconcat(\"a\", \"b\", \"c\"))))>;\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, Dump) { verifyFormat("dump \"Dump\";\n"); } + +TEST_F(FormatTestTableGen, If) { + verifyFormat("if !gt(x, 0) then {\n" + " def : IfThen<x>;\n" + "} else {\n" + " def : IfElse<x>;\n" + "}\n"); +} + +TEST_F(FormatTestTableGen, Assert) { + verifyFormat("assert !le(DefVar1, 0), \"Assert1\";\n"); +} + +TEST_F(FormatTestTableGen, DefAlignment) { + FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); + Style.ColumnLimit = 60; + verifyFormat("def Def : Parent {}\n" + "def DefDef : Parent {}\n" + "def DefDefDef : Parent {}\n", + Style); + Style.AlignConsecutiveTableGenDefinitions.Enabled = true; + verifyFormat("def Def : Parent {}\n" + "def DefDef : Parent {}\n" + "def DefDefDef : Parent {}\n", + Style); +} + +TEST_F(FormatTestTableGen, DAGArgAlignment) { + FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); + Style.ColumnLimit = 60; + Style.TableGenBreakInsideDAGArgList = true; + Style.TableGenBreakingDAGArgOperators = {"ins", "outs"}; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a:$src1,\n" + " aa:$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a :$src1,\n" + " aa :$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); +} + +TEST_F(FormatTestTableGen, CondOperatorAlignment) { + FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); + Style.ColumnLimit = 60; + verifyFormat("let CondOpe1 = !cond(!eq(size, 1): 1,\n" + " !eq(size, 16): 1,\n" + " true: 0);\n", + Style); + Style.AlignConsecutiveTableGenCondOperatorColons.Enabled = true; + verifyFormat("let CondOpe1 = !cond(!eq(size, 1) : 1,\n" + " !eq(size, 16): 1,\n" + " true : 0);\n", + Style); +} + } // namespace format } // end namespace clang diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 8e6935319b2f3d..9533696d010bb9 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2165,6 +2165,56 @@ TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) { EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown); } +TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) { + auto Style = getLLVMStyle(FormatStyle::LK_TableGen); + Style.TableGenBreakingDAGArgOperators = {"ins", "outs"}; + + auto AnnotateValue = [this, &Style](llvm::StringRef Code) { + // Values are annotated only in specific context. + auto Result = annotate(("def X { let V = " + Code + "; }").str(), Style); + return decltype(Result){Result.begin() + 6, Result.end() - 3}; + }; + // Both of bang/cond operators + auto Tokens = AnnotateValue("!cond(!eq(x, 0): 1, true: x)"); + ASSERT_EQ(Tokens.size(), 15u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator); + EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TableGenBangOperator); + EXPECT_TOKEN(Tokens[8], tok::colon, TT_TableGenCondOperatorColon); + EXPECT_TOKEN(Tokens[10], tok::comma, TT_TableGenCondOperatorComma); + EXPECT_TOKEN(Tokens[12], tok::colon, TT_TableGenCondOperatorColon); + // DAGArg values with operator identifier + Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener); + EXPECT_TOKEN(Tokens[1], tok::identifier, TT_TableGenDAGArgOperatorID); + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon); + EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // $src1 + EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon); + EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser); + // List literal + Tokens = AnnotateValue("[1, 2, 3]"); + ASSERT_EQ(Tokens.size(), 7u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::l_square, TT_TableGenListOpener); + EXPECT_TOKEN(Tokens[6], tok::r_square, TT_TableGenListCloser); + // Suffixes of values + Tokens = AnnotateValue("valid.field"); + ASSERT_EQ(Tokens.size(), 3u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::period, TT_TableGenValueSuffix); + // Code + Tokens = AnnotateValue("[{ code is multiline string }]"); + ASSERT_EQ(Tokens.size(), 1u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString); + // The definition + Tokens = annotate("def Def : Parent<Child> {}", Style); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; // This contains eof. + // We use inheritance colon and function brace. They are enough. + EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon); + EXPECT_TOKEN(Tokens[4], tok::less, TT_TableGenParamAngleOpener); + EXPECT_TOKEN(Tokens[6], tok::greater, TT_TableGenParamAngleCloser); + EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace); +} + TEST_F(TokenAnnotatorTest, UnderstandConstructors) { auto Tokens = annotate("Class::Class() : BaseClass(), Member() {}"); _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits