[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.
This revision was automatically updated to reflect the committed changes. Closed by commit rG8c2a61397607: Hand Allocator and IdentifierTable into FormatTokenLexer. (authored by klimek). Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D83218/new/ https://reviews.llvm.org/D83218 Files: clang/lib/Format/FormatTokenLexer.cpp clang/lib/Format/FormatTokenLexer.h clang/lib/Format/TokenAnalyzer.cpp Index: clang/lib/Format/TokenAnalyzer.cpp === --- clang/lib/Format/TokenAnalyzer.cpp +++ clang/lib/Format/TokenAnalyzer.cpp @@ -64,11 +64,16 @@ std::pair TokenAnalyzer::process() { tooling::Replacements Result; - FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), - Env.getFirstStartColumn(), Style, Encoding); + llvm::SpecificBumpPtrAllocator Allocator; + IdentifierTable IdentTable(getFormattingLangOpts(Style)); + FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(), + Env.getFirstStartColumn(), Style, Encoding, Allocator, - UnwrappedLineParser Parser(Style, Tokens.getKeywords(), - Env.getFirstStartColumn(), Tokens.lex(), *this); + IdentTable); + ArrayRef Toks(Lex.lex()); + SmallVector Tokens(Toks.begin(), Toks.end()); + UnwrappedLineParser Parser(Style, Lex.getKeywords(), + Env.getFirstStartColumn(), Tokens, *this); Parser.parse(); assert(UnwrappedLines.rbegin()->empty()); unsigned Penalty = 0; @@ -76,14 +81,14 @@ LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); SmallVector AnnotatedLines; -TokenAnnotator Annotator(Style, Tokens.getKeywords()); +TokenAnnotator Annotator(Style, Lex.getKeywords()); for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) { AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i])); Annotator.annotate(*AnnotatedLines.back()); } std::pair RunResult = -analyze(Annotator, AnnotatedLines, Tokens); +analyze(Annotator, AnnotatedLines, Lex); LLVM_DEBUG({ llvm::dbgs() << "Replacements for run " << Run << ":\n"; Index: clang/lib/Format/FormatTokenLexer.h === --- clang/lib/Format/FormatTokenLexer.h +++ clang/lib/Format/FormatTokenLexer.h @@ -38,7 +38,9 @@ class FormatTokenLexer { public: FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column, - const FormatStyle &Style, encoding::Encoding Encoding); + const FormatStyle &Style, encoding::Encoding Encoding, + llvm::SpecificBumpPtrAllocator &Allocator, + IdentifierTable &IdentTable); ArrayRef lex(); @@ -103,10 +105,10 @@ const SourceManager &SourceMgr; FileID ID; const FormatStyle &Style; - IdentifierTable IdentTable; + IdentifierTable &IdentTable; AdditionalKeywords Keywords; encoding::Encoding Encoding; - llvm::SpecificBumpPtrAllocator Allocator; + llvm::SpecificBumpPtrAllocator &Allocator; // Index (in 'Tokens') of the last token that starts a new line. unsigned FirstInLineIndex; SmallVector Tokens; Index: clang/lib/Format/FormatTokenLexer.cpp === --- clang/lib/Format/FormatTokenLexer.cpp +++ clang/lib/Format/FormatTokenLexer.cpp @@ -22,13 +22,15 @@ namespace clang { namespace format { -FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, - unsigned Column, const FormatStyle &Style, - encoding::Encoding Encoding) +FormatTokenLexer::FormatTokenLexer( +const SourceManager &SourceMgr, FileID ID, unsigned Column, +const FormatStyle &Style, encoding::Encoding Encoding, +llvm::SpecificBumpPtrAllocator &Allocator, +IdentifierTable &IdentTable) : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}), Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID), - Style(Style), IdentTable(getFormattingLangOpts(Style)), - Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0), + Style(Style), IdentTable(IdentTable), Keywords(IdentTable), + Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0), FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin), MacroBlockEndRegex(Style.MacroBlockEnd) { Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr, Index: clang/lib/Format/TokenAnalyzer.cpp === --- clang/lib/Format/TokenAnalyzer.cpp +++ clang/lib/Format/TokenAnalyzer.cpp @@ -64,11 +64,16 @@ std::pair TokenAnalyzer::process() { tooling::Replacements Result; - FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), -
[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.
klimek updated this revision to Diff 275964. klimek added a comment. Address review comment. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D83218/new/ https://reviews.llvm.org/D83218 Files: clang/lib/Format/FormatTokenLexer.cpp clang/lib/Format/FormatTokenLexer.h clang/lib/Format/TokenAnalyzer.cpp Index: clang/lib/Format/TokenAnalyzer.cpp === --- clang/lib/Format/TokenAnalyzer.cpp +++ clang/lib/Format/TokenAnalyzer.cpp @@ -64,11 +64,16 @@ std::pair TokenAnalyzer::process() { tooling::Replacements Result; - FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), - Env.getFirstStartColumn(), Style, Encoding); + llvm::SpecificBumpPtrAllocator Allocator; + IdentifierTable IdentTable(getFormattingLangOpts(Style)); + FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(), + Env.getFirstStartColumn(), Style, Encoding, Allocator, - UnwrappedLineParser Parser(Style, Tokens.getKeywords(), - Env.getFirstStartColumn(), Tokens.lex(), *this); + IdentTable); + ArrayRef Toks(Lex.lex()); + SmallVector Tokens(Toks.begin(), Toks.end()); + UnwrappedLineParser Parser(Style, Lex.getKeywords(), + Env.getFirstStartColumn(), Tokens, *this); Parser.parse(); assert(UnwrappedLines.rbegin()->empty()); unsigned Penalty = 0; @@ -76,14 +81,14 @@ LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); SmallVector AnnotatedLines; -TokenAnnotator Annotator(Style, Tokens.getKeywords()); +TokenAnnotator Annotator(Style, Lex.getKeywords()); for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) { AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i])); Annotator.annotate(*AnnotatedLines.back()); } std::pair RunResult = -analyze(Annotator, AnnotatedLines, Tokens); +analyze(Annotator, AnnotatedLines, Lex); LLVM_DEBUG({ llvm::dbgs() << "Replacements for run " << Run << ":\n"; Index: clang/lib/Format/FormatTokenLexer.h === --- clang/lib/Format/FormatTokenLexer.h +++ clang/lib/Format/FormatTokenLexer.h @@ -38,7 +38,9 @@ class FormatTokenLexer { public: FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column, - const FormatStyle &Style, encoding::Encoding Encoding); + const FormatStyle &Style, encoding::Encoding Encoding, + llvm::SpecificBumpPtrAllocator &Allocator, + IdentifierTable &IdentTable); ArrayRef lex(); @@ -103,10 +105,10 @@ const SourceManager &SourceMgr; FileID ID; const FormatStyle &Style; - IdentifierTable IdentTable; + IdentifierTable &IdentTable; AdditionalKeywords Keywords; encoding::Encoding Encoding; - llvm::SpecificBumpPtrAllocator Allocator; + llvm::SpecificBumpPtrAllocator &Allocator; // Index (in 'Tokens') of the last token that starts a new line. unsigned FirstInLineIndex; SmallVector Tokens; Index: clang/lib/Format/FormatTokenLexer.cpp === --- clang/lib/Format/FormatTokenLexer.cpp +++ clang/lib/Format/FormatTokenLexer.cpp @@ -22,13 +22,15 @@ namespace clang { namespace format { -FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, - unsigned Column, const FormatStyle &Style, - encoding::Encoding Encoding) +FormatTokenLexer::FormatTokenLexer( +const SourceManager &SourceMgr, FileID ID, unsigned Column, +const FormatStyle &Style, encoding::Encoding Encoding, +llvm::SpecificBumpPtrAllocator &Allocator, +IdentifierTable &IdentTable) : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}), Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID), - Style(Style), IdentTable(getFormattingLangOpts(Style)), - Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0), + Style(Style), IdentTable(IdentTable), Keywords(IdentTable), + Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0), FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin), MacroBlockEndRegex(Style.MacroBlockEnd) { Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr, Index: clang/lib/Format/TokenAnalyzer.cpp === --- clang/lib/Format/TokenAnalyzer.cpp +++ clang/lib/Format/TokenAnalyzer.cpp @@ -64,11 +64,16 @@ std::pair TokenAnalyzer::process() { tooling::Replacements Result; - FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), - Env.getFirstStartColumn(), Style, Encoding); + llvm::SpecificBumpPtrAll
[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.
sammccall accepted this revision. sammccall added inline comments. This revision is now accepted and ready to land. Comment at: clang/lib/Format/FormatTokenLexer.h:121 llvm::Regex MacroBlockEndRegex; + llvm::SpecificBumpPtrAllocator &Allocator; this seems a little odd, like it fit better where it was, up to you. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D83218/new/ https://reviews.llvm.org/D83218 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.
klimek created this revision. klimek added a reviewer: sammccall. Herald added a project: clang. Herald added a subscriber: cfe-commits. This allows us to share the allocator in the future so we can create tokens while parsing. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D83218 Files: clang/lib/Format/FormatTokenLexer.cpp clang/lib/Format/FormatTokenLexer.h clang/lib/Format/TokenAnalyzer.cpp Index: clang/lib/Format/TokenAnalyzer.cpp === --- clang/lib/Format/TokenAnalyzer.cpp +++ clang/lib/Format/TokenAnalyzer.cpp @@ -64,11 +64,16 @@ std::pair TokenAnalyzer::process() { tooling::Replacements Result; - FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), - Env.getFirstStartColumn(), Style, Encoding); + llvm::SpecificBumpPtrAllocator Allocator; + IdentifierTable IdentTable(getFormattingLangOpts(Style)); + FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(), + Env.getFirstStartColumn(), Style, Encoding, Allocator, - UnwrappedLineParser Parser(Style, Tokens.getKeywords(), - Env.getFirstStartColumn(), Tokens.lex(), *this); + IdentTable); + ArrayRef Toks(Lex.lex()); + SmallVector Tokens(Toks.begin(), Toks.end()); + UnwrappedLineParser Parser(Style, Lex.getKeywords(), + Env.getFirstStartColumn(), Tokens, *this); Parser.parse(); assert(UnwrappedLines.rbegin()->empty()); unsigned Penalty = 0; @@ -76,14 +81,14 @@ LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); SmallVector AnnotatedLines; -TokenAnnotator Annotator(Style, Tokens.getKeywords()); +TokenAnnotator Annotator(Style, Lex.getKeywords()); for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) { AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i])); Annotator.annotate(*AnnotatedLines.back()); } std::pair RunResult = -analyze(Annotator, AnnotatedLines, Tokens); +analyze(Annotator, AnnotatedLines, Lex); LLVM_DEBUG({ llvm::dbgs() << "Replacements for run " << Run << ":\n"; Index: clang/lib/Format/FormatTokenLexer.h === --- clang/lib/Format/FormatTokenLexer.h +++ clang/lib/Format/FormatTokenLexer.h @@ -38,7 +38,9 @@ class FormatTokenLexer { public: FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column, - const FormatStyle &Style, encoding::Encoding Encoding); + const FormatStyle &Style, encoding::Encoding Encoding, + llvm::SpecificBumpPtrAllocator &Allocator, + IdentifierTable &IdentTable); ArrayRef lex(); @@ -103,10 +105,9 @@ const SourceManager &SourceMgr; FileID ID; const FormatStyle &Style; - IdentifierTable IdentTable; + IdentifierTable &IdentTable; AdditionalKeywords Keywords; encoding::Encoding Encoding; - llvm::SpecificBumpPtrAllocator Allocator; // Index (in 'Tokens') of the last token that starts a new line. unsigned FirstInLineIndex; SmallVector Tokens; @@ -117,6 +118,7 @@ llvm::Regex MacroBlockBeginRegex; llvm::Regex MacroBlockEndRegex; + llvm::SpecificBumpPtrAllocator &Allocator; // Targets that may appear inside a C# attribute. static const llvm::StringSet<> CSharpAttributeTargets; Index: clang/lib/Format/FormatTokenLexer.cpp === --- clang/lib/Format/FormatTokenLexer.cpp +++ clang/lib/Format/FormatTokenLexer.cpp @@ -22,15 +22,17 @@ namespace clang { namespace format { -FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, - unsigned Column, const FormatStyle &Style, - encoding::Encoding Encoding) +FormatTokenLexer::FormatTokenLexer( +const SourceManager &SourceMgr, FileID ID, unsigned Column, +const FormatStyle &Style, encoding::Encoding Encoding, +llvm::SpecificBumpPtrAllocator &Allocator, +IdentifierTable &IdentTable) : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}), Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID), - Style(Style), IdentTable(getFormattingLangOpts(Style)), - Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0), - FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin), - MacroBlockEndRegex(Style.MacroBlockEnd) { + Style(Style), IdentTable(IdentTable), Keywords(IdentTable), + Encoding(Encoding), FirstInLineIndex(0), FormattingDisabled(false), + MacroBlockBeginRegex(Style.MacroBlockBegin), + MacroBlockEndRegex(Style.MacroBlockEnd), Allocator(Allocator) { Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr, getFormattingLangOpts(St