[PATCH] D65628: [clang-doc] Parallelize reducing phase
This revision was automatically updated to reflect the committed changes.
Closed by commit rL368206: [clang-doc] Parallelize reducing phase (authored by
DiegoAstiazaran, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.
Changed prior to commit:
https://reviews.llvm.org/D65628?vs=213986&id=213988#toc
Repository:
rL LLVM
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D65628/new/
https://reviews.llvm.org/D65628
Files:
clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp
Index: clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp
===
--- clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp
+++ clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp
@@ -28,6 +28,7 @@
#include "clang/ASTMatchers/ASTMatchersInternal.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/AllTUsExecution.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Execution.h"
#include "clang/Tooling/Tooling.h"
@@ -38,7 +39,9 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
+#include
#include
using namespace clang::ast_matchers;
@@ -158,30 +161,6 @@
return Path;
}
-// Iterate through tool results and build string map of info vectors from the
-// encoded bitstreams.
-bool bitcodeResultsToInfos(
-tooling::ToolResults &Results,
-llvm::StringMap>> &Output) {
- bool Err = false;
- Results.forEachResult([&](StringRef Key, StringRef Value) {
-llvm::BitstreamCursor Stream(Value);
-doc::ClangDocBitcodeReader Reader(Stream);
-auto Infos = Reader.readBitcode();
-if (!Infos) {
- llvm::errs() << toString(Infos.takeError()) << "\n";
- Err = true;
- return;
-}
-for (auto &I : Infos.get()) {
- auto R =
- Output.try_emplace(Key, std::vector>());
- R.first->second.emplace_back(std::move(I));
-}
- });
- return Err;
-}
-
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::error_code OK;
@@ -256,40 +235,72 @@
// In ToolResults, the Key is the hashed USR and the value is the
// bitcode-encoded representation of the Info object.
llvm::outs() << "Collecting infos...\n";
- llvm::StringMap>> USRToInfos;
- if (bitcodeResultsToInfos(*Exec->get()->getToolResults(), USRToInfos))
-return 1;
+ llvm::StringMap> USRToBitcode;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+auto R = USRToBitcode.try_emplace(Key, std::vector());
+R.first->second.emplace_back(Value);
+ });
// First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToInfos.size() << " infos...\n";
- for (auto &Group : USRToInfos) {
-auto Reduced = doc::mergeInfos(Group.getValue());
-if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- continue;
-}
-
-doc::Info *I = Reduced.get().get();
-auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
- "." + Format);
-if (!InfoPath) {
- llvm::errs() << toString(InfoPath.takeError()) << "\n";
- return 1;
-}
-std::error_code FileErr;
-llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
-llvm::sys::fs::OF_None);
-if (FileErr != OK) {
- llvm::errs() << "Error opening info file: " << FileErr.message() << "\n";
- continue;
-}
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ std::atomic Error;
+ Error = false;
+ // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
+ llvm::ThreadPool Pool(ExecutorConcurrency == 0 ? llvm::hardware_concurrency()
+ : ExecutorConcurrency);
+ for (auto &Group : USRToBitcode) {
+Pool.async([&]() {
+ std::vector> Infos;
+
+ for (auto &Bitcode : Group.getValue()) {
+llvm::BitstreamCursor Stream(Bitcode);
+doc::ClangDocBitcodeReader Reader(Stream);
+auto ReadInfos = Reader.readBitcode();
+if (!ReadInfos) {
+ llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+ Error = true;
+ return;
+}
+std::move(ReadInfos->begin(), ReadInfos->end(),
+ std::back_inserter(Infos));
+ }
+
+ auto Reduced = doc::mergeInfos(Infos);
+ if (!Reduced) {
+llvm::errs() << llvm::toString(Reduced.takeError());
+return;
+ }
+
+ doc::Info *I = Reduced.get().get();
+ auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
+"." + Format);
+ if (!InfoPath) {
+llvm::errs() << toString(InfoPath.take
[PATCH] D65628: [clang-doc] Parallelize reducing phase
DiegoAstiazaran updated this revision to Diff 213986.
DiegoAstiazaran marked an inline comment as done.
DiegoAstiazaran added a comment.
Add comment
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D65628/new/
https://reviews.llvm.org/D65628
Files:
clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
Index: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
===
--- clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -28,6 +28,7 @@
#include "clang/ASTMatchers/ASTMatchersInternal.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/AllTUsExecution.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Execution.h"
#include "clang/Tooling/Tooling.h"
@@ -38,7 +39,9 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
+#include
#include
using namespace clang::ast_matchers;
@@ -158,30 +161,6 @@
return Path;
}
-// Iterate through tool results and build string map of info vectors from the
-// encoded bitstreams.
-bool bitcodeResultsToInfos(
-tooling::ToolResults &Results,
-llvm::StringMap>> &Output) {
- bool Err = false;
- Results.forEachResult([&](StringRef Key, StringRef Value) {
-llvm::BitstreamCursor Stream(Value);
-doc::ClangDocBitcodeReader Reader(Stream);
-auto Infos = Reader.readBitcode();
-if (!Infos) {
- llvm::errs() << toString(Infos.takeError()) << "\n";
- Err = true;
- return;
-}
-for (auto &I : Infos.get()) {
- auto R =
- Output.try_emplace(Key, std::vector>());
- R.first->second.emplace_back(std::move(I));
-}
- });
- return Err;
-}
-
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::error_code OK;
@@ -256,41 +235,73 @@
// In ToolResults, the Key is the hashed USR and the value is the
// bitcode-encoded representation of the Info object.
llvm::outs() << "Collecting infos...\n";
- llvm::StringMap>> USRToInfos;
- if (bitcodeResultsToInfos(*Exec->get()->getToolResults(), USRToInfos))
-return 1;
+ llvm::StringMap> USRToBitcode;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+auto R = USRToBitcode.try_emplace(Key, std::vector());
+R.first->second.emplace_back(Value);
+ });
// First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToInfos.size() << " infos...\n";
- for (auto &Group : USRToInfos) {
-auto Reduced = doc::mergeInfos(Group.getValue());
-if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- continue;
-}
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ std::atomic Error;
+ Error = false;
+ // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
+ llvm::ThreadPool Pool(ExecutorConcurrency == 0 ? llvm::hardware_concurrency()
+ : ExecutorConcurrency);
+ for (auto &Group : USRToBitcode) {
+Pool.async([&]() {
+ std::vector> Infos;
-doc::Info *I = Reduced.get().get();
-auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
- "." + Format);
-if (!InfoPath) {
- llvm::errs() << toString(InfoPath.takeError()) << "\n";
- return 1;
-}
-std::error_code FileErr;
-llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
-llvm::sys::fs::OF_None);
-if (FileErr != OK) {
- llvm::errs() << "Error opening info file: " << FileErr.message() << "\n";
- continue;
-}
+ for (auto &Bitcode : Group.getValue()) {
+llvm::BitstreamCursor Stream(Bitcode);
+doc::ClangDocBitcodeReader Reader(Stream);
+auto ReadInfos = Reader.readBitcode();
+if (!ReadInfos) {
+ llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+ Error = true;
+ return;
+}
+std::move(ReadInfos->begin(), ReadInfos->end(),
+ std::back_inserter(Infos));
+ }
-// Add a reference to this Info in the Index
-clang::doc::Generator::addInfoToIndex(CDCtx.Idx, I);
+ auto Reduced = doc::mergeInfos(Infos);
+ if (!Reduced) {
+llvm::errs() << llvm::toString(Reduced.takeError());
+return;
+ }
-if (auto Err = G->get()->generateDocForInfo(I, InfoOS, CDCtx))
- llvm::errs() << toString(std::move(Err)) << "\n";
+ doc::Info *I = Reduced.get().get();
+ auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
+"." + Format);
+ if (!InfoPath) {
+llvm::errs() << toString(InfoPath.takeError()
[PATCH] D65628: [clang-doc] Parallelize reducing phase
juliehockett accepted this revision. juliehockett added a comment. This revision is now accepted and ready to land. LGTM (after comment nit addressed) Comment at: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp:249 + Error = false; + llvm::ThreadPool Pool(ExecutorConcurrency == 0 ? llvm::hardware_concurrency() + : ExecutorConcurrency); nit: comment about where this var is coming from CHANGES SINCE LAST ACTION https://reviews.llvm.org/D65628/new/ https://reviews.llvm.org/D65628 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D65628: [clang-doc] Parallelize reducing phase
DiegoAstiazaran added inline comments. Comment at: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp:72 +static llvm::cl::opt ThreadCount( +"thread-count", juliehockett wrote: > Can we use the pre-existing concurrency flag instead > (https://github.com/llvm/llvm-project/blob/91e5cdfc93729c61c757db4efd4a82670ac7f929/clang/lib/Tooling/AllTUsExecution.cpp#L150)? > It seems counterintuitive to have to set two different concurrency flags for > one tool. > > You'll have to put up a separate patch to expose that flag in the AllTUs > header, so that you can access it (see rL344335, as an example). Done in D65833. CHANGES SINCE LAST ACTION https://reviews.llvm.org/D65628/new/ https://reviews.llvm.org/D65628 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D65628: [clang-doc] Parallelize reducing phase
DiegoAstiazaran updated this revision to Diff 213968.
DiegoAstiazaran marked 2 inline comments as done.
DiegoAstiazaran added a comment.
Use pre-existing concurrency flag `--execute-concurrency` instead implementing
a new one.
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D65628/new/
https://reviews.llvm.org/D65628
Files:
clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
Index: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
===
--- clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -28,6 +28,7 @@
#include "clang/ASTMatchers/ASTMatchersInternal.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/AllTUsExecution.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Execution.h"
#include "clang/Tooling/Tooling.h"
@@ -38,7 +39,9 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
+#include
#include
using namespace clang::ast_matchers;
@@ -158,30 +161,6 @@
return Path;
}
-// Iterate through tool results and build string map of info vectors from the
-// encoded bitstreams.
-bool bitcodeResultsToInfos(
-tooling::ToolResults &Results,
-llvm::StringMap>> &Output) {
- bool Err = false;
- Results.forEachResult([&](StringRef Key, StringRef Value) {
-llvm::BitstreamCursor Stream(Value);
-doc::ClangDocBitcodeReader Reader(Stream);
-auto Infos = Reader.readBitcode();
-if (!Infos) {
- llvm::errs() << toString(Infos.takeError()) << "\n";
- Err = true;
- return;
-}
-for (auto &I : Infos.get()) {
- auto R =
- Output.try_emplace(Key, std::vector>());
- R.first->second.emplace_back(std::move(I));
-}
- });
- return Err;
-}
-
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::error_code OK;
@@ -256,41 +235,72 @@
// In ToolResults, the Key is the hashed USR and the value is the
// bitcode-encoded representation of the Info object.
llvm::outs() << "Collecting infos...\n";
- llvm::StringMap>> USRToInfos;
- if (bitcodeResultsToInfos(*Exec->get()->getToolResults(), USRToInfos))
-return 1;
+ llvm::StringMap> USRToBitcode;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+auto R = USRToBitcode.try_emplace(Key, std::vector());
+R.first->second.emplace_back(Value);
+ });
// First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToInfos.size() << " infos...\n";
- for (auto &Group : USRToInfos) {
-auto Reduced = doc::mergeInfos(Group.getValue());
-if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- continue;
-}
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ std::atomic Error;
+ Error = false;
+ llvm::ThreadPool Pool(ExecutorConcurrency == 0 ? llvm::hardware_concurrency()
+ : ExecutorConcurrency);
+ for (auto &Group : USRToBitcode) {
+Pool.async([&]() {
+ std::vector> Infos;
-doc::Info *I = Reduced.get().get();
-auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
- "." + Format);
-if (!InfoPath) {
- llvm::errs() << toString(InfoPath.takeError()) << "\n";
- return 1;
-}
-std::error_code FileErr;
-llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
-llvm::sys::fs::OF_None);
-if (FileErr != OK) {
- llvm::errs() << "Error opening info file: " << FileErr.message() << "\n";
- continue;
-}
+ for (auto &Bitcode : Group.getValue()) {
+llvm::BitstreamCursor Stream(Bitcode);
+doc::ClangDocBitcodeReader Reader(Stream);
+auto ReadInfos = Reader.readBitcode();
+if (!ReadInfos) {
+ llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+ Error = true;
+ return;
+}
+std::move(ReadInfos->begin(), ReadInfos->end(),
+ std::back_inserter(Infos));
+ }
-// Add a reference to this Info in the Index
-clang::doc::Generator::addInfoToIndex(CDCtx.Idx, I);
+ auto Reduced = doc::mergeInfos(Infos);
+ if (!Reduced) {
+llvm::errs() << llvm::toString(Reduced.takeError());
+return;
+ }
-if (auto Err = G->get()->generateDocForInfo(I, InfoOS, CDCtx))
- llvm::errs() << toString(std::move(Err)) << "\n";
+ doc::Info *I = Reduced.get().get();
+ auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
+"." + Format);
+ if (!InfoPath) {
+llvm::errs() << toString(InfoPa
[PATCH] D65628: [clang-doc] Parallelize reducing phase
juliehockett added inline comments. Comment at: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp:72 +static llvm::cl::opt ThreadCount( +"thread-count", Can we use the pre-existing concurrency flag instead (https://github.com/llvm/llvm-project/blob/91e5cdfc93729c61c757db4efd4a82670ac7f929/clang/lib/Tooling/AllTUsExecution.cpp#L150)? It seems counterintuitive to have to set two different concurrency flags for one tool. You'll have to put up a separate patch to expose that flag in the AllTUs header, so that you can access it (see rL344335, as an example). CHANGES SINCE LAST ACTION https://reviews.llvm.org/D65628/new/ https://reviews.llvm.org/D65628 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D65628: [clang-doc] Parallelize reducing phase
DiegoAstiazaran updated this revision to Diff 213425.
DiegoAstiazaran marked 2 inline comments as done.
DiegoAstiazaran added a comment.
Herald added a subscriber: jfb.
Fix atomicity issues.
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D65628/new/
https://reviews.llvm.org/D65628
Files:
clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
Index: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
===
--- clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -38,7 +38,9 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
+#include
#include
using namespace clang::ast_matchers;
@@ -67,6 +69,12 @@
llvm::cl::desc("CSS stylesheets to extend the default styles."),
llvm::cl::cat(ClangDocCategory));
+static llvm::cl::opt ThreadCount(
+"thread-count",
+llvm::cl::desc("Threads to use for collecting and reducing infos."),
+llvm::cl::init(llvm::hardware_concurrency()),
+llvm::cl::cat(ClangDocCategory));
+
enum OutputFormatTy {
md,
yaml,
@@ -153,30 +161,6 @@
return Path;
}
-// Iterate through tool results and build string map of info vectors from the
-// encoded bitstreams.
-bool bitcodeResultsToInfos(
-tooling::ToolResults &Results,
-llvm::StringMap>> &Output) {
- bool Err = false;
- Results.forEachResult([&](StringRef Key, StringRef Value) {
-llvm::BitstreamCursor Stream(Value);
-doc::ClangDocBitcodeReader Reader(Stream);
-auto Infos = Reader.readBitcode();
-if (!Infos) {
- llvm::errs() << toString(Infos.takeError()) << "\n";
- Err = true;
- return;
-}
-for (auto &I : Infos.get()) {
- auto R =
- Output.try_emplace(Key, std::vector>());
- R.first->second.emplace_back(std::move(I));
-}
- });
- return Err;
-}
-
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::error_code OK;
@@ -237,37 +221,68 @@
// In ToolResults, the Key is the hashed USR and the value is the
// bitcode-encoded representation of the Info object.
llvm::outs() << "Collecting infos...\n";
- llvm::StringMap>> USRToInfos;
- if (bitcodeResultsToInfos(*Exec->get()->getToolResults(), USRToInfos))
-return 1;
+ llvm::StringMap> USRToBitcode;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+auto R = USRToBitcode.try_emplace(Key, std::vector());
+R.first->second.emplace_back(Value);
+ });
// First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToInfos.size() << " infos...\n";
- for (auto &Group : USRToInfos) {
-auto Reduced = doc::mergeInfos(Group.getValue());
-if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- continue;
-}
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ std::atomic Error;
+ Error = false;
+ llvm::ThreadPool Pool(ThreadCount);
+ for (auto &Group : USRToBitcode) {
+Pool.async([&]() {
+ std::vector> Infos;
-doc::Info *I = Reduced.get().get();
-auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
- "." + Format);
-if (!InfoPath) {
- llvm::errs() << toString(InfoPath.takeError()) << "\n";
- return 1;
-}
-std::error_code FileErr;
-llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr, llvm::sys::fs::F_None);
-if (FileErr != OK) {
- llvm::errs() << "Error opening info file: " << FileErr.message() << "\n";
- continue;
-}
+ for (auto &Bitcode : Group.getValue()) {
+llvm::BitstreamCursor Stream(Bitcode);
+doc::ClangDocBitcodeReader Reader(Stream);
+auto ReadInfos = Reader.readBitcode();
+if (!ReadInfos) {
+ llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+ Error = true;
+ return;
+}
+std::move(ReadInfos->begin(), ReadInfos->end(),
+ std::back_inserter(Infos));
+ }
+
+ auto Reduced = doc::mergeInfos(Infos);
+ if (!Reduced) {
+llvm::errs() << llvm::toString(Reduced.takeError());
+return;
+ }
+
+ doc::Info *I = Reduced.get().get();
+ auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
+"." + Format);
+ if (!InfoPath) {
+llvm::errs() << toString(InfoPath.takeError()) << "\n";
+Error = true;
+return;
+ }
+ std::error_code FileErr;
+ llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
+ llvm::sys::fs::F_None);
+ if (FileErr != OK) {
+llvm::errs() << "Error opening info file: " << FileErr.message()
+
[PATCH] D65628: [clang-doc] Parallelize reducing phase
jakehehrlich added a comment. LGTM other than a couple atomicity issues. Comment at: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp:244 + llvm::errs() << toString(ReadInfos.takeError()) << "\n"; + Error = true; + return; use std::atomic Comment at: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp:262 +llvm::errs() << toString(InfoPath.takeError()) << "\n"; +Error = true; +return; This isn't technically thread safe. Use an std::atomic CHANGES SINCE LAST ACTION https://reviews.llvm.org/D65628/new/ https://reviews.llvm.org/D65628 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D65628: [clang-doc] Parallelize reducing phase
DiegoAstiazaran created this revision.
DiegoAstiazaran added reviewers: jakehehrlich, juliehockett.
DiegoAstiazaran added a project: clang-tools-extra.
Herald added subscribers: kadircet, arphaman.
Reduce phase has been parallelized and a execution time was reduced by 60% with
this.
The reading of bitcode (bitcode -> Info) was moved to this segment of code
parallelized so it now happens just before reducing.
https://reviews.llvm.org/D65628
Files:
clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
Index: clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
===
--- clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -38,6 +38,7 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
#include
@@ -67,6 +68,12 @@
llvm::cl::desc("CSS stylesheets to extend the default styles."),
llvm::cl::cat(ClangDocCategory));
+static llvm::cl::opt ThreadCount(
+"thread-count",
+llvm::cl::desc("Threads to use for collecting and reducing infos."),
+llvm::cl::init(llvm::hardware_concurrency()),
+llvm::cl::cat(ClangDocCategory));
+
enum OutputFormatTy {
md,
yaml,
@@ -153,30 +160,6 @@
return Path;
}
-// Iterate through tool results and build string map of info vectors from the
-// encoded bitstreams.
-bool bitcodeResultsToInfos(
-tooling::ToolResults &Results,
-llvm::StringMap>> &Output) {
- bool Err = false;
- Results.forEachResult([&](StringRef Key, StringRef Value) {
-llvm::BitstreamCursor Stream(Value);
-doc::ClangDocBitcodeReader Reader(Stream);
-auto Infos = Reader.readBitcode();
-if (!Infos) {
- llvm::errs() << toString(Infos.takeError()) << "\n";
- Err = true;
- return;
-}
-for (auto &I : Infos.get()) {
- auto R =
- Output.try_emplace(Key, std::vector>());
- R.first->second.emplace_back(std::move(I));
-}
- });
- return Err;
-}
-
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::error_code OK;
@@ -237,37 +220,67 @@
// In ToolResults, the Key is the hashed USR and the value is the
// bitcode-encoded representation of the Info object.
llvm::outs() << "Collecting infos...\n";
- llvm::StringMap>> USRToInfos;
- if (bitcodeResultsToInfos(*Exec->get()->getToolResults(), USRToInfos))
-return 1;
+ llvm::StringMap> USRToBitcode;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+auto R = USRToBitcode.try_emplace(Key, std::vector());
+R.first->second.emplace_back(Value);
+ });
// First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToInfos.size() << " infos...\n";
- for (auto &Group : USRToInfos) {
-auto Reduced = doc::mergeInfos(Group.getValue());
-if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- continue;
-}
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ bool Error = false;
+ llvm::ThreadPool Pool(ThreadCount);
+ for (auto &Group : USRToBitcode) {
+Pool.async([&]() {
+ std::vector> Infos;
-doc::Info *I = Reduced.get().get();
-auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
- "." + Format);
-if (!InfoPath) {
- llvm::errs() << toString(InfoPath.takeError()) << "\n";
- return 1;
-}
-std::error_code FileErr;
-llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr, llvm::sys::fs::F_None);
-if (FileErr != OK) {
- llvm::errs() << "Error opening info file: " << FileErr.message() << "\n";
- continue;
-}
+ for (auto &Bitcode : Group.getValue()) {
+llvm::BitstreamCursor Stream(Bitcode);
+doc::ClangDocBitcodeReader Reader(Stream);
+auto ReadInfos = Reader.readBitcode();
+if (!ReadInfos) {
+ llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+ Error = true;
+ return;
+}
+std::move(ReadInfos->begin(), ReadInfos->end(),
+ std::back_inserter(Infos));
+ }
+
+ auto Reduced = doc::mergeInfos(Infos);
+ if (!Reduced) {
+llvm::errs() << llvm::toString(Reduced.takeError());
+return;
+ }
+
+ doc::Info *I = Reduced.get().get();
+ auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
+"." + Format);
+ if (!InfoPath) {
+llvm::errs() << toString(InfoPath.takeError()) << "\n";
+Error = true;
+return;
+ }
+ std::error_code FileErr;
+ llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
+ llvm::sys::fs::F_None);
+ if (FileErr != OK) {
+
