noajshu updated this revision to Diff 379104. noajshu added a comment. Make cache dir parameter a Twine and avoid unnecessary copy in conversion to SmallString.
Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D111371/new/ https://reviews.llvm.org/D111371 Files: clang/lib/CodeGen/BackendUtil.cpp lld/COFF/LTO.cpp lld/ELF/LTO.cpp lld/MachO/LTO.cpp lld/wasm/LTO.cpp llvm/include/llvm/LTO/Caching.h llvm/include/llvm/LTO/LTO.h llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h llvm/include/llvm/Support/Caching.h llvm/lib/LTO/CMakeLists.txt llvm/lib/LTO/Caching.cpp llvm/lib/LTO/LTOCodeGenerator.cpp llvm/lib/Support/CMakeLists.txt llvm/lib/Support/Caching.cpp llvm/tools/gold/gold-plugin.cpp llvm/tools/llvm-lto/llvm-lto.cpp llvm/tools/llvm-lto2/llvm-lto2.cpp
Index: llvm/tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- llvm/tools/llvm-lto2/llvm-lto2.cpp +++ llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -19,10 +19,10 @@ #include "llvm/CodeGen/CommandFlags.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/LTO.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Remarks/HotnessThresholdParser.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/InitLLVM.h" @@ -362,14 +362,13 @@ if (HasErrors) return 1; - auto AddStream = - [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { + auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> { std::string Path = OutputFilename + "." + utostr(Task); std::error_code EC; auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None); check(EC, Path); - return std::make_unique<lto::NativeObjectStream>(std::move(S)); + return std::make_unique<NativeObjectStream>(std::move(S)); }; auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) { @@ -378,7 +377,8 @@ NativeObjectCache Cache; if (!CacheDir.empty()) - Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache"); + Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer), + "failed to create cache"); check(Lto.run(AddStream, Cache), "LTO::run failed"); return 0; Index: llvm/tools/llvm-lto/llvm-lto.cpp =================================================================== --- llvm/tools/llvm-lto/llvm-lto.cpp +++ llvm/tools/llvm-lto/llvm-lto.cpp @@ -1097,8 +1097,7 @@ error("writing merged module failed."); } - auto AddStream = - [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { + auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> { std::string PartFilename = OutputFilename; if (Parallelism != 1) PartFilename += "." + utostr(Task); @@ -1108,7 +1107,7 @@ std::make_unique<raw_fd_ostream>(PartFilename, EC, sys::fs::OF_None); if (EC) error("error opening the file '" + PartFilename + "': " + EC.message()); - return std::make_unique<lto::NativeObjectStream>(std::move(S)); + return std::make_unique<NativeObjectStream>(std::move(S)); }; if (!CodeGen.compileOptimized(AddStream, Parallelism)) Index: llvm/tools/gold/gold-plugin.cpp =================================================================== --- llvm/tools/gold/gold-plugin.cpp +++ llvm/tools/gold/gold-plugin.cpp @@ -1081,12 +1081,11 @@ size_t MaxTasks = Lto->getMaxTasks(); std::vector<std::pair<SmallString<128>, bool>> Files(MaxTasks); - auto AddStream = - [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { + auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> { Files[Task].second = !SaveTemps; int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps, Files[Task].first, Task); - return std::make_unique<lto::NativeObjectStream>( + return std::make_unique<NativeObjectStream>( std::make_unique<llvm::raw_fd_ostream>(FD, true)); }; @@ -1096,7 +1095,7 @@ NativeObjectCache Cache; if (!options::cache_dir.empty()) - Cache = check(localCache(options::cache_dir, AddBuffer)); + Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer)); check(Lto->run(AddStream, Cache)); Index: llvm/lib/Support/Caching.cpp =================================================================== --- llvm/lib/Support/Caching.cpp +++ llvm/lib/Support/Caching.cpp @@ -1,4 +1,4 @@ -//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===// +//===-Caching.cpp - LLVM File Cache Handling ------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,11 +6,11 @@ // //===----------------------------------------------------------------------===// // -// This file implements the Caching for ThinLTO. +// This file implements the Caching used by ThinLTO. // //===----------------------------------------------------------------------===// -#include "llvm/LTO/Caching.h" +#include "llvm/Support/Caching.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" @@ -26,13 +26,20 @@ #endif using namespace llvm; -using namespace llvm::lto; -Expected<NativeObjectCache> lto::localCache(StringRef CacheDirectoryPath, - AddBufferFn AddBuffer) { - if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath)) +Expected<NativeObjectCache> llvm::localCache(Twine CacheNameRef, + Twine TempFilePrefixRef, + Twine CacheDirectoryPathRef, + AddBufferFn AddBuffer) { + if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPathRef)) return errorCodeToError(EC); + // Create local copies which are safely captured-by-copy in lambdas + SmallString<64> CacheName, TempFilePrefix, CacheDirectoryPath; + CacheNameRef.toVector(CacheName); + TempFilePrefixRef.toVector(TempFilePrefix); + CacheDirectoryPathRef.toVector(CacheDirectoryPath); + return [=](unsigned Task, StringRef Key) -> AddStreamFn { // This choice of file name allows the cache to be pruned (see pruneCache() // in include/llvm/Support/CachePruning.h). @@ -134,12 +141,13 @@ return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> { // Write to a temporary to avoid race condition SmallString<64> TempFilenameModel; - sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o"); + sys::path::append(TempFilenameModel, CacheDirectoryPath, + TempFilePrefix + "-%%%%%%.tmp.o"); Expected<sys::fs::TempFile> Temp = sys::fs::TempFile::create( TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write); if (!Temp) { errs() << "Error: " << toString(Temp.takeError()) << "\n"; - report_fatal_error("ThinLTO: Can't get a temporary file"); + report_fatal_error(CacheName + ": Can't get a temporary file"); } // This CacheStream will move the temporary file into the cache when done. Index: llvm/lib/Support/CMakeLists.txt =================================================================== --- llvm/lib/Support/CMakeLists.txt +++ llvm/lib/Support/CMakeLists.txt @@ -123,6 +123,7 @@ BranchProbability.cpp BuryPointer.cpp CachePruning.cpp + Caching.cpp circular_raw_ostream.cpp Chrono.cpp COM.cpp Index: llvm/lib/LTO/LTOCodeGenerator.cpp =================================================================== --- llvm/lib/LTO/LTOCodeGenerator.cpp +++ llvm/lib/LTO/LTOCodeGenerator.cpp @@ -245,8 +245,7 @@ // make unique temp output file to put generated code SmallString<128> Filename; - auto AddStream = - [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { + auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> { StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o"); int FD; @@ -255,7 +254,7 @@ if (EC) emitError(EC.message()); - return std::make_unique<lto::NativeObjectStream>( + return std::make_unique<NativeObjectStream>( std::make_unique<llvm::raw_fd_ostream>(FD, true)); }; @@ -557,7 +556,7 @@ return true; } -bool LTOCodeGenerator::compileOptimized(lto::AddStreamFn AddStream, +bool LTOCodeGenerator::compileOptimized(AddStreamFn AddStream, unsigned ParallelismLevel) { if (!this->determineTarget()) return false; Index: llvm/lib/LTO/CMakeLists.txt =================================================================== --- llvm/lib/LTO/CMakeLists.txt +++ llvm/lib/LTO/CMakeLists.txt @@ -1,5 +1,4 @@ add_llvm_component_library(LLVMLTO - Caching.cpp LTO.cpp LTOBackend.cpp LTOModule.cpp Index: llvm/include/llvm/Support/Caching.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/Caching.h @@ -0,0 +1,76 @@ +//===- Caching.h - LLVM File Cache Handling Configuration -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the localCache function, which allows clients to add a +// filesystem cache. This is used by ThinLTO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CACHING_H +#define LLVM_SUPPORT_CACHING_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/thread.h" + +namespace llvm { + +/// This class wraps an output stream for a native object. Most clients should +/// just be able to return an instance of this base class from the stream +/// callback, but if a client needs to perform some action after the stream is +/// written to, that can be done by deriving from this class and overriding the +/// destructor. +class NativeObjectStream { +public: + NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS) + : OS(std::move(OS)) {} + std::unique_ptr<raw_pwrite_stream> OS; + virtual ~NativeObjectStream() = default; +}; + +/// This type defines the callback to add a native object that is generated on +/// the fly. +/// +/// Stream callbacks must be thread safe. +using AddStreamFn = + std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>; + +/// This is the type of a native object cache. To request an item from the +/// cache, pass a unique string as the Key. For hits, the cached file will be +/// added to the link and this function will return AddStreamFn(). For misses, +/// the cache will return a stream callback which must be called at most once to +/// produce content for the stream. The native object stream produced by the +/// stream callback will add the file to the link after the stream is written +/// to. +/// +/// Clients generally look like this: +/// +/// if (AddStreamFn AddStream = Cache(Task, Key)) +/// ProduceContent(AddStream); +using NativeObjectCache = + std::function<AddStreamFn(unsigned Task, StringRef Key)>; + +/// This type defines the callback to add a pre-existing native object file +/// (e.g. in a cache). +/// +/// Buffer callbacks must be thread safe. +using AddBufferFn = + std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)>; + +/// Create a local file system cache which uses the given cache name, temporary +/// file prefix, cache directory and file callback. This function also creates +/// the cache directory if it does not already exist. The cache name appears in +/// error messages for errors during caching. The temporary file prefix is used +/// in the temporary file naming scheme used when writing files atomically. +Expected<NativeObjectCache> localCache(Twine CacheNameRef, + Twine TempFilePrefixRef, + Twine CacheDirectoryPathRef, + AddBufferFn AddBuffer); +} // namespace llvm + +#endif Index: llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h =================================================================== --- llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -176,7 +176,7 @@ /// created using the \p AddStream callback. Returns true on success. /// /// Calls \a verifyMergedModuleOnce(). - bool compileOptimized(lto::AddStreamFn AddStream, unsigned ParallelismLevel); + bool compileOptimized(AddStreamFn AddStream, unsigned ParallelismLevel); /// Enable the Freestanding mode: indicate that the optimizer should not /// assume builtins are present on the target. Index: llvm/include/llvm/LTO/LTO.h =================================================================== --- llvm/include/llvm/LTO/LTO.h +++ llvm/include/llvm/LTO/LTO.h @@ -21,6 +21,7 @@ #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/LTO/Config.h" #include "llvm/Object/IRSymtab.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/Error.h" #include "llvm/Support/thread.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" @@ -187,40 +188,6 @@ } }; -/// This class wraps an output stream for a native object. Most clients should -/// just be able to return an instance of this base class from the stream -/// callback, but if a client needs to perform some action after the stream is -/// written to, that can be done by deriving from this class and overriding the -/// destructor. -class NativeObjectStream { -public: - NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {} - std::unique_ptr<raw_pwrite_stream> OS; - virtual ~NativeObjectStream() = default; -}; - -/// This type defines the callback to add a native object that is generated on -/// the fly. -/// -/// Stream callbacks must be thread safe. -using AddStreamFn = - std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>; - -/// This is the type of a native object cache. To request an item from the -/// cache, pass a unique string as the Key. For hits, the cached file will be -/// added to the link and this function will return AddStreamFn(). For misses, -/// the cache will return a stream callback which must be called at most once to -/// produce content for the stream. The native object stream produced by the -/// stream callback will add the file to the link after the stream is written -/// to. -/// -/// Clients generally look like this: -/// -/// if (AddStreamFn AddStream = Cache(Task, Key)) -/// ProduceContent(AddStream); -using NativeObjectCache = - std::function<AddStreamFn(unsigned Task, StringRef Key)>; - /// A ThinBackend defines what happens after the thin-link phase during ThinLTO. /// The details of this type definition aren't important; clients can only /// create a ThinBackend using one of the create*ThinBackend() functions below. Index: llvm/include/llvm/LTO/Caching.h =================================================================== --- llvm/include/llvm/LTO/Caching.h +++ /dev/null @@ -1,38 +0,0 @@ -//===- Caching.h - LLVM Link Time Optimizer Configuration -----------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines the localCache function, which allows clients to add a -// filesystem cache to ThinLTO. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LTO_CACHING_H -#define LLVM_LTO_CACHING_H - -#include "llvm/LTO/LTO.h" - -namespace llvm { -namespace lto { - -/// This type defines the callback to add a pre-existing native object file -/// (e.g. in a cache). -/// -/// Buffer callbacks must be thread safe. -using AddBufferFn = - std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)>; - -/// Create a local file system cache which uses the given cache directory and -/// file callback. This function also creates the cache directory if it does not -/// already exist. -Expected<NativeObjectCache> localCache(StringRef CacheDirectoryPath, - AddBufferFn AddBuffer); - -} // namespace lto -} // namespace llvm - -#endif Index: lld/wasm/LTO.cpp =================================================================== --- lld/wasm/LTO.cpp +++ lld/wasm/LTO.cpp @@ -19,10 +19,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -127,17 +127,17 @@ // The --thinlto-cache-dir option specifies the path to a directory in which // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->thinLTOCacheDir.empty()) - cache = check( - lto::localCache(config->thinLTOCacheDir, - [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { + files[task] = std::move(mb); + })); checkError(ltoObj->run( [&](size_t task) { - return std::make_unique<lto::NativeObjectStream>( + return std::make_unique<NativeObjectStream>( std::make_unique<raw_svector_ostream>(buf[task])); }, cache)); Index: lld/MachO/LTO.cpp =================================================================== --- lld/MachO/LTO.cpp +++ lld/MachO/LTO.cpp @@ -17,9 +17,9 @@ #include "lld/Common/ErrorHandler.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" @@ -105,17 +105,17 @@ // The -cache_path_lto option specifies the path to a directory in which // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->thinLTOCacheDir.empty()) - cache = check( - lto::localCache(config->thinLTOCacheDir, - [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { + files[task] = std::move(mb); + })); checkError(ltoObj->run( [&](size_t task) { - return std::make_unique<lto::NativeObjectStream>( + return std::make_unique<NativeObjectStream>( std::make_unique<raw_svector_ostream>(buf[task])); }, cache)); Index: lld/ELF/LTO.cpp =================================================================== --- lld/ELF/LTO.cpp +++ lld/ELF/LTO.cpp @@ -23,10 +23,10 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -304,18 +304,18 @@ // The --thinlto-cache-dir option specifies the path to a directory in which // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->thinLTOCacheDir.empty()) - cache = check( - lto::localCache(config->thinLTOCacheDir, - [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { + files[task] = std::move(mb); + })); if (!bitcodeFiles.empty()) checkError(ltoObj->run( [&](size_t task) { - return std::make_unique<lto::NativeObjectStream>( + return std::make_unique<NativeObjectStream>( std::make_unique<raw_svector_ostream>(buf[task])); }, cache)); Index: lld/COFF/LTO.cpp =================================================================== --- lld/COFF/LTO.cpp +++ lld/COFF/LTO.cpp @@ -20,10 +20,10 @@ #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" @@ -164,16 +164,17 @@ // The /lldltocache option specifies the path to a directory in which to cache // native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. - lto::NativeObjectCache cache; + NativeObjectCache cache; if (!config->ltoCache.empty()) - cache = check(lto::localCache( - config->ltoCache, [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { - files[task] = std::move(mb); - })); + cache = + check(localCache("ThinLTO", "Thin", config->ltoCache, + [&](size_t task, std::unique_ptr<MemoryBuffer> mb) { + files[task] = std::move(mb); + })); checkError(ltoObj->run( [&](size_t task) { - return std::make_unique<lto::NativeObjectStream>( + return std::make_unique<NativeObjectStream>( std::make_unique<raw_svector_ostream>(buf[task])); }, cache)); Index: clang/lib/CodeGen/BackendUtil.cpp =================================================================== --- clang/lib/CodeGen/BackendUtil.cpp +++ clang/lib/CodeGen/BackendUtil.cpp @@ -1542,7 +1542,7 @@ return; auto AddStream = [&](size_t Task) { - return std::make_unique<lto::NativeObjectStream>(std::move(OS)); + return std::make_unique<NativeObjectStream>(std::move(OS)); }; lto::Config Conf; if (CGOpts.SaveTempsFilePrefix != "") {
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits