noajshu updated this revision to Diff 378316.
noajshu added a comment.
Adjust cache name/prefix customization.
Move customization arguments to be first. Convert to twines, remove unnecessary
copy. Separate customized file prefix from customized cache name (to avoid
chaning any behavior from existing ThinLTO cache).
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D111371/new/
https://reviews.llvm.org/D111371
Files:
clang/lib/CodeGen/BackendUtil.cpp
lld/COFF/LTO.cpp
lld/ELF/LTO.cpp
lld/MachO/LTO.cpp
lld/wasm/LTO.cpp
llvm/include/llvm/LTO/Caching.h
llvm/include/llvm/LTO/LTO.h
llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
llvm/include/llvm/Support/Caching.h
llvm/lib/LTO/CMakeLists.txt
llvm/lib/LTO/Caching.cpp
llvm/lib/LTO/LTOCodeGenerator.cpp
llvm/lib/Support/CMakeLists.txt
llvm/lib/Support/Caching.cpp
llvm/tools/gold/gold-plugin.cpp
llvm/tools/llvm-lto/llvm-lto.cpp
llvm/tools/llvm-lto2/llvm-lto2.cpp
Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===================================================================
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Passes/PassPlugin.h"
#include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
if (HasErrors)
return 1;
- auto AddStream =
- [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
std::string Path = OutputFilename + "." + utostr(Task);
std::error_code EC;
auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None);
check(EC, Path);
- return std::make_unique<lto::NativeObjectStream>(std::move(S));
+ return std::make_unique<NativeObjectStream>(std::move(S));
};
auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
@@ -378,7 +377,8 @@
NativeObjectCache Cache;
if (!CacheDir.empty())
- Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+ Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+ "failed to create cache");
check(Lto.run(AddStream, Cache), "LTO::run failed");
return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===================================================================
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
error("writing merged module failed.");
}
- auto AddStream =
- [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
std::string PartFilename = OutputFilename;
if (Parallelism != 1)
PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
std::make_unique<raw_fd_ostream>(PartFilename, EC, sys::fs::OF_None);
if (EC)
error("error opening the file '" + PartFilename + "': " + EC.message());
- return std::make_unique<lto::NativeObjectStream>(std::move(S));
+ return std::make_unique<NativeObjectStream>(std::move(S));
};
if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===================================================================
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
size_t MaxTasks = Lto->getMaxTasks();
std::vector<std::pair<SmallString<128>, bool>> Files(MaxTasks);
- auto AddStream =
- [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
Files[Task].second = !SaveTemps;
int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
- return std::make_unique<lto::NativeObjectStream>(
+ return std::make_unique<NativeObjectStream>(
std::make_unique<llvm::raw_fd_ostream>(FD, true));
};
@@ -1096,7 +1095,7 @@
NativeObjectCache Cache;
if (!options::cache_dir.empty())
- Cache = check(localCache(options::cache_dir, AddBuffer));
+ Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
check(Lto->run(AddStream, Cache));
Index: llvm/lib/Support/Caching.cpp
===================================================================
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===//
+//===-Caching.cpp - LLVM File Cache Handling ------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
//
//===----------------------------------------------------------------------===//
//
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
//
//===----------------------------------------------------------------------===//
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/FileSystem.h"
@@ -26,10 +26,11 @@
#endif
using namespace llvm;
-using namespace llvm::lto;
-Expected<NativeObjectCache> lto::localCache(StringRef CacheDirectoryPath,
- AddBufferFn AddBuffer) {
+Expected<NativeObjectCache> llvm::localCache(Twine CacheName,
+ Twine TempFilePrefix,
+ StringRef CacheDirectoryPath,
+ AddBufferFn AddBuffer) {
if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))
return errorCodeToError(EC);
@@ -134,12 +135,13 @@
return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
// Write to a temporary to avoid race condition
SmallString<64> TempFilenameModel;
- sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o");
+ sys::path::append(TempFilenameModel, CacheDirectoryPath,
+ TempFilePrefix + "-%%%%%%.tmp.o");
Expected<sys::fs::TempFile> Temp = sys::fs::TempFile::create(
TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write);
if (!Temp) {
errs() << "Error: " << toString(Temp.takeError()) << "\n";
- report_fatal_error("ThinLTO: Can't get a temporary file");
+ report_fatal_error(CacheName + ": Can't get a temporary file");
}
// This CacheStream will move the temporary file into the cache when done.
Index: llvm/lib/Support/CMakeLists.txt
===================================================================
--- llvm/lib/Support/CMakeLists.txt
+++ llvm/lib/Support/CMakeLists.txt
@@ -123,6 +123,7 @@
BranchProbability.cpp
BuryPointer.cpp
CachePruning.cpp
+ Caching.cpp
circular_raw_ostream.cpp
Chrono.cpp
COM.cpp
Index: llvm/lib/LTO/LTOCodeGenerator.cpp
===================================================================
--- llvm/lib/LTO/LTOCodeGenerator.cpp
+++ llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -245,8 +245,7 @@
// make unique temp output file to put generated code
SmallString<128> Filename;
- auto AddStream =
- [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o");
int FD;
@@ -255,7 +254,7 @@
if (EC)
emitError(EC.message());
- return std::make_unique<lto::NativeObjectStream>(
+ return std::make_unique<NativeObjectStream>(
std::make_unique<llvm::raw_fd_ostream>(FD, true));
};
@@ -557,7 +556,7 @@
return true;
}
-bool LTOCodeGenerator::compileOptimized(lto::AddStreamFn AddStream,
+bool LTOCodeGenerator::compileOptimized(AddStreamFn AddStream,
unsigned ParallelismLevel) {
if (!this->determineTarget())
return false;
Index: llvm/lib/LTO/CMakeLists.txt
===================================================================
--- llvm/lib/LTO/CMakeLists.txt
+++ llvm/lib/LTO/CMakeLists.txt
@@ -1,5 +1,4 @@
add_llvm_component_library(LLVMLTO
- Caching.cpp
LTO.cpp
LTOBackend.cpp
LTOModule.cpp
Index: llvm/include/llvm/Support/Caching.h
===================================================================
--- /dev/null
+++ llvm/include/llvm/Support/Caching.h
@@ -0,0 +1,74 @@
+//===- Caching.h - LLVM File Cache Handling Configuration -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the localCache function, which allows clients to add a
+// filesystem cache. This is used by ThinLTO.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_CACHING_H
+#define LLVM_SUPPORT_CACHING_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/thread.h"
+
+namespace llvm {
+
+/// This class wraps an output stream for a native object. Most clients should
+/// just be able to return an instance of this base class from the stream
+/// callback, but if a client needs to perform some action after the stream is
+/// written to, that can be done by deriving from this class and overriding the
+/// destructor.
+class NativeObjectStream {
+public:
+ NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS)
+ : OS(std::move(OS)) {}
+ std::unique_ptr<raw_pwrite_stream> OS;
+ virtual ~NativeObjectStream() = default;
+};
+
+/// This type defines the callback to add a native object that is generated on
+/// the fly.
+///
+/// Stream callbacks must be thread safe.
+using AddStreamFn =
+ std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>;
+
+/// This is the type of a native object cache. To request an item from the
+/// cache, pass a unique string as the Key. For hits, the cached file will be
+/// added to the link and this function will return AddStreamFn(). For misses,
+/// the cache will return a stream callback which must be called at most once to
+/// produce content for the stream. The native object stream produced by the
+/// stream callback will add the file to the link after the stream is written
+/// to.
+///
+/// Clients generally look like this:
+///
+/// if (AddStreamFn AddStream = Cache(Task, Key))
+/// ProduceContent(AddStream);
+using NativeObjectCache =
+ std::function<AddStreamFn(unsigned Task, StringRef Key)>;
+
+/// This type defines the callback to add a pre-existing native object file
+/// (e.g. in a cache).
+///
+/// Buffer callbacks must be thread safe.
+using AddBufferFn =
+ std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)>;
+
+/// Create a local file system cache which uses the given cache directory and
+/// file callback. This function also creates the cache directory if it does not
+/// already exist.
+Expected<NativeObjectCache> localCache(Twine CacheName, Twine TempFilePrefix,
+
+ StringRef CacheDirectoryPath,
+ AddBufferFn AddBuffer);
+} // namespace llvm
+
+#endif
Index: llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
===================================================================
--- llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
+++ llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
@@ -176,7 +176,7 @@
/// created using the \p AddStream callback. Returns true on success.
///
/// Calls \a verifyMergedModuleOnce().
- bool compileOptimized(lto::AddStreamFn AddStream, unsigned ParallelismLevel);
+ bool compileOptimized(AddStreamFn AddStream, unsigned ParallelismLevel);
/// Enable the Freestanding mode: indicate that the optimizer should not
/// assume builtins are present on the target.
Index: llvm/include/llvm/LTO/LTO.h
===================================================================
--- llvm/include/llvm/LTO/LTO.h
+++ llvm/include/llvm/LTO/LTO.h
@@ -21,6 +21,7 @@
#include "llvm/IR/ModuleSummaryIndex.h"
#include "llvm/LTO/Config.h"
#include "llvm/Object/IRSymtab.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/thread.h"
#include "llvm/Transforms/IPO/FunctionAttrs.h"
@@ -187,40 +188,6 @@
}
};
-/// This class wraps an output stream for a native object. Most clients should
-/// just be able to return an instance of this base class from the stream
-/// callback, but if a client needs to perform some action after the stream is
-/// written to, that can be done by deriving from this class and overriding the
-/// destructor.
-class NativeObjectStream {
-public:
- NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}
- std::unique_ptr<raw_pwrite_stream> OS;
- virtual ~NativeObjectStream() = default;
-};
-
-/// This type defines the callback to add a native object that is generated on
-/// the fly.
-///
-/// Stream callbacks must be thread safe.
-using AddStreamFn =
- std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>;
-
-/// This is the type of a native object cache. To request an item from the
-/// cache, pass a unique string as the Key. For hits, the cached file will be
-/// added to the link and this function will return AddStreamFn(). For misses,
-/// the cache will return a stream callback which must be called at most once to
-/// produce content for the stream. The native object stream produced by the
-/// stream callback will add the file to the link after the stream is written
-/// to.
-///
-/// Clients generally look like this:
-///
-/// if (AddStreamFn AddStream = Cache(Task, Key))
-/// ProduceContent(AddStream);
-using NativeObjectCache =
- std::function<AddStreamFn(unsigned Task, StringRef Key)>;
-
/// A ThinBackend defines what happens after the thin-link phase during ThinLTO.
/// The details of this type definition aren't important; clients can only
/// create a ThinBackend using one of the create*ThinBackend() functions below.
Index: llvm/include/llvm/LTO/Caching.h
===================================================================
--- llvm/include/llvm/LTO/Caching.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===- Caching.h - LLVM Link Time Optimizer Configuration -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the localCache function, which allows clients to add a
-// filesystem cache to ThinLTO.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LTO_CACHING_H
-#define LLVM_LTO_CACHING_H
-
-#include "llvm/LTO/LTO.h"
-
-namespace llvm {
-namespace lto {
-
-/// This type defines the callback to add a pre-existing native object file
-/// (e.g. in a cache).
-///
-/// Buffer callbacks must be thread safe.
-using AddBufferFn =
- std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)>;
-
-/// Create a local file system cache which uses the given cache directory and
-/// file callback. This function also creates the cache directory if it does not
-/// already exist.
-Expected<NativeObjectCache> localCache(StringRef CacheDirectoryPath,
- AddBufferFn AddBuffer);
-
-} // namespace lto
-} // namespace llvm
-
-#endif
Index: lld/wasm/LTO.cpp
===================================================================
--- lld/wasm/LTO.cpp
+++ lld/wasm/LTO.cpp
@@ -19,10 +19,10 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
@@ -127,17 +127,17 @@
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- lto::NativeObjectCache cache;
+ NativeObjectCache cache;
if (!config->thinLTOCacheDir.empty())
- cache = check(
- lto::localCache(config->thinLTOCacheDir,
- [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
- files[task] = std::move(mb);
- }));
+ cache =
+ check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
+ [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
+ files[task] = std::move(mb);
+ }));
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<lto::NativeObjectStream>(
+ return std::make_unique<NativeObjectStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: lld/MachO/LTO.cpp
===================================================================
--- lld/MachO/LTO.cpp
+++ lld/MachO/LTO.cpp
@@ -17,9 +17,9 @@
#include "lld/Common/ErrorHandler.h"
#include "lld/Common/Strings.h"
#include "lld/Common/TargetOptionsCommandFlags.h"
-#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
@@ -105,17 +105,17 @@
// The -cache_path_lto option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- lto::NativeObjectCache cache;
+ NativeObjectCache cache;
if (!config->thinLTOCacheDir.empty())
- cache = check(
- lto::localCache(config->thinLTOCacheDir,
- [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
- files[task] = std::move(mb);
- }));
+ cache =
+ check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
+ [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
+ files[task] = std::move(mb);
+ }));
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<lto::NativeObjectStream>(
+ return std::make_unique<NativeObjectStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: lld/ELF/LTO.cpp
===================================================================
--- lld/ELF/LTO.cpp
+++ lld/ELF/LTO.cpp
@@ -23,10 +23,10 @@
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
@@ -304,18 +304,18 @@
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- lto::NativeObjectCache cache;
+ NativeObjectCache cache;
if (!config->thinLTOCacheDir.empty())
- cache = check(
- lto::localCache(config->thinLTOCacheDir,
- [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
- files[task] = std::move(mb);
- }));
+ cache =
+ check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
+ [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
+ files[task] = std::move(mb);
+ }));
if (!bitcodeFiles.empty())
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<lto::NativeObjectStream>(
+ return std::make_unique<NativeObjectStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: lld/COFF/LTO.cpp
===================================================================
--- lld/COFF/LTO.cpp
+++ lld/COFF/LTO.cpp
@@ -20,10 +20,10 @@
#include "llvm/ADT/Twine.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
@@ -164,16 +164,17 @@
// The /lldltocache option specifies the path to a directory in which to cache
// native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- lto::NativeObjectCache cache;
+ NativeObjectCache cache;
if (!config->ltoCache.empty())
- cache = check(lto::localCache(
- config->ltoCache, [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
- files[task] = std::move(mb);
- }));
+ cache =
+ check(localCache("ThinLTO", "Thin", config->ltoCache,
+ [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
+ files[task] = std::move(mb);
+ }));
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<lto::NativeObjectStream>(
+ return std::make_unique<NativeObjectStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: clang/lib/CodeGen/BackendUtil.cpp
===================================================================
--- clang/lib/CodeGen/BackendUtil.cpp
+++ clang/lib/CodeGen/BackendUtil.cpp
@@ -1541,7 +1541,7 @@
return;
auto AddStream = [&](size_t Task) {
- return std::make_unique<lto::NativeObjectStream>(std::move(OS));
+ return std::make_unique<NativeObjectStream>(std::move(OS));
};
lto::Config Conf;
if (CGOpts.SaveTempsFilePrefix != "") {
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
