Author: bruno Date: Thu Dec 22 01:06:03 2016 New Revision: 290326 URL: http://llvm.org/viewvc/llvm-project?rev=290326&view=rev Log: [CrashReproducer] Add support for merging -ivfsoverlay
Merge all VFS mapped files inside -ivfsoverlay inputs into the vfs overlay provided by the crash reproducer. This is the last missing piece to allow crash reproducers to fully work with user frameworks; when combined with headermaps, it allows clang to find additional frameworks. rdar://problem/27913709 Added: cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m Modified: cfe/trunk/include/clang/Basic/VirtualFileSystem.h cfe/trunk/include/clang/Frontend/Utils.h cfe/trunk/lib/Basic/VirtualFileSystem.cpp cfe/trunk/lib/Frontend/CompilerInstance.cpp cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp Modified: cfe/trunk/include/clang/Basic/VirtualFileSystem.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/VirtualFileSystem.h?rev=290326&r1=290325&r2=290326&view=diff ============================================================================== --- cfe/trunk/include/clang/Basic/VirtualFileSystem.h (original) +++ cfe/trunk/include/clang/Basic/VirtualFileSystem.h Thu Dec 22 01:06:03 2016 @@ -362,6 +362,16 @@ struct YAMLVFSEntry { std::string RPath; }; +/// \brief Collect all pairs of <virtual path, real path> entries from the +/// \p YAMLFilePath. This is used by the module dependency collector to forward +/// the entries into the reproducer output VFS YAML file. +void collectVFSFromYAML( + std::unique_ptr<llvm::MemoryBuffer> Buffer, + llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, + SmallVectorImpl<YAMLVFSEntry> &CollectedEntries, + void *DiagContext = nullptr, + IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem()); + class YAMLVFSWriter { std::vector<YAMLVFSEntry> Mappings; Optional<bool> IsCaseSensitive; Modified: cfe/trunk/include/clang/Frontend/Utils.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/Utils.h?rev=290326&r1=290325&r2=290326&view=diff ============================================================================== --- cfe/trunk/include/clang/Frontend/Utils.h (original) +++ cfe/trunk/include/clang/Frontend/Utils.h Thu Dec 22 01:06:03 2016 @@ -128,11 +128,11 @@ class ModuleDependencyCollector : public llvm::StringMap<std::string> SymLinkMap; bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result); - std::error_code copyToRoot(StringRef Src); + std::error_code copyToRoot(StringRef Src, StringRef Dst = ""); public: StringRef getDest() { return DestDir; } bool insertSeen(StringRef Filename) { return Seen.insert(Filename).second; } - void addFile(StringRef Filename); + void addFile(StringRef Filename, StringRef FileDst = ""); void addFileMapping(StringRef VPath, StringRef RPath) { VFSWriter.addFileMapping(VPath, RPath); } Modified: cfe/trunk/lib/Basic/VirtualFileSystem.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/VirtualFileSystem.cpp?rev=290326&r1=290325&r2=290326&view=diff ============================================================================== --- cfe/trunk/lib/Basic/VirtualFileSystem.cpp (original) +++ cfe/trunk/lib/Basic/VirtualFileSystem.cpp Thu Dec 22 01:06:03 2016 @@ -887,9 +887,6 @@ private: RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS) : ExternalFS(std::move(ExternalFS)) {} - /// \brief Looks up \p Path in \c Roots. - ErrorOr<Entry *> lookupPath(const Twine &Path); - /// \brief Looks up the path <tt>[Start, End)</tt> in \p From, possibly /// recursing into the contents of \p From if it is a directory. ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start, @@ -899,6 +896,9 @@ private: ErrorOr<Status> status(const Twine &Path, Entry *E); public: + /// \brief Looks up \p Path in \c Roots. + ErrorOr<Entry *> lookupPath(const Twine &Path); + /// \brief Parses \p Buffer, which is expected to be in YAML format and /// returns a virtual file system representing its contents. static RedirectingFileSystem * @@ -1606,6 +1606,47 @@ vfs::getVFSFromYAML(std::unique_ptr<Memo std::move(ExternalFS)); } +static void getVFSEntries(Entry *SrcE, SmallVectorImpl<StringRef> &Path, + SmallVectorImpl<YAMLVFSEntry> &Entries) { + auto Kind = SrcE->getKind(); + if (Kind == EK_Directory) { + auto *DE = dyn_cast<RedirectingDirectoryEntry>(SrcE); + assert(DE && "Must be a directory"); + for (std::unique_ptr<Entry> &SubEntry : + llvm::make_range(DE->contents_begin(), DE->contents_end())) { + Path.push_back(SubEntry->getName()); + getVFSEntries(SubEntry.get(), Path, Entries); + Path.pop_back(); + } + return; + } + + assert(Kind == EK_File && "Must be a EK_File"); + auto *FE = dyn_cast<RedirectingFileEntry>(SrcE); + assert(FE && "Must be a file"); + SmallString<128> VPath; + for (auto &Comp : Path) + llvm::sys::path::append(VPath, Comp); + Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath())); +} + +void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer, + SourceMgr::DiagHandlerTy DiagHandler, + StringRef YAMLFilePath, + SmallVectorImpl<YAMLVFSEntry> &CollectedEntries, + void *DiagContext, + IntrusiveRefCntPtr<FileSystem> ExternalFS) { + RedirectingFileSystem *VFS = RedirectingFileSystem::create( + std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext, + std::move(ExternalFS)); + ErrorOr<Entry *> RootE = VFS->lookupPath("/"); + if (!RootE) + return; + SmallVector<StringRef, 8> Components; + Components.push_back("/"); + getVFSEntries(*RootE, Components, CollectedEntries); +} + UniqueID vfs::getNextVirtualUniqueID() { static std::atomic<unsigned> UID; unsigned ID = ++UID; Modified: cfe/trunk/lib/Frontend/CompilerInstance.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInstance.cpp?rev=290326&r1=290325&r2=290326&view=diff ============================================================================== --- cfe/trunk/lib/Frontend/CompilerInstance.cpp (original) +++ cfe/trunk/lib/Frontend/CompilerInstance.cpp Thu Dec 22 01:06:03 2016 @@ -181,6 +181,26 @@ static void collectIncludePCH(CompilerIn } } +static void collectVFSEntries(CompilerInstance &CI, + std::shared_ptr<ModuleDependencyCollector> MDC) { + if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty()) + return; + + // Collect all VFS found. + SmallVector<vfs::YAMLVFSEntry, 16> VFSEntries; + for (const std::string &VFSFile : CI.getHeaderSearchOpts().VFSOverlayFiles) { + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer = + llvm::MemoryBuffer::getFile(VFSFile); + if (!Buffer) + return; + vfs::collectVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr, + VFSFile, VFSEntries); + } + + for (auto &E : VFSEntries) + MDC->addFile(E.VPath, E.RPath); +} + // Diagnostics static void SetUpDiagnosticLog(DiagnosticOptions *DiagOpts, const CodeGenOptions *CodeGenOpts, @@ -412,6 +432,7 @@ void CompilerInstance::createPreprocesso addDependencyCollector(ModuleDepCollector); collectHeaderMaps(PP->getHeaderSearchInfo(), ModuleDepCollector); collectIncludePCH(*this, ModuleDepCollector); + collectVFSEntries(*this, ModuleDepCollector); } for (auto &Listener : DependencyCollectors) Modified: cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp?rev=290326&r1=290325&r2=290326&view=diff ============================================================================== --- cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp (original) +++ cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp Thu Dec 22 01:06:03 2016 @@ -201,7 +201,8 @@ bool ModuleDependencyCollector::getRealP return true; } -std::error_code ModuleDependencyCollector::copyToRoot(StringRef Src) { +std::error_code ModuleDependencyCollector::copyToRoot(StringRef Src, + StringRef Dst) { using namespace llvm::sys; // We need an absolute src path to append to the root. @@ -213,23 +214,35 @@ std::error_code ModuleDependencyCollecto AbsoluteSrc = path::remove_leading_dotslash(AbsoluteSrc); // Canonicalize the source path by removing "..", "." components. - SmallString<256> CanonicalPath = AbsoluteSrc; - path::remove_dots(CanonicalPath, /*remove_dot_dot=*/true); + SmallString<256> VirtualPath = AbsoluteSrc; + path::remove_dots(VirtualPath, /*remove_dot_dot=*/true); // If a ".." component is present after a symlink component, remove_dots may // lead to the wrong real destination path. Let the source be canonicalized // like that but make sure we always use the real path for the destination. - SmallString<256> RealPath; - if (!getRealPath(AbsoluteSrc, RealPath)) - RealPath = CanonicalPath; - SmallString<256> Dest = getDest(); - path::append(Dest, path::relative_path(RealPath)); + SmallString<256> CopyFrom; + if (!getRealPath(AbsoluteSrc, CopyFrom)) + CopyFrom = VirtualPath; + SmallString<256> CacheDst = getDest(); + + if (Dst.empty()) { + // The common case is to map the virtual path to the same path inside the + // cache. + path::append(CacheDst, path::relative_path(CopyFrom)); + } else { + // When collecting entries from input vfsoverlays, copy the external + // contents into the cache but still map from the source. + if (!fs::exists(Dst)) + return std::error_code(); + path::append(CacheDst, Dst); + CopyFrom = Dst; + } // Copy the file into place. - if (std::error_code EC = fs::create_directories(path::parent_path(Dest), - /*IgnoreExisting=*/true)) + if (std::error_code EC = fs::create_directories(path::parent_path(CacheDst), + /*IgnoreExisting=*/true)) return EC; - if (std::error_code EC = fs::copy_file(RealPath, Dest)) + if (std::error_code EC = fs::copy_file(CopyFrom, CacheDst)) return EC; // Always map a canonical src path to its real path into the YAML, by doing @@ -237,12 +250,12 @@ std::error_code ModuleDependencyCollecto // overlay, which is a way to emulate symlink inside the VFS; this is also // needed for correctness, not doing that can lead to module redifinition // errors. - addFileMapping(CanonicalPath, Dest); + addFileMapping(VirtualPath, CacheDst); return std::error_code(); } -void ModuleDependencyCollector::addFile(StringRef Filename) { +void ModuleDependencyCollector::addFile(StringRef Filename, StringRef FileDst) { if (insertSeen(Filename)) - if (copyToRoot(Filename)) + if (copyToRoot(Filename, FileDst)) HasErrors = true; } Added: cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m?rev=290326&view=auto ============================================================================== --- cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m (added) +++ cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m Thu Dec 22 01:06:03 2016 @@ -0,0 +1,45 @@ +// REQUIRES: crash-recovery, shell, system-darwin + +// RUN: rm -rf %t +// RUN: mkdir -p %t/m +// RUN: cp %S/../VFS/Inputs/actual_module2.map %t/actual_module2.map +// RUN: sed -e "s:INPUT_DIR:%t:g" -e "s:OUT_DIR:%t/example:g" \ +// RUN: %S/../VFS/Inputs/vfsoverlay2.yaml > %t/srcvfs.yaml + +// RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \ +// RUN: %clang -fsyntax-only -nostdinc %s \ +// RUN: -I %S/Inputs/crash-recovery/usr/include \ +// RUN: -ivfsoverlay %t/srcvfs.yaml \ +// RUN: -fmodules -fmodules-cache-path=%t/m/ 2>&1 | FileCheck %s + +// RUN: FileCheck --check-prefix=CHECKSH %s -input-file %t/crash-vfs-*.sh +// RUN: FileCheck --check-prefix=CHECKYAML %s -input-file \ +// RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml +// RUN: find %t/crash-vfs-*.cache/vfs | \ +// RUN: grep "%t/actual_module2.map" | count 1 + +#include <stdio.h> + +// CHECK: Preprocessed source(s) and associated run script(s) are located at: +// CHECK-NEXT: note: diagnostic msg: {{.*}}.m +// CHECK-NEXT: note: diagnostic msg: {{.*}}.cache + +// CHECKSH: # Crash reproducer +// CHECKSH-NEXT: # Driver args: "-fsyntax-only" +// CHECKSH-NEXT: # Original command: {{.*$}} +// CHECKSH-NEXT: "-cc1" +// CHECKSH: "crash-vfs-{{[^ ]*}}.m" +// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml" +// CHECKSH: "-fmodules-cache-path=crash-vfs-{{[^ ]*}}.cache/repro-modules" + +// CHECKYAML: 'case-sensitive': +// CHECKYAML-NEXT: 'use-external-names': 'false', +// CHECKYAML-NEXT: 'overlay-relative': 'true', +// CHECKYAML-NEXT: 'ignore-non-existent-contents': 'false' +// CHECKYAML: 'type': 'directory' +// CHECKYAML: 'name': "/[[PATH:.*]]/example" +// CHECKYAML: 'contents': [ +// CHECKYAML-NEXT: { +// CHECKYAML-NEXT: 'type': 'file', +// CHECKYAML-NEXT: 'name': "module.map", +// CHECKYAML-NEXT: 'external-contents': "/[[OTHERPATH:.*]]/actual_module2.map" _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits