saiislam updated this revision to Diff 322724.
saiislam added a comment.
Added support for optional TargetID during unbundling of archives.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D93525/new/
https://reviews.llvm.org/D93525
Files:
clang/docs/ClangOffloadBundler.rst
clang/test/Driver/clang-offload-bundler.c
clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
Index: clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
===================================================================
--- clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -14,6 +14,7 @@
///
//===----------------------------------------------------------------------===//
+#include "clang/Basic/TargetID.h"
#include "clang/Basic/Version.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
@@ -22,14 +23,18 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ArchiveWriter.h"
#include "llvm/Object/Binary.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
@@ -82,6 +87,7 @@
" bc - llvm-bc\n"
" s - assembler\n"
" o - object\n"
+ " a - archive of objects\n"
" gch - precompiled-header\n"
" ast - clang AST file"),
cl::cat(ClangOffloadBundlerCategory));
@@ -128,6 +134,21 @@
OffloadKind = KindTriplePair.first;
Triple = KindTriplePair.second;
}
+
+/// In presence of <target-id>, the <target-triple> should contain separator
+/// "-" for all standard four components, even if they are empty.
+static void getTargetIDFromTriple(StringRef Triple, StringRef &TargetID) {
+
+ if (Triple.count('-') <= 3) {
+ TargetID = StringRef();
+ } else {
+ auto Tmp = Triple.split('-').second; // strip triple.architecture
+ Tmp = Tmp.split('-').second; // strip triple.vendor
+ Tmp = Tmp.split('-').second; // strip triple.OS
+ TargetID = Tmp.split('-').second; // strip triple.environment
+ }
+}
+
static bool hasHostKind(StringRef Target) {
StringRef OffloadKind;
StringRef Triple;
@@ -160,7 +181,7 @@
virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
/// Read the current bundle and write the result into the stream \a OS.
- virtual Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
+ virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
/// Write the header of the bundled file to \a OS based on the information
/// gathered from \a Inputs.
@@ -375,7 +396,7 @@
return Error::success();
}
- Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+ Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
StringRef FC = Input.getBuffer();
OS.write(FC.data() + CurBundleInfo->second.Offset,
@@ -538,7 +559,7 @@
Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
- Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+ Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
Expected<StringRef> ContentOrErr = CurrentSection->getContents();
if (!ContentOrErr)
return ContentOrErr.takeError();
@@ -732,7 +753,7 @@
return Error::success();
}
- Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+ Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
StringRef FC = Input.getBuffer();
size_t BundleStart = ReadChars;
@@ -827,6 +848,8 @@
return std::make_unique<TextFileHandler>(/*Comment=*/"#");
if (FilesType == "o")
return CreateObjectFileHandler(FirstInput);
+ if (FilesType == "a")
+ return CreateObjectFileHandler(FirstInput);
if (FilesType == "gch")
return std::make_unique<BinaryFileHandler>();
if (FilesType == "ast")
@@ -1026,6 +1049,279 @@
return Error::success();
}
+static Archive::Kind getDefaultArchiveKindForHost() {
+ return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN
+ : Archive::K_GNU;
+}
+
+/// @brief Checks if a code object \p CodeObjectID is compatible with a given
+/// target \p Target. Both arguments are in bundle-entry-id format:
+/// <bundle-entry-id> ::== <offload-kind> "-" <target-triple> ["-"<target-id>]
+///
+/// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
+///
+static bool isCodeObjectCompatible(StringRef CodeObjectID, StringRef Target) {
+
+ // Compatible in case of exact match
+ if (Target == CodeObjectID) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "Comptabile: Exact match: " << CodeObjectID
+ << "\n");
+ return true;
+ }
+
+ // Incompatible if Kind or Triple mismatch
+ StringRef CurKind, TargetKind;
+ StringRef CurTriple, TargetTriple;
+ getOffloadKindAndTriple(CodeObjectID, CurKind, CurTriple);
+ getOffloadKindAndTriple(Target, TargetKind, TargetTriple);
+
+ llvm::Triple CurTripleT(CurTriple);
+ llvm::Triple TargetTripleT(TargetTriple);
+ if (CurKind != TargetKind || !CurTripleT.isCompatibleWith(TargetTripleT)) {
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: "
+ << CodeObjectID << "]\t:\t[Target: " << Target << "]\n");
+ return false;
+ }
+
+ // Incompatible in case of Processor mismatch
+ StringRef CurTID, TargetTID;
+ llvm::StringMap<bool> CurFeatures, TargetFeatures;
+
+ getTargetIDFromTriple(CurTriple, CurTID);
+ getTargetIDFromTriple(TargetTriple, TargetTID);
+
+ llvm::Optional<StringRef> CurProc =
+ clang::parseTargetID(CurTripleT, CurTID, &CurFeatures);
+ llvm::Optional<StringRef> TargetProc =
+ clang::parseTargetID(CurTripleT, TargetTID, &TargetFeatures);
+
+ if (!TargetProc || !CurProc || CurProc.getValue() != TargetProc.getValue()) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "Incompatible: Processor mismatch \t[CodeObject: "
+ << CodeObjectID << "]\t:\t[Target: " << Target
+ << "]\n");
+ return false;
+ }
+
+ // Compatible if all features of Current Bundle are set to Any and Target has
+ // arbitrary features on/off
+ if (CurFeatures.empty() && !TargetFeatures.empty()) {
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs()
+ << "Compatible: All target features are set to ANY \t[CodeObject: "
+ << CodeObjectID << "]\t:\t[Target: " << Target << "]\n");
+ return true;
+ }
+
+ // Incompatible in case there is a mismatch between any non-ANY value of
+ // Current Bundle with that of Target
+ for (const auto &F : CurFeatures) {
+ auto Loc = TargetFeatures.find(F.getKey());
+ if (Loc == TargetFeatures.end()) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "Incompatible: Code Object's non-ANY feature "
+ "not found in Target Features \t[CodeObject: "
+ << CodeObjectID << "]\t:\t[Target: " << Target
+ << "]\n");
+ return false;
+ } else if (Loc->getValue() != F.getValue()) {
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs() << "Incompatible: Value of Code Object's non-ANY feature is "
+ "not matching with Target feature's value \t[CodeObject: "
+ << CodeObjectID << "]\t:\t[Target: " << Target << "]\n");
+ return false;
+ }
+ }
+
+ // Compatible if all features of Target are:
+ // - either, present in the Code Object's features map with same sign,
+ // - or, the feature is missing from Code Objects's features map i.e. it is
+ // set to ANY
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs() << "Compatible: Target IDs are compatible \t[CodeObject: "
+ << CodeObjectID << "]\t:\t[Target: " << Target << "]\n");
+ return true;
+}
+
+/// @brief Computes a list of targets among all given targets which are
+/// compatible with this code object
+/// @param [in] Code Object \p CodeObjectID
+/// @param [out] List of all compatible targets \p CompatibleTargets among all
+/// given targets
+/// @return false, if no compatible target is found
+static bool
+getCompatibleOffloadTargets(StringRef CodeObjectID,
+ SmallVectorImpl<StringRef> &CompatibleTargets) {
+ if (CodeObjectID.empty()) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "Code Object ID is empty\n");
+ return false;
+ }
+ if (!CompatibleTargets.empty()) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "CompatibleTargets list should be empty\n");
+ return false;
+ }
+ for (auto &Target : TargetNames) {
+ if (isCodeObjectCompatible(CodeObjectID, Target))
+ CompatibleTargets.push_back(Target);
+ }
+ return !CompatibleTargets.empty();
+}
+
+/// UnbundleArchive takes an archive file (".a") as input containing bundled
+/// object files, and a list of offload targets (not host), and extracts the
+/// device code into a new archive file for each offload target. Each resulting
+/// archive file contains all device object files corresponding to that
+/// particular offload target device. The created archive file does not
+/// contain an index of the symbols and code object files are named as follows:
+/// <Parent Bundle Name> "-" <CodeObject's-TargetID>
+static Error UnbundleArchive() {
+ std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
+
+ /// Map of target names with list of object files that will form the device
+ /// specific archive for that target
+ StringMap<std::vector<NewArchiveMember>> OutputArchivesMap;
+
+ // Map of target names and output archive filenames
+ StringMap<StringRef> TargetOutputFileNameMap;
+
+ auto Output = OutputFileNames.begin();
+ for (auto &Target : TargetNames) {
+ TargetOutputFileNameMap[Target] = *Output;
+ ++Output;
+ }
+
+ StringRef IFName = InputFileNames.front();
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+ MemoryBuffer::getFileOrSTDIN(IFName, -1, false);
+ if (std::error_code EC = BufOrErr.getError())
+ return createFileError(InputFileNames.front(), EC);
+
+ ArchiveBuffers.push_back(std::move(*BufOrErr));
+ Expected<std::unique_ptr<llvm::object::Archive>> LibOrErr =
+ Archive::create(ArchiveBuffers.back()->getMemBufferRef());
+ if (!LibOrErr)
+ return LibOrErr.takeError();
+
+ auto Archive = std::move(*LibOrErr);
+
+ Error ArchiveErr = Error::success();
+ auto ChildEnd = Archive->child_end();
+ for (auto ChildIter = Archive->child_begin(ArchiveErr); ChildIter != ChildEnd;
+ ++ChildIter) {
+ if (ArchiveErr)
+ return ArchiveErr;
+ auto ChildNameOrErr = (*ChildIter).getName();
+ if (!ChildNameOrErr)
+ return ChildNameOrErr.takeError();
+
+ StringRef ChildName = sys::path::filename(*ChildNameOrErr);
+
+ auto CodeObjectBufferRefOrErr = (*ChildIter).getMemoryBufferRef();
+ if (!CodeObjectBufferRefOrErr)
+ return CodeObjectBufferRefOrErr.takeError();
+
+ auto CodeObjectBuffer =
+ MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
+
+ Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
+ CreateFileHandler(*CodeObjectBuffer);
+ if (!FileHandlerOrErr)
+ return FileHandlerOrErr.takeError();
+
+ std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
+ assert(FileHandler);
+
+ if (Error ReadErr = FileHandler.get()->ReadHeader(*CodeObjectBuffer))
+ return ReadErr;
+
+ Expected<Optional<StringRef>> CurBundleIDOrErr =
+ FileHandler->ReadBundleStart(*CodeObjectBuffer);
+ if (!CurBundleIDOrErr)
+ return CurBundleIDOrErr.takeError();
+
+ Optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr;
+ // No device code in this child, skip
+ if (!OptionalCurBundleID.hasValue())
+ continue;
+ StringRef CurBundleID = *OptionalCurBundleID;
+
+ // Process all the bundle entries found in this child of input archive
+ while (!CurBundleID.empty()) {
+ SmallVector<StringRef> CompatibleTargets;
+ if (hasHostKind(CurBundleID)) {
+ // Do nothing, we don't extract host code yet
+ } else if (getCompatibleOffloadTargets(CurBundleID, CompatibleTargets)) {
+ std::string BundleData;
+ raw_string_ostream DataStream(BundleData);
+ if (Error Err =
+ FileHandler.get()->ReadBundle(DataStream, *CodeObjectBuffer))
+ return Err;
+
+ for (auto &CompatibleTarget : CompatibleTargets) {
+ SmallString<128> StatsFile;
+ StatsFile.assign(ChildName);
+ std::string OutputBundleName =
+ Twine(llvm::sys::path::stem(StatsFile) + "-" + CurBundleID).str();
+ std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy(
+ DataStream.str(), OutputBundleName);
+ ArchiveBuffers.push_back(std::move(MemBuf));
+ llvm::MemoryBufferRef MemBufRef =
+ MemoryBufferRef(*(ArchiveBuffers.back()));
+
+ // For inserting <CompatibleTarget, list<CodeObject>> entry in
+ // OutputArchivesMap
+ if (OutputArchivesMap.find(CompatibleTarget) ==
+ OutputArchivesMap.end()) {
+
+ std::vector<NewArchiveMember> ArchiveMembers;
+ ArchiveMembers.push_back(NewArchiveMember(MemBufRef));
+ OutputArchivesMap.insert_or_assign(CompatibleTarget,
+ std::move(ArchiveMembers));
+ } else {
+ OutputArchivesMap[CompatibleTarget].push_back(
+ NewArchiveMember(MemBufRef));
+ }
+ }
+ }
+
+ if (Error Err = FileHandler.get()->ReadBundleEnd(*CodeObjectBuffer))
+ return Err;
+
+ Expected<Optional<StringRef>> NextTripleOrErr =
+ FileHandler->ReadBundleStart(*CodeObjectBuffer);
+ if (!NextTripleOrErr)
+ return NextTripleOrErr.takeError();
+
+ CurBundleID = ((*NextTripleOrErr).hasValue()) ? **NextTripleOrErr : "";
+ } // end of processing of all bundle entries of this child of input archive
+ } // end of while over children of input archive
+
+ assert(!ArchiveErr);
+
+ /// Write out an archive for each target
+ for (auto &Target : TargetNames) {
+ StringRef FileName = TargetOutputFileNameMap[Target];
+ StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers =
+ OutputArchivesMap.find(Target);
+ if (CurArchiveMembers != OutputArchivesMap.end()) {
+ if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(),
+ true, getDefaultArchiveKindForHost(),
+ true, false, nullptr))
+ return WriteErr;
+ }
+ }
+
+ return Error::success();
+}
+
static void PrintVersion(raw_ostream &OS) {
OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
}
@@ -1110,6 +1406,11 @@
"match in unbundling mode"));
}
} else {
+ if (FilesType == "a") {
+ reportError(createStringError(errc::invalid_argument,
+ "Archive files are only supported "
+ "for unbundling"));
+ }
if (OutputFileNames.size() != 1) {
reportError(createStringError(
errc::invalid_argument,
@@ -1177,6 +1478,11 @@
Twine(HostTargetNum)));
}
- doWork([]() { return Unbundle ? UnbundleFiles() : BundleFiles(); });
+ doWork([]() {
+ return (Unbundle && FilesType == "a")
+ ? UnbundleArchive()
+ : (Unbundle) ? UnbundleFiles() : BundleFiles();
+ });
+
return 0;
}
Index: clang/test/Driver/clang-offload-bundler.c
===================================================================
--- clang/test/Driver/clang-offload-bundler.c
+++ clang/test/Driver/clang-offload-bundler.c
@@ -103,6 +103,10 @@
// RUN: not clang-offload-bundler -type=i -targets=host-%itanium_abi_triple,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR9B
// CK-ERR9B: error: Duplicate targets are not allowed
+// RUN: not clang-offload-bundler -type=a -targets=hxst-powerpcxxle-ibm-linux-gnu,openxp-pxxerpc64le-ibm-linux-gnu,xpenmp-x86_xx-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR10A
+
+// CK-ERR10A: error: Archive files are only supported for unbundling
+
//
// Check text bundle. This is a readable format, so we check for the format we expect to find.
//
@@ -362,6 +366,37 @@
// CKLST2-NOT: openmp-powerpc64le-ibm-linux-gnu
// CKLST2-NOT: openmp-x86_64-pc-linux-gnu
+//
+// Check archive unbundling
+//
+// Create few code object bundles and archive them to create an input archive
+// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-amdgcn-amd-amdhsa--gfx906,openmp-amdgcn-amd-amdhsa--gfx908 -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.simple.bundle
+// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack+,openmp-amdgcn-amd-amdhsa--gfx908:sramecc+:xnack+ -inputs=%t.o,%t.tgt1,%t.tgt1 -outputs=%t.targetID1.bundle
+// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack-,openmp-amdgcn-amd-amdhsa--gfx908:sramecc+:xnack- -inputs=%t.o,%t.tgt1,%t.tgt1 -outputs=%t.targetID2.bundle
+// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-amdgcn-amd-amdhsa--gfx906:xnack-,openmp-amdgcn-amd-amdhsa--gfx908:xnack- -inputs=%t.o,%t.tgt1,%t.tgt1 -outputs=%t.targetID3.bundle
+// RUN: llvm-ar cr %t.input-archive.a %t.simple.bundle %t.targetID1.bundle %t.targetID2.bundle %t.targetID3.bundle
+
+// RUN: clang-offload-bundler -unbundle -type=a -targets=openmp-amdgcn-amd-amdhsa--gfx906,openmp-amdgcn-amd-amdhsa--gfx908 -inputs=%t.input-archive.a -outputs=%t-archive-gfx906-simple.a,%t-archive-gfx908-simple.a
+// RUN: llvm-ar t %t-archive-gfx906-simple.a | FileCheck %s -check-prefix=GFX906
+// GFX906: simple-openmp-amdgcn-amd-amdhsa--gfx906
+// RUN: llvm-ar t %t-archive-gfx908-simple.a | FileCheck %s -check-prefix=GFX908
+// GFX908-NOT: {{gfx906|sramecc|xnack}}
+
+// RUN: clang-offload-bundler -unbundle -type=a -targets=openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack+ -inputs=%t.input-archive.a -outputs=%t-archive-gfx906-tid1.a
+// RUN: llvm-ar t %t-archive-gfx906-tid1.a | FileCheck %s -check-prefix=TEST1
+// TEST1: simple-openmp-amdgcn-amd-amdhsa--gfx906
+// TEST1: targetID1-openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack+
+
+// RUN: clang-offload-bundler -unbundle -type=a -targets=openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack- -inputs=%t.input-archive.a -outputs=%t-archive-gfx906-tid2.a
+// RUN: llvm-ar t %t-archive-gfx906-tid2.a | FileCheck %s -check-prefix=TEST2
+// TEST2-NOT: {{gfx908|sramecc\-|xnack\+}}
+
+// RUN: clang-offload-bundler -unbundle -type=a -targets=openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack- -inputs=%t.input-archive.a -outputs=%t-archive-gfx906-tid3.a
+// RUN: llvm-ar t %t-archive-gfx906-tid3.a | FileCheck %s -check-prefix=TEST3
+// TEST3: simple-openmp-amdgcn-amd-amdhsa--gfx906
+// TEST3: targetID2-openmp-amdgcn-amd-amdhsa--gfx906:sramecc+:xnack-
+// TEST3: targetID3-openmp-amdgcn-amd-amdhsa--gfx906:xnack-
+
// Some code so that we can create a binary out of this file.
int A = 0;
void test_func(void) {
Index: clang/docs/ClangOffloadBundler.rst
===================================================================
--- clang/docs/ClangOffloadBundler.rst
+++ clang/docs/ClangOffloadBundler.rst
@@ -121,7 +121,20 @@
============= ==============================================================
**target-triple**
- The target triple of the code object.
+ The target triple of the code object:
+
+.. code::
+
+ <Architecture>-<Vendor>-<OS>-<Environment>
+
+It is required to have all four components present, if target-id is present.
+If a component is not specified then the empty string must be used in its place.
+For example, if ``<Vendor>`` and ``<Environment>`` is unspecified the target
+triple would be:
+
+.. code::
+
+ myarch--myOS-
**target-id**
The canonical target ID of the code object. Present only if the target
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
