Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 63362. sfantao added a comment. - Add tests for when the host does not come first. - Fix format of comments and add variable to trace the position of the host input. http://reviews.llvm.org/D13909 Files: test/CMakeLists.txt test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,680 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// The index of the host input in the list of inputs. +static unsigned HostInputIndex = ~0u; + +/// Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// Generic file handler interface. +class FileHandler { +public: + /// Update the file handler with information from the header of the bundled + /// file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// Read the marker of the next bundled to be read in the file. The triple of + /// the target associated with that bundled is returned. An empty string is + /// returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Inpu
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 66023. sfantao added a comment. - Fix bug in conditional. - Rebase. https://reviews.llvm.org/D13909 Files: test/CMakeLists.txt test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,681 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// The index of the host input in the list of inputs. +static unsigned HostInputIndex = ~0u; + +/// Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// Generic file handler interface. +class FileHandler { +public: + /// Update the file handler with information from the header of the bundled + /// file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// Read the marker of the next bundled to be read in the file. The triple of + /// the target associated with that bundled is returned. An empty string is + /// returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// Read the current bundle and write the result into the stream \a OS. + virtual void
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 39590. sfantao added a comment. Rebase. http://reviews.llvm.org/D13909 Files: tools/CMakeLists.txt tools/Makefile tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp tools/clang-offload-bundler/Makefile Index: tools/clang-offload-bundler/Makefile === --- /dev/null +++ tools/clang-offload-bundler/Makefile @@ -0,0 +1,21 @@ +##===- clang-offload-bundler/Makefile --*- Makefile -*-===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===--===## + +CLANG_LEVEL := ../.. + +TOOLNAME = clang-offload-bundler + +# No plugins, optimize startup time. +TOOL_NO_EXPORTS = 1 + +include $(CLANG_LEVEL)/../../Makefile.config +LINK_COMPONENTS := $(TARGETS_TO_BUILD) support option +USEDLIBS = clangBasic.a + +include $(CLANG_LEVEL)/Makefile Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,548 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list TargetNames("targets", cl::CommaSeparated, + cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// \brief Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// \brief Generic file handler interface. +class FileHandler { +protected: + /// \brief Update the file handler with information from the header of the + /// bundled file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// \brief Read the marker of the next bundled to be read in the file. The + /// triple of the target associated with that bundled is returned. An empty + /// string is returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// \brief Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// \brief Read the current bundle and write the result into the stream \a OS. + virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// \brief Write the header of the bundled file to \a OS based on the + //
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 40999. sfantao updated the summary for this revision. sfantao added a comment. Rebase. http://reviews.llvm.org/D13909 Files: tools/CMakeLists.txt tools/Makefile tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp tools/clang-offload-bundler/Makefile Index: tools/clang-offload-bundler/Makefile === --- /dev/null +++ tools/clang-offload-bundler/Makefile @@ -0,0 +1,21 @@ +##===- clang-offload-bundler/Makefile --*- Makefile -*-===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===--===## + +CLANG_LEVEL := ../.. + +TOOLNAME = clang-offload-bundler + +# No plugins, optimize startup time. +TOOL_NO_EXPORTS = 1 + +include $(CLANG_LEVEL)/../../Makefile.config +LINK_COMPONENTS := $(TARGETS_TO_BUILD) support option +USEDLIBS = clangBasic.a + +include $(CLANG_LEVEL)/Makefile Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,548 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list TargetNames("targets", cl::CommaSeparated, + cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// \brief Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// \brief Generic file handler interface. +class FileHandler { +protected: + /// \brief Update the file handler with information from the header of the + /// bundled file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// \brief Read the marker of the next bundled to be read in the file. The + /// triple of the target associated with that bundled is returned. An empty + /// string is returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// \brief Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// \brief Read the current bundle and write the result into the stream \a OS. + virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// \brief Write the header
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
Hahnfeld accepted this revision. Hahnfeld added a reviewer: Hahnfeld. Hahnfeld added a comment. This revision is now accepted and ready to land. LGTM with only some minor nits on some of the comments and a CMake question Comment at: test/CMakeLists.txt:27-33 @@ -26,8 +26,9 @@ list(APPEND CLANG_TEST_DEPS clang clang-headers clang-format c-index-test diagtool clang-tblgen + clang-offload-bundler ) I think `clang-offload-bundler` needs to be added as dependency for the `clang` target because it will really need the bundler at runtime, not only when testing... (Disclaimer: I'm no CMake expert) Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:102-104 @@ +101,5 @@ + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// Read the marker of the next bundled to be read in the file. The triple of + /// the target associated with that bundled is returned. An empty string is + /// returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; s/bundled/bundle/? Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:151 @@ +150,3 @@ + +/// Read 8-byte integers to/from a buffer in little-endian format. +static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) { `to/from`? Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:164 @@ +163,3 @@ + +/// Write and write 8-byte integers to/from a buffer in little-endian format. +static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) { Duplicate `and write`? `to/from`? Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:568 @@ +567,3 @@ + if (!FoundHostBundle) { +llvm::errs() << "error: Can't find bundles for all requested targets\n"; +return true; Better say that we haven't found the bundle for the host? https://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 68086. sfantao marked 5 inline comments as done. sfantao added a comment. - Fix comments and diagnostics. https://reviews.llvm.org/D13909 Files: test/CMakeLists.txt test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,681 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// The index of the host input in the list of inputs. +static unsigned HostInputIndex = ~0u; + +/// Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// Generic file handler interface. +class FileHandler { +public: + /// Update the file handler with information from the header of the bundled + /// file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// Read the marker of the next bundled to be read in the file. The triple of + /// the target associated with that bundle is returned. An empty string is + /// returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// Read the current bundle and write the result i
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao added a comment. Hi Jonas, Thanks for the review! Comment at: test/CMakeLists.txt:27-33 @@ -26,8 +26,9 @@ list(APPEND CLANG_TEST_DEPS clang clang-headers clang-format c-index-test diagtool clang-tblgen + clang-offload-bundler ) Hahnfeld wrote: > I think `clang-offload-bundler` needs to be added as dependency for the > `clang` target because it will really need the bundler at runtime, not only > when testing... > > (Disclaimer: I'm no CMake expert) The bundler tool already depends on clang, so that would cause a circular dependency. I think that in general not building the bundler is fine - the user may not be interested in doing offloading, so if he attempts to do so, that would fail as, say, ld was not in the system. I'm adding it only for testing because there are tests that will exercise the bundler that will fail if the driver does not detect the tool. Should we ask someone in specific for an opinion? Let me know your thoughts. Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:151 @@ +150,3 @@ + +/// Read 8-byte integers to/from a buffer in little-endian format. +static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) { Hahnfeld wrote: > `to/from`? Thanks for catching this! Fixed in the last diff, it should be `from`. Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:164 @@ +163,3 @@ + +/// Write and write 8-byte integers to/from a buffer in little-endian format. +static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) { Hahnfeld wrote: > Duplicate `and write`? `to/from`? Fixed in the last diff, it should be `to`. Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:568 @@ +567,3 @@ + if (!FoundHostBundle) { +llvm::errs() << "error: Can't find bundles for all requested targets\n"; +return true; Hahnfeld wrote: > Better say that we haven't found the bundle for the host? Makes sense, I changed the message in the last diff. https://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
Hahnfeld added inline comments. Comment at: test/CMakeLists.txt:27-33 @@ -26,8 +26,9 @@ list(APPEND CLANG_TEST_DEPS clang clang-headers clang-format c-index-test diagtool clang-tblgen + clang-offload-bundler ) Most users will get it anyway because it is built for the `install` target and I think the build system should do its best to build and install all needed dependencies. I think this currently only fails when using `make clang` and then trying to invoke the compiler from the build directory. I agree that this should be quite rare but that could be fixed by `add_dependencies(clang clang-offload-bundler)` which has worked for me. However I now don't have a really strong opinion here because it works with the `install` target. https://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 69124. sfantao marked an inline comment as done. sfantao added a comment. - Add clang-offload bundler as dependency to clang. https://reviews.llvm.org/D13909 Files: test/CMakeLists.txt test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,681 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// The index of the host input in the list of inputs. +static unsigned HostInputIndex = ~0u; + +/// Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// Generic file handler interface. +class FileHandler { +public: + /// Update the file handler with information from the header of the bundled + /// file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// Read the marker of the next bundled to be read in the file. The triple of + /// the target associated with that bundle is returned. An empty string is + /// returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// Read the current bundle an
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao added a comment. Hi Jonas, Thanks again for the review! Comment at: test/CMakeLists.txt:27-33 @@ -26,8 +26,9 @@ list(APPEND CLANG_TEST_DEPS clang clang-headers clang-format c-index-test diagtool clang-tblgen + clang-offload-bundler ) Hahnfeld wrote: > Most users will get it anyway because it is built for the `install` target > and I think the build system should do its best to build and install all > needed dependencies. > > I think this currently only fails when using `make clang` and then trying to > invoke the compiler from the build directory. I agree that this should be > quite rare but that could be fixed by `add_dependencies(clang > clang-offload-bundler)` which has worked for me. However I now don't have a > really strong opinion here because it works with the `install` target. Ok, I added the line `add_dependencies(clang clang-offload-bundler) ` as you suggest. I was worried that referring to clang libs would cause a circular dependency, but it seems to work just fine. https://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
Hahnfeld added a comment. SGTM https://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 62249. sfantao added a comment. Herald added a subscriber: mehdi_amini. - Reorganize file handlers. Generate empty files if no components exist in the bundle instead of just failing. http://reviews.llvm.org/D13909 Files: test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,683 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// \brief Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// \brief Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// \brief Generic file handler interface. +class FileHandler { +public: + /// \brief Update the file handler with information from the header of the + /// bundled file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// \brief Read the marker of the next bundled to be read in the file. The + /// triple of the target associated with that bundled is returned. An empty + /// string is returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// \brief Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// \brief Read the current bundle and write the
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
ABataev added a comment. Remove empty 'return' and ';' statements where they are not required. Comment at: test/Driver/clang-offload-bundler.c:14 @@ +13,3 @@ +// +// RUN: touch %t.empty + Hmm, will it work on Windows? Maybe it is better just to add an empty test file? Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:174 @@ +173,3 @@ + /// \brief Information about the bundles extracted from the header. + struct BundleInfo { +/// \brief Size of the bundle. 1. 'final' 2. Default field initializers instead of default constructor Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:304-305 @@ +303,4 @@ + } + void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) { return; } + void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) { return; } + void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) { remove empty 'return's Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:308 @@ +307,3 @@ +OS.write(Input.getBufferStart(), Input.getBufferSize()); +return; + } Remove it Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:335 @@ +334,3 @@ + /// \brief Number of chars read from input. + size_t ReadChars; + Default initializer Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:453 @@ +452,3 @@ + unsigned Idx = 0; + for (auto I : InputFileNames) { +ErrorOr> CodeOrErr = auto &File instead of auto I? Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:477-478 @@ +476,4 @@ + auto Input = InputBuffers.begin(); + for (auto Triple = TargetNames.begin(); Triple < TargetNames.end(); + ++Triple, ++Input) { +FH.get()->WriteBundleStart(OutputFile, *Triple); for (auto &Triple : TargetNames) Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:513-514 @@ +512,4 @@ + auto Output = OutputFileNames.begin(); + for (auto Triple = TargetNames.begin(); Triple < TargetNames.end(); + ++Triple, ++Output) +Worklist[*Triple] = *Output; for (auto &Triple : TargetNames) Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:677-680 @@ +676,6 @@ + + if (Unbundle) +return UnbundleFiles(); + else +return BundleFiles(); + Maybe 'return Unbundle ? UnbundleFiles() : BundleFiles();'? http://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 62538. sfantao marked 9 inline comments as done. sfantao added a comment. - Merge branch 'master' into patch-D13909 - Remove unecessary returns and fix iterators. http://reviews.llvm.org/D13909 Files: test/CMakeLists.txt test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,671 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// \brief Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// \brief Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// \brief Generic file handler interface. +class FileHandler { +public: + /// \brief Update the file handler with information from the header of the + /// bundled file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// \brief Read the marker of the next bundled to be read in the file. The + /// triple of the target associated with that bundled is returned. An empty + /// string is returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// \brief Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// \brief Read the current bundle and write the
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao added a comment. Hi Alexey, Thanks for the review! Comment at: test/Driver/clang-offload-bundler.c:14 @@ +13,3 @@ +// +// RUN: touch %t.empty + ABataev wrote: > Hmm, will it work on Windows? Maybe it is better just to add an empty test > file? I see many other tests in clang using it. There used to be a directive `//requires-shell` but I don't think that is being used now. Should I go ahead and create an empty file? Unfortunately, it is not easy for me to try these things on Windows. Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:477-478 @@ +476,4 @@ + auto Input = InputBuffers.begin(); + for (auto Triple = TargetNames.begin(); Triple < TargetNames.end(); + ++Triple, ++Input) { +FH.get()->WriteBundleStart(OutputFile, *Triple); ABataev wrote: > for (auto &Triple : TargetNames) > Ok, moved the iterator `Input` to the end of the loop body. Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:513-514 @@ +512,4 @@ + auto Output = OutputFileNames.begin(); + for (auto Triple = TargetNames.begin(); Triple < TargetNames.end(); + ++Triple, ++Output) +Worklist[*Triple] = *Output; ABataev wrote: > for (auto &Triple : TargetNames) Ok, moved `Output` iterator to the bottom of the loop body. http://reviews.llvm.org/D13909 ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool
sfantao updated this revision to Diff 62540. sfantao added a comment. - Remove \brief when not needed. http://reviews.llvm.org/D13909 Files: test/CMakeLists.txt test/Driver/clang-offload-bundler.c tools/CMakeLists.txt tools/clang-offload-bundler/CMakeLists.txt tools/clang-offload-bundler/ClangOffloadBundler.cpp Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp === --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,670 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===--===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory +ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list +InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list +OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::list +TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, +cl::desc("[-,...]"), +cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt +Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, +StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// Generic file handler interface. +class FileHandler { +public: + /// Update the file handler with information from the header of the bundled + /// file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// Read the marker of the next bundled to be read in the file. The triple of + /// the target associated with that bundled is returned. An empty string is + /// returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// Read the current bundle and write the result into the stream \a OS. + virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// Write the header of the bundled fi