Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 63362.
sfantao added a comment.

- Add tests for when the host does not come first.
- Fix format of comments and add variable to trace the position of the host 
input.


http://reviews.llvm.org/D13909

Files:
  test/CMakeLists.txt
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,680 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// The index of the host input in the list of inputs.
+static unsigned HostInputIndex = ~0u;
+
+/// Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// Generic file handler interface.
+class FileHandler {
+public:
+  /// Update the file handler with information from the header of the bundled
+  /// file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// Read the marker of the next bundled to be read in the file. The triple of
+  /// the target associated with that bundled is returned. An empty string is
+  /// returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Inpu

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 66023.
sfantao added a comment.

- Fix bug in conditional.
- Rebase.


https://reviews.llvm.org/D13909

Files:
  test/CMakeLists.txt
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,681 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// The index of the host input in the list of inputs.
+static unsigned HostInputIndex = ~0u;
+
+/// Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// Generic file handler interface.
+class FileHandler {
+public:
+  /// Update the file handler with information from the header of the bundled
+  /// file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// Read the marker of the next bundled to be read in the file. The triple of
+  /// the target associated with that bundled is returned. An empty string is
+  /// returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// Read the current bundle and write the result into the stream \a OS.
+  virtual void

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 39590.
sfantao added a comment.

Rebase.


http://reviews.llvm.org/D13909

Files:
  tools/CMakeLists.txt
  tools/Makefile
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp
  tools/clang-offload-bundler/Makefile

Index: tools/clang-offload-bundler/Makefile
===
--- /dev/null
+++ tools/clang-offload-bundler/Makefile
@@ -0,0 +1,21 @@
+##===- clang-offload-bundler/Makefile --*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===--===##
+
+CLANG_LEVEL := ../..
+
+TOOLNAME = clang-offload-bundler
+
+# No plugins, optimize startup time.
+TOOL_NO_EXPORTS = 1
+
+include $(CLANG_LEVEL)/../../Makefile.config
+LINK_COMPONENTS := $(TARGETS_TO_BUILD) support option
+USEDLIBS = clangBasic.a
+
+include $(CLANG_LEVEL)/Makefile
Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,548 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list TargetNames("targets", cl::CommaSeparated,
+ cl::OneOrMore,
+ cl::desc("[,...]"),
+ cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// \brief Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// \brief Generic file handler interface.
+class FileHandler {
+protected:
+  /// \brief Update the file handler with information from the header of the
+  /// bundled file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker of the next bundled to be read in the file. The
+  /// triple of the target associated with that bundled is returned. An empty
+  /// string is returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// \brief Read the current bundle and write the result into the stream \a OS.
+  virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
+
+  /// \brief Write the header of the bundled file to \a OS based on the
+  //

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 40999.
sfantao updated the summary for this revision.
sfantao added a comment.

Rebase.


http://reviews.llvm.org/D13909

Files:
  tools/CMakeLists.txt
  tools/Makefile
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp
  tools/clang-offload-bundler/Makefile

Index: tools/clang-offload-bundler/Makefile
===
--- /dev/null
+++ tools/clang-offload-bundler/Makefile
@@ -0,0 +1,21 @@
+##===- clang-offload-bundler/Makefile --*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===--===##
+
+CLANG_LEVEL := ../..
+
+TOOLNAME = clang-offload-bundler
+
+# No plugins, optimize startup time.
+TOOL_NO_EXPORTS = 1
+
+include $(CLANG_LEVEL)/../../Makefile.config
+LINK_COMPONENTS := $(TARGETS_TO_BUILD) support option
+USEDLIBS = clangBasic.a
+
+include $(CLANG_LEVEL)/Makefile
Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,548 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list TargetNames("targets", cl::CommaSeparated,
+ cl::OneOrMore,
+ cl::desc("[,...]"),
+ cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// \brief Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// \brief Generic file handler interface.
+class FileHandler {
+protected:
+  /// \brief Update the file handler with information from the header of the
+  /// bundled file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker of the next bundled to be read in the file. The
+  /// triple of the target associated with that bundled is returned. An empty
+  /// string is returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// \brief Read the current bundle and write the result into the stream \a OS.
+  virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
+
+  /// \brief Write the header 

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Jonas Hahnfeld via cfe-commits]

Hahnfeld accepted this revision.
Hahnfeld added a reviewer: Hahnfeld.
Hahnfeld added a comment.
This revision is now accepted and ready to land.

LGTM with only some minor nits on some of the comments and a CMake question



Comment at: test/CMakeLists.txt:27-33
@@ -26,8 +26,9 @@
 
 list(APPEND CLANG_TEST_DEPS
   clang clang-headers
   clang-format
   c-index-test diagtool
   clang-tblgen
+  clang-offload-bundler
   )
   

I think `clang-offload-bundler` needs to be added as dependency for the `clang` 
target because it will really need the bundler at runtime, not only when 
testing...

(Disclaimer: I'm no CMake expert)


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:102-104
@@ +101,5 @@
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// Read the marker of the next bundled to be read in the file. The triple of
+  /// the target associated with that bundled is returned. An empty string is
+  /// returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;

s/bundled/bundle/?


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:151
@@ +150,3 @@
+
+/// Read 8-byte integers to/from a buffer in little-endian format.
+static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {

`to/from`?


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:164
@@ +163,3 @@
+
+/// Write and write 8-byte integers to/from a buffer in little-endian format.
+static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) {

Duplicate `and write`? `to/from`?


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:568
@@ +567,3 @@
+  if (!FoundHostBundle) {
+llvm::errs() << "error: Can't find bundles for all requested targets\n";
+return true;

Better say that we haven't found the bundle for the host?


https://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 68086.
sfantao marked 5 inline comments as done.
sfantao added a comment.

- Fix comments and diagnostics.


https://reviews.llvm.org/D13909

Files:
  test/CMakeLists.txt
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,681 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// The index of the host input in the list of inputs.
+static unsigned HostInputIndex = ~0u;
+
+/// Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// Generic file handler interface.
+class FileHandler {
+public:
+  /// Update the file handler with information from the header of the bundled
+  /// file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// Read the marker of the next bundled to be read in the file. The triple of
+  /// the target associated with that bundle is returned. An empty string is
+  /// returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// Read the current bundle and write the result i

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao added a comment.

Hi Jonas,

Thanks for the review!



Comment at: test/CMakeLists.txt:27-33
@@ -26,8 +26,9 @@
 
 list(APPEND CLANG_TEST_DEPS
   clang clang-headers
   clang-format
   c-index-test diagtool
   clang-tblgen
+  clang-offload-bundler
   )
   

Hahnfeld wrote:
> I think `clang-offload-bundler` needs to be added as dependency for the 
> `clang` target because it will really need the bundler at runtime, not only 
> when testing...
> 
> (Disclaimer: I'm no CMake expert)
The bundler tool already depends on clang, so that would cause a circular 
dependency. I think that in general not building the bundler is fine - the user 
may not be interested in doing offloading, so if he attempts to do so, that 
would fail as, say, ld was not in the system.

I'm adding it only for testing because there are tests that will exercise the 
bundler that will fail if the driver does not detect the tool. 

Should we ask someone in specific for an opinion? Let me know your thoughts. 


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:151
@@ +150,3 @@
+
+/// Read 8-byte integers to/from a buffer in little-endian format.
+static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {

Hahnfeld wrote:
> `to/from`?
Thanks for catching this! Fixed in the last diff, it should be `from`.


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:164
@@ +163,3 @@
+
+/// Write and write 8-byte integers to/from a buffer in little-endian format.
+static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) {

Hahnfeld wrote:
> Duplicate `and write`? `to/from`?
Fixed in the last diff, it should be `to`.


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:568
@@ +567,3 @@
+  if (!FoundHostBundle) {
+llvm::errs() << "error: Can't find bundles for all requested targets\n";
+return true;

Hahnfeld wrote:
> Better say that we haven't found the bundle for the host?
Makes sense, I changed the message in the last diff.


https://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Jonas Hahnfeld via cfe-commits]

Hahnfeld added inline comments.


Comment at: test/CMakeLists.txt:27-33
@@ -26,8 +26,9 @@
 
 list(APPEND CLANG_TEST_DEPS
   clang clang-headers
   clang-format
   c-index-test diagtool
   clang-tblgen
+  clang-offload-bundler
   )
   

Most users will get it anyway because it is built for the `install` target and 
I think the build system should do its best to build and install all needed 
dependencies.

I think this currently only fails when using `make clang` and then trying to 
invoke the compiler from the build directory. I agree that this should be quite 
rare but that could be fixed by `add_dependencies(clang clang-offload-bundler)` 
which has worked for me. However I now don't have a really strong opinion here 
because it works with the `install` target.


https://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 69124.
sfantao marked an inline comment as done.
sfantao added a comment.

- Add clang-offload bundler as dependency to clang.


https://reviews.llvm.org/D13909

Files:
  test/CMakeLists.txt
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,681 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// The index of the host input in the list of inputs.
+static unsigned HostInputIndex = ~0u;
+
+/// Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// Generic file handler interface.
+class FileHandler {
+public:
+  /// Update the file handler with information from the header of the bundled
+  /// file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// Read the marker of the next bundled to be read in the file. The triple of
+  /// the target associated with that bundle is returned. An empty string is
+  /// returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// Read the current bundle an

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao added a comment.

Hi Jonas,

Thanks again for the review!



Comment at: test/CMakeLists.txt:27-33
@@ -26,8 +26,9 @@
 
 list(APPEND CLANG_TEST_DEPS
   clang clang-headers
   clang-format
   c-index-test diagtool
   clang-tblgen
+  clang-offload-bundler
   )
   

Hahnfeld wrote:
> Most users will get it anyway because it is built for the `install` target 
> and I think the build system should do its best to build and install all 
> needed dependencies.
> 
> I think this currently only fails when using `make clang` and then trying to 
> invoke the compiler from the build directory. I agree that this should be 
> quite rare but that could be fixed by `add_dependencies(clang 
> clang-offload-bundler)` which has worked for me. However I now don't have a 
> really strong opinion here because it works with the `install` target.
Ok, I added the line `add_dependencies(clang clang-offload-bundler) ` as you 
suggest. I was worried that referring to clang libs would cause a circular 
dependency, but it seems to work just fine.


https://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Jonas Hahnfeld via cfe-commits]

Hahnfeld added a comment.

SGTM


https://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 62249.
sfantao added a comment.
Herald added a subscriber: mehdi_amini.

- Reorganize file handlers. Generate empty files if no components exist in the 
bundle instead of just failing.


http://reviews.llvm.org/D13909

Files:
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,683 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// \brief Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// \brief Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// \brief Generic file handler interface.
+class FileHandler {
+public:
+  /// \brief Update the file handler with information from the header of the
+  /// bundled file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker of the next bundled to be read in the file. The
+  /// triple of the target associated with that bundled is returned. An empty
+  /// string is returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// \brief Read the current bundle and write the 

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Alexey Bataev via cfe-commits]

ABataev added a comment.

Remove empty 'return' and ';' statements where they are not required.



Comment at: test/Driver/clang-offload-bundler.c:14
@@ +13,3 @@
+//
+// RUN: touch %t.empty
+

Hmm, will it work on Windows? Maybe it is better just to add an empty test file?


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:174
@@ +173,3 @@
+  /// \brief Information about the bundles extracted from the header.
+  struct BundleInfo {
+/// \brief Size of the bundle.

1. 'final'
2. Default field initializers instead of default constructor


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:304-305
@@ +303,4 @@
+  }
+  void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) { return; }
+  void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) { return; }
+  void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {

remove empty 'return's


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:308
@@ +307,3 @@
+OS.write(Input.getBufferStart(), Input.getBufferSize());
+return;
+  }

Remove it


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:335
@@ +334,3 @@
+  /// \brief Number of chars read from input.
+  size_t ReadChars;
+

Default initializer


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:453
@@ +452,3 @@
+  unsigned Idx = 0;
+  for (auto I : InputFileNames) {
+ErrorOr> CodeOrErr =

auto &File instead of auto I?


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:477-478
@@ +476,4 @@
+  auto Input = InputBuffers.begin();
+  for (auto Triple = TargetNames.begin(); Triple < TargetNames.end();
+   ++Triple, ++Input) {
+FH.get()->WriteBundleStart(OutputFile, *Triple);

for (auto &Triple : TargetNames)



Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:513-514
@@ +512,4 @@
+  auto Output = OutputFileNames.begin();
+  for (auto Triple = TargetNames.begin(); Triple < TargetNames.end();
+   ++Triple, ++Output)
+Worklist[*Triple] = *Output;

for (auto &Triple : TargetNames)


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:677-680
@@ +676,6 @@
+
+  if (Unbundle)
+return UnbundleFiles();
+  else
+return BundleFiles();
+

Maybe 'return Unbundle ? UnbundleFiles() : BundleFiles();'?


http://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 62538.
sfantao marked 9 inline comments as done.
sfantao added a comment.

- Merge branch 'master' into patch-D13909
- Remove unecessary returns and fix iterators.


http://reviews.llvm.org/D13909

Files:
  test/CMakeLists.txt
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,671 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// \brief Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// \brief Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// \brief Generic file handler interface.
+class FileHandler {
+public:
+  /// \brief Update the file handler with information from the header of the
+  /// bundled file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker of the next bundled to be read in the file. The
+  /// triple of the target associated with that bundled is returned. An empty
+  /// string is returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// \brief Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// \brief Read the current bundle and write the

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao added a comment.

Hi Alexey,

Thanks for the review!



Comment at: test/Driver/clang-offload-bundler.c:14
@@ +13,3 @@
+//
+// RUN: touch %t.empty
+

ABataev wrote:
> Hmm, will it work on Windows? Maybe it is better just to add an empty test 
> file?
I see many other tests in clang using it. There used to be a directive 
`//requires-shell` but I don't think that is being used now. Should I go ahead 
and create an empty file? Unfortunately, it is not easy for me to try these 
things on Windows.


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:477-478
@@ +476,4 @@
+  auto Input = InputBuffers.begin();
+  for (auto Triple = TargetNames.begin(); Triple < TargetNames.end();
+   ++Triple, ++Input) {
+FH.get()->WriteBundleStart(OutputFile, *Triple);

ABataev wrote:
> for (auto &Triple : TargetNames)
> 
Ok, moved the iterator `Input` to the end of the loop body.


Comment at: tools/clang-offload-bundler/ClangOffloadBundler.cpp:513-514
@@ +512,4 @@
+  auto Output = OutputFileNames.begin();
+  for (auto Triple = TargetNames.begin(); Triple < TargetNames.end();
+   ++Triple, ++Output)
+Worklist[*Triple] = *Output;

ABataev wrote:
> for (auto &Triple : TargetNames)
Ok, moved `Output` iterator to the bottom of the loop body.


http://reviews.llvm.org/D13909



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D13909: clang-offload-bundler - offload files bundling/unbundling tool

[Samuel Antao via cfe-commits]

sfantao updated this revision to Diff 62540.
sfantao added a comment.

- Remove \brief when not needed.


http://reviews.llvm.org/D13909

Files:
  test/CMakeLists.txt
  test/Driver/clang-offload-bundler.c
  tools/CMakeLists.txt
  tools/clang-offload-bundler/CMakeLists.txt
  tools/clang-offload-bundler/ClangOffloadBundler.cpp

Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===
--- /dev/null
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -0,0 +1,670 @@
+//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+///
+/// \file
+/// \brief This file implements a clang-offload-bundler that bundles different
+/// files that relate with the same source code but different targets into a
+/// single one. Also the implements the opposite functionality, i.e. unbundle
+/// files previous created by this tool.
+///
+//===--===//
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+// Mark all our options with this category, everything else (except for -version
+// and -help) will be hidden.
+static cl::OptionCategory
+ClangOffloadBundlerCategory("clang-offload-bundler options");
+
+static cl::list
+InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
+   cl::desc("[,...]"),
+   cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::list
+TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
+cl::desc("[-,...]"),
+cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+FilesType("type", cl::Required,
+  cl::desc("Type of the files to be bundled/unbundled.\n"
+   "Current supported types are:\n"
+   "  i   - cpp-output\n"
+   "  ii  - c++-cpp-output\n"
+   "  ll  - llvm\n"
+   "  bc  - llvm-bc\n"
+   "  s   - assembler\n"
+   "  o   - object\n"
+   "  gch - precompiled-header\n"
+   "  ast - clang AST file"),
+  cl::cat(ClangOffloadBundlerCategory));
+static cl::opt
+Unbundle("unbundle",
+ cl::desc("Unbundle bundled file into several output files.\n"),
+ cl::init(false), cl::cat(ClangOffloadBundlerCategory));
+
+/// Magic string that marks the existence of offloading data.
+#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+
+/// Obtain the offload kind and real machine triple out of the target
+/// information specified by the user.
+static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
+StringRef &Triple) {
+  auto KindTriplePair = Target.split('-');
+  OffloadKind = KindTriplePair.first;
+  Triple = KindTriplePair.second;
+}
+static bool hasHostKind(StringRef Target) {
+  StringRef OffloadKind;
+  StringRef Triple;
+  getOffloadKindAndTriple(Target, OffloadKind, Triple);
+  return OffloadKind == "host";
+}
+
+/// Generic file handler interface.
+class FileHandler {
+public:
+  /// Update the file handler with information from the header of the bundled
+  /// file
+  virtual void ReadHeader(MemoryBuffer &Input) = 0;
+  /// Read the marker of the next bundled to be read in the file. The triple of
+  /// the target associated with that bundled is returned. An empty string is
+  /// returned if there are no more bundles to be read.
+  virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
+  /// Read the marker that closes the current bundle.
+  virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
+  /// Read the current bundle and write the result into the stream \a OS.
+  virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
+
+  /// Write the header of the bundled fi