From: Junyan He <junyan...@intel.com> We use compiler API to generate the new type of ELF binary for runtime usage.
Signed-off-by: Junyan He <junyan...@intel.com> --- backend/src/gbe_bin_generater.cpp | 669 ++++++++++++++++++-------------------- 1 file changed, 314 insertions(+), 355 deletions(-) diff --git a/backend/src/gbe_bin_generater.cpp b/backend/src/gbe_bin_generater.cpp index 8e42891..f621c0d 100644 --- a/backend/src/gbe_bin_generater.cpp +++ b/backend/src/gbe_bin_generater.cpp @@ -36,420 +36,379 @@ #include <stdlib.h> #include <stdio.h> -#include "backend/program.h" -#include "backend/program.hpp" -#include "backend/src/sys/platform.hpp" -#include "src/cl_device_data.h" +#include "elfio/elfio.hpp" using namespace std; +using namespace ELFIO; #define FILE_NOT_FIND_ERR 1 #define FILE_MAP_ERR 2 #define FILE_BUILD_FAILED 3 #define FILE_SERIALIZATION_FAILED 4 +extern "C" bool +GenBuildProgram(uint32_t deviceID, const char *source, size_t src_length, const char *options, + size_t errBufSize, char *err, size_t *errRetSize, char **binary, size_t *binarySize); +extern "C" bool +GenCompileProgram(uint32_t deviceID, const char *source, size_t src_length, const char **headers, + size_t *header_length, const char **header_names, int headerNum, const char *options, + size_t errBufSize, char *err, size_t *errRetSize, char **binary, size_t *binarySize); + static uint32_t gen_pci_id = 0; -class program_build_instance { +class program_build_instance +{ protected: - string prog_path; - string build_opt; - static string bin_path; - static bool str_fmt_out; - int fd; - int file_len; - const char* code; - gbe::Program* gbe_prog; + string prog_path; + string build_opt; + static string bin_path; + static bool str_fmt_out; + int fd; + int file_len; + const char *code; + char *binary; + size_t binary_sz; + char build_log[4096]; + char *elf_bin; + size_t elf_bin_sz; public: - program_build_instance (void) : fd(-1), file_len(0), code(NULL), gbe_prog(NULL) { } - explicit program_build_instance (const char* file_path, const char* option = NULL) - : prog_path(file_path), build_opt(option), fd(-1), file_len(0), - code(NULL), gbe_prog(NULL) { } - - ~program_build_instance () { - if (code) { - munmap((void *)(code), file_len); - code = NULL; - } - - if (fd >= 0) - close(fd); - - if (gbe_prog) - gbe_program_delete(reinterpret_cast<gbe_program>(gbe_prog)); - } - - program_build_instance(program_build_instance&& other) = default; -#if 0 - { -#define SWAP(ELT) \ - do { \ - auto elt = this->ELT; \ - this->ELT = other.ELT; \ - other.ELT = elt; \ - } while(0) - - SWAP(fd); - SWAP(code); - SWAP(file_len); - SWAP(prog_path); - SWAP(build_opt); -#undef SWAP - } -#endif - - explicit program_build_instance(const program_build_instance& other) = delete; - program_build_instance& operator= (const program_build_instance& other) { - /* we do not want to be Lvalue copied, but operator is needed to instance the - template of vector<program_build_instance>. */ - assert(1); - return *this; - } - - const char* file_map_open (void) throw (int); - - const char* get_code (void) { - return code; - } - - const string& get_program_path (void) { - return prog_path; - } - - int get_size (void) { - return file_len; - } - - void print_file (void) { - cout << code << endl; + program_build_instance(void) : fd(-1), file_len(0), code(NULL), binary(NULL), binary_sz(0) {} + explicit program_build_instance(const char *file_path, const char *option = NULL) + : prog_path(file_path), build_opt(option), fd(-1), file_len(0), + code(NULL), binary(NULL), binary_sz(0), elf_bin(NULL), elf_bin_sz(0) {} + + ~program_build_instance() + { + if (code) { + munmap((void *)(code), file_len); + code = NULL; } - void dump (void) { - cout << "program path: " << prog_path << endl; - cout << "Build option: " << build_opt << endl; - print_file(); - } - - static void set_str_fmt_out (bool flag) { - str_fmt_out = flag; - } - - static int set_bin_path (const char* path) { - if (bin_path.size()) - return 0; - - bin_path = path; - return 1; - } - - void build_program(void) throw(int); - void serialize_program(void) throw(int); + if (fd >= 0) + close(fd); + + if (binary) + free(binary); + if (elf_bin) + free(elf_bin); + } + + program_build_instance(program_build_instance &&other) = default; + explicit program_build_instance(const program_build_instance &other) = delete; + program_build_instance &operator=(const program_build_instance &other) + { + /* we do not want to be Lvalue copied, but operator is needed to instance the + template of vector<program_build_instance>. */ + assert(1); + return *this; + } + + const char *file_map_open(void) throw(int); + const char *get_code(void) { return code; } + const string &get_program_path(void) { return prog_path; } + int get_size(void) { return file_len; } + void print_file(void) { cout << code << endl; } + + void dump(void) + { + cout << "program path: " << prog_path << endl; + cout << "Build option: " << build_opt << endl; + print_file(); + } + + static void set_str_fmt_out(bool flag) { str_fmt_out = flag; } + + static int set_bin_path(const char *path) + { + if (bin_path.size()) + return 0; + + bin_path = path; + return 1; + } + + void build_program(void) throw(int); + void serialize_program(void) throw(int); }; string program_build_instance::bin_path; bool program_build_instance::str_fmt_out = false; -#define OUTS_UPDATE_SZ(elt) SERIALIZE_OUT(elt, oss, header_sz) -#define OUTF_UPDATE_SZ(elt) SERIALIZE_OUT(elt, ofs, header_sz) void program_build_instance::serialize_program(void) throw(int) { - ofstream ofs; - ostringstream oss; - size_t sz = 0, header_sz = 0; - ofs.open(bin_path, ofstream::out | ofstream::trunc | ofstream::binary); - - char src_hw_info[4]=""; - if(IS_IVYBRIDGE(gen_pci_id)){ - src_hw_info[0]='I'; - src_hw_info[1]='V'; - src_hw_info[2]='B'; - if(IS_BAYTRAIL_T(gen_pci_id)){ - src_hw_info[0]='B'; - src_hw_info[1]='Y'; - src_hw_info[2]='T'; + ofstream ofs; + ostringstream oss; + size_t sz = 0; + ofs.open(bin_path, ofstream::out | ofstream::trunc | ofstream::binary); + + if (str_fmt_out) { + string array_name = "Unknown_name_array"; + unsigned long last_slash = bin_path.rfind("/"); + unsigned long last_dot = bin_path.rfind("."); + + if (last_slash != string::npos && last_dot != string::npos) + array_name = bin_path.substr(last_slash + 1, last_dot - 1 - last_slash); + + ofs << "#include <stddef.h>" + << "\n"; + + std::string all_kernel_names = ""; + std::istringstream elf_str(std::string(elf_bin, elf_bin_sz)); + elfio reader; + if (reader.load(elf_str) == false) + assert(0); + + Elf_Half sec_num = reader.sections.size(); + for (int i = 0; i < sec_num; ++i) { + section *psec = reader.sections[i]; + // Check section type + if (psec->get_type() == SHT_SYMTAB) { + const symbol_section_accessor symbols(reader, psec); + for (unsigned int j = 0; j < symbols.get_symbols_num(); ++j) { + std::string name; + Elf64_Addr value; + Elf_Xword size; + unsigned char bind; + unsigned char type; + Elf_Half section_index; + unsigned char other; + + // Read symbol properties + symbols.get_symbol(j, name, value, size, bind, + type, section_index, other); + if (type != STT_FUNC) + continue; + if (bind != STB_GLOBAL) + continue; + + all_kernel_names += name; + all_kernel_names += ";"; + } } - }else if(IS_HASWELL(gen_pci_id)){ - src_hw_info[0]='H'; - src_hw_info[1]='S'; - src_hw_info[2]='W'; - }else if(IS_BROADWELL(gen_pci_id)){ - src_hw_info[0]='B'; - src_hw_info[1]='D'; - src_hw_info[2]='W'; - }else if(IS_CHERRYVIEW(gen_pci_id)){ - src_hw_info[0]='C'; - src_hw_info[1]='H'; - src_hw_info[2]='V'; - }else if(IS_SKYLAKE(gen_pci_id)){ - src_hw_info[0]='S'; - src_hw_info[1]='K'; - src_hw_info[2]='L'; - }else if(IS_BROXTON(gen_pci_id)){ - src_hw_info[0]='B'; - src_hw_info[1]='X'; - src_hw_info[2]='T'; } + all_kernel_names.erase(all_kernel_names.end() - 1); + ofs << "char *" << array_name << "_kernels = \"" + << all_kernel_names << "\";\n"; - if (str_fmt_out) { - - if(gen_pci_id){ - //add header to differeciate from llvm bitcode binary. - // (5 bytes: 1 byte for binary version, 4 byte for bc code, 'GENC' is for gen binary.) - char gen_header[6] = "\1GENC"; - OUTS_UPDATE_SZ(gen_header[0]); - OUTS_UPDATE_SZ(gen_header[1]); - OUTS_UPDATE_SZ(gen_header[2]); - OUTS_UPDATE_SZ(gen_header[3]); - OUTS_UPDATE_SZ(gen_header[4]); - OUTS_UPDATE_SZ(src_hw_info[0]); - OUTS_UPDATE_SZ(src_hw_info[1]); - OUTS_UPDATE_SZ(src_hw_info[2]); - } - - string array_name = "Unknown_name_array"; - unsigned long last_slash = bin_path.rfind("/"); - unsigned long last_dot = bin_path.rfind("."); - - if (last_slash != string::npos && last_dot != string::npos) - array_name = bin_path.substr(last_slash + 1, last_dot - 1 - last_slash); - - ofs << "#include <stddef.h>" << "\n"; - ofs << "char " << array_name << "[] = {" << "\n"; - - if(gen_pci_id){ - sz = gbe_prog->serializeToBin(oss); - sz += header_sz; - }else{ - char *llvm_binary; - size_t bin_length = gbe_program_serialize_to_binary((gbe_program)gbe_prog, &llvm_binary, 1); - oss.write(llvm_binary, bin_length); - sz += bin_length; - free(llvm_binary); - } + ofs << "char " << array_name << "[] = {" + << "\n"; - for (size_t i = 0; i < sz; i++) { - unsigned char c = oss.str().c_str()[i]; - char asic_str[9]; - sprintf(asic_str, "%2.2x", c); - ofs << "0x"; - ofs << asic_str << ((i == sz - 1) ? "" : ", "); - } - ofs << "};\n"; + if (gen_pci_id != 0) { + oss.write(elf_bin, elf_bin_sz); + sz += elf_bin_sz; + } else { + oss.write(binary, binary_sz); + sz += binary_sz; + } - string array_size = array_name + "_size"; - ofs << "size_t " << array_size << " = " << sz << ";" << "\n"; + for (size_t i = 0; i < sz; i++) { + unsigned char c = oss.str().c_str()[i]; + char asic_str[9]; + sprintf(asic_str, "%2.2x", c); + ofs << "0x"; + ofs << asic_str << ((i == sz - 1) ? "" : ", "); + } + ofs << "};\n"; + + string array_size = array_name + "_size"; + ofs << "size_t " << array_size << " = " << sz << ";" + << "\n"; + } else { + if (gen_pci_id != 0) { + ofs.write(elf_bin, elf_bin_sz); + sz += elf_bin_sz; } else { - if(gen_pci_id){ - //add header to differeciate from llvm bitcode binary. - // (5 bytes: 1 byte for binary version, 4 byte for bc code, 'GENC' is for gen binary.) - char gen_header[6] = "\1GENC"; - OUTF_UPDATE_SZ(gen_header[0]); - OUTF_UPDATE_SZ(gen_header[1]); - OUTF_UPDATE_SZ(gen_header[2]); - OUTF_UPDATE_SZ(gen_header[3]); - OUTF_UPDATE_SZ(gen_header[4]); - OUTF_UPDATE_SZ(src_hw_info[0]); - OUTF_UPDATE_SZ(src_hw_info[1]); - OUTF_UPDATE_SZ(src_hw_info[2]); - sz = gbe_prog->serializeToBin(ofs); - }else{ - char *llvm_binary; - size_t bin_length = gbe_program_serialize_to_binary((gbe_program)gbe_prog, &llvm_binary, 1); - ofs.write(llvm_binary, bin_length); - sz+=bin_length; - free(llvm_binary); - } + ofs.write(binary, binary_sz); + sz += binary_sz; } + } - ofs.close(); + ofs.close(); - if (!sz) { - throw FILE_SERIALIZATION_FAILED; - } + if (!sz) { + throw FILE_SERIALIZATION_FAILED; + } } - void program_build_instance::build_program(void) throw(int) { - gbe_program opaque = NULL; - if(gen_pci_id){ - opaque = gbe_program_new_from_source(gen_pci_id, code, 0, build_opt.c_str(), NULL, NULL); - }else{ - opaque = gbe_program_compile_from_source(0, code, NULL, 0, build_opt.c_str(), NULL, NULL); + size_t err_log_sz = 0; + bool ret = false; + + if (gen_pci_id) { + ret = GenBuildProgram(gen_pci_id, code, strlen(code) + 1, build_opt.c_str(), sizeof(build_log), + build_log, &err_log_sz, &elf_bin, &elf_bin_sz); + } else { + ret = GenCompileProgram(gen_pci_id, code, strlen(code) + 1, NULL, NULL, NULL, 0, build_opt.c_str(), + sizeof(build_log), + build_log, &err_log_sz, &binary, &binary_sz); + if (ret && str_fmt_out) { + err_log_sz = 0; + ret = GenBuildProgram(0x5A84, code, strlen(code) + 1, build_opt.c_str(), sizeof(build_log), + build_log, &err_log_sz, &elf_bin, &elf_bin_sz); // Just build to get kernel name } - if (!opaque) - throw FILE_BUILD_FAILED; + } - gbe_prog = reinterpret_cast<gbe::Program*>(opaque); + if (ret == false) + throw FILE_BUILD_FAILED; +} - if(gen_pci_id){ - assert(gbe_program_get_kernel_num(opaque)); - } +const char *program_build_instance::file_map_open(void) throw(int) +{ + void *address; + + /* Open the file */ + fd = ::open(prog_path.c_str(), O_RDONLY); + if (fd < 0) { + throw FILE_NOT_FIND_ERR; + } + + /* Map it */ + file_len = lseek(fd, 0, SEEK_END); + lseek(fd, 0, SEEK_SET); + address = mmap(0, file_len, PROT_READ, MAP_SHARED, fd, 0); + if (address == NULL) { + throw FILE_MAP_ERR; + } + + code = reinterpret_cast<const char *>(address); + return code; } -const char* program_build_instance::file_map_open(void) throw(int) +typedef vector<program_build_instance> prog_vector; + +int main(int argc, const char **argv) { - void * address; + prog_vector prog_insts; + vector<string> argv_saved; + const char *build_opt; + const char *file_path; + int i; + int oc; + deque<int> used_index; + + if (argc < 2) { + cout << "Usage: kernel_path [-pbuild_parameter] [-obin_path] [-tgen_pci_id]" << endl; + return 0; + } - /* Open the file */ - fd = ::open(prog_path.c_str(), O_RDONLY); - if (fd < 0) { - throw FILE_NOT_FIND_ERR; - } + used_index.assign(argc, 0); - /* Map it */ - file_len = lseek(fd, 0, SEEK_END); - lseek(fd, 0, SEEK_SET); - address = mmap(0, file_len, PROT_READ, MAP_SHARED, fd, 0); - if (address == NULL) { - throw FILE_MAP_ERR; - } + /* because getopt will re-sort the argv, so we save here. */ + for (i = 0; i < argc; i++) { + argv_saved.push_back(string(argv[i])); + } - code = reinterpret_cast<const char*>(address); - return code; -} + while ((oc = getopt(argc, (char *const *)argv, "t:o:p:s")) != -1) { + switch (oc) { + case 'p': { + int opt_index; -typedef vector<program_build_instance> prog_vector; + if (argv[optind - 1][0] == '-') { // -pXXX like + opt_index = optind - 1; + } else { // Must be -p XXXX mode + opt_index = optind - 2; + used_index[opt_index + 1] = 1; + } -int main (int argc, const char **argv) -{ - prog_vector prog_insts; - vector<string> argv_saved; - const char* build_opt; - const char* file_path; - int i; - int oc; - deque<int> used_index; - - if (argc < 2) { - cout << "Usage: kernel_path [-pbuild_parameter] [-obin_path] [-tgen_pci_id]" << endl; - return 0; - } + /* opt must follow the file name.*/ + if ((opt_index < 2) || argv[opt_index - 1][0] == '-') { + cout << "Usage note: Building option must follow file name" << endl; + return 1; + } - used_index.assign(argc, 0); + file_path = argv[opt_index - 1]; + build_opt = optarg; - /* because getopt will re-sort the argv, so we save here. */ - for (i=0; i< argc; i++) { - argv_saved.push_back(string(argv[i])); + prog_insts.push_back(program_build_instance(file_path, build_opt)); + break; } - while ( (oc = getopt(argc, (char * const *)argv, "t:o:p:s")) != -1 ) { - switch (oc) { - case 'p': - { - int opt_index; - - if (argv[optind-1][0] == '-') {// -pXXX like - opt_index = optind - 1; - } else { // Must be -p XXXX mode - opt_index = optind - 2; - used_index[opt_index + 1] = 1; - } - - /* opt must follow the file name.*/ - if ((opt_index < 2 ) || argv[opt_index-1][0] == '-') { - cout << "Usage note: Building option must follow file name" << endl; - return 1; - } - - file_path = argv[opt_index - 1]; - build_opt = optarg; - - prog_insts.push_back(program_build_instance(file_path, build_opt)); - break; - } + case 'o': + if (!program_build_instance::set_bin_path(optarg)) { + cout << "Can not specify the bin path more than once." << endl; + return 1; + } + used_index[optind - 1] = 1; + break; - case 'o': - if (!program_build_instance::set_bin_path(optarg)) { - cout << "Can not specify the bin path more than once." << endl; - return 1; - } - used_index[optind-1] = 1; - break; - - case 't': - { - char *s = optarg; - if (optarg[0] == '0' && (optarg[1] == 'x' || optarg[1] == 'X')) - s += 2; - - if (s[0] < '0' || s[0] > '9') { - cout << "Invalid target option argument" << endl; - return 1; - } - - std::stringstream str(s); - str >> std::hex >> gen_pci_id; - - used_index[optind-1] = 1; - break; - } + case 't': { + char *s = optarg; + if (optarg[0] == '0' && (optarg[1] == 'x' || optarg[1] == 'X')) + s += 2; - case 's': - program_build_instance::set_str_fmt_out(true); - used_index[optind-1] = 1; - break; + if (s[0] < '0' || s[0] > '9') { + cout << "Invalid target option argument" << endl; + return 1; + } - case ':': - cout << "Miss the file option argument" << endl; - return 1; + std::stringstream str(s); + str >> std::hex >> gen_pci_id; - default: - cout << "Unknown opt" << endl; - } + used_index[optind - 1] = 1; + break; } - for (i=1; i < argc; i++) { - //cout << argv_saved[i] << endl; - if (argv_saved[i].size() && argv_saved[i][0] != '-') { - if (used_index[i]) - continue; - - string file_name = argv_saved[i]; - prog_vector::iterator result = find_if(prog_insts.begin(), prog_insts.end(), - [&](program_build_instance & prog_inst)-> bool { - bool result = false; - if (prog_inst.get_program_path() == file_name) - result = true; - - return result; - }); - - if (result == prog_insts.end()) { - prog_insts.push_back(program_build_instance(file_name.c_str(), "")); - } - } - } + case 's': + program_build_instance::set_str_fmt_out(true); + used_index[optind - 1] = 1; + break; - for (auto& inst : prog_insts) { - try { - inst.file_map_open(); - inst.build_program(); - inst.serialize_program(); - } - catch (int & err_no) { - if (err_no == FILE_NOT_FIND_ERR) { - cout << "can not open the file " << - inst.get_program_path() << endl; - } else if (err_no == FILE_MAP_ERR) { - cout << "map the file " << - inst.get_program_path() << " failed" << endl; - } else if (err_no == FILE_BUILD_FAILED) { - cout << "build the file " << - inst.get_program_path() << " failed" << endl; - } else if (err_no == FILE_SERIALIZATION_FAILED) { - cout << "Serialize the file " << - inst.get_program_path() << " failed" << endl; - } - return -1; - } + case ':': + cout << "Miss the file option argument" << endl; + return 1; + + default: + cout << "Unknown opt" << endl; + } + } + + for (i = 1; i < argc; i++) { + //cout << argv_saved[i] << endl; + if (argv_saved[i].size() && argv_saved[i][0] != '-') { + if (used_index[i]) + continue; + + string file_name = argv_saved[i]; + prog_vector::iterator result = find_if(prog_insts.begin(), prog_insts.end(), + [&](program_build_instance &prog_inst) -> bool { + bool result = false; + if (prog_inst.get_program_path() == file_name) + result = true; + + return result; + }); + + if (result == prog_insts.end()) { + prog_insts.push_back(program_build_instance(file_name.c_str(), "")); + } + } + } + + for (auto &inst : prog_insts) { + try { + inst.file_map_open(); + inst.build_program(); + inst.serialize_program(); + } catch (int &err_no) { + if (err_no == FILE_NOT_FIND_ERR) { + cout << "can not open the file " << inst.get_program_path() << endl; + } else if (err_no == FILE_MAP_ERR) { + cout << "map the file " << inst.get_program_path() << " failed" << endl; + } else if (err_no == FILE_BUILD_FAILED) { + cout << "build the file " << inst.get_program_path() << " failed" << endl; + } else if (err_no == FILE_SERIALIZATION_FAILED) { + cout << "Serialize the file " << inst.get_program_path() << " failed" << endl; + } + return -1; } + } - //for (auto& inst : prog_insts) { - // inst.dump(); - //} + //for (auto& inst : prog_insts) { + // inst.dump(); + //} - return 0; + return 0; } -- 2.7.4 _______________________________________________ Beignet mailing list Beignet@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/beignet