Signed-off-by: Pierre Moreau <pierre.mor...@free.fr> --- src/gallium/state_trackers/clover/Makefile.am | 10 +- src/gallium/state_trackers/clover/Makefile.sources | 4 + .../state_trackers/clover/spirv/invocation.cpp | 481 +++++++++++++++++++++ .../state_trackers/clover/spirv/invocation.hpp | 40 ++ 4 files changed, 533 insertions(+), 2 deletions(-) create mode 100644 src/gallium/state_trackers/clover/spirv/invocation.cpp create mode 100644 src/gallium/state_trackers/clover/spirv/invocation.hpp
diff --git a/src/gallium/state_trackers/clover/Makefile.am b/src/gallium/state_trackers/clover/Makefile.am index 321393536d..e29457e948 100644 --- a/src/gallium/state_trackers/clover/Makefile.am +++ b/src/gallium/state_trackers/clover/Makefile.am @@ -28,7 +28,7 @@ cl_HEADERS = \ $(top_srcdir)/include/CL/opencl.h endif -noinst_LTLIBRARIES = libclover.la libcltgsi.la libclllvm.la +noinst_LTLIBRARIES = libclover.la libcltgsi.la libclllvm.la libspirv.la libcltgsi_la_CXXFLAGS = \ -std=c++11 \ @@ -50,13 +50,19 @@ libclllvm_la_CXXFLAGS = \ libclllvm_la_SOURCES = $(LLVM_SOURCES) +libspirv_la_CXXFLAGS = \ + -std=c++11 \ + $(VISIBILITY_CXXFLAGS) + +libspirv_la_SOURCES = $(SPIRV_SOURCES) + libclover_la_CXXFLAGS = \ -std=c++11 \ $(CLOVER_STD_OVERRIDE) \ $(VISIBILITY_CXXFLAGS) libclover_la_LIBADD = \ - libcltgsi.la libclllvm.la + libcltgsi.la libclllvm.la libspirv.la libclover_la_SOURCES = $(CPP_SOURCES) diff --git a/src/gallium/state_trackers/clover/Makefile.sources b/src/gallium/state_trackers/clover/Makefile.sources index e9828b107b..f223bebcd3 100644 --- a/src/gallium/state_trackers/clover/Makefile.sources +++ b/src/gallium/state_trackers/clover/Makefile.sources @@ -66,3 +66,7 @@ LLVM_SOURCES := \ TGSI_SOURCES := \ tgsi/compiler.cpp \ tgsi/invocation.hpp + +SPIRV_SOURCES := \ + spirv/invocation.cpp \ + spirv/invocation.hpp diff --git a/src/gallium/state_trackers/clover/spirv/invocation.cpp b/src/gallium/state_trackers/clover/spirv/invocation.cpp new file mode 100644 index 0000000000..3e740eb998 --- /dev/null +++ b/src/gallium/state_trackers/clover/spirv/invocation.cpp @@ -0,0 +1,481 @@ +// +// Copyright 2017 Pierre Moreau +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include <vector> +#include <unordered_map> + +#include <spirv-tools/libspirv.hpp> + +#include "core/error.hpp" +#include "invocation.hpp" +#include "llvm/util.hpp" +#include "spirv/spirv_linker.h" +#include "spirv/spirv_utils.h" +#include "util/algorithm.hpp" +#include "util/functional.hpp" +#include "util/u_debug.h" + +#include "spirv.hpp11" + +using namespace clover; + +namespace { + + template<typename T> + T get(const std::vector<char>& source, size_t index) { + if (index * sizeof(spirv_word) + 3u > source.size()) + return static_cast<T>(0); + return static_cast<T>(spirv_get_word(source.data(), index)); + } + + enum module::argument::type + convertStorageClass(spv::StorageClass storage_class) { + switch (storage_class) { + case spv::StorageClass::UniformConstant: + return module::argument::constant; + case spv::StorageClass::Workgroup: + return module::argument::local; + case spv::StorageClass::CrossWorkgroup: + return module::argument::global; + default: + throw build_error(); + } + } + + enum module::argument::type + convertImageType(spv::Id id, spv::Dim dim, spv::AccessQualifier access, + std::string &err) { +#define APPEND_DIM(d) \ + switch(access) { \ + case spv::AccessQualifier::ReadOnly: \ + return module::argument::image##d##_rd; \ + case spv::AccessQualifier::WriteOnly: \ + return module::argument::image##d##_wr; \ + default: \ + err += "Invalid access qualifier " #d " for image " + \ + std::to_string(static_cast<int>(id)); \ + throw build_error(); \ + } + + switch (dim) { + case spv::Dim::Dim2D: + APPEND_DIM(2d) + case spv::Dim::Dim3D: + APPEND_DIM(3d) + default: + err += "Invalid dimension " + std::to_string(static_cast<int>(dim)) + + " for image " + std::to_string(static_cast<int>(id)); + throw build_error(); + } + +#undef APPEND_DIM + } + +#define GET_OPERAND(type, operand_id) get<type>(source, i + operand_id) + + void + add_kernels_data(module &m, std::string &err, + const std::vector<char> &source) { + const unsigned int length = source.size() / sizeof(spirv_word); + unsigned int i = 5u; // Skip header + + std::string kernel_name; + uint32_t kernel_nb = 0u; + std::vector<module::argument> args; + uint32_t pointer_size = 0u; + + std::unordered_map<spv::Id, std::string> kernels; + std::unordered_map<spv::Id, module::argument> types; + + while (i < length) { + const auto desc_word = get<spirv_word>(source, i); + const auto opcode = static_cast<spv::Op>(desc_word & spv::OpCodeMask); + const unsigned int num_operands = desc_word >> spv::WordCountShift; + + switch (opcode) { + case spv::Op::OpEntryPoint: + if (GET_OPERAND(spv::ExecutionModel, 1) != spv::ExecutionModel::Kernel) + break; + kernels.emplace(GET_OPERAND(spv::Id, 2), + &source[(i + 3) * sizeof(spirv_word)]); + break; + + case spv::Op::OpMemoryModel: + switch (GET_OPERAND(spv::AddressingModel, 1)) { + case spv::AddressingModel::Physical32: + pointer_size = 32u / 8u; + break; + case spv::AddressingModel::Physical64: + pointer_size = 64u / 8u; + break; + case spv::AddressingModel::Logical: + err += "Addressing model 'Logical' is not valid for OpenCL\n"; + // FALLTHROUGH + case spv::AddressingModel::Max: + throw build_error(); + } + break; + + case spv::Op::OpTypeInt: // FALLTHROUGH + case spv::Op::OpTypeFloat: + types[GET_OPERAND(spv::Id, 1)] = { module::argument::scalar, + GET_OPERAND(spirv_word, 2) / 8u }; + break; + + case spv::Op::OpTypeVector: { + const auto type_id = GET_OPERAND(spv::Id, 2); + const auto types_iter = types.find(type_id); + if (types_iter == types.end()) { + err += "Type " + std::to_string(type_id) + " is missing\n"; + throw build_error(); + } + types[GET_OPERAND(spv::Id, 1)] = { module::argument::scalar, + types_iter->second.size * GET_OPERAND(spirv_word, 3) }; + break; + } + + case spv::Op::OpTypeSampler: + types[GET_OPERAND(spv::Id, 1)] = { module::argument::sampler, + sizeof(cl_sampler) }; + break; + + case spv::Op::OpTypeImage: { + const auto id = GET_OPERAND(spv::Id, 1); + const auto dim = GET_OPERAND(spv::Dim, 3); + const auto access = GET_OPERAND(spv::AccessQualifier, 9); + types[id] = { convertImageType(id, dim, access, err), 0 }; + break; + } + + + case spv::Op::OpTypePointer: { + const auto id = GET_OPERAND(spv::Id, 1); + const auto storage_class = GET_OPERAND(spv::StorageClass, 2); + types[id] = { convertStorageClass(storage_class), pointer_size }; + break; + } + + // case spv::Op::OpTypeBool: not allowed + // case spv::Op::OpTypeArray: expressed as pointer to underlying type + // case spv::Op::OpTypeMatrix: there is no native matrix type in OpenCL + // case spv::Op::OpTypePipe: OpenCL 2.0 + // case spv::Op::OpTypeQueue: OpenCL 2.0 + + case spv::Op::OpFunction: { + const auto kernels_iter = kernels.find(GET_OPERAND(spv::Id, 2)); + if (kernels_iter != kernels.end()) + kernel_name = kernels_iter->second; + break; + } + + case spv::Op::OpFunctionParameter: { + if (kernel_name.empty()) + break; + const auto type_id = GET_OPERAND(spv::Id, 1); + const auto types_iter = types.find(type_id); + if (types_iter == types.end()) { + err += "Type " + std::to_string(type_id) + " is missing\n"; + throw build_error(); + } + args.push_back(types_iter->second); + break; + } + + case spv::Op::OpFunctionEnd: + if (kernel_name.empty()) + break; + m.syms.emplace_back(kernel_name, 0, kernel_nb++, args); + kernel_name.clear(); + args.clear(); + break; + + default: + break; + } + + i += num_operands; + } + } + + enum module::section::type + guess_binary_type(const std::vector<char> &source, std::string &err) { + const unsigned int length = source.size() / sizeof(spirv_word); + unsigned int i = 5u; // Skip header + bool has_linkage_capability = false; + bool processed_decorations = false; + + while (i < length) { + const auto desc_word = get<spirv_word>(source, i); + const auto opcode = static_cast<spv::Op>(desc_word & spv::OpCodeMask); + const unsigned int num_operands = desc_word >> spv::WordCountShift; + + switch (opcode) { + case spv::Op::OpCapability: + if (GET_OPERAND(spv::Capability, 1) == spv::Capability::Linkage) + has_linkage_capability = true; + break; + + case spv::Op::OpDecorate: // FALLTHROUGH + case spv::Op::OpMemberDecorate: { + processed_decorations = true; + const auto deco_index = (opcode == spv::Op::OpDecorate) ? 2u : 3u; + const auto decoration = GET_OPERAND(spv::Decoration, deco_index); + if (decoration != spv::Decoration::LinkageAttributes) + break; + const auto linkage_type = GET_OPERAND(spv::LinkageType, + num_operands - 1); + if (linkage_type == spv::LinkageType::Import) + return module::section::text_intermediate; + break; + } + + case spv::Op::OpGroupDecorate: + case spv::Op::OpGroupMemberDecorate: + case spv::Op::OpDecorationGroup: + processed_decorations = true; + break; + + default: + // OpCapability are the first instructions following the header, so + // when reaching the first non-OpCapability, if we have not seen + // any Capability::Linkage, the module is an executable (no missing + // symbols, and not exporting any symbols so not a library). + if (!has_linkage_capability) + return module::section::text_executable; + + // All decorations are grouped together, so if we are reaching a + // non-decoration instruction but have seen some before, then we + // have processed all decoration instructions. And if we are still + // here, that means that only export type linkage were found. + if (processed_decorations) + return module::section::text_library; + break; + } + + i += num_operands; + } + + return module::section::text_intermediate; + } + + std::string + version_to_string(unsigned version) { + return std::to_string((version >> 16u) & 0xff) + "." + + std::to_string((version >> 8u) & 0xff); + } + + void + check_spirv_version(const char *binary, std::string &r_log) { + const auto binary_version = spirv_get_word(binary, 1u); + if (binary_version <= spv::Version) + return; + + r_log += "SPIR-V version " + version_to_string(binary_version) + + " is not supported; supported versions <= " + + version_to_string(spv::Version); + _debug_printf("%s\n", r_log.c_str()); + throw build_error(); + } + + std::string + format_validator_msg(spv_message_level_t level, + const spv_position_t& position, const char* message) { + auto const level_to_string = [](spv_message_level_t level){ +#define LVL2STR(lvl) case SPV_MSG_##lvl: return std::string(#lvl) + switch (level) { + LVL2STR(FATAL); + LVL2STR(INTERNAL_ERROR); + LVL2STR(ERROR); + LVL2STR(WARNINING); + LVL2STR(INFO); + LVL2STR(DEBUG); + } +#undef LVL2STR + return std::string(); + }; + return "[" + level_to_string(level) + "] At word No." + + std::to_string(position.index) + ": \"" + message + "\"\n"; + } +} + +module +clover::spirv::process_program(const void *il, const size_t length, + bool guess_section_type, std::string &r_log) { + auto c_il = reinterpret_cast<const char*>(il); + char *cpu_endianness_binary = spirv_spirv_to_cpu(c_il, length); + + auto const validator_consumer = [&r_log](spv_message_level_t level, + const char* /* source */, + const spv_position_t& position, + const char* message) { + r_log += format_validator_msg(level, position, message); + }; + spvtools::SpirvTools spvTool(SPV_ENV_OPENCL_2_1); + spvTool.SetMessageConsumer(validator_consumer); + if (!spvTool.Validate(reinterpret_cast<const uint32_t*>(cpu_endianness_binary), length / 4u)) + throw build_error(); + + check_spirv_version(cpu_endianness_binary, r_log); + + std::vector<char> source(cpu_endianness_binary, cpu_endianness_binary + length); + + // It is our responsability to free the converted result. + free(cpu_endianness_binary); + + module m; + add_kernels_data(m, r_log, source); + auto section_type = module::section::text_intermediate; + if (guess_section_type) + section_type = guess_binary_type(source, r_log); + m.secs.push_back(module::section(0, section_type, source.size(), source)); + + return m; +} + +module +clover::spirv::link_program(const std::vector<module> &modules, + const std::string &opts, std::string &r_log) { + std::vector<std::string> options = clover::llvm::tokenize(opts); + + const bool create_library = count("-create-library", options); + erase_if(equals("-create-library"), options); + + const bool enable_link = count("-enable-link-options", options); + erase_if(equals("-enable-link-options"), options); + if (enable_link && !create_library) { + r_log += "SPIR-V linker: '-enable-link-options' can't be used without '-create-library'\n"; + throw invalid_build_options_error(); + } + + const auto get_opt = [&options,enable_link,&r_log](const std::string &name){ + bool var = count(name, options); + erase_if(equals(name), options); + if (var && !enable_link) { + r_log += "SPIR-V linker: '" + name + "' can't be used without '-enable-link-options'\n"; + throw invalid_build_options_error(); + } + return var; + }; + + auto denorms_are_zero = get_opt("-cl-no-signed-zeroes"); + auto no_signed_zeroes = get_opt("-cl-denorms_are_zero"); + auto unsafe_math = get_opt("-cl-unsafe-math-optimizations"); + auto finite_math = get_opt("-cl-finite-math-only"); + auto fast_relaxed = get_opt("-cl-fast-relaxed-math"); + + if (!options.empty()) { + r_log += "SPIR-V linker: Invalid linker options: "; + for (const auto &opt : options) + r_log += "'" + opt + "' "; + throw invalid_build_options_error(); + } + + unsafe_math |= fast_relaxed; + finite_math |= fast_relaxed; + no_signed_zeroes |= unsafe_math; + bool enable_mad = unsafe_math; + + module m; + + const auto section_type = create_library ? module::section::text_library : + module::section::text_executable; + + std::vector<const char *> sections; + sections.reserve(modules.size()); + std::vector<unsigned> lengths; + lengths.reserve(modules.size()); + + auto const validator_consumer = [&r_log](spv_message_level_t level, + const char* /* source */, + const spv_position_t& position, + const char* message) { + r_log += format_validator_msg(level, position, message); + }; + spvtools::SpirvTools spvTool(SPV_ENV_OPENCL_2_1); + spvTool.SetMessageConsumer(validator_consumer); + + auto found_binary = false; + for (const auto &mod : modules) { + const module::section *msec = nullptr; + try { + msec = &find(type_equals(module::section::text_library), mod.secs); + } catch (const std::out_of_range &e) { + try { + msec = &find(type_equals(module::section::text_intermediate), + mod.secs); + } catch (const std::out_of_range &e) { + } + } + found_binary |= msec != nullptr; + + // OpenCL 1.2 Specification, Section 5.6.3: + // > * All programs specified by input_programs contain a compiled binary + // > or library for the device. In this case, a link is performed to + // > generate a program executable for this device. + // > * None of the programs contain a compiled binary or library for that + // > device. In this case, no link is performed and there will be no + // > program executable generated for this device. + // > * All other cases will return a CL_INVALID_OPERATION error. + if (found_binary != (msec != nullptr)) { + r_log += "SPIR-V linker: Some programs have binaries/libraries and some have nothing\n"; + throw error(CL_INVALID_OPERATION); + } + + const auto c_il = msec->data.data(); + const auto length = msec->data.size(); + + if (!spvTool.Validate(reinterpret_cast<const uint32_t*>(c_il), length / 4u)) + throw build_error(); + + check_spirv_version(c_il, r_log); + + sections.push_back(c_il); + lengths.push_back(length / sizeof(spirv_word)); + } + + if (!found_binary) + return m; + + unsigned final_size; + char *error_msg; + unsigned error_msg_length; + auto final_data = spirv_link_binaries(sections.data(), lengths.data(), + modules.size(), create_library, + &final_size, &error_msg, + &error_msg_length); + if (!final_data) { + r_log += error_msg; + free(error_msg); + throw build_error(); + } + +// if (!spvTool.Validate(reinterpret_cast<const uint32_t*>(final_data), final_size)) +// throw build_error(); + + for (const auto &mod : modules) + m.syms.insert(m.syms.end(), mod.syms.begin(), mod.syms.end()); + + m.secs.emplace_back(module::section{ 0, section_type, final_size, + { final_data, final_data + final_size * sizeof(spirv_word) } }); + + return m; +} diff --git a/src/gallium/state_trackers/clover/spirv/invocation.hpp b/src/gallium/state_trackers/clover/spirv/invocation.hpp new file mode 100644 index 0000000000..06e8f5bda8 --- /dev/null +++ b/src/gallium/state_trackers/clover/spirv/invocation.hpp @@ -0,0 +1,40 @@ +// +// Copyright 2017 Pierre Moreau +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_SPIRV_INVOCATION_HPP +#define CLOVER_SPIRV_INVOCATION_HPP + +#include "core/module.hpp" + +namespace clover { + namespace spirv { + module process_program(const void *il, + const size_t length, + bool guess_section_type, + std::string &r_log); + + module link_program(const std::vector<module> &modules, + const std::string &opts, std::string &r_log); + } +} + +#endif -- 2.12.2 _______________________________________________ mesa-dev mailing list mesa-dev@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/mesa-dev