From: Pierre Moreau <pierre.mor...@free.fr> v2 (Karol Herbst): silence warnings about unhandled enum values --- .../clover/spirv/invocation.cpp | 598 ++++++++++++++++++ .../clover/spirv/invocation.hpp | 12 + 2 files changed, 610 insertions(+)
diff --git a/src/gallium/state_trackers/clover/spirv/invocation.cpp b/src/gallium/state_trackers/clover/spirv/invocation.cpp index b874f2f061c..62886e77495 100644 --- a/src/gallium/state_trackers/clover/spirv/invocation.cpp +++ b/src/gallium/state_trackers/clover/spirv/invocation.cpp @@ -22,10 +22,24 @@ #include "invocation.hpp" +#include <stack> +#include <tuple> +#include <unordered_map> +#include <unordered_set> +#include <vector> + #ifdef CLOVER_ALLOW_SPIRV #include <spirv-tools/libspirv.hpp> +#include <spirv-tools/linker.hpp> #endif +#include "core/error.hpp" +#include "core/platform.hpp" +#include "invocation.hpp" +#include "llvm/util.hpp" +#include "pipe/p_state.h" +#include "util/algorithm.hpp" +#include "util/functional.hpp" #include "util/u_math.h" #include "compiler/spirv/spirv.h" @@ -34,6 +48,472 @@ using namespace clover; namespace { + template<typename T> + T get(const char *source, size_t index) { + const uint32_t *word_ptr = reinterpret_cast<const uint32_t *>(source); + return static_cast<T>(word_ptr[index]); + } + + enum module::argument::type + convertStorageClass(SpvStorageClass storage_class, std::string &err) { + switch (storage_class) { + case SpvStorageClassFunction: + return module::argument::scalar; + case SpvStorageClassUniformConstant: + return module::argument::constant; + case SpvStorageClassWorkgroup: + return module::argument::local; + case SpvStorageClassCrossWorkgroup: + return module::argument::global; + default: + err += "Invalid storage type " + std::to_string(storage_class) + "\n"; + throw build_error(); + } + } + + enum module::argument::type + convertImageType(SpvId id, SpvDim dim, SpvAccessQualifier access, + std::string &err) { +#define APPEND_DIM(d) \ + switch(access) { \ + case SpvAccessQualifierReadOnly: \ + return module::argument::image##d##_rd; \ + case SpvAccessQualifierWriteOnly: \ + return module::argument::image##d##_wr; \ + default: \ + err += "Unsupported access qualifier " #d " for image " + \ + std::to_string(id); \ + throw build_error(); \ + } + + switch (dim) { + case SpvDim2D: + APPEND_DIM(2d) + case SpvDim3D: + APPEND_DIM(3d) + default: + err += "Unsupported dimension " + std::to_string(dim) + " for image " + + std::to_string(id); + throw build_error(); + } + +#undef APPEND_DIM + } + + module::section + make_text_section(const std::vector<char> &code, + enum module::section::type section_type) { + const pipe_llvm_program_header header { uint32_t(code.size()) }; + module::section text { 0, section_type, header.num_bytes, {} }; + + text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header), + reinterpret_cast<const char *>(&header) + sizeof(header)); + text.data.insert(text.data.end(), code.begin(), code.end()); + + return text; + } + + module + create_module_from_spirv(const std::vector<char> &source, + size_t pointer_byte_size, + std::string &err) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = 5u; // Skip header + + std::string kernel_name; + size_t kernel_nb = 0u; + std::vector<module::argument> args; + + module m; + + std::unordered_map<SpvId, std::string> kernels; + std::unordered_map<SpvId, module::argument> types; + std::unordered_map<SpvId, SpvId> pointer_types; + std::unordered_map<SpvId, unsigned int> constants; + std::unordered_set<SpvId> packed_structures; + std::unordered_map<SpvId, std::vector<SpvFunctionParameterAttribute>> + func_param_attr_map; + +#define GET_OPERAND(type, operand_id) get<type>(source.data(), i + operand_id) + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + switch (opcode) { + case SpvOpEntryPoint: + if (GET_OPERAND(SpvExecutionModel, 1) == SpvExecutionModelKernel) + kernels.emplace(GET_OPERAND(SpvId, 2), + source.data() + (i + 3u) * sizeof(uint32_t)); + break; + + case SpvOpDecorate: { + const auto id = GET_OPERAND(SpvId, 1); + const auto decoration = GET_OPERAND(SpvDecoration, 2); + if (decoration == SpvDecorationCPacked) + packed_structures.emplace(id); + else if (decoration == SpvDecorationFuncParamAttr) + func_param_attr_map[id].push_back(GET_OPERAND(SpvFunctionParameterAttribute, 3u)); + break; + } + + case SpvOpGroupDecorate: { + const auto group_id = GET_OPERAND(SpvId, 1); + if (packed_structures.count(group_id)) { + for (unsigned int i = 2u; i < num_operands; ++i) + packed_structures.emplace(GET_OPERAND(SpvId, i)); + } + const auto func_param_attr_iter = + func_param_attr_map.find(group_id); + if (func_param_attr_iter != func_param_attr_map.end()) { + for (unsigned int i = 2u; i < num_operands; ++i) + func_param_attr_map.emplace(GET_OPERAND(SpvId, i), + func_param_attr_iter->second); + } + break; + } + + case SpvOpConstant: + // We only care about constants that represents the size of arrays. + // If they are passed as argument, they will never be more than + // 4GB-wide, and even if they did, a clover::module::argument size + // is represented by an int. + constants[GET_OPERAND(SpvId, 2)] = GET_OPERAND(unsigned int, 3u); + break; + + case SpvOpTypeInt: // FALLTHROUGH + case SpvOpTypeFloat: { + const auto size = GET_OPERAND(uint32_t, 2) / 8u; + types[GET_OPERAND(SpvId, 1)] = { module::argument::scalar, + size, size, size, + module::argument::zero_ext }; + break; + } + + case SpvOpTypeArray: { + const auto id = GET_OPERAND(SpvId, 1); + const auto type_id = GET_OPERAND(SpvId, 2); + const auto types_iter = types.find(type_id); + if (types_iter == types.end()) + break; + + const auto constant_id = GET_OPERAND(SpvId, 3); + const auto constants_iter = constants.find(constant_id); + if (constants_iter == constants.end()) { + err += "Constant " + std::to_string(constant_id) + + " is missing\n"; + throw build_error(); + } + const auto elem_size = types_iter->second.size; + const auto elem_nbs = constants_iter->second; + const auto size = elem_size * elem_nbs; + types[id] = { module::argument::scalar, size, size, + types_iter->second.target_align, + module::argument::zero_ext }; + break; + } + + case SpvOpTypeStruct: { + const auto id = GET_OPERAND(SpvId, 1); + const bool is_packed = packed_structures.count(id); + + unsigned struct_size = 0u; + unsigned max_elem_size = 0u; + for (unsigned j = 2u; j < num_operands; ++j) { + const auto type_id = GET_OPERAND(SpvId, j); + const auto types_iter = types.find(type_id); + if (types_iter == types.end()) + break; + + const auto alignment = is_packed ? 1u + : types_iter->second.target_align; + const auto padding = (-struct_size) & (alignment - 1u); + struct_size += padding + types_iter->second.target_size; + max_elem_size = std::max(max_elem_size, + types_iter->second.target_align); + } + if (!is_packed) + struct_size += (-struct_size) & (max_elem_size - 1u); + + types[id] = { module::argument::scalar, struct_size, struct_size, + is_packed ? 1u + : max_elem_size, module::argument::zero_ext + }; + break; + } + + case SpvOpTypeVector: { + const auto id = GET_OPERAND(SpvId, 1); + const auto type_id = GET_OPERAND(SpvId, 2); + const auto types_iter = types.find(type_id); + if (types_iter == types.end()) + break; + + const auto elem_size = types_iter->second.size; + const auto elem_nbs = GET_OPERAND(uint32_t, 3); + const auto size = elem_size * elem_nbs; + types[id] = { module::argument::scalar, size, size, size, + module::argument::zero_ext }; + break; + } + + case SpvOpTypeForwardPointer: // FALLTHROUGH + case SpvOpTypePointer: { + const auto id = GET_OPERAND(SpvId, 1); + const auto storage_class = GET_OPERAND(SpvStorageClass, 2); + // Input means this is for a builtin variable, which can not be + // passed as an argument to a kernel. + if (storage_class == SpvStorageClassInput) + break; + types[id] = { convertStorageClass(storage_class, err), + sizeof(cl_mem), + static_cast<module::size_t>(pointer_byte_size), + static_cast<module::size_t>(pointer_byte_size), + module::argument::zero_ext + }; + if (opcode == SpvOpTypePointer) + pointer_types[id] = GET_OPERAND(SpvId, 3); + break; + } + + case SpvOpTypeSampler: + types[GET_OPERAND(SpvId, 1)] = { module::argument::sampler, + sizeof(cl_sampler) + }; + break; + + case SpvOpTypeImage: { + const auto id = GET_OPERAND(SpvId, 1); + const auto dim = GET_OPERAND(SpvDim, 3); + const auto access = GET_OPERAND(SpvAccessQualifier, 9); + types[id] = { convertImageType(id, dim, access, err), + sizeof(cl_mem), sizeof(cl_mem), sizeof(cl_mem), + module::argument::zero_ext + }; + break; + } + + case SpvOpTypePipe: // FALLTHROUGH + case SpvOpTypeQueue: { + err += "TypePipe and TypeQueue are valid SPIR-V 1.0 types, but are " + "not available in the currently supported OpenCL C version." + "\n"; + throw build_error(); + } + + case SpvOpFunction: { + const auto kernels_iter = kernels.find(GET_OPERAND(SpvId, 2)); + if (kernels_iter != kernels.end()) + kernel_name = kernels_iter->second; + break; + } + + case SpvOpFunctionParameter: { + if (kernel_name.empty()) + break; + + const auto type_id = GET_OPERAND(SpvId, 1); + auto arg = types.find(type_id)->second; + const auto &func_param_attr_iter = + func_param_attr_map.find(GET_OPERAND(SpvId, 2)); + if (func_param_attr_iter != func_param_attr_map.end()) { + for (auto &i : func_param_attr_iter->second) { + switch (i) { + case SpvFunctionParameterAttributeSext: + arg.ext_type = module::argument::sign_ext; + break; + case SpvFunctionParameterAttributeZext: + arg.ext_type = module::argument::zero_ext; + break; + case SpvFunctionParameterAttributeByVal: { + const SpvId ptr_type_id = + pointer_types.find(type_id)->second; + arg = types.find(ptr_type_id)->second; + break; + } + default: + break; + } + } + } + args.emplace_back(arg); + break; + } + + case SpvOpFunctionEnd: + if (kernel_name.empty()) + break; + m.syms.emplace_back(kernel_name, 0, kernel_nb, args); + ++kernel_nb; + kernel_name.clear(); + args.clear(); + break; + + default: + break; + } + + i += num_operands; + } + +#undef GET_OPERAND + + m.secs.push_back(make_text_section(source, + module::section::text_intermediate)); + return m; + } + + bool + check_capabilities(const device &dev, const std::vector<char> &source, + std::string &r_log) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = 5u; // Skip header + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + if (opcode != SpvOpCapability) + break; + + const auto capability = get<SpvCapability>(source.data(), i + 1u); + switch (capability) { + // Mandatory capabilities + case SpvCapabilityAddresses: + case SpvCapabilityFloat16Buffer: + case SpvCapabilityGroups: + case SpvCapabilityInt64: + case SpvCapabilityInt16: + case SpvCapabilityInt8: + case SpvCapabilityKernel: + case SpvCapabilityLinkage: + case SpvCapabilityVector16: + break; + // Optional capabilities + case SpvCapabilityImageBasic: + case SpvCapabilityLiteralSampler: + case SpvCapabilitySampled1D: + case SpvCapabilityImage1D: + case SpvCapabilitySampledBuffer: + case SpvCapabilityImageBuffer: + if (!dev.image_support()) { + r_log += "Capability 'ImageBasic' is not supported.\n"; + return false; + } + break; + case SpvCapabilityFloat64: + if (!dev.has_doubles()) { + r_log += "Capability 'Float64' is not supported.\n"; + return false; + } + break; + // Enabled through extensions + case SpvCapabilityFloat16: + if (!dev.has_halves()) { + r_log += "Capability 'Float16' is not supported.\n"; + return false; + } + break; + default: + r_log += "Capability '" + std::to_string(capability) + + "' is not supported.\n"; + return false; + } + + i += num_operands; + } + + return true; + } + + bool + check_extensions(const device &dev, const std::vector<char> &source, + std::string &r_log) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = 5u; // Skip header + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + if (opcode == SpvOpCapability) { + i += num_operands; + continue; + } + if (opcode != SpvOpExtension) + break; + + const char *extension = source.data() + (i + 1u) * sizeof(uint32_t); + const std::string device_extensions = dev.supported_extensions(); + const std::string platform_extensions = + dev.platform.supported_extensions(); + if (device_extensions.find(extension) == std::string::npos && + platform_extensions.find(extension) == std::string::npos) { + r_log += "Extension '" + std::string(extension) + + "' is not supported.\n"; + return false; + } + + i += num_operands; + } + + return true; + } + + bool + check_memory_model(const device &dev, const std::vector<char> &source, + std::string &r_log) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = 5u; // Skip header + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + switch (opcode) { + case SpvOpMemoryModel: + switch (get<SpvAddressingModel>(source.data(), i + 1u)) { + case SpvAddressingModelPhysical32: + return dev.address_bits() == 32; + case SpvAddressingModelPhysical64: + return dev.address_bits() == 64; + default: + unreachable("Only Physical32 and Physical64 are valid for OpenCL, and the binary was already validated"); + return false; + } + break; + default: + break; + } + + i += num_operands; + } + + return false; + } + + // Copies the input binary and convert it to the endianness of the host CPU. + std::vector<char> + spirv_to_cpu(const std::vector<char> &binary) + { + const uint32_t first_word = get<uint32_t>(binary.data(), 0u); + if (first_word == SpvMagicNumber) + return binary; + + std::vector<char> cpu_endianness_binary(binary.size()); + for (size_t i = 0; i < (binary.size() / 4u); ++i) { + const uint32_t word = get<uint32_t>(binary.data(), i); + reinterpret_cast<uint32_t *>(cpu_endianness_binary.data())[i] = + util_bswap32(word); + } + + return cpu_endianness_binary; + } + #ifdef CLOVER_ALLOW_SPIRV std::string format_validator_msg(spv_message_level_t level, const char * /* source */, @@ -96,7 +576,117 @@ clover::spirv::is_binary_spirv(const char *il, size_t length) (util_bswap32(first_word) == SpvMagicNumber); } +module +clover::spirv::process_program(const std::vector<char> &binary, + const device &dev, bool validate, + std::string &r_log) { + std::vector<char> source = spirv_to_cpu(binary); + + if (validate && !is_valid_spirv( + reinterpret_cast<const uint32_t *>(source.data()), + source.size() / 4u, dev.device_version(), + [&r_log](const char *log){ r_log += std::string(log); })) + throw build_error(); + + if (!check_capabilities(dev, source, r_log)) + throw build_error(); + if (!check_extensions(dev, source, r_log)) + throw build_error(); + if (!check_memory_model(dev, source, r_log)) + throw build_error(); + + return create_module_from_spirv(source, + dev.address_bits() == 32 ? 4u : 8u, r_log); +} + #ifdef CLOVER_ALLOW_SPIRV +module +clover::spirv::link_program(const std::vector<module> &modules, + const device &dev, const std::string &opts, + std::string &r_log) { + std::vector<std::string> options = clover::llvm::tokenize(opts); + + const bool create_library = count("-create-library", options); + erase_if(equals("-create-library"), options); + erase_if(equals("-enable-link-options"), options); + + if (!options.empty()) { + r_log += "SPIR-V linker: Ignoring the following link options: "; + for (const auto &opt : options) + r_log += "'" + opt + "' "; + } + + spvtools::LinkerOptions linker_options; + linker_options.SetCreateLibrary(create_library); + + module m; + + const auto section_type = create_library ? module::section::text_library : + module::section::text_executable; + + std::vector<const uint32_t *> sections; + sections.reserve(modules.size()); + std::vector<size_t> lengths; + lengths.reserve(modules.size()); + + auto const validator_consumer = [&r_log](spv_message_level_t level, + const char *source, + const spv_position_t &position, + const char *message) { + r_log += format_validator_msg(level, source, position, message); + }; + + for (const auto &mod : modules) { + const module::section *msec = nullptr; + try { + msec = &find([&](const module::section &sec) { + return sec.type == module::section::text_intermediate || + sec.type == module::section::text_library; + }, mod.secs); + } catch (const std::out_of_range &e) { + // We should never reach this as validate_link_devices already checked + // that a binary was present. + assert(false); + } + + const auto c_il = msec->data.data() + + sizeof(struct pipe_llvm_program_header); + const auto length = msec->size; + + sections.push_back(reinterpret_cast<const uint32_t *>(c_il)); + lengths.push_back(length / sizeof(uint32_t)); + } + + std::vector<uint32_t> linked_binary; + + const std::string opencl_version = dev.device_version(); + const spv_target_env target_env = + convert_opencl_str_to_target_env(opencl_version); + + const spvtools::MessageConsumer consumer = validator_consumer; + spvtools::Context context(target_env); + context.SetMessageConsumer(std::move(consumer)); + + if (Link(context, sections.data(), lengths.data(), sections.size(), + &linked_binary, linker_options) != SPV_SUCCESS) + throw error(CL_LINK_PROGRAM_FAILURE); + + if (!is_valid_spirv(linked_binary.data(), linked_binary.size(), + opencl_version, + [&r_log](const char *log){ r_log += std::string(log); })) + throw error(CL_LINK_PROGRAM_FAILURE); + + for (const auto &mod : modules) + m.syms.insert(m.syms.end(), mod.syms.begin(), mod.syms.end()); + + m.secs.emplace_back(make_text_section({ + reinterpret_cast<char *>(linked_binary.data()), + reinterpret_cast<char *>(linked_binary.data()) + + linked_binary.size() * sizeof(uint32_t) }, section_type)); + + return m; +} + bool clover::spirv::is_valid_spirv(const uint32_t *binary, size_t length, const std::string &opencl_version, @@ -120,6 +710,14 @@ clover::spirv::is_valid_spirv(const uint32_t *binary, size_t length, return spvTool.Validate(binary, length); } #else +module +clover::spirv::link_program(const std::vector<module> &/*modules*/, + const device &/*dev*/, const std::string &/*opts*/, + std::string &r_log) { + r_log += "SPIRV-Tools and llvm-spirv are required for linking SPIR-V binaries.\n"; + throw error(CL_LINKER_NOT_AVAILABLE); +} + bool clover::spirv::is_valid_spirv(const uint32_t * /*binary*/, size_t /*length*/, const std::string &/*opencl_version*/, diff --git a/src/gallium/state_trackers/clover/spirv/invocation.hpp b/src/gallium/state_trackers/clover/spirv/invocation.hpp index 92255a68a8e..37cd1377cb2 100644 --- a/src/gallium/state_trackers/clover/spirv/invocation.hpp +++ b/src/gallium/state_trackers/clover/spirv/invocation.hpp @@ -24,6 +24,8 @@ #define CLOVER_SPIRV_INVOCATION_HPP #include "core/context.hpp" +#include "core/module.hpp" +#include "core/program.hpp" namespace clover { namespace spirv { @@ -41,6 +43,16 @@ namespace clover { bool is_valid_spirv(const uint32_t *binary, size_t length, const std::string &opencl_version, const context::notify_action ¬ify); + + // Creates a clover module out of the given SPIR-V binary. + module process_program(const std::vector<char> &binary, + const device &dev, bool validate, + std::string &r_log); + + // Combines multiple clover modules into a single one, resolving + // link dependencies between them. + module link_program(const std::vector<module> &modules, const device &dev, + const std::string &opts, std::string &r_log); } } -- 2.21.0 _______________________________________________ mesa-dev mailing list mesa-dev@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/mesa-dev