1 // 2 // Copyright 2018 Pierre Moreau 3 // 4 // Permission is hereby granted, free of charge, to any person obtaining a 5 // copy of this software and associated documentation files (the "Software"), 6 // to deal in the Software without restriction, including without limitation 7 // the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 // and/or sell copies of the Software, and to permit persons to whom the 9 // Software is furnished to do so, subject to the following conditions: 10 // 11 // The above copyright notice and this permission notice shall be included in 12 // all copies or substantial portions of the Software. 13 // 14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 // OTHER DEALINGS IN THE SOFTWARE. 21 // 22 23 #include "invocation.hpp" 24 25 #include <limits> 26 #include <unordered_map> 27 #include <unordered_set> 28 #include <vector> 29 30 #ifdef HAVE_CLOVER_SPIRV 31 #include <spirv-tools/libspirv.hpp> 32 #include <spirv-tools/linker.hpp> 33 #endif 34 35 #include "core/error.hpp" 36 #include "core/platform.hpp" 37 #include "invocation.hpp" 38 #include "llvm/util.hpp" 39 #include "pipe/p_state.h" 40 #include "util/algorithm.hpp" 41 #include "util/functional.hpp" 42 #include "util/u_math.h" 43 44 #include "compiler/spirv/spirv.h" 45 46 #define SPIRV_HEADER_WORD_SIZE 5 47 48 using namespace clover; 49 50 using clover::detokenize; 51 52 #ifdef HAVE_CLOVER_SPIRV 53 namespace { 54 55 static const std::array<std::string,7> type_strs = { 56 "uchar", "ushort", "uint", "ulong", "half", "float", "double" 57 }; 58 59 template<typename T> 60 T get(const char *source, size_t index) { 61 const uint32_t *word_ptr = reinterpret_cast<const uint32_t *>(source); 62 return static_cast<T>(word_ptr[index]); 63 } 64 65 enum binary::argument::type 66 convert_storage_class(SpvStorageClass storage_class, std::string &err) { 67 switch (storage_class) { 68 case SpvStorageClassFunction: 69 return binary::argument::scalar; 70 case SpvStorageClassUniformConstant: 71 return binary::argument::global; 72 case SpvStorageClassWorkgroup: 73 return binary::argument::local; 74 case SpvStorageClassCrossWorkgroup: 75 return binary::argument::global; 76 default: 77 err += "Invalid storage type " + std::to_string(storage_class) + "\n"; 78 throw build_error(); 79 } 80 } 81 82 cl_kernel_arg_address_qualifier 83 convert_storage_class_to_cl(SpvStorageClass storage_class) { 84 switch (storage_class) { 85 case SpvStorageClassUniformConstant: 86 return CL_KERNEL_ARG_ADDRESS_CONSTANT; 87 case SpvStorageClassWorkgroup: 88 return CL_KERNEL_ARG_ADDRESS_LOCAL; 89 case SpvStorageClassCrossWorkgroup: 90 return CL_KERNEL_ARG_ADDRESS_GLOBAL; 91 case SpvStorageClassFunction: 92 default: 93 return CL_KERNEL_ARG_ADDRESS_PRIVATE; 94 } 95 } 96 97 enum binary::argument::type 98 convert_image_type(SpvId id, SpvDim dim, SpvAccessQualifier access, 99 std::string &err) { 100 switch (dim) { 101 case SpvDim1D: 102 case SpvDim2D: 103 case SpvDim3D: 104 case SpvDimBuffer: 105 switch (access) { 106 case SpvAccessQualifierReadOnly: 107 return binary::argument::image_rd; 108 case SpvAccessQualifierWriteOnly: 109 return binary::argument::image_wr; 110 default: 111 err += "Unknown access qualifier " + std::to_string(access) + " for image " 112 + std::to_string(id) + ".\n"; 113 throw build_error(); 114 } 115 default: 116 err += "Unknown dimension " + std::to_string(dim) + " for image " 117 + std::to_string(id) + ".\n"; 118 throw build_error(); 119 } 120 } 121 122 binary::section 123 make_text_section(const std::string &code, 124 enum binary::section::type section_type) { 125 const pipe_binary_program_header header { uint32_t(code.size()) }; 126 binary::section text { 0, section_type, header.num_bytes, {} }; 127 128 text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header), 129 reinterpret_cast<const char *>(&header) + sizeof(header)); 130 text.data.insert(text.data.end(), code.begin(), code.end()); 131 132 return text; 133 } 134 135 binary 136 create_binary_from_spirv(const std::string &source, 137 size_t pointer_byte_size, 138 std::string &err) { 139 const size_t length = source.size() / sizeof(uint32_t); 140 size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header 141 142 std::string kernel_name; 143 size_t kernel_nb = 0u; 144 std::vector<binary::argument> args; 145 std::vector<size_t> req_local_size; 146 147 binary b; 148 149 std::vector<std::string> attributes; 150 std::unordered_map<SpvId, std::vector<size_t> > req_local_sizes; 151 std::unordered_map<SpvId, std::string> kernels; 152 std::unordered_map<SpvId, binary::argument> types; 153 std::unordered_map<SpvId, SpvId> pointer_types; 154 std::unordered_map<SpvId, unsigned int> constants; 155 std::unordered_set<SpvId> packed_structures; 156 std::unordered_map<SpvId, std::vector<SpvFunctionParameterAttribute>> 157 func_param_attr_map; 158 std::unordered_map<SpvId, std::string> names; 159 std::unordered_map<SpvId, cl_kernel_arg_type_qualifier> qualifiers; 160 std::unordered_map<std::string, std::vector<std::string> > param_type_names; 161 162 while (i < length) { 163 const auto inst = &source[i * sizeof(uint32_t)]; 164 const auto desc_word = get<uint32_t>(inst, 0); 165 const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); 166 const unsigned int num_operands = desc_word >> SpvWordCountShift; 167 168 switch (opcode) { 169 case SpvOpName: { 170 names.emplace(get<SpvId>(inst, 1), 171 source.data() + (i + 2u) * sizeof(uint32_t)); 172 break; 173 } 174 175 case SpvOpString: { 176 // SPIRV-LLVM-Translator stores param type names as OpStrings 177 std::string str(source.data() + (i + 2u) * sizeof(uint32_t)); 178 if (str.find("kernel_arg_type.") != 0) 179 break; 180 181 std::string line; 182 std::istringstream istream(str.substr(16)); 183 184 std::getline(istream, line, '.'); 185 186 std::string k = line; 187 while (std::getline(istream, line, ',')) 188 param_type_names[k].push_back(line); 189 break; 190 } 191 192 case SpvOpEntryPoint: 193 if (get<SpvExecutionModel>(inst, 1) == SpvExecutionModelKernel) 194 kernels.emplace(get<SpvId>(inst, 2), 195 source.data() + (i + 3u) * sizeof(uint32_t)); 196 break; 197 198 case SpvOpExecutionMode: 199 switch (get<SpvExecutionMode>(inst, 2)) { 200 case SpvExecutionModeLocalSize: { 201 req_local_sizes[get<SpvId>(inst, 1)] = { 202 get<uint32_t>(inst, 3), 203 get<uint32_t>(inst, 4), 204 get<uint32_t>(inst, 5) 205 }; 206 std::string s = "reqd_work_group_size("; 207 s += std::to_string(get<uint32_t>(inst, 3)); 208 s += ","; 209 s += std::to_string(get<uint32_t>(inst, 4)); 210 s += ","; 211 s += std::to_string(get<uint32_t>(inst, 5)); 212 s += ")"; 213 attributes.emplace_back(s); 214 break; 215 } 216 case SpvExecutionModeLocalSizeHint: { 217 std::string s = "work_group_size_hint("; 218 s += std::to_string(get<uint32_t>(inst, 3)); 219 s += ","; 220 s += std::to_string(get<uint32_t>(inst, 4)); 221 s += ","; 222 s += std::to_string(get<uint32_t>(inst, 5)); 223 s += ")"; 224 attributes.emplace_back(s); 225 break; 226 } 227 case SpvExecutionModeVecTypeHint: { 228 uint32_t val = get<uint32_t>(inst, 3); 229 uint32_t size = val >> 16; 230 231 val &= 0xf; 232 if (val > 6) 233 val = 0; 234 std::string s = "vec_type_hint("; 235 s += type_strs[val]; 236 s += std::to_string(size); 237 s += ")"; 238 attributes.emplace_back(s); 239 break; 240 } 241 default: 242 break; 243 } 244 break; 245 246 case SpvOpDecorate: { 247 const auto id = get<SpvId>(inst, 1); 248 const auto decoration = get<SpvDecoration>(inst, 2); 249 switch (decoration) { 250 case SpvDecorationCPacked: 251 packed_structures.emplace(id); 252 break; 253 case SpvDecorationFuncParamAttr: { 254 const auto attribute = 255 get<SpvFunctionParameterAttribute>(inst, 3u); 256 func_param_attr_map[id].push_back(attribute); 257 break; 258 } 259 case SpvDecorationVolatile: 260 qualifiers[id] |= CL_KERNEL_ARG_TYPE_VOLATILE; 261 break; 262 default: 263 break; 264 } 265 break; 266 } 267 268 case SpvOpGroupDecorate: { 269 const auto group_id = get<SpvId>(inst, 1); 270 if (packed_structures.count(group_id)) { 271 for (unsigned int i = 2u; i < num_operands; ++i) 272 packed_structures.emplace(get<SpvId>(inst, i)); 273 } 274 const auto func_param_attr_iter = 275 func_param_attr_map.find(group_id); 276 if (func_param_attr_iter != func_param_attr_map.end()) { 277 for (unsigned int i = 2u; i < num_operands; ++i) { 278 auto &attrs = func_param_attr_map[get<SpvId>(inst, i)]; 279 attrs.insert(attrs.begin(), 280 func_param_attr_iter->second.begin(), 281 func_param_attr_iter->second.end()); 282 } 283 } 284 if (qualifiers.count(group_id)) { 285 for (unsigned int i = 2u; i < num_operands; ++i) 286 qualifiers[get<SpvId>(inst, i)] |= qualifiers[group_id]; 287 } 288 break; 289 } 290 291 case SpvOpConstant: 292 // We only care about constants that represent the size of arrays. 293 // If they are passed as argument, they will never be more than 294 // 4GB-wide, and even if they did, a clover::binary::argument size 295 // is represented by an int. 296 constants[get<SpvId>(inst, 2)] = get<unsigned int>(inst, 3u); 297 break; 298 299 case SpvOpTypeInt: 300 case SpvOpTypeFloat: { 301 const auto size = get<uint32_t>(inst, 2) / 8u; 302 const auto id = get<SpvId>(inst, 1); 303 types[id] = { binary::argument::scalar, size, size, size, 304 binary::argument::zero_ext }; 305 types[id].info.address_qualifier = CL_KERNEL_ARG_ADDRESS_PRIVATE; 306 break; 307 } 308 309 case SpvOpTypeArray: { 310 const auto id = get<SpvId>(inst, 1); 311 const auto type_id = get<SpvId>(inst, 2); 312 const auto types_iter = types.find(type_id); 313 if (types_iter == types.end()) 314 break; 315 316 const auto constant_id = get<SpvId>(inst, 3); 317 const auto constants_iter = constants.find(constant_id); 318 if (constants_iter == constants.end()) { 319 err += "Constant " + std::to_string(constant_id) + 320 " is missing\n"; 321 throw build_error(); 322 } 323 const auto elem_size = types_iter->second.size; 324 const auto elem_nbs = constants_iter->second; 325 const auto size = elem_size * elem_nbs; 326 types[id] = { binary::argument::scalar, size, size, 327 types_iter->second.target_align, 328 binary::argument::zero_ext }; 329 break; 330 } 331 332 case SpvOpTypeStruct: { 333 const auto id = get<SpvId>(inst, 1); 334 const bool is_packed = packed_structures.count(id); 335 336 unsigned struct_size = 0u; 337 unsigned struct_align = 1u; 338 for (unsigned j = 2u; j < num_operands; ++j) { 339 const auto type_id = get<SpvId>(inst, j); 340 const auto types_iter = types.find(type_id); 341 342 // If a type was not found, that means it is not one of the 343 // types allowed as kernel arguments. And since the binary has 344 // been validated, this means this type is not used for kernel 345 // arguments, and therefore can be ignored. 346 if (types_iter == types.end()) 347 break; 348 349 const auto alignment = is_packed ? 1u 350 : types_iter->second.target_align; 351 const auto padding = (-struct_size) & (alignment - 1u); 352 struct_size += padding + types_iter->second.target_size; 353 struct_align = std::max(struct_align, alignment); 354 } 355 struct_size += (-struct_size) & (struct_align - 1u); 356 types[id] = { binary::argument::scalar, struct_size, struct_size, 357 struct_align, binary::argument::zero_ext }; 358 break; 359 } 360 361 case SpvOpTypeVector: { 362 const auto id = get<SpvId>(inst, 1); 363 const auto type_id = get<SpvId>(inst, 2); 364 const auto types_iter = types.find(type_id); 365 366 // If a type was not found, that means it is not one of the 367 // types allowed as kernel arguments. And since the binary has 368 // been validated, this means this type is not used for kernel 369 // arguments, and therefore can be ignored. 370 if (types_iter == types.end()) 371 break; 372 373 const auto elem_size = types_iter->second.size; 374 const auto elem_nbs = get<uint32_t>(inst, 3); 375 const auto size = elem_size * (elem_nbs != 3 ? elem_nbs : 4); 376 types[id] = { binary::argument::scalar, size, size, size, 377 binary::argument::zero_ext }; 378 types[id].info.address_qualifier = CL_KERNEL_ARG_ADDRESS_PRIVATE; 379 break; 380 } 381 382 case SpvOpTypeForwardPointer: // FALLTHROUGH 383 case SpvOpTypePointer: { 384 const auto id = get<SpvId>(inst, 1); 385 const auto storage_class = get<SpvStorageClass>(inst, 2); 386 // Input means this is for a builtin variable, which can not be 387 // passed as an argument to a kernel. 388 if (storage_class == SpvStorageClassInput) 389 break; 390 391 if (opcode == SpvOpTypePointer) 392 pointer_types[id] = get<SpvId>(inst, 3); 393 394 binary::size_t alignment; 395 if (storage_class == SpvStorageClassWorkgroup) 396 alignment = opcode == SpvOpTypePointer ? types[pointer_types[id]].target_align : 0; 397 else 398 alignment = pointer_byte_size; 399 400 types[id] = { convert_storage_class(storage_class, err), 401 sizeof(cl_mem), 402 static_cast<binary::size_t>(pointer_byte_size), 403 alignment, 404 binary::argument::zero_ext }; 405 types[id].info.address_qualifier = convert_storage_class_to_cl(storage_class); 406 break; 407 } 408 409 case SpvOpTypeSampler: 410 types[get<SpvId>(inst, 1)] = { binary::argument::sampler, 411 sizeof(cl_sampler) }; 412 break; 413 414 case SpvOpTypeImage: { 415 const auto id = get<SpvId>(inst, 1); 416 const auto dim = get<SpvDim>(inst, 3); 417 const auto access = get<SpvAccessQualifier>(inst, 9); 418 types[id] = { convert_image_type(id, dim, access, err), 419 sizeof(cl_mem), sizeof(cl_mem), sizeof(cl_mem), 420 binary::argument::zero_ext }; 421 break; 422 } 423 424 case SpvOpTypePipe: // FALLTHROUGH 425 case SpvOpTypeQueue: { 426 err += "TypePipe and TypeQueue are valid SPIR-V 1.0 types, but are " 427 "not available in the currently supported OpenCL C version." 428 "\n"; 429 throw build_error(); 430 } 431 432 case SpvOpFunction: { 433 auto id = get<SpvId>(inst, 2); 434 const auto kernels_iter = kernels.find(id); 435 if (kernels_iter != kernels.end()) 436 kernel_name = kernels_iter->second; 437 438 const auto req_local_size_iter = req_local_sizes.find(id); 439 if (req_local_size_iter != req_local_sizes.end()) 440 req_local_size = (*req_local_size_iter).second; 441 else 442 req_local_size = { 0, 0, 0 }; 443 444 break; 445 } 446 447 case SpvOpFunctionParameter: { 448 if (kernel_name.empty()) 449 break; 450 451 const auto id = get<SpvId>(inst, 2); 452 const auto type_id = get<SpvId>(inst, 1); 453 auto arg = types.find(type_id)->second; 454 const auto &func_param_attr_iter = 455 func_param_attr_map.find(get<SpvId>(inst, 2)); 456 if (func_param_attr_iter != func_param_attr_map.end()) { 457 for (auto &i : func_param_attr_iter->second) { 458 switch (i) { 459 case SpvFunctionParameterAttributeSext: 460 arg.ext_type = binary::argument::sign_ext; 461 break; 462 case SpvFunctionParameterAttributeZext: 463 arg.ext_type = binary::argument::zero_ext; 464 break; 465 case SpvFunctionParameterAttributeByVal: { 466 const SpvId ptr_type_id = 467 pointer_types.find(type_id)->second; 468 arg = types.find(ptr_type_id)->second; 469 break; 470 } 471 case SpvFunctionParameterAttributeNoAlias: 472 arg.info.type_qualifier |= CL_KERNEL_ARG_TYPE_RESTRICT; 473 break; 474 case SpvFunctionParameterAttributeNoWrite: 475 arg.info.type_qualifier |= CL_KERNEL_ARG_TYPE_CONST; 476 break; 477 default: 478 break; 479 } 480 } 481 } 482 483 auto name_it = names.find(id); 484 if (name_it != names.end()) 485 arg.info.arg_name = (*name_it).second; 486 487 arg.info.type_qualifier |= qualifiers[id]; 488 arg.info.address_qualifier = types[type_id].info.address_qualifier; 489 arg.info.access_qualifier = CL_KERNEL_ARG_ACCESS_NONE; 490 args.emplace_back(arg); 491 break; 492 } 493 494 case SpvOpFunctionEnd: 495 if (kernel_name.empty()) 496 break; 497 498 for (size_t i = 0; i < param_type_names[kernel_name].size(); i++) 499 args[i].info.type_name = param_type_names[kernel_name][i]; 500 501 b.syms.emplace_back(kernel_name, detokenize(attributes, " "), 502 req_local_size, 0, kernel_nb, args); 503 ++kernel_nb; 504 kernel_name.clear(); 505 args.clear(); 506 attributes.clear(); 507 break; 508 509 default: 510 break; 511 } 512 513 i += num_operands; 514 } 515 516 b.secs.push_back(make_text_section(source, 517 binary::section::text_intermediate)); 518 return b; 519 } 520 521 bool 522 check_spirv_version(const device &dev, const char *binary, 523 std::string &r_log) { 524 const auto spirv_version = get<uint32_t>(binary, 1u); 525 const auto supported_spirv_versions = clover::spirv::supported_versions(); 526 const auto compare_versions = 527 [module_version = 528 clover::spirv::to_opencl_version_encoding(spirv_version)](const cl_name_version &supported){ 529 return supported.version == module_version; 530 }; 531 532 if (std::find_if(supported_spirv_versions.cbegin(), 533 supported_spirv_versions.cend(), 534 compare_versions) != supported_spirv_versions.cend()) 535 return true; 536 537 r_log += "SPIR-V version " + 538 clover::spirv::version_to_string(spirv_version) + 539 " is not supported; supported versions:"; 540 for (const auto &version : supported_spirv_versions) { 541 r_log += " " + clover::spirv::version_to_string(version.version); 542 } 543 r_log += "\n"; 544 return false; 545 } 546 547 bool 548 check_capabilities(const device &dev, const std::string &source, 549 std::string &r_log) { 550 const size_t length = source.size() / sizeof(uint32_t); 551 size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header 552 553 while (i < length) { 554 const auto desc_word = get<uint32_t>(source.data(), i); 555 const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); 556 const unsigned int num_operands = desc_word >> SpvWordCountShift; 557 558 if (opcode != SpvOpCapability) 559 break; 560 561 const auto capability = get<SpvCapability>(source.data(), i + 1u); 562 switch (capability) { 563 // Mandatory capabilities 564 case SpvCapabilityAddresses: 565 case SpvCapabilityFloat16Buffer: 566 case SpvCapabilityGroups: 567 case SpvCapabilityInt64: 568 case SpvCapabilityInt16: 569 case SpvCapabilityInt8: 570 case SpvCapabilityKernel: 571 case SpvCapabilityLinkage: 572 case SpvCapabilityVector16: 573 break; 574 // Optional capabilities 575 case SpvCapabilityImageBasic: 576 case SpvCapabilityLiteralSampler: 577 case SpvCapabilitySampled1D: 578 case SpvCapabilityImage1D: 579 case SpvCapabilitySampledBuffer: 580 case SpvCapabilityImageBuffer: 581 if (!dev.image_support()) { 582 r_log += "Capability 'ImageBasic' is not supported.\n"; 583 return false; 584 } 585 break; 586 case SpvCapabilityFloat64: 587 if (!dev.has_doubles()) { 588 r_log += "Capability 'Float64' is not supported.\n"; 589 return false; 590 } 591 break; 592 // Enabled through extensions 593 case SpvCapabilityFloat16: 594 if (!dev.has_halves()) { 595 r_log += "Capability 'Float16' is not supported.\n"; 596 return false; 597 } 598 break; 599 case SpvCapabilityInt64Atomics: 600 if (!dev.has_int64_atomics()) { 601 r_log += "Capability 'Int64Atomics' is not supported.\n"; 602 return false; 603 } 604 break; 605 default: 606 r_log += "Capability '" + std::to_string(capability) + 607 "' is not supported.\n"; 608 return false; 609 } 610 611 i += num_operands; 612 } 613 614 return true; 615 } 616 617 bool 618 check_extensions(const device &dev, const std::string &source, 619 std::string &r_log) { 620 const size_t length = source.size() / sizeof(uint32_t); 621 size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header 622 const auto spirv_extensions = spirv::supported_extensions(); 623 624 while (i < length) { 625 const auto desc_word = get<uint32_t>(source.data(), i); 626 const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); 627 const unsigned int num_operands = desc_word >> SpvWordCountShift; 628 629 if (opcode == SpvOpCapability) { 630 i += num_operands; 631 continue; 632 } 633 if (opcode != SpvOpExtension) 634 break; 635 636 const std::string extension = source.data() + (i + 1u) * sizeof(uint32_t); 637 if (spirv_extensions.count(extension) == 0) { 638 r_log += "Extension '" + extension + "' is not supported.\n"; 639 return false; 640 } 641 642 i += num_operands; 643 } 644 645 return true; 646 } 647 648 bool 649 check_memory_model(const device &dev, const std::string &source, 650 std::string &r_log) { 651 const size_t length = source.size() / sizeof(uint32_t); 652 size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header 653 654 while (i < length) { 655 const auto desc_word = get<uint32_t>(source.data(), i); 656 const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); 657 const unsigned int num_operands = desc_word >> SpvWordCountShift; 658 659 switch (opcode) { 660 case SpvOpMemoryModel: 661 switch (get<SpvAddressingModel>(source.data(), i + 1u)) { 662 case SpvAddressingModelPhysical32: 663 return dev.address_bits() == 32; 664 case SpvAddressingModelPhysical64: 665 return dev.address_bits() == 64; 666 default: 667 unreachable("Only Physical32 and Physical64 are valid for OpenCL, and the binary was already validated"); 668 return false; 669 } 670 break; 671 default: 672 break; 673 } 674 675 i += num_operands; 676 } 677 678 return false; 679 } 680 681 // Copies the input binary and convert it to the endianness of the host CPU. 682 std::string 683 spirv_to_cpu(const std::string &binary) 684 { 685 const uint32_t first_word = get<uint32_t>(binary.data(), 0u); 686 if (first_word == SpvMagicNumber) 687 return binary; 688 689 std::vector<char> cpu_endianness_binary(binary.size()); 690 for (size_t i = 0; i < (binary.size() / 4u); ++i) { 691 const uint32_t word = get<uint32_t>(binary.data(), i); 692 reinterpret_cast<uint32_t *>(cpu_endianness_binary.data())[i] = 693 util_bswap32(word); 694 } 695 696 return std::string(cpu_endianness_binary.begin(), 697 cpu_endianness_binary.end()); 698 } 699 700 #ifdef HAVE_CLOVER_SPIRV 701 std::string 702 format_validator_msg(spv_message_level_t level, const char * /* source */, 703 const spv_position_t &position, const char *message) { 704 std::string level_str; 705 switch (level) { 706 case SPV_MSG_FATAL: 707 level_str = "Fatal"; 708 break; 709 case SPV_MSG_INTERNAL_ERROR: 710 level_str = "Internal error"; 711 break; 712 case SPV_MSG_ERROR: 713 level_str = "Error"; 714 break; 715 case SPV_MSG_WARNING: 716 level_str = "Warning"; 717 break; 718 case SPV_MSG_INFO: 719 level_str = "Info"; 720 break; 721 case SPV_MSG_DEBUG: 722 level_str = "Debug"; 723 break; 724 } 725 return "[" + level_str + "] At word No." + 726 std::to_string(position.index) + ": \"" + message + "\"\n"; 727 } 728 729 spv_target_env 730 convert_opencl_version_to_target_env(const cl_version opencl_version) { 731 // Pick 1.2 for 3.0 for now 732 if (opencl_version == CL_MAKE_VERSION(3, 0, 0)) { 733 return SPV_ENV_OPENCL_1_2; 734 } else if (opencl_version == CL_MAKE_VERSION(2, 2, 0)) { 735 return SPV_ENV_OPENCL_2_2; 736 } else if (opencl_version == CL_MAKE_VERSION(2, 1, 0)) { 737 return SPV_ENV_OPENCL_2_1; 738 } else if (opencl_version == CL_MAKE_VERSION(2, 0, 0)) { 739 return SPV_ENV_OPENCL_2_0; 740 } else if (opencl_version == CL_MAKE_VERSION(1, 2, 0) || 741 opencl_version == CL_MAKE_VERSION(1, 1, 0) || 742 opencl_version == CL_MAKE_VERSION(1, 0, 0)) { 743 // SPIR-V is only defined for OpenCL >= 1.2, however some drivers 744 // might use it with OpenCL 1.0 and 1.1. 745 return SPV_ENV_OPENCL_1_2; 746 } else { 747 throw build_error("Invalid OpenCL version"); 748 } 749 } 750 #endif 751 752 } 753 754 bool 755 clover::spirv::is_binary_spirv(const std::string &binary) 756 { 757 // A SPIR-V binary is at the very least 5 32-bit words, which represent the 758 // SPIR-V header. 759 if (binary.size() < 20u) 760 return false; 761 762 const uint32_t first_word = 763 reinterpret_cast<const uint32_t *>(binary.data())[0u]; 764 return (first_word == SpvMagicNumber) || 765 (util_bswap32(first_word) == SpvMagicNumber); 766 } 767 768 std::string 769 clover::spirv::version_to_string(uint32_t version) { 770 const uint32_t major_version = (version >> 16) & 0xff; 771 const uint32_t minor_version = (version >> 8) & 0xff; 772 return std::to_string(major_version) + '.' + 773 std::to_string(minor_version); 774 } 775 776 binary 777 clover::spirv::compile_program(const std::string &binary, 778 const device &dev, std::string &r_log, 779 bool validate) { 780 std::string source = spirv_to_cpu(binary); 781 782 if (validate && !is_valid_spirv(source, dev.device_version(), r_log)) 783 throw build_error(); 784 785 if (!check_spirv_version(dev, source.data(), r_log)) 786 throw build_error(); 787 if (!check_capabilities(dev, source, r_log)) 788 throw build_error(); 789 if (!check_extensions(dev, source, r_log)) 790 throw build_error(); 791 if (!check_memory_model(dev, source, r_log)) 792 throw build_error(); 793 794 return create_binary_from_spirv(source, 795 dev.address_bits() == 32 ? 4u : 8u, r_log); 796 } 797 798 binary 799 clover::spirv::link_program(const std::vector<binary> &binaries, 800 const device &dev, const std::string &opts, 801 std::string &r_log) { 802 std::vector<std::string> options = tokenize(opts); 803 804 bool create_library = false; 805 806 std::string ignored_options; 807 for (const std::string &option : options) { 808 if (option == "-create-library") { 809 create_library = true; 810 } else { 811 ignored_options += "'" + option + "' "; 812 } 813 } 814 if (!ignored_options.empty()) { 815 r_log += "Ignoring the following link options: " + ignored_options 816 + "\n"; 817 } 818 819 spvtools::LinkerOptions linker_options; 820 linker_options.SetCreateLibrary(create_library); 821 822 binary b; 823 824 const auto section_type = create_library ? binary::section::text_library : 825 binary::section::text_executable; 826 827 std::vector<const uint32_t *> sections; 828 sections.reserve(binaries.size()); 829 std::vector<size_t> lengths; 830 lengths.reserve(binaries.size()); 831 832 auto const validator_consumer = [&r_log](spv_message_level_t level, 833 const char *source, 834 const spv_position_t &position, 835 const char *message) { 836 r_log += format_validator_msg(level, source, position, message); 837 }; 838 839 for (const auto &bin : binaries) { 840 const auto &bsec = find([](const binary::section &sec) { 841 return sec.type == binary::section::text_intermediate || 842 sec.type == binary::section::text_library; 843 }, bin.secs); 844 845 const auto c_il = ((struct pipe_binary_program_header*)bsec.data.data())->blob; 846 const auto length = bsec.size; 847 848 if (!check_spirv_version(dev, c_il, r_log)) 849 throw error(CL_LINK_PROGRAM_FAILURE); 850 851 sections.push_back(reinterpret_cast<const uint32_t *>(c_il)); 852 lengths.push_back(length / sizeof(uint32_t)); 853 } 854 855 std::vector<uint32_t> linked_binary; 856 857 const cl_version opencl_version = dev.device_version(); 858 const spv_target_env target_env = 859 convert_opencl_version_to_target_env(opencl_version); 860 861 const spvtools::MessageConsumer consumer = validator_consumer; 862 spvtools::Context context(target_env); 863 context.SetMessageConsumer(std::move(consumer)); 864 865 if (Link(context, sections.data(), lengths.data(), sections.size(), 866 &linked_binary, linker_options) != SPV_SUCCESS) 867 throw error(CL_LINK_PROGRAM_FAILURE); 868 869 std::string final_binary{ 870 reinterpret_cast<char *>(linked_binary.data()), 871 reinterpret_cast<char *>(linked_binary.data() + 872 linked_binary.size()) }; 873 if (!is_valid_spirv(final_binary, opencl_version, r_log)) 874 throw error(CL_LINK_PROGRAM_FAILURE); 875 876 if (has_flag(llvm::debug::spirv)) 877 llvm::debug::log(".spvasm", spirv::print_module(final_binary, dev.device_version())); 878 879 for (const auto &bin : binaries) 880 b.syms.insert(b.syms.end(), bin.syms.begin(), bin.syms.end()); 881 882 b.secs.emplace_back(make_text_section(final_binary, section_type)); 883 884 return b; 885 } 886 887 bool 888 clover::spirv::is_valid_spirv(const std::string &binary, 889 const cl_version opencl_version, 890 std::string &r_log) { 891 auto const validator_consumer = 892 [&r_log](spv_message_level_t level, const char *source, 893 const spv_position_t &position, const char *message) { 894 r_log += format_validator_msg(level, source, position, message); 895 }; 896 897 const spv_target_env target_env = 898 convert_opencl_version_to_target_env(opencl_version); 899 spvtools::SpirvTools spvTool(target_env); 900 spvTool.SetMessageConsumer(validator_consumer); 901 902 spvtools::ValidatorOptions validator_options; 903 validator_options.SetUniversalLimit(spv_validator_limit_max_function_args, 904 std::numeric_limits<uint32_t>::max()); 905 906 return spvTool.Validate(reinterpret_cast<const uint32_t *>(binary.data()), 907 binary.size() / 4u, validator_options); 908 } 909 910 std::string 911 clover::spirv::print_module(const std::string &binary, 912 const cl_version opencl_version) { 913 const spv_target_env target_env = 914 convert_opencl_version_to_target_env(opencl_version); 915 spvtools::SpirvTools spvTool(target_env); 916 spv_context spvContext = spvContextCreate(target_env); 917 if (!spvContext) 918 return "Failed to create an spv_context for disassembling the binary."; 919 920 spv_text disassembly; 921 spvBinaryToText(spvContext, 922 reinterpret_cast<const uint32_t *>(binary.data()), 923 binary.size() / 4u, SPV_BINARY_TO_TEXT_OPTION_NONE, 924 &disassembly, nullptr); 925 spvContextDestroy(spvContext); 926 927 const std::string disassemblyStr = disassembly->str; 928 spvTextDestroy(disassembly); 929 930 return disassemblyStr; 931 } 932 933 std::unordered_set<std::string> 934 clover::spirv::supported_extensions() { 935 return { 936 /* this is only a hint so all devices support that */ 937 "SPV_KHR_no_integer_wrap_decoration" 938 }; 939 } 940 941 std::vector<cl_name_version> 942 clover::spirv::supported_versions() { 943 return { cl_name_version { CL_MAKE_VERSION(1u, 0u, 0u), "SPIR-V" } }; 944 } 945 946 cl_version 947 clover::spirv::to_opencl_version_encoding(uint32_t version) { 948 return CL_MAKE_VERSION((version >> 16u) & 0xff, 949 (version >> 8u) & 0xff, 0u); 950 } 951 952 uint32_t 953 clover::spirv::to_spirv_version_encoding(cl_version version) { 954 return ((CL_VERSION_MAJOR(version) & 0xff) << 16u) | 955 ((CL_VERSION_MINOR(version) & 0xff) << 8u); 956 } 957 958 #else 959 bool 960 clover::spirv::is_binary_spirv(const std::string &binary) 961 { 962 return false; 963 } 964 965 bool 966 clover::spirv::is_valid_spirv(const std::string &/*binary*/, 967 const cl_version opencl_version, 968 std::string &/*r_log*/) { 969 return false; 970 } 971 972 std::string 973 clover::spirv::version_to_string(uint32_t version) { 974 return ""; 975 } 976 977 binary 978 clover::spirv::compile_program(const std::string &binary, 979 const device &dev, std::string &r_log, 980 bool validate) { 981 r_log += "SPIR-V support in clover is not enabled.\n"; 982 throw build_error(); 983 } 984 985 binary 986 clover::spirv::link_program(const std::vector<binary> &/*binaries*/, 987 const device &/*dev*/, const std::string &/*opts*/, 988 std::string &r_log) { 989 r_log += "SPIR-V support in clover is not enabled.\n"; 990 throw error(CL_LINKER_NOT_AVAILABLE); 991 } 992 993 std::string 994 clover::spirv::print_module(const std::string &binary, 995 const cl_version opencl_version) { 996 return std::string(); 997 } 998 999 std::unordered_set<std::string> 1000 clover::spirv::supported_extensions() { 1001 return {}; 1002 } 1003 1004 std::vector<cl_name_version> 1005 clover::spirv::supported_versions() { 1006 return {}; 1007 } 1008 1009 cl_version 1010 clover::spirv::to_opencl_version_encoding(uint32_t version) { 1011 return CL_MAKE_VERSION(0u, 0u, 0u); 1012 } 1013 1014 uint32_t 1015 clover::spirv::to_spirv_version_encoding(cl_version version) { 1016 return 0u; 1017 } 1018 #endif 1019
Indexes created Sat Aug 08 00:24:54 UTC 2026