1 //===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/MC/MCWin64EH.h" 10 #include "llvm/ADT/Twine.h" 11 #include "llvm/MC/MCContext.h" 12 #include "llvm/MC/MCExpr.h" 13 #include "llvm/MC/MCObjectFileInfo.h" 14 #include "llvm/MC/MCObjectStreamer.h" 15 #include "llvm/MC/MCSectionCOFF.h" 16 #include "llvm/MC/MCStreamer.h" 17 #include "llvm/MC/MCSymbol.h" 18 #include "llvm/Support/Win64EH.h" 19 20 using namespace llvm; 21 22 // NOTE: All relocations generated here are 4-byte image-relative. 23 24 static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) { 25 uint8_t Count = 0; 26 for (const auto &I : Insns) { 27 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 28 default: 29 llvm_unreachable("Unsupported unwind code"); 30 case Win64EH::UOP_PushNonVol: 31 case Win64EH::UOP_AllocSmall: 32 case Win64EH::UOP_SetFPReg: 33 case Win64EH::UOP_PushMachFrame: 34 Count += 1; 35 break; 36 case Win64EH::UOP_SaveNonVol: 37 case Win64EH::UOP_SaveXMM128: 38 Count += 2; 39 break; 40 case Win64EH::UOP_SaveNonVolBig: 41 case Win64EH::UOP_SaveXMM128Big: 42 Count += 3; 43 break; 44 case Win64EH::UOP_AllocLarge: 45 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2; 46 break; 47 } 48 } 49 return Count; 50 } 51 52 static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 53 const MCSymbol *RHS) { 54 MCContext &Context = Streamer.getContext(); 55 const MCExpr *Diff = 56 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 57 MCSymbolRefExpr::create(RHS, Context), Context); 58 Streamer.emitValue(Diff, 1); 59 } 60 61 static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin, 62 WinEH::Instruction &inst) { 63 uint8_t b2; 64 uint16_t w; 65 b2 = (inst.Operation & 0x0F); 66 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 67 default: 68 llvm_unreachable("Unsupported unwind code"); 69 case Win64EH::UOP_PushNonVol: 70 EmitAbsDifference(streamer, inst.Label, begin); 71 b2 |= (inst.Register & 0x0F) << 4; 72 streamer.emitInt8(b2); 73 break; 74 case Win64EH::UOP_AllocLarge: 75 EmitAbsDifference(streamer, inst.Label, begin); 76 if (inst.Offset > 512 * 1024 - 8) { 77 b2 |= 0x10; 78 streamer.emitInt8(b2); 79 w = inst.Offset & 0xFFF8; 80 streamer.emitInt16(w); 81 w = inst.Offset >> 16; 82 } else { 83 streamer.emitInt8(b2); 84 w = inst.Offset >> 3; 85 } 86 streamer.emitInt16(w); 87 break; 88 case Win64EH::UOP_AllocSmall: 89 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4; 90 EmitAbsDifference(streamer, inst.Label, begin); 91 streamer.emitInt8(b2); 92 break; 93 case Win64EH::UOP_SetFPReg: 94 EmitAbsDifference(streamer, inst.Label, begin); 95 streamer.emitInt8(b2); 96 break; 97 case Win64EH::UOP_SaveNonVol: 98 case Win64EH::UOP_SaveXMM128: 99 b2 |= (inst.Register & 0x0F) << 4; 100 EmitAbsDifference(streamer, inst.Label, begin); 101 streamer.emitInt8(b2); 102 w = inst.Offset >> 3; 103 if (inst.Operation == Win64EH::UOP_SaveXMM128) 104 w >>= 1; 105 streamer.emitInt16(w); 106 break; 107 case Win64EH::UOP_SaveNonVolBig: 108 case Win64EH::UOP_SaveXMM128Big: 109 b2 |= (inst.Register & 0x0F) << 4; 110 EmitAbsDifference(streamer, inst.Label, begin); 111 streamer.emitInt8(b2); 112 if (inst.Operation == Win64EH::UOP_SaveXMM128Big) 113 w = inst.Offset & 0xFFF0; 114 else 115 w = inst.Offset & 0xFFF8; 116 streamer.emitInt16(w); 117 w = inst.Offset >> 16; 118 streamer.emitInt16(w); 119 break; 120 case Win64EH::UOP_PushMachFrame: 121 if (inst.Offset == 1) 122 b2 |= 0x10; 123 EmitAbsDifference(streamer, inst.Label, begin); 124 streamer.emitInt8(b2); 125 break; 126 } 127 } 128 129 static void EmitSymbolRefWithOfs(MCStreamer &streamer, 130 const MCSymbol *Base, 131 const MCSymbol *Other) { 132 MCContext &Context = streamer.getContext(); 133 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Base, Context); 134 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Other, Context); 135 const MCExpr *Ofs = MCBinaryExpr::createSub(OtherRef, BaseRef, Context); 136 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base, 137 MCSymbolRefExpr::VK_COFF_IMGREL32, 138 Context); 139 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, Ofs, Context), 4); 140 } 141 142 static void EmitRuntimeFunction(MCStreamer &streamer, 143 const WinEH::FrameInfo *info) { 144 MCContext &context = streamer.getContext(); 145 146 streamer.emitValueToAlignment(4); 147 EmitSymbolRefWithOfs(streamer, info->Function, info->Begin); 148 EmitSymbolRefWithOfs(streamer, info->Function, info->End); 149 streamer.emitValue(MCSymbolRefExpr::create(info->Symbol, 150 MCSymbolRefExpr::VK_COFF_IMGREL32, 151 context), 4); 152 } 153 154 static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { 155 // If this UNWIND_INFO already has a symbol, it's already been emitted. 156 if (info->Symbol) 157 return; 158 159 MCContext &context = streamer.getContext(); 160 MCSymbol *Label = context.createTempSymbol(); 161 162 streamer.emitValueToAlignment(4); 163 streamer.emitLabel(Label); 164 info->Symbol = Label; 165 166 // Upper 3 bits are the version number (currently 1). 167 uint8_t flags = 0x01; 168 if (info->ChainedParent) 169 flags |= Win64EH::UNW_ChainInfo << 3; 170 else { 171 if (info->HandlesUnwind) 172 flags |= Win64EH::UNW_TerminateHandler << 3; 173 if (info->HandlesExceptions) 174 flags |= Win64EH::UNW_ExceptionHandler << 3; 175 } 176 streamer.emitInt8(flags); 177 178 if (info->PrologEnd) 179 EmitAbsDifference(streamer, info->PrologEnd, info->Begin); 180 else 181 streamer.emitInt8(0); 182 183 uint8_t numCodes = CountOfUnwindCodes(info->Instructions); 184 streamer.emitInt8(numCodes); 185 186 uint8_t frame = 0; 187 if (info->LastFrameInst >= 0) { 188 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst]; 189 assert(frameInst.Operation == Win64EH::UOP_SetFPReg); 190 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0); 191 } 192 streamer.emitInt8(frame); 193 194 // Emit unwind instructions (in reverse order). 195 uint8_t numInst = info->Instructions.size(); 196 for (uint8_t c = 0; c < numInst; ++c) { 197 WinEH::Instruction inst = info->Instructions.back(); 198 info->Instructions.pop_back(); 199 EmitUnwindCode(streamer, info->Begin, inst); 200 } 201 202 // For alignment purposes, the instruction array will always have an even 203 // number of entries, with the final entry potentially unused (in which case 204 // the array will be one longer than indicated by the count of unwind codes 205 // field). 206 if (numCodes & 1) { 207 streamer.emitInt16(0); 208 } 209 210 if (flags & (Win64EH::UNW_ChainInfo << 3)) 211 EmitRuntimeFunction(streamer, info->ChainedParent); 212 else if (flags & 213 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3)) 214 streamer.emitValue(MCSymbolRefExpr::create(info->ExceptionHandler, 215 MCSymbolRefExpr::VK_COFF_IMGREL32, 216 context), 4); 217 else if (numCodes == 0) { 218 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not 219 // a chained unwind info, if there is no handler, and if there are fewer 220 // than 2 slots used in the unwind code array, we have to pad to 8 bytes. 221 streamer.emitInt32(0); 222 } 223 } 224 225 void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const { 226 // Emit the unwind info structs first. 227 for (const auto &CFI : Streamer.getWinFrameInfos()) { 228 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 229 Streamer.SwitchSection(XData); 230 ::EmitUnwindInfo(Streamer, CFI.get()); 231 } 232 233 // Now emit RUNTIME_FUNCTION entries. 234 for (const auto &CFI : Streamer.getWinFrameInfos()) { 235 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 236 Streamer.SwitchSection(PData); 237 EmitRuntimeFunction(Streamer, CFI.get()); 238 } 239 } 240 241 void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 242 WinEH::FrameInfo *info, 243 bool HandlerData) const { 244 // Switch sections (the static function above is meant to be called from 245 // here and from Emit(). 246 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 247 Streamer.SwitchSection(XData); 248 249 ::EmitUnwindInfo(Streamer, info); 250 } 251 252 static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 253 const MCSymbol *RHS) { 254 MCContext &Context = Streamer.getContext(); 255 const MCExpr *Diff = 256 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 257 MCSymbolRefExpr::create(RHS, Context), Context); 258 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer); 259 // It should normally be possible to calculate the length of a function 260 // at this point, but it might not be possible in the presence of certain 261 // unusual constructs, like an inline asm with an alignment directive. 262 int64_t value; 263 if (!Diff->evaluateAsAbsolute(value, OS->getAssembler())) 264 report_fatal_error("Failed to evaluate function length in SEH unwind info"); 265 return value; 266 } 267 268 static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 269 uint32_t Count = 0; 270 for (const auto &I : Insns) { 271 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 272 default: 273 llvm_unreachable("Unsupported ARM64 unwind code"); 274 case Win64EH::UOP_AllocSmall: 275 Count += 1; 276 break; 277 case Win64EH::UOP_AllocMedium: 278 Count += 2; 279 break; 280 case Win64EH::UOP_AllocLarge: 281 Count += 4; 282 break; 283 case Win64EH::UOP_SaveR19R20X: 284 Count += 1; 285 break; 286 case Win64EH::UOP_SaveFPLRX: 287 Count += 1; 288 break; 289 case Win64EH::UOP_SaveFPLR: 290 Count += 1; 291 break; 292 case Win64EH::UOP_SaveReg: 293 Count += 2; 294 break; 295 case Win64EH::UOP_SaveRegP: 296 Count += 2; 297 break; 298 case Win64EH::UOP_SaveRegPX: 299 Count += 2; 300 break; 301 case Win64EH::UOP_SaveRegX: 302 Count += 2; 303 break; 304 case Win64EH::UOP_SaveLRPair: 305 Count += 2; 306 break; 307 case Win64EH::UOP_SaveFReg: 308 Count += 2; 309 break; 310 case Win64EH::UOP_SaveFRegP: 311 Count += 2; 312 break; 313 case Win64EH::UOP_SaveFRegX: 314 Count += 2; 315 break; 316 case Win64EH::UOP_SaveFRegPX: 317 Count += 2; 318 break; 319 case Win64EH::UOP_SetFP: 320 Count += 1; 321 break; 322 case Win64EH::UOP_AddFP: 323 Count += 2; 324 break; 325 case Win64EH::UOP_Nop: 326 Count += 1; 327 break; 328 case Win64EH::UOP_End: 329 Count += 1; 330 break; 331 case Win64EH::UOP_SaveNext: 332 Count += 1; 333 break; 334 case Win64EH::UOP_TrapFrame: 335 Count += 1; 336 break; 337 case Win64EH::UOP_PushMachFrame: 338 Count += 1; 339 break; 340 case Win64EH::UOP_Context: 341 Count += 1; 342 break; 343 case Win64EH::UOP_ClearUnwoundToCall: 344 Count += 1; 345 break; 346 } 347 } 348 return Count; 349 } 350 351 // Unwind opcode encodings and restrictions are documented at 352 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 353 static void ARM64EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin, 354 WinEH::Instruction &inst) { 355 uint8_t b, reg; 356 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 357 default: 358 llvm_unreachable("Unsupported ARM64 unwind code"); 359 case Win64EH::UOP_AllocSmall: 360 b = (inst.Offset >> 4) & 0x1F; 361 streamer.emitInt8(b); 362 break; 363 case Win64EH::UOP_AllocMedium: { 364 uint16_t hw = (inst.Offset >> 4) & 0x7FF; 365 b = 0xC0; 366 b |= (hw >> 8); 367 streamer.emitInt8(b); 368 b = hw & 0xFF; 369 streamer.emitInt8(b); 370 break; 371 } 372 case Win64EH::UOP_AllocLarge: { 373 uint32_t w; 374 b = 0xE0; 375 streamer.emitInt8(b); 376 w = inst.Offset >> 4; 377 b = (w & 0x00FF0000) >> 16; 378 streamer.emitInt8(b); 379 b = (w & 0x0000FF00) >> 8; 380 streamer.emitInt8(b); 381 b = w & 0x000000FF; 382 streamer.emitInt8(b); 383 break; 384 } 385 case Win64EH::UOP_SetFP: 386 b = 0xE1; 387 streamer.emitInt8(b); 388 break; 389 case Win64EH::UOP_AddFP: 390 b = 0xE2; 391 streamer.emitInt8(b); 392 b = (inst.Offset >> 3); 393 streamer.emitInt8(b); 394 break; 395 case Win64EH::UOP_Nop: 396 b = 0xE3; 397 streamer.emitInt8(b); 398 break; 399 case Win64EH::UOP_SaveR19R20X: 400 b = 0x20; 401 b |= (inst.Offset >> 3) & 0x1F; 402 streamer.emitInt8(b); 403 break; 404 case Win64EH::UOP_SaveFPLRX: 405 b = 0x80; 406 b |= ((inst.Offset - 1) >> 3) & 0x3F; 407 streamer.emitInt8(b); 408 break; 409 case Win64EH::UOP_SaveFPLR: 410 b = 0x40; 411 b |= (inst.Offset >> 3) & 0x3F; 412 streamer.emitInt8(b); 413 break; 414 case Win64EH::UOP_SaveReg: 415 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 416 reg = inst.Register - 19; 417 b = 0xD0 | ((reg & 0xC) >> 2); 418 streamer.emitInt8(b); 419 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 420 streamer.emitInt8(b); 421 break; 422 case Win64EH::UOP_SaveRegX: 423 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 424 reg = inst.Register - 19; 425 b = 0xD4 | ((reg & 0x8) >> 3); 426 streamer.emitInt8(b); 427 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 428 streamer.emitInt8(b); 429 break; 430 case Win64EH::UOP_SaveRegP: 431 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 432 reg = inst.Register - 19; 433 b = 0xC8 | ((reg & 0xC) >> 2); 434 streamer.emitInt8(b); 435 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 436 streamer.emitInt8(b); 437 break; 438 case Win64EH::UOP_SaveRegPX: 439 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 440 reg = inst.Register - 19; 441 b = 0xCC | ((reg & 0xC) >> 2); 442 streamer.emitInt8(b); 443 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 444 streamer.emitInt8(b); 445 break; 446 case Win64EH::UOP_SaveLRPair: 447 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 448 reg = inst.Register - 19; 449 assert((reg % 2) == 0 && "Saved reg must be 19+2*X"); 450 reg /= 2; 451 b = 0xD6 | ((reg & 0x7) >> 2); 452 streamer.emitInt8(b); 453 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 454 streamer.emitInt8(b); 455 break; 456 case Win64EH::UOP_SaveFReg: 457 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 458 reg = inst.Register - 8; 459 b = 0xDC | ((reg & 0x4) >> 2); 460 streamer.emitInt8(b); 461 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 462 streamer.emitInt8(b); 463 break; 464 case Win64EH::UOP_SaveFRegX: 465 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 466 reg = inst.Register - 8; 467 b = 0xDE; 468 streamer.emitInt8(b); 469 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 470 streamer.emitInt8(b); 471 break; 472 case Win64EH::UOP_SaveFRegP: 473 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 474 reg = inst.Register - 8; 475 b = 0xD8 | ((reg & 0x4) >> 2); 476 streamer.emitInt8(b); 477 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 478 streamer.emitInt8(b); 479 break; 480 case Win64EH::UOP_SaveFRegPX: 481 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 482 reg = inst.Register - 8; 483 b = 0xDA | ((reg & 0x4) >> 2); 484 streamer.emitInt8(b); 485 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 486 streamer.emitInt8(b); 487 break; 488 case Win64EH::UOP_End: 489 b = 0xE4; 490 streamer.emitInt8(b); 491 break; 492 case Win64EH::UOP_SaveNext: 493 b = 0xE6; 494 streamer.emitInt8(b); 495 break; 496 case Win64EH::UOP_TrapFrame: 497 b = 0xE8; 498 streamer.emitInt8(b); 499 break; 500 case Win64EH::UOP_PushMachFrame: 501 b = 0xE9; 502 streamer.emitInt8(b); 503 break; 504 case Win64EH::UOP_Context: 505 b = 0xEA; 506 streamer.emitInt8(b); 507 break; 508 case Win64EH::UOP_ClearUnwoundToCall: 509 b = 0xEC; 510 streamer.emitInt8(b); 511 break; 512 } 513 } 514 515 // Returns the epilog symbol of an epilog with the exact same unwind code 516 // sequence, if it exists. Otherwise, returns nulltpr. 517 // EpilogInstrs - Unwind codes for the current epilog. 518 // Epilogs - Epilogs that potentialy match the current epilog. 519 static MCSymbol* 520 FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs, 521 const std::vector<MCSymbol *>& Epilogs, 522 const WinEH::FrameInfo *info) { 523 for (auto *EpilogStart : Epilogs) { 524 auto InstrsIter = info->EpilogMap.find(EpilogStart); 525 assert(InstrsIter != info->EpilogMap.end() && 526 "Epilog not found in EpilogMap"); 527 const auto &Instrs = InstrsIter->second; 528 529 if (Instrs.size() != EpilogInstrs.size()) 530 continue; 531 532 bool Match = true; 533 for (unsigned i = 0; i < Instrs.size(); ++i) 534 if (Instrs[i].Operation != EpilogInstrs[i].Operation || 535 Instrs[i].Offset != EpilogInstrs[i].Offset || 536 Instrs[i].Register != EpilogInstrs[i].Register) { 537 Match = false; 538 break; 539 } 540 541 if (Match) 542 return EpilogStart; 543 } 544 return nullptr; 545 } 546 547 static void simplifyOpcodes(std::vector<WinEH::Instruction> &Instructions, 548 bool Reverse) { 549 unsigned PrevOffset = -1; 550 unsigned PrevRegister = -1; 551 552 auto VisitInstruction = [&](WinEH::Instruction &Inst) { 553 // Convert 2-byte opcodes into equivalent 1-byte ones. 554 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) { 555 Inst.Operation = Win64EH::UOP_SaveFPLR; 556 Inst.Register = -1; 557 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 558 Inst.Register == 29) { 559 Inst.Operation = Win64EH::UOP_SaveFPLRX; 560 Inst.Register = -1; 561 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 562 Inst.Register == 19 && Inst.Offset <= 248) { 563 Inst.Operation = Win64EH::UOP_SaveR19R20X; 564 Inst.Register = -1; 565 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) { 566 Inst.Operation = Win64EH::UOP_SetFP; 567 } else if (Inst.Operation == Win64EH::UOP_SaveRegP && 568 Inst.Register == PrevRegister + 2 && 569 Inst.Offset == PrevOffset + 16) { 570 Inst.Operation = Win64EH::UOP_SaveNext; 571 Inst.Register = -1; 572 Inst.Offset = 0; 573 // Intentionally not creating UOP_SaveNext for float register pairs, 574 // as current versions of Windows (up to at least 20.04) is buggy 575 // regarding SaveNext for float pairs. 576 } 577 // Update info about the previous instruction, for detecting if 578 // the next one can be made a UOP_SaveNext 579 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) { 580 PrevOffset = 0; 581 PrevRegister = 19; 582 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) { 583 PrevOffset = 0; 584 PrevRegister = Inst.Register; 585 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) { 586 PrevOffset = Inst.Offset; 587 PrevRegister = Inst.Register; 588 } else if (Inst.Operation == Win64EH::UOP_SaveNext) { 589 PrevRegister += 2; 590 PrevOffset += 16; 591 } else { 592 PrevRegister = -1; 593 PrevOffset = -1; 594 } 595 }; 596 597 // Iterate over instructions in a forward order (for prologues), 598 // backwards for epilogues (i.e. always reverse compared to how the 599 // opcodes are stored). 600 if (Reverse) { 601 for (auto It = Instructions.rbegin(); It != Instructions.rend(); It++) 602 VisitInstruction(*It); 603 } else { 604 for (WinEH::Instruction &Inst : Instructions) 605 VisitInstruction(Inst); 606 } 607 } 608 609 static int checkPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 610 int PrologCodeBytes) { 611 // Can only pack if there's one single epilog 612 if (info->EpilogMap.size() != 1) 613 return -1; 614 615 const std::vector<WinEH::Instruction> &Epilog = 616 info->EpilogMap.begin()->second; 617 618 // Can pack if the epilog is a subset of the prolog but not vice versa 619 if (Epilog.size() > info->Instructions.size()) 620 return -1; 621 622 // Check that the epilog actually is a perfect match for the end (backwrds) 623 // of the prolog. 624 for (int I = Epilog.size() - 1; I >= 0; I--) { 625 if (info->Instructions[I] != Epilog[Epilog.size() - 1 - I]) 626 return -1; 627 } 628 629 // Check that the epilog actually is at the very end of the function, 630 // otherwise it can't be packed. 631 uint32_t DistanceFromEnd = (uint32_t)GetAbsDifference( 632 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 633 if (DistanceFromEnd / 4 != Epilog.size()) 634 return -1; 635 636 int Offset = Epilog.size() == info->Instructions.size() 637 ? 0 638 : ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 639 &info->Instructions[Epilog.size()], 640 info->Instructions.size() - Epilog.size())); 641 642 // Check that the offset and prolog size fits in the first word; it's 643 // unclear whether the epilog count in the extension word can be taken 644 // as packed epilog offset. 645 if (Offset > 31 || PrologCodeBytes > 124) 646 return -1; 647 648 info->EpilogMap.clear(); 649 return Offset; 650 } 651 652 static bool tryPackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength, 653 int PackedEpilogOffset) { 654 if (PackedEpilogOffset == 0) { 655 // Fully symmetric prolog and epilog, should be ok for packed format. 656 // For CR=3, the corresponding synthesized epilog actually lacks the 657 // SetFP opcode, but unwinding should work just fine despite that 658 // (if at the SetFP opcode, the unwinder considers it as part of the 659 // function body and just unwinds the full prolog instead). 660 } else if (PackedEpilogOffset == 1) { 661 // One single case of differences between prolog and epilog is allowed: 662 // The epilog can lack a single SetFP that is the last opcode in the 663 // prolog, for the CR=3 case. 664 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP) 665 return false; 666 } else { 667 // Too much difference between prolog and epilog. 668 return false; 669 } 670 unsigned RegI = 0, RegF = 0; 671 int Predecrement = 0; 672 enum { 673 Start, 674 Start2, 675 IntRegs, 676 FloatRegs, 677 InputArgs, 678 StackAdjust, 679 FrameRecord, 680 End 681 } Location = Start; 682 bool StandaloneLR = false, FPLRPair = false; 683 int StackOffset = 0; 684 int Nops = 0; 685 // Iterate over the prolog and check that all opcodes exactly match 686 // the canonical order and form. A more lax check could verify that 687 // all saved registers are in the expected locations, but not enforce 688 // the order - that would work fine when unwinding from within 689 // functions, but not be exactly right if unwinding happens within 690 // prologs/epilogs. 691 for (const WinEH::Instruction &Inst : info->Instructions) { 692 switch (Inst.Operation) { 693 case Win64EH::UOP_End: 694 if (Location != Start) 695 return false; 696 Location = Start2; 697 break; 698 case Win64EH::UOP_SaveR19R20X: 699 if (Location != Start2) 700 return false; 701 Predecrement = Inst.Offset; 702 RegI = 2; 703 Location = IntRegs; 704 break; 705 case Win64EH::UOP_SaveRegX: 706 if (Location != Start2) 707 return false; 708 Predecrement = Inst.Offset; 709 if (Inst.Register == 19) 710 RegI += 1; 711 else if (Inst.Register == 30) 712 StandaloneLR = true; 713 else 714 return false; 715 // Odd register; can't be any further int registers. 716 Location = FloatRegs; 717 break; 718 case Win64EH::UOP_SaveRegPX: 719 // Can't have this in a canonical prologue. Either this has been 720 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported 721 // register pair. 722 // It can't be canonicalized into SaveR19R20X if the offset is 723 // larger than 248 bytes, but even with the maximum case with 724 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should 725 // fit into SaveR19R20X. 726 // The unwinding opcodes can't describe the otherwise seemingly valid 727 // case for RegI=1 CR=1, that would start with a 728 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor 729 // SaveLRPair. 730 return false; 731 case Win64EH::UOP_SaveRegP: 732 if (Location != IntRegs || Inst.Offset != 8 * RegI || 733 Inst.Register != 19 + RegI) 734 return false; 735 RegI += 2; 736 break; 737 case Win64EH::UOP_SaveReg: 738 if (Location != IntRegs || Inst.Offset != 8 * RegI) 739 return false; 740 if (Inst.Register == 19 + RegI) 741 RegI += 1; 742 else if (Inst.Register == 30) 743 StandaloneLR = true; 744 else 745 return false; 746 // Odd register; can't be any further int registers. 747 Location = FloatRegs; 748 break; 749 case Win64EH::UOP_SaveLRPair: 750 if (Location != IntRegs || Inst.Offset != 8 * RegI || 751 Inst.Register != 19 + RegI) 752 return false; 753 RegI += 1; 754 StandaloneLR = true; 755 Location = FloatRegs; 756 break; 757 case Win64EH::UOP_SaveFRegX: 758 // Packed unwind can't handle prologs that only save one single 759 // float register. 760 return false; 761 case Win64EH::UOP_SaveFReg: 762 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF || 763 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 764 return false; 765 RegF += 1; 766 Location = InputArgs; 767 break; 768 case Win64EH::UOP_SaveFRegPX: 769 if (Location != Start2 || Inst.Register != 8) 770 return false; 771 Predecrement = Inst.Offset; 772 RegF = 2; 773 Location = FloatRegs; 774 break; 775 case Win64EH::UOP_SaveFRegP: 776 if ((Location != IntRegs && Location != FloatRegs) || 777 Inst.Register != 8 + RegF || 778 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 779 return false; 780 RegF += 2; 781 Location = FloatRegs; 782 break; 783 case Win64EH::UOP_SaveNext: 784 if (Location == IntRegs) 785 RegI += 2; 786 else if (Location == FloatRegs) 787 RegF += 2; 788 else 789 return false; 790 break; 791 case Win64EH::UOP_Nop: 792 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs) 793 return false; 794 Location = InputArgs; 795 Nops++; 796 break; 797 case Win64EH::UOP_AllocSmall: 798 case Win64EH::UOP_AllocMedium: 799 if (Location != Start2 && Location != IntRegs && Location != FloatRegs && 800 Location != InputArgs && Location != StackAdjust) 801 return false; 802 // Can have either a single decrement, or a pair of decrements with 803 // 4080 and another decrement. 804 if (StackOffset == 0) 805 StackOffset = Inst.Offset; 806 else if (StackOffset != 4080) 807 return false; 808 else 809 StackOffset += Inst.Offset; 810 Location = StackAdjust; 811 break; 812 case Win64EH::UOP_SaveFPLRX: 813 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it 814 // should be followed by a FPLR instead. 815 if (Location != Start2 && Location != IntRegs && Location != FloatRegs && 816 Location != InputArgs) 817 return false; 818 StackOffset = Inst.Offset; 819 Location = FrameRecord; 820 FPLRPair = true; 821 break; 822 case Win64EH::UOP_SaveFPLR: 823 // This can only follow after a StackAdjust 824 if (Location != StackAdjust || Inst.Offset != 0) 825 return false; 826 Location = FrameRecord; 827 FPLRPair = true; 828 break; 829 case Win64EH::UOP_SetFP: 830 if (Location != FrameRecord) 831 return false; 832 Location = End; 833 break; 834 } 835 } 836 if (RegI > 10 || RegF > 8) 837 return false; 838 if (StandaloneLR && FPLRPair) 839 return false; 840 if (FPLRPair && Location != End) 841 return false; 842 if (Nops != 0 && Nops != 4) 843 return false; 844 int H = Nops == 4; 845 int IntSZ = 8 * RegI; 846 if (StandaloneLR) 847 IntSZ += 8; 848 int FpSZ = 8 * RegF; // RegF not yet decremented 849 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF; 850 if (Predecrement != SavSZ) 851 return false; 852 if (FPLRPair && StackOffset < 16) 853 return false; 854 if (StackOffset % 16) 855 return false; 856 uint32_t FrameSize = (StackOffset + SavSZ) / 16; 857 if (FrameSize > 0x1FF) 858 return false; 859 assert(RegF != 1 && "One single float reg not allowed"); 860 if (RegF > 0) 861 RegF--; // Convert from actual number of registers, to value stored 862 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 863 int Flag = 0x01; // Function segments not supported yet 864 int CR = FPLRPair ? 3 : StandaloneLR ? 1 : 0; 865 info->PackedInfo |= Flag << 0; 866 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 867 info->PackedInfo |= (RegF & 0x7) << 13; 868 info->PackedInfo |= (RegI & 0xF) << 16; 869 info->PackedInfo |= (H & 0x1) << 20; 870 info->PackedInfo |= (CR & 0x3) << 21; 871 info->PackedInfo |= (FrameSize & 0x1FF) << 23; 872 return true; 873 } 874 875 // Populate the .xdata section. The format of .xdata on ARM64 is documented at 876 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 877 static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 878 bool TryPacked = true) { 879 // If this UNWIND_INFO already has a symbol, it's already been emitted. 880 if (info->Symbol) 881 return; 882 // If there's no unwind info here (not even a terminating UOP_End), the 883 // unwind info is considered bogus and skipped. If this was done in 884 // response to an explicit .seh_handlerdata, the associated trailing 885 // handler data is left orphaned in the xdata section. 886 if (info->empty()) { 887 info->EmitAttempted = true; 888 return; 889 } 890 if (info->EmitAttempted) { 891 // If we tried to emit unwind info before (due to an explicit 892 // .seh_handlerdata directive), but skipped it (because there was no 893 // valid information to emit at the time), and it later got valid unwind 894 // opcodes, we can't emit it here, because the trailing handler data 895 // was already emitted elsewhere in the xdata section. 896 streamer.getContext().reportError( 897 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 898 " skipped due to no unwind info at the time " 899 "(.seh_handlerdata too early?), but the function later " 900 "did get unwind info that can't be emitted"); 901 return; 902 } 903 904 simplifyOpcodes(info->Instructions, false); 905 for (auto &I : info->EpilogMap) 906 simplifyOpcodes(I.second, true); 907 908 MCContext &context = streamer.getContext(); 909 MCSymbol *Label = context.createTempSymbol(); 910 911 streamer.emitValueToAlignment(4); 912 streamer.emitLabel(Label); 913 info->Symbol = Label; 914 915 int64_t RawFuncLength; 916 if (!info->FuncletOrFuncEnd) { 917 report_fatal_error("FuncletOrFuncEnd not set"); 918 } else { 919 // FIXME: GetAbsDifference tries to compute the length of the function 920 // immediately, before the whole file is emitted, but in general 921 // that's impossible: the size in bytes of certain assembler directives 922 // like .align and .fill is not known until the whole file is parsed and 923 // relaxations are applied. Currently, GetAbsDifference fails with a fatal 924 // error in that case. (We mostly don't hit this because inline assembly 925 // specifying those directives is rare, and we don't normally try to 926 // align loops on AArch64.) 927 // 928 // There are two potential approaches to delaying the computation. One, 929 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000", 930 // as long as we have some conservative estimate we could use to prove 931 // that we don't need to split the unwind data. Emitting the constant 932 // is straightforward, but there's no existing code for estimating the 933 // size of the function. 934 // 935 // The other approach would be to use a dedicated, relaxable fragment, 936 // which could grow to accommodate splitting the unwind data if 937 // necessary. This is more straightforward, since it automatically works 938 // without any new infrastructure, and it's consistent with how we handle 939 // relaxation in other contexts. But it would require some refactoring 940 // to move parts of the pdata/xdata emission into the implementation of 941 // a fragment. We could probably continue to encode the unwind codes 942 // here, but we'd have to emit the pdata, the xdata header, and the 943 // epilogue scopes later, since they depend on whether the we need to 944 // split the unwind data. 945 RawFuncLength = GetAbsDifference(streamer, info->FuncletOrFuncEnd, 946 info->Begin); 947 } 948 if (RawFuncLength > 0xFFFFF) 949 report_fatal_error("SEH unwind data splitting not yet implemented"); 950 uint32_t FuncLength = (uint32_t)RawFuncLength / 4; 951 uint32_t PrologCodeBytes = ARM64CountOfUnwindCodes(info->Instructions); 952 uint32_t TotalCodeBytes = PrologCodeBytes; 953 954 int PackedEpilogOffset = checkPackedEpilog(streamer, info, PrologCodeBytes); 955 956 if (PackedEpilogOffset >= 0 && !info->HandlesExceptions && 957 FuncLength <= 0x7ff && TryPacked) { 958 // Matching prolog/epilog and no exception handlers; check if the 959 // prolog matches the patterns that can be described by the packed 960 // format. 961 962 // info->Symbol was already set even if we didn't actually write any 963 // unwind info there. Keep using that as indicator that this unwind 964 // info has been generated already. 965 966 if (tryPackedUnwind(info, FuncLength, PackedEpilogOffset)) 967 return; 968 } 969 970 // Process epilogs. 971 MapVector<MCSymbol *, uint32_t> EpilogInfo; 972 // Epilogs processed so far. 973 std::vector<MCSymbol *> AddedEpilogs; 974 975 for (auto &I : info->EpilogMap) { 976 MCSymbol *EpilogStart = I.first; 977 auto &EpilogInstrs = I.second; 978 uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs); 979 980 MCSymbol* MatchingEpilog = 981 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 982 if (MatchingEpilog) { 983 assert(EpilogInfo.find(MatchingEpilog) != EpilogInfo.end() && 984 "Duplicate epilog not found"); 985 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 986 // Clear the unwind codes in the EpilogMap, so that they don't get output 987 // in the logic below. 988 EpilogInstrs.clear(); 989 } else { 990 EpilogInfo[EpilogStart] = TotalCodeBytes; 991 TotalCodeBytes += CodeBytes; 992 AddedEpilogs.push_back(EpilogStart); 993 } 994 } 995 996 // Code Words, Epilog count, E, X, Vers, Function Length 997 uint32_t row1 = 0x0; 998 uint32_t CodeWords = TotalCodeBytes / 4; 999 uint32_t CodeWordsMod = TotalCodeBytes % 4; 1000 if (CodeWordsMod) 1001 CodeWords++; 1002 uint32_t EpilogCount = 1003 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size(); 1004 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124; 1005 if (!ExtensionWord) { 1006 row1 |= (EpilogCount & 0x1F) << 22; 1007 row1 |= (CodeWords & 0x1F) << 27; 1008 } 1009 if (info->HandlesExceptions) // X 1010 row1 |= 1 << 20; 1011 if (PackedEpilogOffset >= 0) // E 1012 row1 |= 1 << 21; 1013 row1 |= FuncLength & 0x3FFFF; 1014 streamer.emitInt32(row1); 1015 1016 // Extended Code Words, Extended Epilog Count 1017 if (ExtensionWord) { 1018 // FIXME: We should be able to split unwind info into multiple sections. 1019 // FIXME: We should share epilog codes across epilogs, where possible, 1020 // which would make this issue show up less frequently. 1021 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 1022 report_fatal_error("SEH unwind data splitting not yet implemented"); 1023 uint32_t row2 = 0x0; 1024 row2 |= (CodeWords & 0xFF) << 16; 1025 row2 |= (EpilogCount & 0xFFFF); 1026 streamer.emitInt32(row2); 1027 } 1028 1029 // Epilog Start Index, Epilog Start Offset 1030 for (auto &I : EpilogInfo) { 1031 MCSymbol *EpilogStart = I.first; 1032 uint32_t EpilogIndex = I.second; 1033 uint32_t EpilogOffset = 1034 (uint32_t)GetAbsDifference(streamer, EpilogStart, info->Begin); 1035 if (EpilogOffset) 1036 EpilogOffset /= 4; 1037 uint32_t row3 = EpilogOffset; 1038 row3 |= (EpilogIndex & 0x3FF) << 22; 1039 streamer.emitInt32(row3); 1040 } 1041 1042 // Emit prolog unwind instructions (in reverse order). 1043 uint8_t numInst = info->Instructions.size(); 1044 for (uint8_t c = 0; c < numInst; ++c) { 1045 WinEH::Instruction inst = info->Instructions.back(); 1046 info->Instructions.pop_back(); 1047 ARM64EmitUnwindCode(streamer, info->Begin, inst); 1048 } 1049 1050 // Emit epilog unwind instructions 1051 for (auto &I : info->EpilogMap) { 1052 auto &EpilogInstrs = I.second; 1053 for (uint32_t i = 0; i < EpilogInstrs.size(); i++) { 1054 WinEH::Instruction inst = EpilogInstrs[i]; 1055 ARM64EmitUnwindCode(streamer, info->Begin, inst); 1056 } 1057 } 1058 1059 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 1060 assert(BytesMod >= 0); 1061 for (int i = 0; i < BytesMod; i++) 1062 streamer.emitInt8(0xE3); 1063 1064 if (info->HandlesExceptions) 1065 streamer.emitValue( 1066 MCSymbolRefExpr::create(info->ExceptionHandler, 1067 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 1068 4); 1069 } 1070 1071 static void ARM64EmitRuntimeFunction(MCStreamer &streamer, 1072 const WinEH::FrameInfo *info) { 1073 MCContext &context = streamer.getContext(); 1074 1075 streamer.emitValueToAlignment(4); 1076 EmitSymbolRefWithOfs(streamer, info->Function, info->Begin); 1077 if (info->PackedInfo) 1078 streamer.emitInt32(info->PackedInfo); 1079 else 1080 streamer.emitValue( 1081 MCSymbolRefExpr::create(info->Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32, 1082 context), 1083 4); 1084 } 1085 1086 void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const { 1087 // Emit the unwind info structs first. 1088 for (const auto &CFI : Streamer.getWinFrameInfos()) { 1089 WinEH::FrameInfo *Info = CFI.get(); 1090 if (Info->empty()) 1091 continue; 1092 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 1093 Streamer.SwitchSection(XData); 1094 ARM64EmitUnwindInfo(Streamer, Info); 1095 } 1096 1097 // Now emit RUNTIME_FUNCTION entries. 1098 for (const auto &CFI : Streamer.getWinFrameInfos()) { 1099 WinEH::FrameInfo *Info = CFI.get(); 1100 // ARM64EmitUnwindInfo above clears the info struct, so we can't check 1101 // empty here. But if a Symbol is set, we should create the corresponding 1102 // pdata entry. 1103 if (!Info->Symbol) 1104 continue; 1105 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 1106 Streamer.SwitchSection(PData); 1107 ARM64EmitRuntimeFunction(Streamer, Info); 1108 } 1109 } 1110 1111 void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 1112 WinEH::FrameInfo *info, 1113 bool HandlerData) const { 1114 // Called if there's an .seh_handlerdata directive before the end of the 1115 // function. This forces writing the xdata record already here - and 1116 // in this case, the function isn't actually ended already, but the xdata 1117 // record needs to know the function length. In these cases, if the funclet 1118 // end hasn't been marked yet, the xdata function length won't cover the 1119 // whole function, only up to this point. 1120 if (!info->FuncletOrFuncEnd) { 1121 Streamer.SwitchSection(info->TextSection); 1122 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 1123 } 1124 // Switch sections (the static function above is meant to be called from 1125 // here and from Emit(). 1126 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 1127 Streamer.SwitchSection(XData); 1128 ARM64EmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 1129 } 1130
Indexes created Tue Feb 24 08:35:24 UTC 2026