1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file declares the API for the instruction selector. 10 /// This class is responsible for selecting machine instructions. 11 /// It's implemented by the target. It's used by the InstructionSelect pass. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H 16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H 17 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" 21 #include "llvm/CodeGen/GlobalISel/Utils.h" 22 #include "llvm/CodeGen/MachineInstrBuilder.h" 23 #include "llvm/CodeGen/MachineOperand.h" 24 #include "llvm/CodeGen/MachineRegisterInfo.h" 25 #include "llvm/CodeGen/TargetInstrInfo.h" 26 #include "llvm/CodeGen/TargetOpcodes.h" 27 #include "llvm/CodeGen/TargetRegisterInfo.h" 28 #include "llvm/IR/Constants.h" 29 #include "llvm/IR/DataLayout.h" 30 #include "llvm/Support/Debug.h" 31 #include "llvm/Support/ErrorHandling.h" 32 #include "llvm/Support/raw_ostream.h" 33 #include <cassert> 34 #include <cstddef> 35 #include <cstdint> 36 37 namespace llvm { 38 39 /// GlobalISel PatFrag Predicates 40 enum { 41 GIPFP_I64_Invalid = 0, 42 GIPFP_APInt_Invalid = 0, 43 GIPFP_APFloat_Invalid = 0, 44 GIPFP_MI_Invalid = 0, 45 }; 46 47 template <class TgtInstructionSelector, class PredicateBitset, 48 class ComplexMatcherMemFn, class CustomRendererFn> 49 bool InstructionSelector::executeMatchTable( 50 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, 51 const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> 52 &ISelInfo, 53 const int64_t *MatchTable, const TargetInstrInfo &TII, 54 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, 55 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, 56 CodeGenCoverage &CoverageInfo) const { 57 58 uint64_t CurrentIdx = 0; 59 SmallVector<uint64_t, 4> OnFailResumeAt; 60 61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc. 62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); 63 64 const uint16_t Flags = State.MIs[0]->getFlags(); 65 66 enum RejectAction { RejectAndGiveUp, RejectAndResume }; 67 auto handleReject = [&]() -> RejectAction { 68 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 69 dbgs() << CurrentIdx << ": Rejected\n"); 70 if (OnFailResumeAt.empty()) 71 return RejectAndGiveUp; 72 CurrentIdx = OnFailResumeAt.pop_back_val(); 73 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" 75 << OnFailResumeAt.size() << " try-blocks remain)\n"); 76 return RejectAndResume; 77 }; 78 79 auto propagateFlags = [=](NewMIVector &OutMIs) { 80 for (auto MIB : OutMIs) { 81 // Set the NoFPExcept flag when no original matched instruction could 82 // raise an FP exception, but the new instruction potentially might. 83 uint16_t MIBFlags = Flags; 84 if (NoFPException && MIB->mayRaiseFPException()) 85 MIBFlags |= MachineInstr::NoFPExcept; 86 MIB.setMIFlags(MIBFlags); 87 } 88 89 return true; 90 }; 91 92 while (true) { 93 assert(CurrentIdx != ~0u && "Invalid MatchTable index"); 94 int64_t MatcherOpcode = MatchTable[CurrentIdx++]; 95 switch (MatcherOpcode) { 96 case GIM_Try: { 97 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 98 dbgs() << CurrentIdx << ": Begin try-block\n"); 99 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); 100 break; 101 } 102 103 case GIM_RecordInsn: { 104 int64_t NewInsnID = MatchTable[CurrentIdx++]; 105 int64_t InsnID = MatchTable[CurrentIdx++]; 106 int64_t OpIdx = MatchTable[CurrentIdx++]; 107 108 // As an optimisation we require that MIs[0] is always the root. Refuse 109 // any attempt to modify it. 110 assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); 111 112 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 113 if (!MO.isReg()) { 114 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 115 dbgs() << CurrentIdx << ": Not a register\n"); 116 if (handleReject() == RejectAndGiveUp) 117 return false; 118 break; 119 } 120 if (Register::isPhysicalRegister(MO.getReg())) { 121 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 122 dbgs() << CurrentIdx << ": Is a physical register\n"); 123 if (handleReject() == RejectAndGiveUp) 124 return false; 125 break; 126 } 127 128 MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); 129 if ((size_t)NewInsnID < State.MIs.size()) 130 State.MIs[NewInsnID] = NewMI; 131 else { 132 assert((size_t)NewInsnID == State.MIs.size() && 133 "Expected to store MIs in order"); 134 State.MIs.push_back(NewMI); 135 } 136 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 137 dbgs() << CurrentIdx << ": MIs[" << NewInsnID 138 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx 139 << ")\n"); 140 break; 141 } 142 143 case GIM_CheckFeatures: { 144 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; 145 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 146 dbgs() << CurrentIdx 147 << ": GIM_CheckFeatures(ExpectedBitsetID=" 148 << ExpectedBitsetID << ")\n"); 149 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != 150 ISelInfo.FeatureBitsets[ExpectedBitsetID]) { 151 if (handleReject() == RejectAndGiveUp) 152 return false; 153 } 154 break; 155 } 156 157 case GIM_CheckOpcode: 158 case GIM_CheckOpcodeIsEither: { 159 int64_t InsnID = MatchTable[CurrentIdx++]; 160 int64_t Expected0 = MatchTable[CurrentIdx++]; 161 int64_t Expected1 = -1; 162 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 163 Expected1 = MatchTable[CurrentIdx++]; 164 165 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 166 unsigned Opcode = State.MIs[InsnID]->getOpcode(); 167 168 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 169 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID 170 << "], ExpectedOpcode=" << Expected0; 171 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 172 dbgs() << " || " << Expected1; 173 dbgs() << ") // Got=" << Opcode << "\n"; 174 ); 175 176 if (Opcode != Expected0 && Opcode != Expected1) { 177 if (handleReject() == RejectAndGiveUp) 178 return false; 179 } 180 break; 181 } 182 case GIM_SwitchOpcode: { 183 int64_t InsnID = MatchTable[CurrentIdx++]; 184 int64_t LowerBound = MatchTable[CurrentIdx++]; 185 int64_t UpperBound = MatchTable[CurrentIdx++]; 186 int64_t Default = MatchTable[CurrentIdx++]; 187 188 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 189 const int64_t Opcode = State.MIs[InsnID]->getOpcode(); 190 191 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { 192 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" 193 << LowerBound << ", " << UpperBound << "), Default=" << Default 194 << ", JumpTable...) // Got=" << Opcode << "\n"; 195 }); 196 if (Opcode < LowerBound || UpperBound <= Opcode) { 197 CurrentIdx = Default; 198 break; 199 } 200 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; 201 if (!CurrentIdx) { 202 CurrentIdx = Default; 203 break; 204 } 205 OnFailResumeAt.push_back(Default); 206 break; 207 } 208 209 case GIM_SwitchType: { 210 int64_t InsnID = MatchTable[CurrentIdx++]; 211 int64_t OpIdx = MatchTable[CurrentIdx++]; 212 int64_t LowerBound = MatchTable[CurrentIdx++]; 213 int64_t UpperBound = MatchTable[CurrentIdx++]; 214 int64_t Default = MatchTable[CurrentIdx++]; 215 216 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 217 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 218 219 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { 220 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID 221 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " 222 << UpperBound << "), Default=" << Default 223 << ", JumpTable...) // Got="; 224 if (!MO.isReg()) 225 dbgs() << "Not a VReg\n"; 226 else 227 dbgs() << MRI.getType(MO.getReg()) << "\n"; 228 }); 229 if (!MO.isReg()) { 230 CurrentIdx = Default; 231 break; 232 } 233 const LLT Ty = MRI.getType(MO.getReg()); 234 const auto TyI = ISelInfo.TypeIDMap.find(Ty); 235 if (TyI == ISelInfo.TypeIDMap.end()) { 236 CurrentIdx = Default; 237 break; 238 } 239 const int64_t TypeID = TyI->second; 240 if (TypeID < LowerBound || UpperBound <= TypeID) { 241 CurrentIdx = Default; 242 break; 243 } 244 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; 245 if (!CurrentIdx) { 246 CurrentIdx = Default; 247 break; 248 } 249 OnFailResumeAt.push_back(Default); 250 break; 251 } 252 253 case GIM_CheckNumOperands: { 254 int64_t InsnID = MatchTable[CurrentIdx++]; 255 int64_t Expected = MatchTable[CurrentIdx++]; 256 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 257 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" 258 << InsnID << "], Expected=" << Expected << ")\n"); 259 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 260 if (State.MIs[InsnID]->getNumOperands() != Expected) { 261 if (handleReject() == RejectAndGiveUp) 262 return false; 263 } 264 break; 265 } 266 case GIM_CheckI64ImmPredicate: 267 case GIM_CheckImmOperandPredicate: { 268 int64_t InsnID = MatchTable[CurrentIdx++]; 269 int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate 270 ? MatchTable[CurrentIdx++] 271 : 1; 272 int64_t Predicate = MatchTable[CurrentIdx++]; 273 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 274 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" 275 << InsnID << "]->getOperand(" << OpIdx 276 << "), Predicate=" << Predicate << ")\n"); 277 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 278 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || 279 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && 280 "Expected immediate operand"); 281 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); 282 int64_t Value = 0; 283 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) 284 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); 285 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) 286 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); 287 else 288 llvm_unreachable("Expected Imm or CImm operand"); 289 290 if (!testImmPredicate_I64(Predicate, Value)) 291 if (handleReject() == RejectAndGiveUp) 292 return false; 293 break; 294 } 295 case GIM_CheckAPIntImmPredicate: { 296 int64_t InsnID = MatchTable[CurrentIdx++]; 297 int64_t Predicate = MatchTable[CurrentIdx++]; 298 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 299 dbgs() 300 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" 301 << InsnID << "], Predicate=" << Predicate << ")\n"); 302 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 303 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 304 "Expected G_CONSTANT"); 305 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); 306 APInt Value; 307 if (State.MIs[InsnID]->getOperand(1).isCImm()) 308 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); 309 else 310 llvm_unreachable("Expected Imm or CImm operand"); 311 312 if (!testImmPredicate_APInt(Predicate, Value)) 313 if (handleReject() == RejectAndGiveUp) 314 return false; 315 break; 316 } 317 case GIM_CheckAPFloatImmPredicate: { 318 int64_t InsnID = MatchTable[CurrentIdx++]; 319 int64_t Predicate = MatchTable[CurrentIdx++]; 320 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 321 dbgs() 322 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" 323 << InsnID << "], Predicate=" << Predicate << ")\n"); 324 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 325 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 326 "Expected G_FCONSTANT"); 327 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); 328 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); 329 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); 330 331 if (!testImmPredicate_APFloat(Predicate, Value)) 332 if (handleReject() == RejectAndGiveUp) 333 return false; 334 break; 335 } 336 case GIM_CheckIsBuildVectorAllOnes: 337 case GIM_CheckIsBuildVectorAllZeros: { 338 int64_t InsnID = MatchTable[CurrentIdx++]; 339 340 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 341 dbgs() << CurrentIdx 342 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" 343 << InsnID << "])\n"); 344 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 345 346 const MachineInstr *MI = State.MIs[InsnID]; 347 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || 348 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && 349 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC"); 350 351 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { 352 if (!isBuildVectorAllOnes(*MI, MRI)) { 353 if (handleReject() == RejectAndGiveUp) 354 return false; 355 } 356 } else { 357 if (!isBuildVectorAllZeros(*MI, MRI)) { 358 if (handleReject() == RejectAndGiveUp) 359 return false; 360 } 361 } 362 363 break; 364 } 365 case GIM_CheckCxxInsnPredicate: { 366 int64_t InsnID = MatchTable[CurrentIdx++]; 367 int64_t Predicate = MatchTable[CurrentIdx++]; 368 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 369 dbgs() 370 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" 371 << InsnID << "], Predicate=" << Predicate << ")\n"); 372 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 373 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); 374 375 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], 376 State.RecordedOperands)) 377 if (handleReject() == RejectAndGiveUp) 378 return false; 379 break; 380 } 381 case GIM_CheckAtomicOrdering: { 382 int64_t InsnID = MatchTable[CurrentIdx++]; 383 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 384 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 385 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" 386 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 387 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 388 if (!State.MIs[InsnID]->hasOneMemOperand()) 389 if (handleReject() == RejectAndGiveUp) 390 return false; 391 392 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 393 if (MMO->getOrdering() != Ordering) 394 if (handleReject() == RejectAndGiveUp) 395 return false; 396 break; 397 } 398 case GIM_CheckAtomicOrderingOrStrongerThan: { 399 int64_t InsnID = MatchTable[CurrentIdx++]; 400 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 401 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 402 dbgs() << CurrentIdx 403 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" 404 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 405 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 406 if (!State.MIs[InsnID]->hasOneMemOperand()) 407 if (handleReject() == RejectAndGiveUp) 408 return false; 409 410 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 411 if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) 412 if (handleReject() == RejectAndGiveUp) 413 return false; 414 break; 415 } 416 case GIM_CheckAtomicOrderingWeakerThan: { 417 int64_t InsnID = MatchTable[CurrentIdx++]; 418 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 419 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 420 dbgs() << CurrentIdx 421 << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" 422 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 423 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 424 if (!State.MIs[InsnID]->hasOneMemOperand()) 425 if (handleReject() == RejectAndGiveUp) 426 return false; 427 428 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 429 if (!isStrongerThan(Ordering, MMO->getOrdering())) 430 if (handleReject() == RejectAndGiveUp) 431 return false; 432 break; 433 } 434 case GIM_CheckMemoryAddressSpace: { 435 int64_t InsnID = MatchTable[CurrentIdx++]; 436 int64_t MMOIdx = MatchTable[CurrentIdx++]; 437 // This accepts a list of possible address spaces. 438 const int NumAddrSpace = MatchTable[CurrentIdx++]; 439 440 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 441 if (handleReject() == RejectAndGiveUp) 442 return false; 443 break; 444 } 445 446 // Need to still jump to the end of the list of address spaces if we find 447 // a match earlier. 448 const uint64_t LastIdx = CurrentIdx + NumAddrSpace; 449 450 const MachineMemOperand *MMO 451 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 452 const unsigned MMOAddrSpace = MMO->getAddrSpace(); 453 454 bool Success = false; 455 for (int I = 0; I != NumAddrSpace; ++I) { 456 unsigned AddrSpace = MatchTable[CurrentIdx++]; 457 DEBUG_WITH_TYPE( 458 TgtInstructionSelector::getName(), 459 dbgs() << "addrspace(" << MMOAddrSpace << ") vs " 460 << AddrSpace << '\n'); 461 462 if (AddrSpace == MMOAddrSpace) { 463 Success = true; 464 break; 465 } 466 } 467 468 CurrentIdx = LastIdx; 469 if (!Success && handleReject() == RejectAndGiveUp) 470 return false; 471 break; 472 } 473 case GIM_CheckMemoryAlignment: { 474 int64_t InsnID = MatchTable[CurrentIdx++]; 475 int64_t MMOIdx = MatchTable[CurrentIdx++]; 476 unsigned MinAlign = MatchTable[CurrentIdx++]; 477 478 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 479 480 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 481 if (handleReject() == RejectAndGiveUp) 482 return false; 483 break; 484 } 485 486 MachineMemOperand *MMO 487 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 488 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 489 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" 490 << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 491 << ")->getAlignment() >= " << MinAlign << ")\n"); 492 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) 493 return false; 494 495 break; 496 } 497 case GIM_CheckMemorySizeEqualTo: { 498 int64_t InsnID = MatchTable[CurrentIdx++]; 499 int64_t MMOIdx = MatchTable[CurrentIdx++]; 500 uint64_t Size = MatchTable[CurrentIdx++]; 501 502 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 503 dbgs() << CurrentIdx 504 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID 505 << "]->memoperands() + " << MMOIdx 506 << ", Size=" << Size << ")\n"); 507 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 508 509 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 510 if (handleReject() == RejectAndGiveUp) 511 return false; 512 break; 513 } 514 515 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 516 517 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 518 dbgs() << MMO->getSize() << " bytes vs " << Size 519 << " bytes\n"); 520 if (MMO->getSize() != Size) 521 if (handleReject() == RejectAndGiveUp) 522 return false; 523 524 break; 525 } 526 case GIM_CheckMemorySizeEqualToLLT: 527 case GIM_CheckMemorySizeLessThanLLT: 528 case GIM_CheckMemorySizeGreaterThanLLT: { 529 int64_t InsnID = MatchTable[CurrentIdx++]; 530 int64_t MMOIdx = MatchTable[CurrentIdx++]; 531 int64_t OpIdx = MatchTable[CurrentIdx++]; 532 533 DEBUG_WITH_TYPE( 534 TgtInstructionSelector::getName(), 535 dbgs() << CurrentIdx << ": GIM_CheckMemorySize" 536 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT 537 ? "EqualTo" 538 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT 539 ? "GreaterThan" 540 : "LessThan") 541 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 542 << ", OpIdx=" << OpIdx << ")\n"); 543 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 544 545 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 546 if (!MO.isReg()) { 547 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 548 dbgs() << CurrentIdx << ": Not a register\n"); 549 if (handleReject() == RejectAndGiveUp) 550 return false; 551 break; 552 } 553 554 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 555 if (handleReject() == RejectAndGiveUp) 556 return false; 557 break; 558 } 559 560 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 561 562 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); 563 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && 564 MMO->getSizeInBits() != Size) { 565 if (handleReject() == RejectAndGiveUp) 566 return false; 567 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && 568 MMO->getSizeInBits() >= Size) { 569 if (handleReject() == RejectAndGiveUp) 570 return false; 571 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && 572 MMO->getSizeInBits() <= Size) 573 if (handleReject() == RejectAndGiveUp) 574 return false; 575 576 break; 577 } 578 case GIM_CheckType: { 579 int64_t InsnID = MatchTable[CurrentIdx++]; 580 int64_t OpIdx = MatchTable[CurrentIdx++]; 581 int64_t TypeID = MatchTable[CurrentIdx++]; 582 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 583 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID 584 << "]->getOperand(" << OpIdx 585 << "), TypeID=" << TypeID << ")\n"); 586 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 587 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 588 if (!MO.isReg() || 589 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { 590 if (handleReject() == RejectAndGiveUp) 591 return false; 592 } 593 break; 594 } 595 case GIM_CheckPointerToAny: { 596 int64_t InsnID = MatchTable[CurrentIdx++]; 597 int64_t OpIdx = MatchTable[CurrentIdx++]; 598 int64_t SizeInBits = MatchTable[CurrentIdx++]; 599 600 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 601 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" 602 << InsnID << "]->getOperand(" << OpIdx 603 << "), SizeInBits=" << SizeInBits << ")\n"); 604 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 605 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 606 const LLT Ty = MRI.getType(MO.getReg()); 607 608 // iPTR must be looked up in the target. 609 if (SizeInBits == 0) { 610 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); 611 const unsigned AddrSpace = Ty.getAddressSpace(); 612 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); 613 } 614 615 assert(SizeInBits != 0 && "Pointer size must be known"); 616 617 if (MO.isReg()) { 618 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) 619 if (handleReject() == RejectAndGiveUp) 620 return false; 621 } else if (handleReject() == RejectAndGiveUp) 622 return false; 623 624 break; 625 } 626 case GIM_RecordNamedOperand: { 627 int64_t InsnID = MatchTable[CurrentIdx++]; 628 int64_t OpIdx = MatchTable[CurrentIdx++]; 629 uint64_t StoreIdx = MatchTable[CurrentIdx++]; 630 631 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 632 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" 633 << InsnID << "]->getOperand(" << OpIdx 634 << "), StoreIdx=" << StoreIdx << ")\n"); 635 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 636 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); 637 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); 638 break; 639 } 640 case GIM_CheckRegBankForClass: { 641 int64_t InsnID = MatchTable[CurrentIdx++]; 642 int64_t OpIdx = MatchTable[CurrentIdx++]; 643 int64_t RCEnum = MatchTable[CurrentIdx++]; 644 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 645 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" 646 << InsnID << "]->getOperand(" << OpIdx 647 << "), RCEnum=" << RCEnum << ")\n"); 648 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 649 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 650 if (!MO.isReg() || 651 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), 652 MRI.getType(MO.getReg())) != 653 RBI.getRegBank(MO.getReg(), MRI, TRI)) { 654 if (handleReject() == RejectAndGiveUp) 655 return false; 656 } 657 break; 658 } 659 660 case GIM_CheckComplexPattern: { 661 int64_t InsnID = MatchTable[CurrentIdx++]; 662 int64_t OpIdx = MatchTable[CurrentIdx++]; 663 int64_t RendererID = MatchTable[CurrentIdx++]; 664 int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; 665 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 666 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID 667 << "] = GIM_CheckComplexPattern(MIs[" << InsnID 668 << "]->getOperand(" << OpIdx 669 << "), ComplexPredicateID=" << ComplexPredicateID 670 << ")\n"); 671 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 672 // FIXME: Use std::invoke() when it's available. 673 ComplexRendererFns Renderer = 674 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( 675 State.MIs[InsnID]->getOperand(OpIdx)); 676 if (Renderer.hasValue()) 677 State.Renderers[RendererID] = Renderer.getValue(); 678 else 679 if (handleReject() == RejectAndGiveUp) 680 return false; 681 break; 682 } 683 684 case GIM_CheckConstantInt: { 685 int64_t InsnID = MatchTable[CurrentIdx++]; 686 int64_t OpIdx = MatchTable[CurrentIdx++]; 687 int64_t Value = MatchTable[CurrentIdx++]; 688 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 689 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" 690 << InsnID << "]->getOperand(" << OpIdx 691 << "), Value=" << Value << ")\n"); 692 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 693 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 694 if (MO.isReg()) { 695 // isOperandImmEqual() will sign-extend to 64-bits, so should we. 696 LLT Ty = MRI.getType(MO.getReg()); 697 Value = SignExtend64(Value, Ty.getSizeInBits()); 698 699 if (!isOperandImmEqual(MO, Value, MRI)) { 700 if (handleReject() == RejectAndGiveUp) 701 return false; 702 } 703 } else if (handleReject() == RejectAndGiveUp) 704 return false; 705 706 break; 707 } 708 709 case GIM_CheckLiteralInt: { 710 int64_t InsnID = MatchTable[CurrentIdx++]; 711 int64_t OpIdx = MatchTable[CurrentIdx++]; 712 int64_t Value = MatchTable[CurrentIdx++]; 713 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 714 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" 715 << InsnID << "]->getOperand(" << OpIdx 716 << "), Value=" << Value << ")\n"); 717 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 718 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 719 if (MO.isImm() && MO.getImm() == Value) 720 break; 721 722 if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) 723 break; 724 725 if (handleReject() == RejectAndGiveUp) 726 return false; 727 728 break; 729 } 730 731 case GIM_CheckIntrinsicID: { 732 int64_t InsnID = MatchTable[CurrentIdx++]; 733 int64_t OpIdx = MatchTable[CurrentIdx++]; 734 int64_t Value = MatchTable[CurrentIdx++]; 735 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 736 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" 737 << InsnID << "]->getOperand(" << OpIdx 738 << "), Value=" << Value << ")\n"); 739 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 740 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 741 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) 742 if (handleReject() == RejectAndGiveUp) 743 return false; 744 break; 745 } 746 case GIM_CheckCmpPredicate: { 747 int64_t InsnID = MatchTable[CurrentIdx++]; 748 int64_t OpIdx = MatchTable[CurrentIdx++]; 749 int64_t Value = MatchTable[CurrentIdx++]; 750 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 751 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" 752 << InsnID << "]->getOperand(" << OpIdx 753 << "), Value=" << Value << ")\n"); 754 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 755 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 756 if (!MO.isPredicate() || MO.getPredicate() != Value) 757 if (handleReject() == RejectAndGiveUp) 758 return false; 759 break; 760 } 761 case GIM_CheckIsMBB: { 762 int64_t InsnID = MatchTable[CurrentIdx++]; 763 int64_t OpIdx = MatchTable[CurrentIdx++]; 764 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 765 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID 766 << "]->getOperand(" << OpIdx << "))\n"); 767 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 768 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { 769 if (handleReject() == RejectAndGiveUp) 770 return false; 771 } 772 break; 773 } 774 case GIM_CheckIsImm: { 775 int64_t InsnID = MatchTable[CurrentIdx++]; 776 int64_t OpIdx = MatchTable[CurrentIdx++]; 777 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 778 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID 779 << "]->getOperand(" << OpIdx << "))\n"); 780 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 781 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { 782 if (handleReject() == RejectAndGiveUp) 783 return false; 784 } 785 break; 786 } 787 case GIM_CheckIsSafeToFold: { 788 int64_t InsnID = MatchTable[CurrentIdx++]; 789 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 790 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" 791 << InsnID << "])\n"); 792 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 793 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { 794 if (handleReject() == RejectAndGiveUp) 795 return false; 796 } 797 break; 798 } 799 case GIM_CheckIsSameOperand: { 800 int64_t InsnID = MatchTable[CurrentIdx++]; 801 int64_t OpIdx = MatchTable[CurrentIdx++]; 802 int64_t OtherInsnID = MatchTable[CurrentIdx++]; 803 int64_t OtherOpIdx = MatchTable[CurrentIdx++]; 804 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 805 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" 806 << InsnID << "][" << OpIdx << "], MIs[" 807 << OtherInsnID << "][" << OtherOpIdx << "])\n"); 808 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 809 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); 810 if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( 811 State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { 812 if (handleReject() == RejectAndGiveUp) 813 return false; 814 } 815 break; 816 } 817 case GIM_Reject: 818 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 819 dbgs() << CurrentIdx << ": GIM_Reject\n"); 820 if (handleReject() == RejectAndGiveUp) 821 return false; 822 break; 823 824 case GIR_MutateOpcode: { 825 int64_t OldInsnID = MatchTable[CurrentIdx++]; 826 uint64_t NewInsnID = MatchTable[CurrentIdx++]; 827 int64_t NewOpcode = MatchTable[CurrentIdx++]; 828 if (NewInsnID >= OutMIs.size()) 829 OutMIs.resize(NewInsnID + 1); 830 831 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), 832 State.MIs[OldInsnID]); 833 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); 834 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 835 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" 836 << NewInsnID << "], MIs[" << OldInsnID << "], " 837 << NewOpcode << ")\n"); 838 break; 839 } 840 841 case GIR_BuildMI: { 842 uint64_t NewInsnID = MatchTable[CurrentIdx++]; 843 int64_t Opcode = MatchTable[CurrentIdx++]; 844 if (NewInsnID >= OutMIs.size()) 845 OutMIs.resize(NewInsnID + 1); 846 847 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], 848 State.MIs[0]->getDebugLoc(), TII.get(Opcode)); 849 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 850 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" 851 << NewInsnID << "], " << Opcode << ")\n"); 852 break; 853 } 854 855 case GIR_Copy: { 856 int64_t NewInsnID = MatchTable[CurrentIdx++]; 857 int64_t OldInsnID = MatchTable[CurrentIdx++]; 858 int64_t OpIdx = MatchTable[CurrentIdx++]; 859 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 860 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); 861 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 862 dbgs() 863 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID 864 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); 865 break; 866 } 867 868 case GIR_CopyOrAddZeroReg: { 869 int64_t NewInsnID = MatchTable[CurrentIdx++]; 870 int64_t OldInsnID = MatchTable[CurrentIdx++]; 871 int64_t OpIdx = MatchTable[CurrentIdx++]; 872 int64_t ZeroReg = MatchTable[CurrentIdx++]; 873 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 874 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); 875 if (isOperandImmEqual(MO, 0, MRI)) 876 OutMIs[NewInsnID].addReg(ZeroReg); 877 else 878 OutMIs[NewInsnID].add(MO); 879 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 880 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" 881 << NewInsnID << "], MIs[" << OldInsnID << "], " 882 << OpIdx << ", " << ZeroReg << ")\n"); 883 break; 884 } 885 886 case GIR_CopySubReg: { 887 int64_t NewInsnID = MatchTable[CurrentIdx++]; 888 int64_t OldInsnID = MatchTable[CurrentIdx++]; 889 int64_t OpIdx = MatchTable[CurrentIdx++]; 890 int64_t SubRegIdx = MatchTable[CurrentIdx++]; 891 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 892 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 893 0, SubRegIdx); 894 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 895 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" 896 << NewInsnID << "], MIs[" << OldInsnID << "], " 897 << OpIdx << ", " << SubRegIdx << ")\n"); 898 break; 899 } 900 901 case GIR_AddImplicitDef: { 902 int64_t InsnID = MatchTable[CurrentIdx++]; 903 int64_t RegNum = MatchTable[CurrentIdx++]; 904 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 905 OutMIs[InsnID].addDef(RegNum, RegState::Implicit); 906 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 907 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" 908 << InsnID << "], " << RegNum << ")\n"); 909 break; 910 } 911 912 case GIR_AddImplicitUse: { 913 int64_t InsnID = MatchTable[CurrentIdx++]; 914 int64_t RegNum = MatchTable[CurrentIdx++]; 915 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 916 OutMIs[InsnID].addUse(RegNum, RegState::Implicit); 917 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 918 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" 919 << InsnID << "], " << RegNum << ")\n"); 920 break; 921 } 922 923 case GIR_AddRegister: { 924 int64_t InsnID = MatchTable[CurrentIdx++]; 925 int64_t RegNum = MatchTable[CurrentIdx++]; 926 uint64_t RegFlags = MatchTable[CurrentIdx++]; 927 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 928 OutMIs[InsnID].addReg(RegNum, RegFlags); 929 DEBUG_WITH_TYPE( 930 TgtInstructionSelector::getName(), 931 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" 932 << InsnID << "], " << RegNum << ", " << RegFlags << ")\n"); 933 break; 934 } 935 936 case GIR_AddTempRegister: 937 case GIR_AddTempSubRegister: { 938 int64_t InsnID = MatchTable[CurrentIdx++]; 939 int64_t TempRegID = MatchTable[CurrentIdx++]; 940 uint64_t TempRegFlags = MatchTable[CurrentIdx++]; 941 unsigned SubReg = 0; 942 if (MatcherOpcode == GIR_AddTempSubRegister) 943 SubReg = MatchTable[CurrentIdx++]; 944 945 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 946 947 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg); 948 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 949 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" 950 << InsnID << "], TempRegisters[" << TempRegID 951 << "]"; 952 if (SubReg) 953 dbgs() << '.' << TRI.getSubRegIndexName(SubReg); 954 dbgs() << ", " << TempRegFlags << ")\n"); 955 break; 956 } 957 958 case GIR_AddImm: { 959 int64_t InsnID = MatchTable[CurrentIdx++]; 960 int64_t Imm = MatchTable[CurrentIdx++]; 961 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 962 OutMIs[InsnID].addImm(Imm); 963 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 964 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID 965 << "], " << Imm << ")\n"); 966 break; 967 } 968 969 case GIR_ComplexRenderer: { 970 int64_t InsnID = MatchTable[CurrentIdx++]; 971 int64_t RendererID = MatchTable[CurrentIdx++]; 972 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 973 for (const auto &RenderOpFn : State.Renderers[RendererID]) 974 RenderOpFn(OutMIs[InsnID]); 975 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 976 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" 977 << InsnID << "], " << RendererID << ")\n"); 978 break; 979 } 980 case GIR_ComplexSubOperandRenderer: { 981 int64_t InsnID = MatchTable[CurrentIdx++]; 982 int64_t RendererID = MatchTable[CurrentIdx++]; 983 int64_t RenderOpID = MatchTable[CurrentIdx++]; 984 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 985 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); 986 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 987 dbgs() << CurrentIdx 988 << ": GIR_ComplexSubOperandRenderer(OutMIs[" 989 << InsnID << "], " << RendererID << ", " 990 << RenderOpID << ")\n"); 991 break; 992 } 993 994 case GIR_CopyConstantAsSImm: { 995 int64_t NewInsnID = MatchTable[CurrentIdx++]; 996 int64_t OldInsnID = MatchTable[CurrentIdx++]; 997 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 998 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); 999 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { 1000 OutMIs[NewInsnID].addImm( 1001 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); 1002 } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) 1003 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); 1004 else 1005 llvm_unreachable("Expected Imm or CImm operand"); 1006 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1007 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" 1008 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1009 break; 1010 } 1011 1012 // TODO: Needs a test case once we have a pattern that uses this. 1013 case GIR_CopyFConstantAsFPImm: { 1014 int64_t NewInsnID = MatchTable[CurrentIdx++]; 1015 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1016 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1017 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); 1018 if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) 1019 OutMIs[NewInsnID].addFPImm( 1020 State.MIs[OldInsnID]->getOperand(1).getFPImm()); 1021 else 1022 llvm_unreachable("Expected FPImm operand"); 1023 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1024 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" 1025 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1026 break; 1027 } 1028 1029 case GIR_CustomRenderer: { 1030 int64_t InsnID = MatchTable[CurrentIdx++]; 1031 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1032 int64_t RendererFnID = MatchTable[CurrentIdx++]; 1033 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1034 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1035 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" 1036 << InsnID << "], MIs[" << OldInsnID << "], " 1037 << RendererFnID << ")\n"); 1038 (ISel.*ISelInfo.CustomRenderers[RendererFnID])( 1039 OutMIs[InsnID], *State.MIs[OldInsnID], 1040 -1); // Not a source operand of the old instruction. 1041 break; 1042 } 1043 case GIR_CustomOperandRenderer: { 1044 int64_t InsnID = MatchTable[CurrentIdx++]; 1045 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1046 int64_t OpIdx = MatchTable[CurrentIdx++]; 1047 int64_t RendererFnID = MatchTable[CurrentIdx++]; 1048 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1049 1050 DEBUG_WITH_TYPE( 1051 TgtInstructionSelector::getName(), 1052 dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs[" 1053 << InsnID << "], MIs[" << OldInsnID << "]->getOperand(" 1054 << OpIdx << "), " 1055 << RendererFnID << ")\n"); 1056 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], 1057 *State.MIs[OldInsnID], 1058 OpIdx); 1059 break; 1060 } 1061 case GIR_ConstrainOperandRC: { 1062 int64_t InsnID = MatchTable[CurrentIdx++]; 1063 int64_t OpIdx = MatchTable[CurrentIdx++]; 1064 int64_t RCEnum = MatchTable[CurrentIdx++]; 1065 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1066 MachineInstr &I = *OutMIs[InsnID].getInstr(); 1067 MachineFunction &MF = *I.getParent()->getParent(); 1068 MachineRegisterInfo &MRI = MF.getRegInfo(); 1069 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum); 1070 MachineOperand &MO = I.getOperand(OpIdx); 1071 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO); 1072 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1073 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" 1074 << InsnID << "], " << OpIdx << ", " << RCEnum 1075 << ")\n"); 1076 break; 1077 } 1078 1079 case GIR_ConstrainSelectedInstOperands: { 1080 int64_t InsnID = MatchTable[CurrentIdx++]; 1081 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1082 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, 1083 RBI); 1084 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1085 dbgs() << CurrentIdx 1086 << ": GIR_ConstrainSelectedInstOperands(OutMIs[" 1087 << InsnID << "])\n"); 1088 break; 1089 } 1090 1091 case GIR_MergeMemOperands: { 1092 int64_t InsnID = MatchTable[CurrentIdx++]; 1093 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1094 1095 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1096 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" 1097 << InsnID << "]"); 1098 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; 1099 while ((MergeInsnID = MatchTable[CurrentIdx++]) != 1100 GIU_MergeMemOperands_EndOfList) { 1101 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1102 dbgs() << ", MIs[" << MergeInsnID << "]"); 1103 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) 1104 OutMIs[InsnID].addMemOperand(MMO); 1105 } 1106 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); 1107 break; 1108 } 1109 1110 case GIR_EraseFromParent: { 1111 int64_t InsnID = MatchTable[CurrentIdx++]; 1112 assert(State.MIs[InsnID] && 1113 "Attempted to erase an undefined instruction"); 1114 State.MIs[InsnID]->eraseFromParent(); 1115 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1116 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" 1117 << InsnID << "])\n"); 1118 break; 1119 } 1120 1121 case GIR_MakeTempReg: { 1122 int64_t TempRegID = MatchTable[CurrentIdx++]; 1123 int64_t TypeID = MatchTable[CurrentIdx++]; 1124 1125 State.TempRegisters[TempRegID] = 1126 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); 1127 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1128 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID 1129 << "] = GIR_MakeTempReg(" << TypeID << ")\n"); 1130 break; 1131 } 1132 1133 case GIR_Coverage: { 1134 int64_t RuleID = MatchTable[CurrentIdx++]; 1135 CoverageInfo.setCovered(RuleID); 1136 1137 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1138 dbgs() 1139 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); 1140 break; 1141 } 1142 1143 case GIR_Done: 1144 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1145 dbgs() << CurrentIdx << ": GIR_Done\n"); 1146 propagateFlags(OutMIs); 1147 return true; 1148 1149 default: 1150 llvm_unreachable("Unexpected command"); 1151 } 1152 } 1153 } 1154 1155 } // end namespace llvm 1156 1157 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H 1158
Indexes created Tue Sep 22 00:26:15 UTC 2026