1 //===- X86InstructionSelector.cpp -----------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the targeting of the InstructionSelector class for 10 /// X86. 11 /// \todo This should be generated by TableGen. 12 //===----------------------------------------------------------------------===// 13 14 #include "MCTargetDesc/X86BaseInfo.h" 15 #include "X86.h" 16 #include "X86InstrBuilder.h" 17 #include "X86InstrInfo.h" 18 #include "X86RegisterBankInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "X86TargetMachine.h" 22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" 24 #include "llvm/CodeGen/GlobalISel/RegisterBank.h" 25 #include "llvm/CodeGen/GlobalISel/Utils.h" 26 #include "llvm/CodeGen/MachineBasicBlock.h" 27 #include "llvm/CodeGen/MachineConstantPool.h" 28 #include "llvm/CodeGen/MachineFunction.h" 29 #include "llvm/CodeGen/MachineInstr.h" 30 #include "llvm/CodeGen/MachineInstrBuilder.h" 31 #include "llvm/CodeGen/MachineMemOperand.h" 32 #include "llvm/CodeGen/MachineOperand.h" 33 #include "llvm/CodeGen/MachineRegisterInfo.h" 34 #include "llvm/CodeGen/TargetOpcodes.h" 35 #include "llvm/CodeGen/TargetRegisterInfo.h" 36 #include "llvm/IR/DataLayout.h" 37 #include "llvm/IR/InstrTypes.h" 38 #include "llvm/IR/IntrinsicsX86.h" 39 #include "llvm/Support/AtomicOrdering.h" 40 #include "llvm/Support/CodeGen.h" 41 #include "llvm/Support/Debug.h" 42 #include "llvm/Support/ErrorHandling.h" 43 #include "llvm/Support/LowLevelTypeImpl.h" 44 #include "llvm/Support/MathExtras.h" 45 #include "llvm/Support/raw_ostream.h" 46 #include <cassert> 47 #include <cstdint> 48 #include <tuple> 49 50 #define DEBUG_TYPE "X86-isel" 51 52 using namespace llvm; 53 54 namespace { 55 56 #define GET_GLOBALISEL_PREDICATE_BITSET 57 #include "X86GenGlobalISel.inc" 58 #undef GET_GLOBALISEL_PREDICATE_BITSET 59 60 class X86InstructionSelector : public InstructionSelector { 61 public: 62 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, 63 const X86RegisterBankInfo &RBI); 64 65 bool select(MachineInstr &I) override; 66 static const char *getName() { return DEBUG_TYPE; } 67 68 private: 69 /// tblgen-erated 'select' implementation, used as the initial selector for 70 /// the patterns that don't require complex C++. 71 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 72 73 // TODO: remove after supported by Tablegen-erated instruction selection. 74 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc, 75 Align Alignment) const; 76 77 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI, 78 MachineFunction &MF) const; 79 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, 80 MachineFunction &MF) const; 81 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI, 82 MachineFunction &MF) const; 83 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, 84 MachineFunction &MF) const; 85 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI, 86 MachineFunction &MF) const; 87 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI, 88 MachineFunction &MF) const; 89 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI, 90 MachineFunction &MF) const; 91 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI, 92 MachineFunction &MF) const; 93 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI, 94 MachineFunction &MF) const; 95 bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, 96 MachineFunction &MF) const; 97 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; 98 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 99 MachineFunction &MF); 100 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 101 MachineFunction &MF); 102 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, 103 MachineFunction &MF) const; 104 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, 105 MachineFunction &MF) const; 106 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI, 107 MachineFunction &MF) const; 108 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI, 109 const unsigned DstReg, 110 const TargetRegisterClass *DstRC, 111 const unsigned SrcReg, 112 const TargetRegisterClass *SrcRC) const; 113 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI, 114 MachineFunction &MF) const; 115 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const; 116 bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI, 117 MachineFunction &MF) const; 118 bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI, 119 MachineFunction &MF) const; 120 121 // emit insert subreg instruction and insert it before MachineInstr &I 122 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 123 MachineRegisterInfo &MRI, MachineFunction &MF) const; 124 // emit extract subreg instruction and insert it before MachineInstr &I 125 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 126 MachineRegisterInfo &MRI, MachineFunction &MF) const; 127 128 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const; 129 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg, 130 MachineRegisterInfo &MRI) const; 131 132 const X86TargetMachine &TM; 133 const X86Subtarget &STI; 134 const X86InstrInfo &TII; 135 const X86RegisterInfo &TRI; 136 const X86RegisterBankInfo &RBI; 137 138 #define GET_GLOBALISEL_PREDICATES_DECL 139 #include "X86GenGlobalISel.inc" 140 #undef GET_GLOBALISEL_PREDICATES_DECL 141 142 #define GET_GLOBALISEL_TEMPORARIES_DECL 143 #include "X86GenGlobalISel.inc" 144 #undef GET_GLOBALISEL_TEMPORARIES_DECL 145 }; 146 147 } // end anonymous namespace 148 149 #define GET_GLOBALISEL_IMPL 150 #include "X86GenGlobalISel.inc" 151 #undef GET_GLOBALISEL_IMPL 152 153 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM, 154 const X86Subtarget &STI, 155 const X86RegisterBankInfo &RBI) 156 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), 157 TRI(*STI.getRegisterInfo()), RBI(RBI), 158 #define GET_GLOBALISEL_PREDICATES_INIT 159 #include "X86GenGlobalISel.inc" 160 #undef GET_GLOBALISEL_PREDICATES_INIT 161 #define GET_GLOBALISEL_TEMPORARIES_INIT 162 #include "X86GenGlobalISel.inc" 163 #undef GET_GLOBALISEL_TEMPORARIES_INIT 164 { 165 } 166 167 // FIXME: This should be target-independent, inferred from the types declared 168 // for each class in the bank. 169 const TargetRegisterClass * 170 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const { 171 if (RB.getID() == X86::GPRRegBankID) { 172 if (Ty.getSizeInBits() <= 8) 173 return &X86::GR8RegClass; 174 if (Ty.getSizeInBits() == 16) 175 return &X86::GR16RegClass; 176 if (Ty.getSizeInBits() == 32) 177 return &X86::GR32RegClass; 178 if (Ty.getSizeInBits() == 64) 179 return &X86::GR64RegClass; 180 } 181 if (RB.getID() == X86::VECRRegBankID) { 182 if (Ty.getSizeInBits() == 32) 183 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; 184 if (Ty.getSizeInBits() == 64) 185 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; 186 if (Ty.getSizeInBits() == 128) 187 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass; 188 if (Ty.getSizeInBits() == 256) 189 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass; 190 if (Ty.getSizeInBits() == 512) 191 return &X86::VR512RegClass; 192 } 193 194 llvm_unreachable("Unknown RegBank!"); 195 } 196 197 const TargetRegisterClass * 198 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg, 199 MachineRegisterInfo &MRI) const { 200 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI); 201 return getRegClass(Ty, RegBank); 202 } 203 204 static unsigned getSubRegIndex(const TargetRegisterClass *RC) { 205 unsigned SubIdx = X86::NoSubRegister; 206 if (RC == &X86::GR32RegClass) { 207 SubIdx = X86::sub_32bit; 208 } else if (RC == &X86::GR16RegClass) { 209 SubIdx = X86::sub_16bit; 210 } else if (RC == &X86::GR8RegClass) { 211 SubIdx = X86::sub_8bit; 212 } 213 214 return SubIdx; 215 } 216 217 static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) { 218 assert(Reg.isPhysical()); 219 if (X86::GR64RegClass.contains(Reg)) 220 return &X86::GR64RegClass; 221 if (X86::GR32RegClass.contains(Reg)) 222 return &X86::GR32RegClass; 223 if (X86::GR16RegClass.contains(Reg)) 224 return &X86::GR16RegClass; 225 if (X86::GR8RegClass.contains(Reg)) 226 return &X86::GR8RegClass; 227 228 llvm_unreachable("Unknown RegClass for PhysReg!"); 229 } 230 231 // Set X86 Opcode and constrain DestReg. 232 bool X86InstructionSelector::selectCopy(MachineInstr &I, 233 MachineRegisterInfo &MRI) const { 234 Register DstReg = I.getOperand(0).getReg(); 235 const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); 236 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); 237 238 Register SrcReg = I.getOperand(1).getReg(); 239 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); 240 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); 241 242 if (DstReg.isPhysical()) { 243 assert(I.isCopy() && "Generic operators do not allow physical registers"); 244 245 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID && 246 DstRegBank.getID() == X86::GPRRegBankID) { 247 248 const TargetRegisterClass *SrcRC = 249 getRegClass(MRI.getType(SrcReg), SrcRegBank); 250 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg); 251 252 if (SrcRC != DstRC) { 253 // This case can be generated by ABI lowering, performe anyext 254 Register ExtSrc = MRI.createVirtualRegister(DstRC); 255 BuildMI(*I.getParent(), I, I.getDebugLoc(), 256 TII.get(TargetOpcode::SUBREG_TO_REG)) 257 .addDef(ExtSrc) 258 .addImm(0) 259 .addReg(SrcReg) 260 .addImm(getSubRegIndex(SrcRC)); 261 262 I.getOperand(1).setReg(ExtSrc); 263 } 264 } 265 266 return true; 267 } 268 269 assert((!SrcReg.isPhysical() || I.isCopy()) && 270 "No phys reg on generic operators"); 271 assert((DstSize == SrcSize || 272 // Copies are a mean to setup initial types, the number of 273 // bits may not exactly match. 274 (SrcReg.isPhysical() && 275 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) && 276 "Copy with different width?!"); 277 278 const TargetRegisterClass *DstRC = 279 getRegClass(MRI.getType(DstReg), DstRegBank); 280 281 if (SrcRegBank.getID() == X86::GPRRegBankID && 282 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize && 283 SrcReg.isPhysical()) { 284 // Change the physical register to performe truncate. 285 286 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg); 287 288 if (DstRC != SrcRC) { 289 I.getOperand(1).setSubReg(getSubRegIndex(DstRC)); 290 I.getOperand(1).substPhysReg(SrcReg, TRI); 291 } 292 } 293 294 // No need to constrain SrcReg. It will get constrained when 295 // we hit another of its use or its defs. 296 // Copies do not have constraints. 297 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg); 298 if (!OldRC || !DstRC->hasSubClassEq(OldRC)) { 299 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 300 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 301 << " operand\n"); 302 return false; 303 } 304 } 305 I.setDesc(TII.get(X86::COPY)); 306 return true; 307 } 308 309 bool X86InstructionSelector::select(MachineInstr &I) { 310 assert(I.getParent() && "Instruction should be in a basic block!"); 311 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 312 313 MachineBasicBlock &MBB = *I.getParent(); 314 MachineFunction &MF = *MBB.getParent(); 315 MachineRegisterInfo &MRI = MF.getRegInfo(); 316 317 unsigned Opcode = I.getOpcode(); 318 if (!isPreISelGenericOpcode(Opcode)) { 319 // Certain non-generic instructions also need some special handling. 320 321 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) 322 return false; 323 324 if (I.isCopy()) 325 return selectCopy(I, MRI); 326 327 return true; 328 } 329 330 assert(I.getNumOperands() == I.getNumExplicitOperands() && 331 "Generic instruction has unexpected implicit operands\n"); 332 333 if (selectImpl(I, *CoverageInfo)) 334 return true; 335 336 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); 337 338 // TODO: This should be implemented by tblgen. 339 switch (I.getOpcode()) { 340 default: 341 return false; 342 case TargetOpcode::G_STORE: 343 case TargetOpcode::G_LOAD: 344 return selectLoadStoreOp(I, MRI, MF); 345 case TargetOpcode::G_PTR_ADD: 346 case TargetOpcode::G_FRAME_INDEX: 347 return selectFrameIndexOrGep(I, MRI, MF); 348 case TargetOpcode::G_GLOBAL_VALUE: 349 return selectGlobalValue(I, MRI, MF); 350 case TargetOpcode::G_CONSTANT: 351 return selectConstant(I, MRI, MF); 352 case TargetOpcode::G_FCONSTANT: 353 return materializeFP(I, MRI, MF); 354 case TargetOpcode::G_PTRTOINT: 355 case TargetOpcode::G_TRUNC: 356 return selectTruncOrPtrToInt(I, MRI, MF); 357 case TargetOpcode::G_INTTOPTR: 358 return selectCopy(I, MRI); 359 case TargetOpcode::G_ZEXT: 360 return selectZext(I, MRI, MF); 361 case TargetOpcode::G_ANYEXT: 362 return selectAnyext(I, MRI, MF); 363 case TargetOpcode::G_ICMP: 364 return selectCmp(I, MRI, MF); 365 case TargetOpcode::G_FCMP: 366 return selectFCmp(I, MRI, MF); 367 case TargetOpcode::G_UADDE: 368 return selectUadde(I, MRI, MF); 369 case TargetOpcode::G_UNMERGE_VALUES: 370 return selectUnmergeValues(I, MRI, MF); 371 case TargetOpcode::G_MERGE_VALUES: 372 case TargetOpcode::G_CONCAT_VECTORS: 373 return selectMergeValues(I, MRI, MF); 374 case TargetOpcode::G_EXTRACT: 375 return selectExtract(I, MRI, MF); 376 case TargetOpcode::G_INSERT: 377 return selectInsert(I, MRI, MF); 378 case TargetOpcode::G_BRCOND: 379 return selectCondBranch(I, MRI, MF); 380 case TargetOpcode::G_IMPLICIT_DEF: 381 case TargetOpcode::G_PHI: 382 return selectImplicitDefOrPHI(I, MRI); 383 case TargetOpcode::G_SDIV: 384 case TargetOpcode::G_UDIV: 385 case TargetOpcode::G_SREM: 386 case TargetOpcode::G_UREM: 387 return selectDivRem(I, MRI, MF); 388 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 389 return selectIntrinsicWSideEffects(I, MRI, MF); 390 } 391 392 return false; 393 } 394 395 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty, 396 const RegisterBank &RB, 397 unsigned Opc, 398 Align Alignment) const { 399 bool Isload = (Opc == TargetOpcode::G_LOAD); 400 bool HasAVX = STI.hasAVX(); 401 bool HasAVX512 = STI.hasAVX512(); 402 bool HasVLX = STI.hasVLX(); 403 404 if (Ty == LLT::scalar(8)) { 405 if (X86::GPRRegBankID == RB.getID()) 406 return Isload ? X86::MOV8rm : X86::MOV8mr; 407 } else if (Ty == LLT::scalar(16)) { 408 if (X86::GPRRegBankID == RB.getID()) 409 return Isload ? X86::MOV16rm : X86::MOV16mr; 410 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) { 411 if (X86::GPRRegBankID == RB.getID()) 412 return Isload ? X86::MOV32rm : X86::MOV32mr; 413 if (X86::VECRRegBankID == RB.getID()) 414 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt : 415 HasAVX ? X86::VMOVSSrm_alt : 416 X86::MOVSSrm_alt) 417 : (HasAVX512 ? X86::VMOVSSZmr : 418 HasAVX ? X86::VMOVSSmr : 419 X86::MOVSSmr); 420 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { 421 if (X86::GPRRegBankID == RB.getID()) 422 return Isload ? X86::MOV64rm : X86::MOV64mr; 423 if (X86::VECRRegBankID == RB.getID()) 424 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt : 425 HasAVX ? X86::VMOVSDrm_alt : 426 X86::MOVSDrm_alt) 427 : (HasAVX512 ? X86::VMOVSDZmr : 428 HasAVX ? X86::VMOVSDmr : 429 X86::MOVSDmr); 430 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) { 431 if (Alignment >= Align(16)) 432 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm 433 : HasAVX512 434 ? X86::VMOVAPSZ128rm_NOVLX 435 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) 436 : (HasVLX ? X86::VMOVAPSZ128mr 437 : HasAVX512 438 ? X86::VMOVAPSZ128mr_NOVLX 439 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr); 440 else 441 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm 442 : HasAVX512 443 ? X86::VMOVUPSZ128rm_NOVLX 444 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) 445 : (HasVLX ? X86::VMOVUPSZ128mr 446 : HasAVX512 447 ? X86::VMOVUPSZ128mr_NOVLX 448 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr); 449 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) { 450 if (Alignment >= Align(32)) 451 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm 452 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX 453 : X86::VMOVAPSYrm) 454 : (HasVLX ? X86::VMOVAPSZ256mr 455 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX 456 : X86::VMOVAPSYmr); 457 else 458 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm 459 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX 460 : X86::VMOVUPSYrm) 461 : (HasVLX ? X86::VMOVUPSZ256mr 462 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX 463 : X86::VMOVUPSYmr); 464 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) { 465 if (Alignment >= Align(64)) 466 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 467 else 468 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 469 } 470 return Opc; 471 } 472 473 // Fill in an address from the given instruction. 474 static void X86SelectAddress(const MachineInstr &I, 475 const MachineRegisterInfo &MRI, 476 X86AddressMode &AM) { 477 assert(I.getOperand(0).isReg() && "unsupported opperand."); 478 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() && 479 "unsupported type."); 480 481 if (I.getOpcode() == TargetOpcode::G_PTR_ADD) { 482 if (auto COff = getConstantVRegSExtVal(I.getOperand(2).getReg(), MRI)) { 483 int64_t Imm = *COff; 484 if (isInt<32>(Imm)) { // Check for displacement overflow. 485 AM.Disp = static_cast<int32_t>(Imm); 486 AM.Base.Reg = I.getOperand(1).getReg(); 487 return; 488 } 489 } 490 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) { 491 AM.Base.FrameIndex = I.getOperand(1).getIndex(); 492 AM.BaseType = X86AddressMode::FrameIndexBase; 493 return; 494 } 495 496 // Default behavior. 497 AM.Base.Reg = I.getOperand(0).getReg(); 498 } 499 500 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, 501 MachineRegisterInfo &MRI, 502 MachineFunction &MF) const { 503 unsigned Opc = I.getOpcode(); 504 505 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) && 506 "unexpected instruction"); 507 508 const Register DefReg = I.getOperand(0).getReg(); 509 LLT Ty = MRI.getType(DefReg); 510 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); 511 512 assert(I.hasOneMemOperand()); 513 auto &MemOp = **I.memoperands_begin(); 514 if (MemOp.isAtomic()) { 515 // Note: for unordered operations, we rely on the fact the appropriate MMO 516 // is already on the instruction we're mutating, and thus we don't need to 517 // make any changes. So long as we select an opcode which is capable of 518 // loading or storing the appropriate size atomically, the rest of the 519 // backend is required to respect the MMO state. 520 if (!MemOp.isUnordered()) { 521 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n"); 522 return false; 523 } 524 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) { 525 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n"); 526 return false; 527 } 528 } 529 530 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign()); 531 if (NewOpc == Opc) 532 return false; 533 534 X86AddressMode AM; 535 X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM); 536 537 I.setDesc(TII.get(NewOpc)); 538 MachineInstrBuilder MIB(MF, I); 539 if (Opc == TargetOpcode::G_LOAD) { 540 I.RemoveOperand(1); 541 addFullAddress(MIB, AM); 542 } else { 543 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL) 544 I.RemoveOperand(1); 545 I.RemoveOperand(0); 546 addFullAddress(MIB, AM).addUse(DefReg); 547 } 548 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 549 } 550 551 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) { 552 if (Ty == LLT::pointer(0, 64)) 553 return X86::LEA64r; 554 else if (Ty == LLT::pointer(0, 32)) 555 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; 556 else 557 llvm_unreachable("Can't get LEA opcode. Unsupported type."); 558 } 559 560 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, 561 MachineRegisterInfo &MRI, 562 MachineFunction &MF) const { 563 unsigned Opc = I.getOpcode(); 564 565 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) && 566 "unexpected instruction"); 567 568 const Register DefReg = I.getOperand(0).getReg(); 569 LLT Ty = MRI.getType(DefReg); 570 571 // Use LEA to calculate frame index and GEP 572 unsigned NewOpc = getLeaOP(Ty, STI); 573 I.setDesc(TII.get(NewOpc)); 574 MachineInstrBuilder MIB(MF, I); 575 576 if (Opc == TargetOpcode::G_FRAME_INDEX) { 577 addOffset(MIB, 0); 578 } else { 579 MachineOperand &InxOp = I.getOperand(2); 580 I.addOperand(InxOp); // set IndexReg 581 InxOp.ChangeToImmediate(1); // set Scale 582 MIB.addImm(0).addReg(0); 583 } 584 585 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 586 } 587 588 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I, 589 MachineRegisterInfo &MRI, 590 MachineFunction &MF) const { 591 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) && 592 "unexpected instruction"); 593 594 auto GV = I.getOperand(1).getGlobal(); 595 if (GV->isThreadLocal()) { 596 return false; // TODO: we don't support TLS yet. 597 } 598 599 // Can't handle alternate code models yet. 600 if (TM.getCodeModel() != CodeModel::Small) 601 return false; 602 603 X86AddressMode AM; 604 AM.GV = GV; 605 AM.GVOpFlags = STI.classifyGlobalReference(GV); 606 607 // TODO: The ABI requires an extra load. not supported yet. 608 if (isGlobalStubReference(AM.GVOpFlags)) 609 return false; 610 611 // TODO: This reference is relative to the pic base. not supported yet. 612 if (isGlobalRelativeToPICBase(AM.GVOpFlags)) 613 return false; 614 615 if (STI.isPICStyleRIPRel()) { 616 // Use rip-relative addressing. 617 assert(AM.Base.Reg == 0 && AM.IndexReg == 0); 618 AM.Base.Reg = X86::RIP; 619 } 620 621 const Register DefReg = I.getOperand(0).getReg(); 622 LLT Ty = MRI.getType(DefReg); 623 unsigned NewOpc = getLeaOP(Ty, STI); 624 625 I.setDesc(TII.get(NewOpc)); 626 MachineInstrBuilder MIB(MF, I); 627 628 I.RemoveOperand(1); 629 addFullAddress(MIB, AM); 630 631 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 632 } 633 634 bool X86InstructionSelector::selectConstant(MachineInstr &I, 635 MachineRegisterInfo &MRI, 636 MachineFunction &MF) const { 637 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) && 638 "unexpected instruction"); 639 640 const Register DefReg = I.getOperand(0).getReg(); 641 LLT Ty = MRI.getType(DefReg); 642 643 if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID) 644 return false; 645 646 uint64_t Val = 0; 647 if (I.getOperand(1).isCImm()) { 648 Val = I.getOperand(1).getCImm()->getZExtValue(); 649 I.getOperand(1).ChangeToImmediate(Val); 650 } else if (I.getOperand(1).isImm()) { 651 Val = I.getOperand(1).getImm(); 652 } else 653 llvm_unreachable("Unsupported operand type."); 654 655 unsigned NewOpc; 656 switch (Ty.getSizeInBits()) { 657 case 8: 658 NewOpc = X86::MOV8ri; 659 break; 660 case 16: 661 NewOpc = X86::MOV16ri; 662 break; 663 case 32: 664 NewOpc = X86::MOV32ri; 665 break; 666 case 64: 667 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used 668 if (isInt<32>(Val)) 669 NewOpc = X86::MOV64ri32; 670 else 671 NewOpc = X86::MOV64ri; 672 break; 673 default: 674 llvm_unreachable("Can't select G_CONSTANT, unsupported type."); 675 } 676 677 I.setDesc(TII.get(NewOpc)); 678 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 679 } 680 681 // Helper function for selectTruncOrPtrToInt and selectAnyext. 682 // Returns true if DstRC lives on a floating register class and 683 // SrcRC lives on a 128-bit vector class. 684 static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC, 685 const TargetRegisterClass *SrcRC) { 686 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass || 687 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) && 688 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass); 689 } 690 691 bool X86InstructionSelector::selectTurnIntoCOPY( 692 MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg, 693 const TargetRegisterClass *DstRC, const unsigned SrcReg, 694 const TargetRegisterClass *SrcRC) const { 695 696 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 697 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 698 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 699 << " operand\n"); 700 return false; 701 } 702 I.setDesc(TII.get(X86::COPY)); 703 return true; 704 } 705 706 bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I, 707 MachineRegisterInfo &MRI, 708 MachineFunction &MF) const { 709 assert((I.getOpcode() == TargetOpcode::G_TRUNC || 710 I.getOpcode() == TargetOpcode::G_PTRTOINT) && 711 "unexpected instruction"); 712 713 const Register DstReg = I.getOperand(0).getReg(); 714 const Register SrcReg = I.getOperand(1).getReg(); 715 716 const LLT DstTy = MRI.getType(DstReg); 717 const LLT SrcTy = MRI.getType(SrcReg); 718 719 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 720 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 721 722 if (DstRB.getID() != SrcRB.getID()) { 723 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) 724 << " input/output on different banks\n"); 725 return false; 726 } 727 728 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 729 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 730 731 if (!DstRC || !SrcRC) 732 return false; 733 734 // If that's truncation of the value that lives on the vector class and goes 735 // into the floating class, just replace it with copy, as we are able to 736 // select it as a regular move. 737 if (canTurnIntoCOPY(DstRC, SrcRC)) 738 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC); 739 740 if (DstRB.getID() != X86::GPRRegBankID) 741 return false; 742 743 unsigned SubIdx; 744 if (DstRC == SrcRC) { 745 // Nothing to be done 746 SubIdx = X86::NoSubRegister; 747 } else if (DstRC == &X86::GR32RegClass) { 748 SubIdx = X86::sub_32bit; 749 } else if (DstRC == &X86::GR16RegClass) { 750 SubIdx = X86::sub_16bit; 751 } else if (DstRC == &X86::GR8RegClass) { 752 SubIdx = X86::sub_8bit; 753 } else { 754 return false; 755 } 756 757 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 758 759 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 760 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 761 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 762 << "\n"); 763 return false; 764 } 765 766 I.getOperand(1).setSubReg(SubIdx); 767 768 I.setDesc(TII.get(X86::COPY)); 769 return true; 770 } 771 772 bool X86InstructionSelector::selectZext(MachineInstr &I, 773 MachineRegisterInfo &MRI, 774 MachineFunction &MF) const { 775 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction"); 776 777 const Register DstReg = I.getOperand(0).getReg(); 778 const Register SrcReg = I.getOperand(1).getReg(); 779 780 const LLT DstTy = MRI.getType(DstReg); 781 const LLT SrcTy = MRI.getType(SrcReg); 782 783 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) && 784 "8=>16 Zext is handled by tablegen"); 785 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) && 786 "8=>32 Zext is handled by tablegen"); 787 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) && 788 "16=>32 Zext is handled by tablegen"); 789 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) && 790 "8=>64 Zext is handled by tablegen"); 791 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) && 792 "16=>64 Zext is handled by tablegen"); 793 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) && 794 "32=>64 Zext is handled by tablegen"); 795 796 if (SrcTy != LLT::scalar(1)) 797 return false; 798 799 unsigned AndOpc; 800 if (DstTy == LLT::scalar(8)) 801 AndOpc = X86::AND8ri; 802 else if (DstTy == LLT::scalar(16)) 803 AndOpc = X86::AND16ri8; 804 else if (DstTy == LLT::scalar(32)) 805 AndOpc = X86::AND32ri8; 806 else if (DstTy == LLT::scalar(64)) 807 AndOpc = X86::AND64ri8; 808 else 809 return false; 810 811 Register DefReg = SrcReg; 812 if (DstTy != LLT::scalar(8)) { 813 Register ImpDefReg = 814 MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); 815 BuildMI(*I.getParent(), I, I.getDebugLoc(), 816 TII.get(TargetOpcode::IMPLICIT_DEF), ImpDefReg); 817 818 DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); 819 BuildMI(*I.getParent(), I, I.getDebugLoc(), 820 TII.get(TargetOpcode::INSERT_SUBREG), DefReg) 821 .addReg(ImpDefReg) 822 .addReg(SrcReg) 823 .addImm(X86::sub_8bit); 824 } 825 826 MachineInstr &AndInst = 827 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) 828 .addReg(DefReg) 829 .addImm(1); 830 831 constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); 832 833 I.eraseFromParent(); 834 return true; 835 } 836 837 bool X86InstructionSelector::selectAnyext(MachineInstr &I, 838 MachineRegisterInfo &MRI, 839 MachineFunction &MF) const { 840 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction"); 841 842 const Register DstReg = I.getOperand(0).getReg(); 843 const Register SrcReg = I.getOperand(1).getReg(); 844 845 const LLT DstTy = MRI.getType(DstReg); 846 const LLT SrcTy = MRI.getType(SrcReg); 847 848 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 849 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 850 851 assert(DstRB.getID() == SrcRB.getID() && 852 "G_ANYEXT input/output on different banks\n"); 853 854 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() && 855 "G_ANYEXT incorrect operand size"); 856 857 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 858 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 859 860 // If that's ANY_EXT of the value that lives on the floating class and goes 861 // into the vector class, just replace it with copy, as we are able to select 862 // it as a regular move. 863 if (canTurnIntoCOPY(SrcRC, DstRC)) 864 return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC); 865 866 if (DstRB.getID() != X86::GPRRegBankID) 867 return false; 868 869 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 870 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 871 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 872 << " operand\n"); 873 return false; 874 } 875 876 if (SrcRC == DstRC) { 877 I.setDesc(TII.get(X86::COPY)); 878 return true; 879 } 880 881 BuildMI(*I.getParent(), I, I.getDebugLoc(), 882 TII.get(TargetOpcode::SUBREG_TO_REG)) 883 .addDef(DstReg) 884 .addImm(0) 885 .addReg(SrcReg) 886 .addImm(getSubRegIndex(SrcRC)); 887 888 I.eraseFromParent(); 889 return true; 890 } 891 892 bool X86InstructionSelector::selectCmp(MachineInstr &I, 893 MachineRegisterInfo &MRI, 894 MachineFunction &MF) const { 895 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction"); 896 897 X86::CondCode CC; 898 bool SwapArgs; 899 std::tie(CC, SwapArgs) = X86::getX86ConditionCode( 900 (CmpInst::Predicate)I.getOperand(1).getPredicate()); 901 902 Register LHS = I.getOperand(2).getReg(); 903 Register RHS = I.getOperand(3).getReg(); 904 905 if (SwapArgs) 906 std::swap(LHS, RHS); 907 908 unsigned OpCmp; 909 LLT Ty = MRI.getType(LHS); 910 911 switch (Ty.getSizeInBits()) { 912 default: 913 return false; 914 case 8: 915 OpCmp = X86::CMP8rr; 916 break; 917 case 16: 918 OpCmp = X86::CMP16rr; 919 break; 920 case 32: 921 OpCmp = X86::CMP32rr; 922 break; 923 case 64: 924 OpCmp = X86::CMP64rr; 925 break; 926 } 927 928 MachineInstr &CmpInst = 929 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 930 .addReg(LHS) 931 .addReg(RHS); 932 933 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 934 TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC); 935 936 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 937 constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI); 938 939 I.eraseFromParent(); 940 return true; 941 } 942 943 bool X86InstructionSelector::selectFCmp(MachineInstr &I, 944 MachineRegisterInfo &MRI, 945 MachineFunction &MF) const { 946 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction"); 947 948 Register LhsReg = I.getOperand(2).getReg(); 949 Register RhsReg = I.getOperand(3).getReg(); 950 CmpInst::Predicate Predicate = 951 (CmpInst::Predicate)I.getOperand(1).getPredicate(); 952 953 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction. 954 static const uint16_t SETFOpcTable[2][3] = { 955 {X86::COND_E, X86::COND_NP, X86::AND8rr}, 956 {X86::COND_NE, X86::COND_P, X86::OR8rr}}; 957 const uint16_t *SETFOpc = nullptr; 958 switch (Predicate) { 959 default: 960 break; 961 case CmpInst::FCMP_OEQ: 962 SETFOpc = &SETFOpcTable[0][0]; 963 break; 964 case CmpInst::FCMP_UNE: 965 SETFOpc = &SETFOpcTable[1][0]; 966 break; 967 } 968 969 // Compute the opcode for the CMP instruction. 970 unsigned OpCmp; 971 LLT Ty = MRI.getType(LhsReg); 972 switch (Ty.getSizeInBits()) { 973 default: 974 return false; 975 case 32: 976 OpCmp = X86::UCOMISSrr; 977 break; 978 case 64: 979 OpCmp = X86::UCOMISDrr; 980 break; 981 } 982 983 Register ResultReg = I.getOperand(0).getReg(); 984 RBI.constrainGenericRegister( 985 ResultReg, 986 *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI); 987 if (SETFOpc) { 988 MachineInstr &CmpInst = 989 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 990 .addReg(LhsReg) 991 .addReg(RhsReg); 992 993 Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass); 994 Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass); 995 MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 996 TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]); 997 MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 998 TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]); 999 MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1000 TII.get(SETFOpc[2]), ResultReg) 1001 .addReg(FlagReg1) 1002 .addReg(FlagReg2); 1003 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 1004 constrainSelectedInstRegOperands(Set1, TII, TRI, RBI); 1005 constrainSelectedInstRegOperands(Set2, TII, TRI, RBI); 1006 constrainSelectedInstRegOperands(Set3, TII, TRI, RBI); 1007 1008 I.eraseFromParent(); 1009 return true; 1010 } 1011 1012 X86::CondCode CC; 1013 bool SwapArgs; 1014 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate); 1015 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code."); 1016 1017 if (SwapArgs) 1018 std::swap(LhsReg, RhsReg); 1019 1020 // Emit a compare of LHS/RHS. 1021 MachineInstr &CmpInst = 1022 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 1023 .addReg(LhsReg) 1024 .addReg(RhsReg); 1025 1026 MachineInstr &Set = 1027 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC); 1028 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 1029 constrainSelectedInstRegOperands(Set, TII, TRI, RBI); 1030 I.eraseFromParent(); 1031 return true; 1032 } 1033 1034 bool X86InstructionSelector::selectUadde(MachineInstr &I, 1035 MachineRegisterInfo &MRI, 1036 MachineFunction &MF) const { 1037 assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction"); 1038 1039 const Register DstReg = I.getOperand(0).getReg(); 1040 const Register CarryOutReg = I.getOperand(1).getReg(); 1041 const Register Op0Reg = I.getOperand(2).getReg(); 1042 const Register Op1Reg = I.getOperand(3).getReg(); 1043 Register CarryInReg = I.getOperand(4).getReg(); 1044 1045 const LLT DstTy = MRI.getType(DstReg); 1046 1047 if (DstTy != LLT::scalar(32)) 1048 return false; 1049 1050 // find CarryIn def instruction. 1051 MachineInstr *Def = MRI.getVRegDef(CarryInReg); 1052 while (Def->getOpcode() == TargetOpcode::G_TRUNC) { 1053 CarryInReg = Def->getOperand(1).getReg(); 1054 Def = MRI.getVRegDef(CarryInReg); 1055 } 1056 1057 unsigned Opcode; 1058 if (Def->getOpcode() == TargetOpcode::G_UADDE) { 1059 // carry set by prev ADD. 1060 1061 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS) 1062 .addReg(CarryInReg); 1063 1064 if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI)) 1065 return false; 1066 1067 Opcode = X86::ADC32rr; 1068 } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) { 1069 // carry is constant, support only 0. 1070 if (*val != 0) 1071 return false; 1072 1073 Opcode = X86::ADD32rr; 1074 } else 1075 return false; 1076 1077 MachineInstr &AddInst = 1078 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg) 1079 .addReg(Op0Reg) 1080 .addReg(Op1Reg); 1081 1082 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg) 1083 .addReg(X86::EFLAGS); 1084 1085 if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) || 1086 !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI)) 1087 return false; 1088 1089 I.eraseFromParent(); 1090 return true; 1091 } 1092 1093 bool X86InstructionSelector::selectExtract(MachineInstr &I, 1094 MachineRegisterInfo &MRI, 1095 MachineFunction &MF) const { 1096 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) && 1097 "unexpected instruction"); 1098 1099 const Register DstReg = I.getOperand(0).getReg(); 1100 const Register SrcReg = I.getOperand(1).getReg(); 1101 int64_t Index = I.getOperand(2).getImm(); 1102 1103 const LLT DstTy = MRI.getType(DstReg); 1104 const LLT SrcTy = MRI.getType(SrcReg); 1105 1106 // Meanwile handle vector type only. 1107 if (!DstTy.isVector()) 1108 return false; 1109 1110 if (Index % DstTy.getSizeInBits() != 0) 1111 return false; // Not extract subvector. 1112 1113 if (Index == 0) { 1114 // Replace by extract subreg copy. 1115 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF)) 1116 return false; 1117 1118 I.eraseFromParent(); 1119 return true; 1120 } 1121 1122 bool HasAVX = STI.hasAVX(); 1123 bool HasAVX512 = STI.hasAVX512(); 1124 bool HasVLX = STI.hasVLX(); 1125 1126 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) { 1127 if (HasVLX) 1128 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr)); 1129 else if (HasAVX) 1130 I.setDesc(TII.get(X86::VEXTRACTF128rr)); 1131 else 1132 return false; 1133 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) { 1134 if (DstTy.getSizeInBits() == 128) 1135 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr)); 1136 else if (DstTy.getSizeInBits() == 256) 1137 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr)); 1138 else 1139 return false; 1140 } else 1141 return false; 1142 1143 // Convert to X86 VEXTRACT immediate. 1144 Index = Index / DstTy.getSizeInBits(); 1145 I.getOperand(2).setImm(Index); 1146 1147 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 1148 } 1149 1150 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg, 1151 MachineInstr &I, 1152 MachineRegisterInfo &MRI, 1153 MachineFunction &MF) const { 1154 const LLT DstTy = MRI.getType(DstReg); 1155 const LLT SrcTy = MRI.getType(SrcReg); 1156 unsigned SubIdx = X86::NoSubRegister; 1157 1158 if (!DstTy.isVector() || !SrcTy.isVector()) 1159 return false; 1160 1161 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() && 1162 "Incorrect Src/Dst register size"); 1163 1164 if (DstTy.getSizeInBits() == 128) 1165 SubIdx = X86::sub_xmm; 1166 else if (DstTy.getSizeInBits() == 256) 1167 SubIdx = X86::sub_ymm; 1168 else 1169 return false; 1170 1171 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 1172 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 1173 1174 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 1175 1176 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 1177 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 1178 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n"); 1179 return false; 1180 } 1181 1182 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg) 1183 .addReg(SrcReg, 0, SubIdx); 1184 1185 return true; 1186 } 1187 1188 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg, 1189 MachineInstr &I, 1190 MachineRegisterInfo &MRI, 1191 MachineFunction &MF) const { 1192 const LLT DstTy = MRI.getType(DstReg); 1193 const LLT SrcTy = MRI.getType(SrcReg); 1194 unsigned SubIdx = X86::NoSubRegister; 1195 1196 // TODO: support scalar types 1197 if (!DstTy.isVector() || !SrcTy.isVector()) 1198 return false; 1199 1200 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() && 1201 "Incorrect Src/Dst register size"); 1202 1203 if (SrcTy.getSizeInBits() == 128) 1204 SubIdx = X86::sub_xmm; 1205 else if (SrcTy.getSizeInBits() == 256) 1206 SubIdx = X86::sub_ymm; 1207 else 1208 return false; 1209 1210 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 1211 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 1212 1213 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 1214 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 1215 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n"); 1216 return false; 1217 } 1218 1219 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY)) 1220 .addReg(DstReg, RegState::DefineNoRead, SubIdx) 1221 .addReg(SrcReg); 1222 1223 return true; 1224 } 1225 1226 bool X86InstructionSelector::selectInsert(MachineInstr &I, 1227 MachineRegisterInfo &MRI, 1228 MachineFunction &MF) const { 1229 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction"); 1230 1231 const Register DstReg = I.getOperand(0).getReg(); 1232 const Register SrcReg = I.getOperand(1).getReg(); 1233 const Register InsertReg = I.getOperand(2).getReg(); 1234 int64_t Index = I.getOperand(3).getImm(); 1235 1236 const LLT DstTy = MRI.getType(DstReg); 1237 const LLT InsertRegTy = MRI.getType(InsertReg); 1238 1239 // Meanwile handle vector type only. 1240 if (!DstTy.isVector()) 1241 return false; 1242 1243 if (Index % InsertRegTy.getSizeInBits() != 0) 1244 return false; // Not insert subvector. 1245 1246 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) { 1247 // Replace by subreg copy. 1248 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF)) 1249 return false; 1250 1251 I.eraseFromParent(); 1252 return true; 1253 } 1254 1255 bool HasAVX = STI.hasAVX(); 1256 bool HasAVX512 = STI.hasAVX512(); 1257 bool HasVLX = STI.hasVLX(); 1258 1259 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) { 1260 if (HasVLX) 1261 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr)); 1262 else if (HasAVX) 1263 I.setDesc(TII.get(X86::VINSERTF128rr)); 1264 else 1265 return false; 1266 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) { 1267 if (InsertRegTy.getSizeInBits() == 128) 1268 I.setDesc(TII.get(X86::VINSERTF32x4Zrr)); 1269 else if (InsertRegTy.getSizeInBits() == 256) 1270 I.setDesc(TII.get(X86::VINSERTF64x4Zrr)); 1271 else 1272 return false; 1273 } else 1274 return false; 1275 1276 // Convert to X86 VINSERT immediate. 1277 Index = Index / InsertRegTy.getSizeInBits(); 1278 1279 I.getOperand(3).setImm(Index); 1280 1281 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 1282 } 1283 1284 bool X86InstructionSelector::selectUnmergeValues( 1285 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { 1286 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) && 1287 "unexpected instruction"); 1288 1289 // Split to extracts. 1290 unsigned NumDefs = I.getNumOperands() - 1; 1291 Register SrcReg = I.getOperand(NumDefs).getReg(); 1292 unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); 1293 1294 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { 1295 MachineInstr &ExtrInst = 1296 *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1297 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg()) 1298 .addReg(SrcReg) 1299 .addImm(Idx * DefSize); 1300 1301 if (!select(ExtrInst)) 1302 return false; 1303 } 1304 1305 I.eraseFromParent(); 1306 return true; 1307 } 1308 1309 bool X86InstructionSelector::selectMergeValues( 1310 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { 1311 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES || 1312 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) && 1313 "unexpected instruction"); 1314 1315 // Split to inserts. 1316 Register DstReg = I.getOperand(0).getReg(); 1317 Register SrcReg0 = I.getOperand(1).getReg(); 1318 1319 const LLT DstTy = MRI.getType(DstReg); 1320 const LLT SrcTy = MRI.getType(SrcReg0); 1321 unsigned SrcSize = SrcTy.getSizeInBits(); 1322 1323 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 1324 1325 // For the first src use insertSubReg. 1326 Register DefReg = MRI.createGenericVirtualRegister(DstTy); 1327 MRI.setRegBank(DefReg, RegBank); 1328 if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF)) 1329 return false; 1330 1331 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) { 1332 Register Tmp = MRI.createGenericVirtualRegister(DstTy); 1333 MRI.setRegBank(Tmp, RegBank); 1334 1335 MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1336 TII.get(TargetOpcode::G_INSERT), Tmp) 1337 .addReg(DefReg) 1338 .addReg(I.getOperand(Idx).getReg()) 1339 .addImm((Idx - 1) * SrcSize); 1340 1341 DefReg = Tmp; 1342 1343 if (!select(InsertInst)) 1344 return false; 1345 } 1346 1347 MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1348 TII.get(TargetOpcode::COPY), DstReg) 1349 .addReg(DefReg); 1350 1351 if (!select(CopyInst)) 1352 return false; 1353 1354 I.eraseFromParent(); 1355 return true; 1356 } 1357 1358 bool X86InstructionSelector::selectCondBranch(MachineInstr &I, 1359 MachineRegisterInfo &MRI, 1360 MachineFunction &MF) const { 1361 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction"); 1362 1363 const Register CondReg = I.getOperand(0).getReg(); 1364 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); 1365 1366 MachineInstr &TestInst = 1367 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri)) 1368 .addReg(CondReg) 1369 .addImm(1); 1370 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1)) 1371 .addMBB(DestMBB).addImm(X86::COND_NE); 1372 1373 constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI); 1374 1375 I.eraseFromParent(); 1376 return true; 1377 } 1378 1379 bool X86InstructionSelector::materializeFP(MachineInstr &I, 1380 MachineRegisterInfo &MRI, 1381 MachineFunction &MF) const { 1382 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) && 1383 "unexpected instruction"); 1384 1385 // Can't handle alternate code models yet. 1386 CodeModel::Model CM = TM.getCodeModel(); 1387 if (CM != CodeModel::Small && CM != CodeModel::Large) 1388 return false; 1389 1390 const Register DstReg = I.getOperand(0).getReg(); 1391 const LLT DstTy = MRI.getType(DstReg); 1392 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 1393 Align Alignment = Align(DstTy.getSizeInBytes()); 1394 const DebugLoc &DbgLoc = I.getDebugLoc(); 1395 1396 unsigned Opc = 1397 getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment); 1398 1399 // Create the load from the constant pool. 1400 const ConstantFP *CFP = I.getOperand(1).getFPImm(); 1401 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment); 1402 MachineInstr *LoadInst = nullptr; 1403 unsigned char OpFlag = STI.classifyLocalReference(nullptr); 1404 1405 if (CM == CodeModel::Large && STI.is64Bit()) { 1406 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 1407 // they cannot be folded into immediate fields. 1408 1409 Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass); 1410 BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg) 1411 .addConstantPoolIndex(CPI, 0, OpFlag); 1412 1413 MachineMemOperand *MMO = MF.getMachineMemOperand( 1414 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, 1415 MF.getDataLayout().getPointerSize(), Alignment); 1416 1417 LoadInst = 1418 addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), 1419 AddrReg) 1420 .addMemOperand(MMO); 1421 1422 } else if (CM == CodeModel::Small || !STI.is64Bit()) { 1423 // Handle the case when globals fit in our immediate field. 1424 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode. 1425 1426 // x86-32 PIC requires a PIC base register for constant pools. 1427 unsigned PICBase = 0; 1428 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) { 1429 // PICBase can be allocated by TII.getGlobalBaseReg(&MF). 1430 // In DAGISEL the code that initialize it generated by the CGBR pass. 1431 return false; // TODO support the mode. 1432 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small) 1433 PICBase = X86::RIP; 1434 1435 LoadInst = addConstantPoolReference( 1436 BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase, 1437 OpFlag); 1438 } else 1439 return false; 1440 1441 constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI); 1442 I.eraseFromParent(); 1443 return true; 1444 } 1445 1446 bool X86InstructionSelector::selectImplicitDefOrPHI( 1447 MachineInstr &I, MachineRegisterInfo &MRI) const { 1448 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF || 1449 I.getOpcode() == TargetOpcode::G_PHI) && 1450 "unexpected instruction"); 1451 1452 Register DstReg = I.getOperand(0).getReg(); 1453 1454 if (!MRI.getRegClassOrNull(DstReg)) { 1455 const LLT DstTy = MRI.getType(DstReg); 1456 const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI); 1457 1458 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { 1459 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 1460 << " operand\n"); 1461 return false; 1462 } 1463 } 1464 1465 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1466 I.setDesc(TII.get(X86::IMPLICIT_DEF)); 1467 else 1468 I.setDesc(TII.get(X86::PHI)); 1469 1470 return true; 1471 } 1472 1473 bool X86InstructionSelector::selectDivRem(MachineInstr &I, 1474 MachineRegisterInfo &MRI, 1475 MachineFunction &MF) const { 1476 // The implementation of this function is taken from X86FastISel. 1477 assert((I.getOpcode() == TargetOpcode::G_SDIV || 1478 I.getOpcode() == TargetOpcode::G_SREM || 1479 I.getOpcode() == TargetOpcode::G_UDIV || 1480 I.getOpcode() == TargetOpcode::G_UREM) && 1481 "unexpected instruction"); 1482 1483 const Register DstReg = I.getOperand(0).getReg(); 1484 const Register Op1Reg = I.getOperand(1).getReg(); 1485 const Register Op2Reg = I.getOperand(2).getReg(); 1486 1487 const LLT RegTy = MRI.getType(DstReg); 1488 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) && 1489 "Arguments and return value types must match"); 1490 1491 const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI); 1492 if (!RegRB || RegRB->getID() != X86::GPRRegBankID) 1493 return false; 1494 1495 const static unsigned NumTypes = 4; // i8, i16, i32, i64 1496 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem 1497 const static bool S = true; // IsSigned 1498 const static bool U = false; // !IsSigned 1499 const static unsigned Copy = TargetOpcode::COPY; 1500 // For the X86 IDIV instruction, in most cases the dividend 1501 // (numerator) must be in a specific register pair highreg:lowreg, 1502 // producing the quotient in lowreg and the remainder in highreg. 1503 // For most data types, to set up the instruction, the dividend is 1504 // copied into lowreg, and lowreg is sign-extended into highreg. The 1505 // exception is i8, where the dividend is defined as a single register rather 1506 // than a register pair, and we therefore directly sign-extend the dividend 1507 // into lowreg, instead of copying, and ignore the highreg. 1508 const static struct DivRemEntry { 1509 // The following portion depends only on the data type. 1510 unsigned SizeInBits; 1511 unsigned LowInReg; // low part of the register pair 1512 unsigned HighInReg; // high part of the register pair 1513 // The following portion depends on both the data type and the operation. 1514 struct DivRemResult { 1515 unsigned OpDivRem; // The specific DIV/IDIV opcode to use. 1516 unsigned OpSignExtend; // Opcode for sign-extending lowreg into 1517 // highreg, or copying a zero into highreg. 1518 unsigned OpCopy; // Opcode for copying dividend into lowreg, or 1519 // zero/sign-extending into lowreg for i8. 1520 unsigned DivRemResultReg; // Register containing the desired result. 1521 bool IsOpSigned; // Whether to use signed or unsigned form. 1522 } ResultTable[NumOps]; 1523 } OpTable[NumTypes] = { 1524 {8, 1525 X86::AX, 1526 0, 1527 { 1528 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv 1529 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem 1530 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U}, // UDiv 1531 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U}, // URem 1532 }}, // i8 1533 {16, 1534 X86::AX, 1535 X86::DX, 1536 { 1537 {X86::IDIV16r, X86::CWD, Copy, X86::AX, S}, // SDiv 1538 {X86::IDIV16r, X86::CWD, Copy, X86::DX, S}, // SRem 1539 {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv 1540 {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem 1541 }}, // i16 1542 {32, 1543 X86::EAX, 1544 X86::EDX, 1545 { 1546 {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S}, // SDiv 1547 {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S}, // SRem 1548 {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv 1549 {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem 1550 }}, // i32 1551 {64, 1552 X86::RAX, 1553 X86::RDX, 1554 { 1555 {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S}, // SDiv 1556 {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S}, // SRem 1557 {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv 1558 {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem 1559 }}, // i64 1560 }; 1561 1562 auto OpEntryIt = llvm::find_if(OpTable, [RegTy](const DivRemEntry &El) { 1563 return El.SizeInBits == RegTy.getSizeInBits(); 1564 }); 1565 if (OpEntryIt == std::end(OpTable)) 1566 return false; 1567 1568 unsigned OpIndex; 1569 switch (I.getOpcode()) { 1570 default: 1571 llvm_unreachable("Unexpected div/rem opcode"); 1572 case TargetOpcode::G_SDIV: 1573 OpIndex = 0; 1574 break; 1575 case TargetOpcode::G_SREM: 1576 OpIndex = 1; 1577 break; 1578 case TargetOpcode::G_UDIV: 1579 OpIndex = 2; 1580 break; 1581 case TargetOpcode::G_UREM: 1582 OpIndex = 3; 1583 break; 1584 } 1585 1586 const DivRemEntry &TypeEntry = *OpEntryIt; 1587 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex]; 1588 1589 const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB); 1590 if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) || 1591 !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) || 1592 !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) { 1593 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 1594 << " operand\n"); 1595 return false; 1596 } 1597 1598 // Move op1 into low-order input register. 1599 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy), 1600 TypeEntry.LowInReg) 1601 .addReg(Op1Reg); 1602 // Zero-extend or sign-extend into high-order input register. 1603 if (OpEntry.OpSignExtend) { 1604 if (OpEntry.IsOpSigned) 1605 BuildMI(*I.getParent(), I, I.getDebugLoc(), 1606 TII.get(OpEntry.OpSignExtend)); 1607 else { 1608 Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass); 1609 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0), 1610 Zero32); 1611 1612 // Copy the zero into the appropriate sub/super/identical physical 1613 // register. Unfortunately the operations needed are not uniform enough 1614 // to fit neatly into the table above. 1615 if (RegTy.getSizeInBits() == 16) { 1616 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), 1617 TypeEntry.HighInReg) 1618 .addReg(Zero32, 0, X86::sub_16bit); 1619 } else if (RegTy.getSizeInBits() == 32) { 1620 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), 1621 TypeEntry.HighInReg) 1622 .addReg(Zero32); 1623 } else if (RegTy.getSizeInBits() == 64) { 1624 BuildMI(*I.getParent(), I, I.getDebugLoc(), 1625 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg) 1626 .addImm(0) 1627 .addReg(Zero32) 1628 .addImm(X86::sub_32bit); 1629 } 1630 } 1631 } 1632 // Generate the DIV/IDIV instruction. 1633 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem)) 1634 .addReg(Op2Reg); 1635 // For i8 remainder, we can't reference ah directly, as we'll end 1636 // up with bogus copies like %r9b = COPY %ah. Reference ax 1637 // instead to prevent ah references in a rex instruction. 1638 // 1639 // The current assumption of the fast register allocator is that isel 1640 // won't generate explicit references to the GR8_NOREX registers. If 1641 // the allocator and/or the backend get enhanced to be more robust in 1642 // that regard, this can be, and should be, removed. 1643 if ((I.getOpcode() == Instruction::SRem || 1644 I.getOpcode() == Instruction::URem) && 1645 OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) { 1646 Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass); 1647 Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass); 1648 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg) 1649 .addReg(X86::AX); 1650 1651 // Shift AX right by 8 bits instead of using AH. 1652 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri), 1653 ResultSuperReg) 1654 .addReg(SourceSuperReg) 1655 .addImm(8); 1656 1657 // Now reference the 8-bit subreg of the result. 1658 BuildMI(*I.getParent(), I, I.getDebugLoc(), 1659 TII.get(TargetOpcode::SUBREG_TO_REG)) 1660 .addDef(DstReg) 1661 .addImm(0) 1662 .addReg(ResultSuperReg) 1663 .addImm(X86::sub_8bit); 1664 } else { 1665 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), 1666 DstReg) 1667 .addReg(OpEntry.DivRemResultReg); 1668 } 1669 I.eraseFromParent(); 1670 return true; 1671 } 1672 1673 bool X86InstructionSelector::selectIntrinsicWSideEffects( 1674 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { 1675 1676 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && 1677 "unexpected instruction"); 1678 1679 if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) 1680 return false; 1681 1682 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP)); 1683 1684 I.eraseFromParent(); 1685 return true; 1686 } 1687 1688 InstructionSelector * 1689 llvm::createX86InstructionSelector(const X86TargetMachine &TM, 1690 X86Subtarget &Subtarget, 1691 X86RegisterBankInfo &RBI) { 1692 return new X86InstructionSelector(TM, Subtarget, RBI); 1693 } 1694
Indexes created Sun Jun 14 00:25:39 UTC 2026