1 //===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Custom DAG lowering for SI 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIISelLowering.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "AMDGPUTargetMachine.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "SIRegisterInfo.h" 20 #include "llvm/ADT/Statistic.h" 21 #include "llvm/Analysis/LegacyDivergenceAnalysis.h" 22 #include "llvm/BinaryFormat/ELF.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/FunctionLoweringInfo.h" 25 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" 26 #include "llvm/CodeGen/MachineLoopInfo.h" 27 #include "llvm/IR/DiagnosticInfo.h" 28 #include "llvm/IR/IntrinsicsAMDGPU.h" 29 #include "llvm/IR/IntrinsicsR600.h" 30 #include "llvm/Support/CommandLine.h" 31 #include "llvm/Support/KnownBits.h" 32 33 using namespace llvm; 34 35 #define DEBUG_TYPE "si-lower" 36 37 STATISTIC(NumTailCalls, "Number of tail calls"); 38 39 static cl::opt<bool> DisableLoopAlignment( 40 "amdgpu-disable-loop-alignment", 41 cl::desc("Do not align and prefetch loops"), 42 cl::init(false)); 43 44 static cl::opt<bool> VGPRReserveforSGPRSpill( 45 "amdgpu-reserve-vgpr-for-sgpr-spill", 46 cl::desc("Allocates one VGPR for future SGPR Spill"), cl::init(true)); 47 48 static cl::opt<bool> UseDivergentRegisterIndexing( 49 "amdgpu-use-divergent-register-indexing", 50 cl::Hidden, 51 cl::desc("Use indirect register addressing for divergent indexes"), 52 cl::init(false)); 53 54 static bool hasFP32Denormals(const MachineFunction &MF) { 55 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 56 return Info->getMode().allFP32Denormals(); 57 } 58 59 static bool hasFP64FP16Denormals(const MachineFunction &MF) { 60 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 61 return Info->getMode().allFP64FP16Denormals(); 62 } 63 64 static unsigned findFirstFreeSGPR(CCState &CCInfo) { 65 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 66 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) { 67 if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) { 68 return AMDGPU::SGPR0 + Reg; 69 } 70 } 71 llvm_unreachable("Cannot allocate sgpr"); 72 } 73 74 SITargetLowering::SITargetLowering(const TargetMachine &TM, 75 const GCNSubtarget &STI) 76 : AMDGPUTargetLowering(TM, STI), 77 Subtarget(&STI) { 78 addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); 79 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); 80 81 addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass); 82 addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); 83 84 addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass); 85 86 const SIRegisterInfo *TRI = STI.getRegisterInfo(); 87 const TargetRegisterClass *V64RegClass = TRI->getVGPR64Class(); 88 89 addRegisterClass(MVT::f64, V64RegClass); 90 addRegisterClass(MVT::v2f32, V64RegClass); 91 92 addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); 93 addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); 94 95 addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass); 96 addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass); 97 98 addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass); 99 addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128)); 100 101 addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass); 102 addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160)); 103 104 addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass); 105 addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256)); 106 107 addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass); 108 addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256)); 109 110 addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass); 111 addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512)); 112 113 addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass); 114 addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512)); 115 116 addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass); 117 addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024)); 118 119 if (Subtarget->has16BitInsts()) { 120 addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass); 121 addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass); 122 123 // Unless there are also VOP3P operations, not operations are really legal. 124 addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass); 125 addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass); 126 addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); 127 addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); 128 } 129 130 addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass); 131 addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024)); 132 133 computeRegisterProperties(Subtarget->getRegisterInfo()); 134 135 // The boolean content concept here is too inflexible. Compares only ever 136 // really produce a 1-bit result. Any copy/extend from these will turn into a 137 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as 138 // it's what most targets use. 139 setBooleanContents(ZeroOrOneBooleanContent); 140 setBooleanVectorContents(ZeroOrOneBooleanContent); 141 142 // We need to custom lower vector stores from local memory 143 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); 144 setOperationAction(ISD::LOAD, MVT::v3i32, Custom); 145 setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 146 setOperationAction(ISD::LOAD, MVT::v5i32, Custom); 147 setOperationAction(ISD::LOAD, MVT::v8i32, Custom); 148 setOperationAction(ISD::LOAD, MVT::v16i32, Custom); 149 setOperationAction(ISD::LOAD, MVT::i1, Custom); 150 setOperationAction(ISD::LOAD, MVT::v32i32, Custom); 151 152 setOperationAction(ISD::STORE, MVT::v2i32, Custom); 153 setOperationAction(ISD::STORE, MVT::v3i32, Custom); 154 setOperationAction(ISD::STORE, MVT::v4i32, Custom); 155 setOperationAction(ISD::STORE, MVT::v5i32, Custom); 156 setOperationAction(ISD::STORE, MVT::v8i32, Custom); 157 setOperationAction(ISD::STORE, MVT::v16i32, Custom); 158 setOperationAction(ISD::STORE, MVT::i1, Custom); 159 setOperationAction(ISD::STORE, MVT::v32i32, Custom); 160 161 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); 162 setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand); 163 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); 164 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand); 165 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand); 166 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand); 167 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand); 168 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand); 169 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand); 170 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand); 171 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand); 172 setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand); 173 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand); 174 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); 175 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand); 176 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand); 177 178 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand); 179 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand); 180 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand); 181 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand); 182 setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand); 183 184 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 185 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 186 187 setOperationAction(ISD::SELECT, MVT::i1, Promote); 188 setOperationAction(ISD::SELECT, MVT::i64, Custom); 189 setOperationAction(ISD::SELECT, MVT::f64, Promote); 190 AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64); 191 192 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); 193 setOperationAction(ISD::SELECT_CC, MVT::i32, Expand); 194 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); 195 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); 196 setOperationAction(ISD::SELECT_CC, MVT::i1, Expand); 197 198 setOperationAction(ISD::SETCC, MVT::i1, Promote); 199 setOperationAction(ISD::SETCC, MVT::v2i1, Expand); 200 setOperationAction(ISD::SETCC, MVT::v4i1, Expand); 201 AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32); 202 203 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Expand); 204 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); 205 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Expand); 206 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Expand); 207 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Expand); 208 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Expand); 209 setOperationAction(ISD::TRUNCATE, MVT::v16i32, Expand); 210 setOperationAction(ISD::FP_ROUND, MVT::v16f32, Expand); 211 212 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Custom); 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Custom); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Custom); 216 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom); 217 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v3i16, Custom); 218 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Custom); 219 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom); 220 221 setOperationAction(ISD::BRCOND, MVT::Other, Custom); 222 setOperationAction(ISD::BR_CC, MVT::i1, Expand); 223 setOperationAction(ISD::BR_CC, MVT::i32, Expand); 224 setOperationAction(ISD::BR_CC, MVT::i64, Expand); 225 setOperationAction(ISD::BR_CC, MVT::f32, Expand); 226 setOperationAction(ISD::BR_CC, MVT::f64, Expand); 227 228 setOperationAction(ISD::UADDO, MVT::i32, Legal); 229 setOperationAction(ISD::USUBO, MVT::i32, Legal); 230 231 setOperationAction(ISD::ADDCARRY, MVT::i32, Legal); 232 setOperationAction(ISD::SUBCARRY, MVT::i32, Legal); 233 234 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 235 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 236 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 237 238 #if 0 239 setOperationAction(ISD::ADDCARRY, MVT::i64, Legal); 240 setOperationAction(ISD::SUBCARRY, MVT::i64, Legal); 241 #endif 242 243 // We only support LOAD/STORE and vector manipulation ops for vectors 244 // with > 4 elements. 245 for (MVT VT : { MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32, 246 MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16, 247 MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, 248 MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32 }) { 249 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) { 250 switch (Op) { 251 case ISD::LOAD: 252 case ISD::STORE: 253 case ISD::BUILD_VECTOR: 254 case ISD::BITCAST: 255 case ISD::EXTRACT_VECTOR_ELT: 256 case ISD::INSERT_VECTOR_ELT: 257 case ISD::INSERT_SUBVECTOR: 258 case ISD::EXTRACT_SUBVECTOR: 259 case ISD::SCALAR_TO_VECTOR: 260 break; 261 case ISD::CONCAT_VECTORS: 262 setOperationAction(Op, VT, Custom); 263 break; 264 default: 265 setOperationAction(Op, VT, Expand); 266 break; 267 } 268 } 269 } 270 271 setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand); 272 273 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that 274 // is expanded to avoid having two separate loops in case the index is a VGPR. 275 276 // Most operations are naturally 32-bit vector operations. We only support 277 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32. 278 for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) { 279 setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); 280 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32); 281 282 setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); 283 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32); 284 285 setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); 286 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32); 287 288 setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); 289 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32); 290 } 291 292 for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) { 293 setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); 294 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32); 295 296 setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); 297 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32); 298 299 setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); 300 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32); 301 302 setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); 303 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32); 304 } 305 306 for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) { 307 setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); 308 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32); 309 310 setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); 311 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32); 312 313 setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); 314 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32); 315 316 setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); 317 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32); 318 } 319 320 for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) { 321 setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); 322 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32); 323 324 setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); 325 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32); 326 327 setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); 328 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32); 329 330 setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); 331 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32); 332 } 333 334 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i32, Expand); 335 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Expand); 336 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i32, Expand); 337 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16f32, Expand); 338 339 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f16, Custom); 340 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 341 342 // Avoid stack access for these. 343 // TODO: Generalize to more vector types. 344 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Custom); 345 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Custom); 346 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 347 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom); 348 349 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); 350 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); 351 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i8, Custom); 352 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom); 353 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i8, Custom); 354 355 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i8, Custom); 356 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom); 357 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i8, Custom); 358 359 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i16, Custom); 360 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f16, Custom); 361 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 362 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom); 363 364 // Deal with vec3 vector operations when widened to vec4. 365 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3i32, Custom); 366 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3f32, Custom); 367 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i32, Custom); 368 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4f32, Custom); 369 370 // Deal with vec5 vector operations when widened to vec8. 371 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5i32, Custom); 372 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5f32, Custom); 373 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i32, Custom); 374 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8f32, Custom); 375 376 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, 377 // and output demarshalling 378 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 379 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); 380 381 // We can't return success/failure, only the old value, 382 // let LLVM add the comparison 383 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand); 384 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Expand); 385 386 if (Subtarget->hasFlatAddressSpace()) { 387 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom); 388 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom); 389 } 390 391 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 392 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); 393 394 // FIXME: This should be narrowed to i32, but that only happens if i64 is 395 // illegal. 396 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32. 397 setOperationAction(ISD::BSWAP, MVT::i64, Legal); 398 setOperationAction(ISD::BSWAP, MVT::i32, Legal); 399 400 // On SI this is s_memtime and s_memrealtime on VI. 401 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); 402 setOperationAction(ISD::TRAP, MVT::Other, Custom); 403 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom); 404 405 if (Subtarget->has16BitInsts()) { 406 setOperationAction(ISD::FPOW, MVT::f16, Promote); 407 setOperationAction(ISD::FPOWI, MVT::f16, Promote); 408 setOperationAction(ISD::FLOG, MVT::f16, Custom); 409 setOperationAction(ISD::FEXP, MVT::f16, Custom); 410 setOperationAction(ISD::FLOG10, MVT::f16, Custom); 411 } 412 413 if (Subtarget->hasMadMacF32Insts()) 414 setOperationAction(ISD::FMAD, MVT::f32, Legal); 415 416 if (!Subtarget->hasBFI()) { 417 // fcopysign can be done in a single instruction with BFI. 418 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 419 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 420 } 421 422 if (!Subtarget->hasBCNT(32)) 423 setOperationAction(ISD::CTPOP, MVT::i32, Expand); 424 425 if (!Subtarget->hasBCNT(64)) 426 setOperationAction(ISD::CTPOP, MVT::i64, Expand); 427 428 if (Subtarget->hasFFBH()) 429 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom); 430 431 if (Subtarget->hasFFBL()) 432 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom); 433 434 // We only really have 32-bit BFE instructions (and 16-bit on VI). 435 // 436 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any 437 // effort to match them now. We want this to be false for i64 cases when the 438 // extraction isn't restricted to the upper or lower half. Ideally we would 439 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that 440 // span the midpoint are probably relatively rare, so don't worry about them 441 // for now. 442 if (Subtarget->hasBFE()) 443 setHasExtractBitsInsn(true); 444 445 // Clamp modifier on add/sub 446 if (Subtarget->hasIntClamp()) { 447 setOperationAction(ISD::UADDSAT, MVT::i32, Legal); 448 setOperationAction(ISD::USUBSAT, MVT::i32, Legal); 449 } 450 451 if (Subtarget->hasAddNoCarry()) { 452 setOperationAction(ISD::SADDSAT, MVT::i16, Legal); 453 setOperationAction(ISD::SSUBSAT, MVT::i16, Legal); 454 setOperationAction(ISD::SADDSAT, MVT::i32, Legal); 455 setOperationAction(ISD::SSUBSAT, MVT::i32, Legal); 456 } 457 458 setOperationAction(ISD::FMINNUM, MVT::f32, Custom); 459 setOperationAction(ISD::FMAXNUM, MVT::f32, Custom); 460 setOperationAction(ISD::FMINNUM, MVT::f64, Custom); 461 setOperationAction(ISD::FMAXNUM, MVT::f64, Custom); 462 463 464 // These are really only legal for ieee_mode functions. We should be avoiding 465 // them for functions that don't have ieee_mode enabled, so just say they are 466 // legal. 467 setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); 468 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); 469 setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); 470 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); 471 472 473 if (Subtarget->haveRoundOpsF64()) { 474 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 475 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 476 setOperationAction(ISD::FRINT, MVT::f64, Legal); 477 } else { 478 setOperationAction(ISD::FCEIL, MVT::f64, Custom); 479 setOperationAction(ISD::FTRUNC, MVT::f64, Custom); 480 setOperationAction(ISD::FRINT, MVT::f64, Custom); 481 setOperationAction(ISD::FFLOOR, MVT::f64, Custom); 482 } 483 484 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 485 486 setOperationAction(ISD::FSIN, MVT::f32, Custom); 487 setOperationAction(ISD::FCOS, MVT::f32, Custom); 488 setOperationAction(ISD::FDIV, MVT::f32, Custom); 489 setOperationAction(ISD::FDIV, MVT::f64, Custom); 490 491 if (Subtarget->has16BitInsts()) { 492 setOperationAction(ISD::Constant, MVT::i16, Legal); 493 494 setOperationAction(ISD::SMIN, MVT::i16, Legal); 495 setOperationAction(ISD::SMAX, MVT::i16, Legal); 496 497 setOperationAction(ISD::UMIN, MVT::i16, Legal); 498 setOperationAction(ISD::UMAX, MVT::i16, Legal); 499 500 setOperationAction(ISD::SIGN_EXTEND, MVT::i16, Promote); 501 AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32); 502 503 setOperationAction(ISD::ROTR, MVT::i16, Expand); 504 setOperationAction(ISD::ROTL, MVT::i16, Expand); 505 506 setOperationAction(ISD::SDIV, MVT::i16, Promote); 507 setOperationAction(ISD::UDIV, MVT::i16, Promote); 508 setOperationAction(ISD::SREM, MVT::i16, Promote); 509 setOperationAction(ISD::UREM, MVT::i16, Promote); 510 setOperationAction(ISD::UADDSAT, MVT::i16, Legal); 511 setOperationAction(ISD::USUBSAT, MVT::i16, Legal); 512 513 setOperationAction(ISD::BITREVERSE, MVT::i16, Promote); 514 515 setOperationAction(ISD::CTTZ, MVT::i16, Promote); 516 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote); 517 setOperationAction(ISD::CTLZ, MVT::i16, Promote); 518 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote); 519 setOperationAction(ISD::CTPOP, MVT::i16, Promote); 520 521 setOperationAction(ISD::SELECT_CC, MVT::i16, Expand); 522 523 setOperationAction(ISD::BR_CC, MVT::i16, Expand); 524 525 setOperationAction(ISD::LOAD, MVT::i16, Custom); 526 527 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 528 529 setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote); 530 AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32); 531 setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote); 532 AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32); 533 534 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom); 535 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Custom); 536 537 // F16 - Constant Actions. 538 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); 539 540 // F16 - Load/Store Actions. 541 setOperationAction(ISD::LOAD, MVT::f16, Promote); 542 AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16); 543 setOperationAction(ISD::STORE, MVT::f16, Promote); 544 AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16); 545 546 // F16 - VOP1 Actions. 547 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); 548 setOperationAction(ISD::FCOS, MVT::f16, Custom); 549 setOperationAction(ISD::FSIN, MVT::f16, Custom); 550 551 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom); 552 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Custom); 553 554 setOperationAction(ISD::FP_TO_SINT, MVT::f16, Promote); 555 setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote); 556 setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote); 557 setOperationAction(ISD::UINT_TO_FP, MVT::f16, Promote); 558 setOperationAction(ISD::FROUND, MVT::f16, Custom); 559 560 // F16 - VOP2 Actions. 561 setOperationAction(ISD::BR_CC, MVT::f16, Expand); 562 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); 563 564 setOperationAction(ISD::FDIV, MVT::f16, Custom); 565 566 // F16 - VOP3 Actions. 567 setOperationAction(ISD::FMA, MVT::f16, Legal); 568 if (STI.hasMadF16()) 569 setOperationAction(ISD::FMAD, MVT::f16, Legal); 570 571 for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) { 572 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) { 573 switch (Op) { 574 case ISD::LOAD: 575 case ISD::STORE: 576 case ISD::BUILD_VECTOR: 577 case ISD::BITCAST: 578 case ISD::EXTRACT_VECTOR_ELT: 579 case ISD::INSERT_VECTOR_ELT: 580 case ISD::INSERT_SUBVECTOR: 581 case ISD::EXTRACT_SUBVECTOR: 582 case ISD::SCALAR_TO_VECTOR: 583 break; 584 case ISD::CONCAT_VECTORS: 585 setOperationAction(Op, VT, Custom); 586 break; 587 default: 588 setOperationAction(Op, VT, Expand); 589 break; 590 } 591 } 592 } 593 594 // v_perm_b32 can handle either of these. 595 setOperationAction(ISD::BSWAP, MVT::i16, Legal); 596 setOperationAction(ISD::BSWAP, MVT::v2i16, Legal); 597 setOperationAction(ISD::BSWAP, MVT::v4i16, Custom); 598 599 // XXX - Do these do anything? Vector constants turn into build_vector. 600 setOperationAction(ISD::Constant, MVT::v2i16, Legal); 601 setOperationAction(ISD::ConstantFP, MVT::v2f16, Legal); 602 603 setOperationAction(ISD::UNDEF, MVT::v2i16, Legal); 604 setOperationAction(ISD::UNDEF, MVT::v2f16, Legal); 605 606 setOperationAction(ISD::STORE, MVT::v2i16, Promote); 607 AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32); 608 setOperationAction(ISD::STORE, MVT::v2f16, Promote); 609 AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32); 610 611 setOperationAction(ISD::LOAD, MVT::v2i16, Promote); 612 AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32); 613 setOperationAction(ISD::LOAD, MVT::v2f16, Promote); 614 AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); 615 616 setOperationAction(ISD::AND, MVT::v2i16, Promote); 617 AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); 618 setOperationAction(ISD::OR, MVT::v2i16, Promote); 619 AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32); 620 setOperationAction(ISD::XOR, MVT::v2i16, Promote); 621 AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32); 622 623 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 624 AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); 625 setOperationAction(ISD::LOAD, MVT::v4f16, Promote); 626 AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); 627 628 setOperationAction(ISD::STORE, MVT::v4i16, Promote); 629 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); 630 setOperationAction(ISD::STORE, MVT::v4f16, Promote); 631 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); 632 633 setOperationAction(ISD::ANY_EXTEND, MVT::v2i32, Expand); 634 setOperationAction(ISD::ZERO_EXTEND, MVT::v2i32, Expand); 635 setOperationAction(ISD::SIGN_EXTEND, MVT::v2i32, Expand); 636 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand); 637 638 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Expand); 639 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i32, Expand); 640 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Expand); 641 642 if (!Subtarget->hasVOP3PInsts()) { 643 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom); 644 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom); 645 } 646 647 setOperationAction(ISD::FNEG, MVT::v2f16, Legal); 648 // This isn't really legal, but this avoids the legalizer unrolling it (and 649 // allows matching fneg (fabs x) patterns) 650 setOperationAction(ISD::FABS, MVT::v2f16, Legal); 651 652 setOperationAction(ISD::FMAXNUM, MVT::f16, Custom); 653 setOperationAction(ISD::FMINNUM, MVT::f16, Custom); 654 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f16, Legal); 655 setOperationAction(ISD::FMINNUM_IEEE, MVT::f16, Legal); 656 657 setOperationAction(ISD::FMINNUM_IEEE, MVT::v4f16, Custom); 658 setOperationAction(ISD::FMAXNUM_IEEE, MVT::v4f16, Custom); 659 660 setOperationAction(ISD::FMINNUM, MVT::v4f16, Expand); 661 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Expand); 662 } 663 664 if (Subtarget->hasVOP3PInsts()) { 665 setOperationAction(ISD::ADD, MVT::v2i16, Legal); 666 setOperationAction(ISD::SUB, MVT::v2i16, Legal); 667 setOperationAction(ISD::MUL, MVT::v2i16, Legal); 668 setOperationAction(ISD::SHL, MVT::v2i16, Legal); 669 setOperationAction(ISD::SRL, MVT::v2i16, Legal); 670 setOperationAction(ISD::SRA, MVT::v2i16, Legal); 671 setOperationAction(ISD::SMIN, MVT::v2i16, Legal); 672 setOperationAction(ISD::UMIN, MVT::v2i16, Legal); 673 setOperationAction(ISD::SMAX, MVT::v2i16, Legal); 674 setOperationAction(ISD::UMAX, MVT::v2i16, Legal); 675 676 setOperationAction(ISD::UADDSAT, MVT::v2i16, Legal); 677 setOperationAction(ISD::USUBSAT, MVT::v2i16, Legal); 678 setOperationAction(ISD::SADDSAT, MVT::v2i16, Legal); 679 setOperationAction(ISD::SSUBSAT, MVT::v2i16, Legal); 680 681 setOperationAction(ISD::FADD, MVT::v2f16, Legal); 682 setOperationAction(ISD::FMUL, MVT::v2f16, Legal); 683 setOperationAction(ISD::FMA, MVT::v2f16, Legal); 684 685 setOperationAction(ISD::FMINNUM_IEEE, MVT::v2f16, Legal); 686 setOperationAction(ISD::FMAXNUM_IEEE, MVT::v2f16, Legal); 687 688 setOperationAction(ISD::FCANONICALIZE, MVT::v2f16, Legal); 689 690 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); 691 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); 692 693 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f16, Custom); 694 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 695 696 setOperationAction(ISD::SHL, MVT::v4i16, Custom); 697 setOperationAction(ISD::SRA, MVT::v4i16, Custom); 698 setOperationAction(ISD::SRL, MVT::v4i16, Custom); 699 setOperationAction(ISD::ADD, MVT::v4i16, Custom); 700 setOperationAction(ISD::SUB, MVT::v4i16, Custom); 701 setOperationAction(ISD::MUL, MVT::v4i16, Custom); 702 703 setOperationAction(ISD::SMIN, MVT::v4i16, Custom); 704 setOperationAction(ISD::SMAX, MVT::v4i16, Custom); 705 setOperationAction(ISD::UMIN, MVT::v4i16, Custom); 706 setOperationAction(ISD::UMAX, MVT::v4i16, Custom); 707 708 setOperationAction(ISD::UADDSAT, MVT::v4i16, Custom); 709 setOperationAction(ISD::SADDSAT, MVT::v4i16, Custom); 710 setOperationAction(ISD::USUBSAT, MVT::v4i16, Custom); 711 setOperationAction(ISD::SSUBSAT, MVT::v4i16, Custom); 712 713 setOperationAction(ISD::FADD, MVT::v4f16, Custom); 714 setOperationAction(ISD::FMUL, MVT::v4f16, Custom); 715 setOperationAction(ISD::FMA, MVT::v4f16, Custom); 716 717 setOperationAction(ISD::FMAXNUM, MVT::v2f16, Custom); 718 setOperationAction(ISD::FMINNUM, MVT::v2f16, Custom); 719 720 setOperationAction(ISD::FMINNUM, MVT::v4f16, Custom); 721 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Custom); 722 setOperationAction(ISD::FCANONICALIZE, MVT::v4f16, Custom); 723 724 setOperationAction(ISD::FEXP, MVT::v2f16, Custom); 725 setOperationAction(ISD::SELECT, MVT::v4i16, Custom); 726 setOperationAction(ISD::SELECT, MVT::v4f16, Custom); 727 728 if (Subtarget->hasPackedFP32Ops()) { 729 setOperationAction(ISD::FADD, MVT::v2f32, Legal); 730 setOperationAction(ISD::FMUL, MVT::v2f32, Legal); 731 setOperationAction(ISD::FMA, MVT::v2f32, Legal); 732 setOperationAction(ISD::FNEG, MVT::v2f32, Legal); 733 734 for (MVT VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32 }) { 735 setOperationAction(ISD::FADD, VT, Custom); 736 setOperationAction(ISD::FMUL, VT, Custom); 737 setOperationAction(ISD::FMA, VT, Custom); 738 } 739 } 740 } 741 742 setOperationAction(ISD::FNEG, MVT::v4f16, Custom); 743 setOperationAction(ISD::FABS, MVT::v4f16, Custom); 744 745 if (Subtarget->has16BitInsts()) { 746 setOperationAction(ISD::SELECT, MVT::v2i16, Promote); 747 AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32); 748 setOperationAction(ISD::SELECT, MVT::v2f16, Promote); 749 AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32); 750 } else { 751 // Legalization hack. 752 setOperationAction(ISD::SELECT, MVT::v2i16, Custom); 753 setOperationAction(ISD::SELECT, MVT::v2f16, Custom); 754 755 setOperationAction(ISD::FNEG, MVT::v2f16, Custom); 756 setOperationAction(ISD::FABS, MVT::v2f16, Custom); 757 } 758 759 for (MVT VT : { MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8 }) { 760 setOperationAction(ISD::SELECT, VT, Custom); 761 } 762 763 setOperationAction(ISD::SMULO, MVT::i64, Custom); 764 setOperationAction(ISD::UMULO, MVT::i64, Custom); 765 766 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 767 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom); 768 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom); 769 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); 770 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f16, Custom); 771 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2i16, Custom); 772 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f16, Custom); 773 774 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom); 775 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2i16, Custom); 776 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v3f16, Custom); 777 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v3i16, Custom); 778 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom); 779 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4i16, Custom); 780 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom); 781 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 782 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::f16, Custom); 783 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom); 784 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom); 785 786 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 787 setOperationAction(ISD::INTRINSIC_VOID, MVT::v2i16, Custom); 788 setOperationAction(ISD::INTRINSIC_VOID, MVT::v2f16, Custom); 789 setOperationAction(ISD::INTRINSIC_VOID, MVT::v3i16, Custom); 790 setOperationAction(ISD::INTRINSIC_VOID, MVT::v3f16, Custom); 791 setOperationAction(ISD::INTRINSIC_VOID, MVT::v4f16, Custom); 792 setOperationAction(ISD::INTRINSIC_VOID, MVT::v4i16, Custom); 793 setOperationAction(ISD::INTRINSIC_VOID, MVT::f16, Custom); 794 setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); 795 setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); 796 797 setTargetDAGCombine(ISD::ADD); 798 setTargetDAGCombine(ISD::ADDCARRY); 799 setTargetDAGCombine(ISD::SUB); 800 setTargetDAGCombine(ISD::SUBCARRY); 801 setTargetDAGCombine(ISD::FADD); 802 setTargetDAGCombine(ISD::FSUB); 803 setTargetDAGCombine(ISD::FMINNUM); 804 setTargetDAGCombine(ISD::FMAXNUM); 805 setTargetDAGCombine(ISD::FMINNUM_IEEE); 806 setTargetDAGCombine(ISD::FMAXNUM_IEEE); 807 setTargetDAGCombine(ISD::FMA); 808 setTargetDAGCombine(ISD::SMIN); 809 setTargetDAGCombine(ISD::SMAX); 810 setTargetDAGCombine(ISD::UMIN); 811 setTargetDAGCombine(ISD::UMAX); 812 setTargetDAGCombine(ISD::SETCC); 813 setTargetDAGCombine(ISD::AND); 814 setTargetDAGCombine(ISD::OR); 815 setTargetDAGCombine(ISD::XOR); 816 setTargetDAGCombine(ISD::SINT_TO_FP); 817 setTargetDAGCombine(ISD::UINT_TO_FP); 818 setTargetDAGCombine(ISD::FCANONICALIZE); 819 setTargetDAGCombine(ISD::SCALAR_TO_VECTOR); 820 setTargetDAGCombine(ISD::ZERO_EXTEND); 821 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); 822 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 823 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 824 825 // All memory operations. Some folding on the pointer operand is done to help 826 // matching the constant offsets in the addressing modes. 827 setTargetDAGCombine(ISD::LOAD); 828 setTargetDAGCombine(ISD::STORE); 829 setTargetDAGCombine(ISD::ATOMIC_LOAD); 830 setTargetDAGCombine(ISD::ATOMIC_STORE); 831 setTargetDAGCombine(ISD::ATOMIC_CMP_SWAP); 832 setTargetDAGCombine(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 833 setTargetDAGCombine(ISD::ATOMIC_SWAP); 834 setTargetDAGCombine(ISD::ATOMIC_LOAD_ADD); 835 setTargetDAGCombine(ISD::ATOMIC_LOAD_SUB); 836 setTargetDAGCombine(ISD::ATOMIC_LOAD_AND); 837 setTargetDAGCombine(ISD::ATOMIC_LOAD_OR); 838 setTargetDAGCombine(ISD::ATOMIC_LOAD_XOR); 839 setTargetDAGCombine(ISD::ATOMIC_LOAD_NAND); 840 setTargetDAGCombine(ISD::ATOMIC_LOAD_MIN); 841 setTargetDAGCombine(ISD::ATOMIC_LOAD_MAX); 842 setTargetDAGCombine(ISD::ATOMIC_LOAD_UMIN); 843 setTargetDAGCombine(ISD::ATOMIC_LOAD_UMAX); 844 setTargetDAGCombine(ISD::ATOMIC_LOAD_FADD); 845 setTargetDAGCombine(ISD::INTRINSIC_VOID); 846 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 847 848 // FIXME: In other contexts we pretend this is a per-function property. 849 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32); 850 851 setSchedulingPreference(Sched::RegPressure); 852 } 853 854 const GCNSubtarget *SITargetLowering::getSubtarget() const { 855 return Subtarget; 856 } 857 858 //===----------------------------------------------------------------------===// 859 // TargetLowering queries 860 //===----------------------------------------------------------------------===// 861 862 // v_mad_mix* support a conversion from f16 to f32. 863 // 864 // There is only one special case when denormals are enabled we don't currently, 865 // where this is OK to use. 866 bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, 867 EVT DestVT, EVT SrcVT) const { 868 return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) || 869 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) && 870 DestVT.getScalarType() == MVT::f32 && 871 SrcVT.getScalarType() == MVT::f16 && 872 // TODO: This probably only requires no input flushing? 873 !hasFP32Denormals(DAG.getMachineFunction()); 874 } 875 876 bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const { 877 // SI has some legal vector types, but no legal vector operations. Say no 878 // shuffles are legal in order to prefer scalarizing some vector operations. 879 return false; 880 } 881 882 MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, 883 CallingConv::ID CC, 884 EVT VT) const { 885 if (CC == CallingConv::AMDGPU_KERNEL) 886 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); 887 888 if (VT.isVector()) { 889 EVT ScalarVT = VT.getScalarType(); 890 unsigned Size = ScalarVT.getSizeInBits(); 891 if (Size == 16) { 892 if (Subtarget->has16BitInsts()) 893 return VT.isInteger() ? MVT::v2i16 : MVT::v2f16; 894 return VT.isInteger() ? MVT::i32 : MVT::f32; 895 } 896 897 if (Size < 16) 898 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32; 899 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32; 900 } 901 902 if (VT.getSizeInBits() > 32) 903 return MVT::i32; 904 905 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); 906 } 907 908 unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context, 909 CallingConv::ID CC, 910 EVT VT) const { 911 if (CC == CallingConv::AMDGPU_KERNEL) 912 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); 913 914 if (VT.isVector()) { 915 unsigned NumElts = VT.getVectorNumElements(); 916 EVT ScalarVT = VT.getScalarType(); 917 unsigned Size = ScalarVT.getSizeInBits(); 918 919 // FIXME: Should probably promote 8-bit vectors to i16. 920 if (Size == 16 && Subtarget->has16BitInsts()) 921 return (NumElts + 1) / 2; 922 923 if (Size <= 32) 924 return NumElts; 925 926 if (Size > 32) 927 return NumElts * ((Size + 31) / 32); 928 } else if (VT.getSizeInBits() > 32) 929 return (VT.getSizeInBits() + 31) / 32; 930 931 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); 932 } 933 934 unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv( 935 LLVMContext &Context, CallingConv::ID CC, 936 EVT VT, EVT &IntermediateVT, 937 unsigned &NumIntermediates, MVT &RegisterVT) const { 938 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) { 939 unsigned NumElts = VT.getVectorNumElements(); 940 EVT ScalarVT = VT.getScalarType(); 941 unsigned Size = ScalarVT.getSizeInBits(); 942 // FIXME: We should fix the ABI to be the same on targets without 16-bit 943 // support, but unless we can properly handle 3-vectors, it will be still be 944 // inconsistent. 945 if (Size == 16 && Subtarget->has16BitInsts()) { 946 RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16; 947 IntermediateVT = RegisterVT; 948 NumIntermediates = (NumElts + 1) / 2; 949 return NumIntermediates; 950 } 951 952 if (Size == 32) { 953 RegisterVT = ScalarVT.getSimpleVT(); 954 IntermediateVT = RegisterVT; 955 NumIntermediates = NumElts; 956 return NumIntermediates; 957 } 958 959 if (Size < 16 && Subtarget->has16BitInsts()) { 960 // FIXME: Should probably form v2i16 pieces 961 RegisterVT = MVT::i16; 962 IntermediateVT = ScalarVT; 963 NumIntermediates = NumElts; 964 return NumIntermediates; 965 } 966 967 968 if (Size != 16 && Size <= 32) { 969 RegisterVT = MVT::i32; 970 IntermediateVT = ScalarVT; 971 NumIntermediates = NumElts; 972 return NumIntermediates; 973 } 974 975 if (Size > 32) { 976 RegisterVT = MVT::i32; 977 IntermediateVT = RegisterVT; 978 NumIntermediates = NumElts * ((Size + 31) / 32); 979 return NumIntermediates; 980 } 981 } 982 983 return TargetLowering::getVectorTypeBreakdownForCallingConv( 984 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT); 985 } 986 987 static EVT memVTFromImageData(Type *Ty, unsigned DMaskLanes) { 988 assert(DMaskLanes != 0); 989 990 if (auto *VT = dyn_cast<FixedVectorType>(Ty)) { 991 unsigned NumElts = std::min(DMaskLanes, VT->getNumElements()); 992 return EVT::getVectorVT(Ty->getContext(), 993 EVT::getEVT(VT->getElementType()), 994 NumElts); 995 } 996 997 return EVT::getEVT(Ty); 998 } 999 1000 // Peek through TFE struct returns to only use the data size. 1001 static EVT memVTFromImageReturn(Type *Ty, unsigned DMaskLanes) { 1002 auto *ST = dyn_cast<StructType>(Ty); 1003 if (!ST) 1004 return memVTFromImageData(Ty, DMaskLanes); 1005 1006 // Some intrinsics return an aggregate type - special case to work out the 1007 // correct memVT. 1008 // 1009 // Only limited forms of aggregate type currently expected. 1010 if (ST->getNumContainedTypes() != 2 || 1011 !ST->getContainedType(1)->isIntegerTy(32)) 1012 return EVT(); 1013 return memVTFromImageData(ST->getContainedType(0), DMaskLanes); 1014 } 1015 1016 bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 1017 const CallInst &CI, 1018 MachineFunction &MF, 1019 unsigned IntrID) const { 1020 if (const AMDGPU::RsrcIntrinsic *RsrcIntr = 1021 AMDGPU::lookupRsrcIntrinsic(IntrID)) { 1022 AttributeList Attr = Intrinsic::getAttributes(CI.getContext(), 1023 (Intrinsic::ID)IntrID); 1024 if (Attr.hasFnAttribute(Attribute::ReadNone)) 1025 return false; 1026 1027 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1028 1029 if (RsrcIntr->IsImage) { 1030 Info.ptrVal = 1031 MFI->getImagePSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); 1032 Info.align.reset(); 1033 } else { 1034 Info.ptrVal = 1035 MFI->getBufferPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); 1036 } 1037 1038 Info.flags = MachineMemOperand::MODereferenceable; 1039 if (Attr.hasFnAttribute(Attribute::ReadOnly)) { 1040 unsigned DMaskLanes = 4; 1041 1042 if (RsrcIntr->IsImage) { 1043 const AMDGPU::ImageDimIntrinsicInfo *Intr 1044 = AMDGPU::getImageDimIntrinsicInfo(IntrID); 1045 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 1046 AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); 1047 1048 if (!BaseOpcode->Gather4) { 1049 // If this isn't a gather, we may have excess loaded elements in the 1050 // IR type. Check the dmask for the real number of elements loaded. 1051 unsigned DMask 1052 = cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue(); 1053 DMaskLanes = DMask == 0 ? 1 : countPopulation(DMask); 1054 } 1055 1056 Info.memVT = memVTFromImageReturn(CI.getType(), DMaskLanes); 1057 } else 1058 Info.memVT = EVT::getEVT(CI.getType()); 1059 1060 // FIXME: What does alignment mean for an image? 1061 Info.opc = ISD::INTRINSIC_W_CHAIN; 1062 Info.flags |= MachineMemOperand::MOLoad; 1063 } else if (Attr.hasFnAttribute(Attribute::WriteOnly)) { 1064 Info.opc = ISD::INTRINSIC_VOID; 1065 1066 Type *DataTy = CI.getArgOperand(0)->getType(); 1067 if (RsrcIntr->IsImage) { 1068 unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue(); 1069 unsigned DMaskLanes = DMask == 0 ? 1 : countPopulation(DMask); 1070 Info.memVT = memVTFromImageData(DataTy, DMaskLanes); 1071 } else 1072 Info.memVT = EVT::getEVT(DataTy); 1073 1074 Info.flags |= MachineMemOperand::MOStore; 1075 } else { 1076 // Atomic 1077 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID : 1078 ISD::INTRINSIC_W_CHAIN; 1079 Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType()); 1080 Info.flags = MachineMemOperand::MOLoad | 1081 MachineMemOperand::MOStore | 1082 MachineMemOperand::MODereferenceable; 1083 1084 // XXX - Should this be volatile without known ordering? 1085 Info.flags |= MachineMemOperand::MOVolatile; 1086 } 1087 return true; 1088 } 1089 1090 switch (IntrID) { 1091 case Intrinsic::amdgcn_atomic_inc: 1092 case Intrinsic::amdgcn_atomic_dec: 1093 case Intrinsic::amdgcn_ds_ordered_add: 1094 case Intrinsic::amdgcn_ds_ordered_swap: 1095 case Intrinsic::amdgcn_ds_fadd: 1096 case Intrinsic::amdgcn_ds_fmin: 1097 case Intrinsic::amdgcn_ds_fmax: { 1098 Info.opc = ISD::INTRINSIC_W_CHAIN; 1099 Info.memVT = MVT::getVT(CI.getType()); 1100 Info.ptrVal = CI.getOperand(0); 1101 Info.align.reset(); 1102 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 1103 1104 const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4)); 1105 if (!Vol->isZero()) 1106 Info.flags |= MachineMemOperand::MOVolatile; 1107 1108 return true; 1109 } 1110 case Intrinsic::amdgcn_buffer_atomic_fadd: { 1111 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1112 1113 Info.opc = ISD::INTRINSIC_W_CHAIN; 1114 Info.memVT = MVT::getVT(CI.getOperand(0)->getType()); 1115 Info.ptrVal = 1116 MFI->getBufferPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); 1117 Info.align.reset(); 1118 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 1119 1120 const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4)); 1121 if (!Vol || !Vol->isZero()) 1122 Info.flags |= MachineMemOperand::MOVolatile; 1123 1124 return true; 1125 } 1126 case Intrinsic::amdgcn_ds_append: 1127 case Intrinsic::amdgcn_ds_consume: { 1128 Info.opc = ISD::INTRINSIC_W_CHAIN; 1129 Info.memVT = MVT::getVT(CI.getType()); 1130 Info.ptrVal = CI.getOperand(0); 1131 Info.align.reset(); 1132 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 1133 1134 const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1)); 1135 if (!Vol->isZero()) 1136 Info.flags |= MachineMemOperand::MOVolatile; 1137 1138 return true; 1139 } 1140 case Intrinsic::amdgcn_global_atomic_csub: { 1141 Info.opc = ISD::INTRINSIC_W_CHAIN; 1142 Info.memVT = MVT::getVT(CI.getType()); 1143 Info.ptrVal = CI.getOperand(0); 1144 Info.align.reset(); 1145 Info.flags = MachineMemOperand::MOLoad | 1146 MachineMemOperand::MOStore | 1147 MachineMemOperand::MOVolatile; 1148 return true; 1149 } 1150 case Intrinsic::amdgcn_image_bvh_intersect_ray: { 1151 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1152 Info.opc = ISD::INTRINSIC_W_CHAIN; 1153 Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT? 1154 Info.ptrVal = 1155 MFI->getImagePSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); 1156 Info.align.reset(); 1157 Info.flags = MachineMemOperand::MOLoad | 1158 MachineMemOperand::MODereferenceable; 1159 return true; 1160 } 1161 case Intrinsic::amdgcn_global_atomic_fadd: 1162 case Intrinsic::amdgcn_global_atomic_fmin: 1163 case Intrinsic::amdgcn_global_atomic_fmax: 1164 case Intrinsic::amdgcn_flat_atomic_fadd: 1165 case Intrinsic::amdgcn_flat_atomic_fmin: 1166 case Intrinsic::amdgcn_flat_atomic_fmax: { 1167 Info.opc = ISD::INTRINSIC_W_CHAIN; 1168 Info.memVT = MVT::getVT(CI.getType()); 1169 Info.ptrVal = CI.getOperand(0); 1170 Info.align.reset(); 1171 Info.flags = MachineMemOperand::MOLoad | 1172 MachineMemOperand::MOStore | 1173 MachineMemOperand::MODereferenceable | 1174 MachineMemOperand::MOVolatile; 1175 return true; 1176 } 1177 case Intrinsic::amdgcn_ds_gws_init: 1178 case Intrinsic::amdgcn_ds_gws_barrier: 1179 case Intrinsic::amdgcn_ds_gws_sema_v: 1180 case Intrinsic::amdgcn_ds_gws_sema_br: 1181 case Intrinsic::amdgcn_ds_gws_sema_p: 1182 case Intrinsic::amdgcn_ds_gws_sema_release_all: { 1183 Info.opc = ISD::INTRINSIC_VOID; 1184 1185 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1186 Info.ptrVal = 1187 MFI->getGWSPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); 1188 1189 // This is an abstract access, but we need to specify a type and size. 1190 Info.memVT = MVT::i32; 1191 Info.size = 4; 1192 Info.align = Align(4); 1193 1194 Info.flags = MachineMemOperand::MOStore; 1195 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier) 1196 Info.flags = MachineMemOperand::MOLoad; 1197 return true; 1198 } 1199 default: 1200 return false; 1201 } 1202 } 1203 1204 bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II, 1205 SmallVectorImpl<Value*> &Ops, 1206 Type *&AccessTy) const { 1207 switch (II->getIntrinsicID()) { 1208 case Intrinsic::amdgcn_atomic_inc: 1209 case Intrinsic::amdgcn_atomic_dec: 1210 case Intrinsic::amdgcn_ds_ordered_add: 1211 case Intrinsic::amdgcn_ds_ordered_swap: 1212 case Intrinsic::amdgcn_ds_append: 1213 case Intrinsic::amdgcn_ds_consume: 1214 case Intrinsic::amdgcn_ds_fadd: 1215 case Intrinsic::amdgcn_ds_fmin: 1216 case Intrinsic::amdgcn_ds_fmax: 1217 case Intrinsic::amdgcn_global_atomic_fadd: 1218 case Intrinsic::amdgcn_flat_atomic_fadd: 1219 case Intrinsic::amdgcn_flat_atomic_fmin: 1220 case Intrinsic::amdgcn_flat_atomic_fmax: 1221 case Intrinsic::amdgcn_global_atomic_csub: { 1222 Value *Ptr = II->getArgOperand(0); 1223 AccessTy = II->getType(); 1224 Ops.push_back(Ptr); 1225 return true; 1226 } 1227 default: 1228 return false; 1229 } 1230 } 1231 1232 bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const { 1233 if (!Subtarget->hasFlatInstOffsets()) { 1234 // Flat instructions do not have offsets, and only have the register 1235 // address. 1236 return AM.BaseOffs == 0 && AM.Scale == 0; 1237 } 1238 1239 return AM.Scale == 0 && 1240 (AM.BaseOffs == 0 || 1241 Subtarget->getInstrInfo()->isLegalFLATOffset( 1242 AM.BaseOffs, AMDGPUAS::FLAT_ADDRESS, SIInstrFlags::FLAT)); 1243 } 1244 1245 bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const { 1246 if (Subtarget->hasFlatGlobalInsts()) 1247 return AM.Scale == 0 && 1248 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset( 1249 AM.BaseOffs, AMDGPUAS::GLOBAL_ADDRESS, 1250 SIInstrFlags::FlatGlobal)); 1251 1252 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) { 1253 // Assume the we will use FLAT for all global memory accesses 1254 // on VI. 1255 // FIXME: This assumption is currently wrong. On VI we still use 1256 // MUBUF instructions for the r + i addressing mode. As currently 1257 // implemented, the MUBUF instructions only work on buffer < 4GB. 1258 // It may be possible to support > 4GB buffers with MUBUF instructions, 1259 // by setting the stride value in the resource descriptor which would 1260 // increase the size limit to (stride * 4GB). However, this is risky, 1261 // because it has never been validated. 1262 return isLegalFlatAddressingMode(AM); 1263 } 1264 1265 return isLegalMUBUFAddressingMode(AM); 1266 } 1267 1268 bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const { 1269 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and 1270 // additionally can do r + r + i with addr64. 32-bit has more addressing 1271 // mode options. Depending on the resource constant, it can also do 1272 // (i64 r0) + (i32 r1) * (i14 i). 1273 // 1274 // Private arrays end up using a scratch buffer most of the time, so also 1275 // assume those use MUBUF instructions. Scratch loads / stores are currently 1276 // implemented as mubuf instructions with offen bit set, so slightly 1277 // different than the normal addr64. 1278 if (!SIInstrInfo::isLegalMUBUFImmOffset(AM.BaseOffs)) 1279 return false; 1280 1281 // FIXME: Since we can split immediate into soffset and immediate offset, 1282 // would it make sense to allow any immediate? 1283 1284 switch (AM.Scale) { 1285 case 0: // r + i or just i, depending on HasBaseReg. 1286 return true; 1287 case 1: 1288 return true; // We have r + r or r + i. 1289 case 2: 1290 if (AM.HasBaseReg) { 1291 // Reject 2 * r + r. 1292 return false; 1293 } 1294 1295 // Allow 2 * r as r + r 1296 // Or 2 * r + i is allowed as r + r + i. 1297 return true; 1298 default: // Don't allow n * r 1299 return false; 1300 } 1301 } 1302 1303 bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL, 1304 const AddrMode &AM, Type *Ty, 1305 unsigned AS, Instruction *I) const { 1306 // No global is ever allowed as a base. 1307 if (AM.BaseGV) 1308 return false; 1309 1310 if (AS == AMDGPUAS::GLOBAL_ADDRESS) 1311 return isLegalGlobalAddressingMode(AM); 1312 1313 if (AS == AMDGPUAS::CONSTANT_ADDRESS || 1314 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT || 1315 AS == AMDGPUAS::BUFFER_FAT_POINTER) { 1316 // If the offset isn't a multiple of 4, it probably isn't going to be 1317 // correctly aligned. 1318 // FIXME: Can we get the real alignment here? 1319 if (AM.BaseOffs % 4 != 0) 1320 return isLegalMUBUFAddressingMode(AM); 1321 1322 // There are no SMRD extloads, so if we have to do a small type access we 1323 // will use a MUBUF load. 1324 // FIXME?: We also need to do this if unaligned, but we don't know the 1325 // alignment here. 1326 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4) 1327 return isLegalGlobalAddressingMode(AM); 1328 1329 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) { 1330 // SMRD instructions have an 8-bit, dword offset on SI. 1331 if (!isUInt<8>(AM.BaseOffs / 4)) 1332 return false; 1333 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) { 1334 // On CI+, this can also be a 32-bit literal constant offset. If it fits 1335 // in 8-bits, it can use a smaller encoding. 1336 if (!isUInt<32>(AM.BaseOffs / 4)) 1337 return false; 1338 } else if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 1339 // On VI, these use the SMEM format and the offset is 20-bit in bytes. 1340 if (!isUInt<20>(AM.BaseOffs)) 1341 return false; 1342 } else 1343 llvm_unreachable("unhandled generation"); 1344 1345 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg. 1346 return true; 1347 1348 if (AM.Scale == 1 && AM.HasBaseReg) 1349 return true; 1350 1351 return false; 1352 1353 } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) { 1354 return isLegalMUBUFAddressingMode(AM); 1355 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || 1356 AS == AMDGPUAS::REGION_ADDRESS) { 1357 // Basic, single offset DS instructions allow a 16-bit unsigned immediate 1358 // field. 1359 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have 1360 // an 8-bit dword offset but we don't know the alignment here. 1361 if (!isUInt<16>(AM.BaseOffs)) 1362 return false; 1363 1364 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg. 1365 return true; 1366 1367 if (AM.Scale == 1 && AM.HasBaseReg) 1368 return true; 1369 1370 return false; 1371 } else if (AS == AMDGPUAS::FLAT_ADDRESS || 1372 AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) { 1373 // For an unknown address space, this usually means that this is for some 1374 // reason being used for pure arithmetic, and not based on some addressing 1375 // computation. We don't have instructions that compute pointers with any 1376 // addressing modes, so treat them as having no offset like flat 1377 // instructions. 1378 return isLegalFlatAddressingMode(AM); 1379 } 1380 1381 // Assume a user alias of global for unknown address spaces. 1382 return isLegalGlobalAddressingMode(AM); 1383 } 1384 1385 bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT, 1386 const SelectionDAG &DAG) const { 1387 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) { 1388 return (MemVT.getSizeInBits() <= 4 * 32); 1389 } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) { 1390 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize(); 1391 return (MemVT.getSizeInBits() <= MaxPrivateBits); 1392 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) { 1393 return (MemVT.getSizeInBits() <= 2 * 32); 1394 } 1395 return true; 1396 } 1397 1398 bool SITargetLowering::allowsMisalignedMemoryAccessesImpl( 1399 unsigned Size, unsigned AddrSpace, Align Alignment, 1400 MachineMemOperand::Flags Flags, bool *IsFast) const { 1401 if (IsFast) 1402 *IsFast = false; 1403 1404 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS || 1405 AddrSpace == AMDGPUAS::REGION_ADDRESS) { 1406 // Check if alignment requirements for ds_read/write instructions are 1407 // disabled. 1408 if (Subtarget->hasUnalignedDSAccessEnabled() && 1409 !Subtarget->hasLDSMisalignedBug()) { 1410 if (IsFast) 1411 *IsFast = Alignment != Align(2); 1412 return true; 1413 } 1414 1415 // Either, the alignment requirements are "enabled", or there is an 1416 // unaligned LDS access related hardware bug though alignment requirements 1417 // are "disabled". In either case, we need to check for proper alignment 1418 // requirements. 1419 // 1420 if (Size == 64) { 1421 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we 1422 // can do a 4 byte aligned, 8 byte access in a single operation using 1423 // ds_read2/write2_b32 with adjacent offsets. 1424 bool AlignedBy4 = Alignment >= Align(4); 1425 if (IsFast) 1426 *IsFast = AlignedBy4; 1427 1428 return AlignedBy4; 1429 } 1430 if (Size == 96) { 1431 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on 1432 // gfx8 and older. 1433 bool AlignedBy16 = Alignment >= Align(16); 1434 if (IsFast) 1435 *IsFast = AlignedBy16; 1436 1437 return AlignedBy16; 1438 } 1439 if (Size == 128) { 1440 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on 1441 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a 1442 // single operation using ds_read2/write2_b64. 1443 bool AlignedBy8 = Alignment >= Align(8); 1444 if (IsFast) 1445 *IsFast = AlignedBy8; 1446 1447 return AlignedBy8; 1448 } 1449 } 1450 1451 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) { 1452 bool AlignedBy4 = Alignment >= Align(4); 1453 if (IsFast) 1454 *IsFast = AlignedBy4; 1455 1456 return AlignedBy4 || 1457 Subtarget->enableFlatScratch() || 1458 Subtarget->hasUnalignedScratchAccess(); 1459 } 1460 1461 // FIXME: We have to be conservative here and assume that flat operations 1462 // will access scratch. If we had access to the IR function, then we 1463 // could determine if any private memory was used in the function. 1464 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS && 1465 !Subtarget->hasUnalignedScratchAccess()) { 1466 bool AlignedBy4 = Alignment >= Align(4); 1467 if (IsFast) 1468 *IsFast = AlignedBy4; 1469 1470 return AlignedBy4; 1471 } 1472 1473 if (Subtarget->hasUnalignedBufferAccessEnabled() && 1474 !(AddrSpace == AMDGPUAS::LOCAL_ADDRESS || 1475 AddrSpace == AMDGPUAS::REGION_ADDRESS)) { 1476 // If we have an uniform constant load, it still requires using a slow 1477 // buffer instruction if unaligned. 1478 if (IsFast) { 1479 // Accesses can really be issued as 1-byte aligned or 4-byte aligned, so 1480 // 2-byte alignment is worse than 1 unless doing a 2-byte accesss. 1481 *IsFast = (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS || 1482 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ? 1483 Alignment >= Align(4) : Alignment != Align(2); 1484 } 1485 1486 return true; 1487 } 1488 1489 // Smaller than dword value must be aligned. 1490 if (Size < 32) 1491 return false; 1492 1493 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the 1494 // byte-address are ignored, thus forcing Dword alignment. 1495 // This applies to private, global, and constant memory. 1496 if (IsFast) 1497 *IsFast = true; 1498 1499 return Size >= 32 && Alignment >= Align(4); 1500 } 1501 1502 bool SITargetLowering::allowsMisalignedMemoryAccesses( 1503 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, 1504 bool *IsFast) const { 1505 if (IsFast) 1506 *IsFast = false; 1507 1508 // TODO: I think v3i32 should allow unaligned accesses on CI with DS_READ_B96, 1509 // which isn't a simple VT. 1510 // Until MVT is extended to handle this, simply check for the size and 1511 // rely on the condition below: allow accesses if the size is a multiple of 4. 1512 if (VT == MVT::Other || (VT != MVT::Other && VT.getSizeInBits() > 1024 && 1513 VT.getStoreSize() > 16)) { 1514 return false; 1515 } 1516 1517 return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace, 1518 Alignment, Flags, IsFast); 1519 } 1520 1521 EVT SITargetLowering::getOptimalMemOpType( 1522 const MemOp &Op, const AttributeList &FuncAttributes) const { 1523 // FIXME: Should account for address space here. 1524 1525 // The default fallback uses the private pointer size as a guess for a type to 1526 // use. Make sure we switch these to 64-bit accesses. 1527 1528 if (Op.size() >= 16 && 1529 Op.isDstAligned(Align(4))) // XXX: Should only do for global 1530 return MVT::v4i32; 1531 1532 if (Op.size() >= 8 && Op.isDstAligned(Align(4))) 1533 return MVT::v2i32; 1534 1535 // Use the default. 1536 return MVT::Other; 1537 } 1538 1539 bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const { 1540 const MemSDNode *MemNode = cast<MemSDNode>(N); 1541 const Value *Ptr = MemNode->getMemOperand()->getValue(); 1542 const Instruction *I = dyn_cast_or_null<Instruction>(Ptr); 1543 return I && I->getMetadata("amdgpu.noclobber"); 1544 } 1545 1546 bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) { 1547 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS || 1548 AS == AMDGPUAS::PRIVATE_ADDRESS; 1549 } 1550 1551 bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS, 1552 unsigned DestAS) const { 1553 // Flat -> private/local is a simple truncate. 1554 // Flat -> global is no-op 1555 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) 1556 return true; 1557 1558 const GCNTargetMachine &TM = 1559 static_cast<const GCNTargetMachine &>(getTargetMachine()); 1560 return TM.isNoopAddrSpaceCast(SrcAS, DestAS); 1561 } 1562 1563 bool SITargetLowering::isMemOpUniform(const SDNode *N) const { 1564 const MemSDNode *MemNode = cast<MemSDNode>(N); 1565 1566 return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand()); 1567 } 1568 1569 TargetLoweringBase::LegalizeTypeAction 1570 SITargetLowering::getPreferredVectorAction(MVT VT) const { 1571 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 && 1572 VT.getScalarType().bitsLE(MVT::i16)) 1573 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector; 1574 return TargetLoweringBase::getPreferredVectorAction(VT); 1575 } 1576 1577 bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 1578 Type *Ty) const { 1579 // FIXME: Could be smarter if called for vector constants. 1580 return true; 1581 } 1582 1583 bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const { 1584 if (Subtarget->has16BitInsts() && VT == MVT::i16) { 1585 switch (Op) { 1586 case ISD::LOAD: 1587 case ISD::STORE: 1588 1589 // These operations are done with 32-bit instructions anyway. 1590 case ISD::AND: 1591 case ISD::OR: 1592 case ISD::XOR: 1593 case ISD::SELECT: 1594 // TODO: Extensions? 1595 return true; 1596 default: 1597 return false; 1598 } 1599 } 1600 1601 // SimplifySetCC uses this function to determine whether or not it should 1602 // create setcc with i1 operands. We don't have instructions for i1 setcc. 1603 if (VT == MVT::i1 && Op == ISD::SETCC) 1604 return false; 1605 1606 return TargetLowering::isTypeDesirableForOp(Op, VT); 1607 } 1608 1609 SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG, 1610 const SDLoc &SL, 1611 SDValue Chain, 1612 uint64_t Offset) const { 1613 const DataLayout &DL = DAG.getDataLayout(); 1614 MachineFunction &MF = DAG.getMachineFunction(); 1615 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1616 1617 const ArgDescriptor *InputPtrReg; 1618 const TargetRegisterClass *RC; 1619 LLT ArgTy; 1620 1621 std::tie(InputPtrReg, RC, ArgTy) = 1622 Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR); 1623 1624 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 1625 MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); 1626 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL, 1627 MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT); 1628 1629 return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Offset)); 1630 } 1631 1632 SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG, 1633 const SDLoc &SL) const { 1634 uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(), 1635 FIRST_IMPLICIT); 1636 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset); 1637 } 1638 1639 SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT, 1640 const SDLoc &SL, SDValue Val, 1641 bool Signed, 1642 const ISD::InputArg *Arg) const { 1643 // First, if it is a widened vector, narrow it. 1644 if (VT.isVector() && 1645 VT.getVectorNumElements() != MemVT.getVectorNumElements()) { 1646 EVT NarrowedVT = 1647 EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 1648 VT.getVectorNumElements()); 1649 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val, 1650 DAG.getConstant(0, SL, MVT::i32)); 1651 } 1652 1653 // Then convert the vector elements or scalar value. 1654 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && 1655 VT.bitsLT(MemVT)) { 1656 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext; 1657 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT)); 1658 } 1659 1660 if (MemVT.isFloatingPoint()) 1661 Val = getFPExtOrFPRound(DAG, Val, SL, VT); 1662 else if (Signed) 1663 Val = DAG.getSExtOrTrunc(Val, SL, VT); 1664 else 1665 Val = DAG.getZExtOrTrunc(Val, SL, VT); 1666 1667 return Val; 1668 } 1669 1670 SDValue SITargetLowering::lowerKernargMemParameter( 1671 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain, 1672 uint64_t Offset, Align Alignment, bool Signed, 1673 const ISD::InputArg *Arg) const { 1674 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); 1675 1676 // Try to avoid using an extload by loading earlier than the argument address, 1677 // and extracting the relevant bits. The load should hopefully be merged with 1678 // the previous argument. 1679 if (MemVT.getStoreSize() < 4 && Alignment < 4) { 1680 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs). 1681 int64_t AlignDownOffset = alignDown(Offset, 4); 1682 int64_t OffsetDiff = Offset - AlignDownOffset; 1683 1684 EVT IntVT = MemVT.changeTypeToInteger(); 1685 1686 // TODO: If we passed in the base kernel offset we could have a better 1687 // alignment than 4, but we don't really need it. 1688 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset); 1689 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), 1690 MachineMemOperand::MODereferenceable | 1691 MachineMemOperand::MOInvariant); 1692 1693 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); 1694 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); 1695 1696 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract); 1697 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal); 1698 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg); 1699 1700 1701 return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL); 1702 } 1703 1704 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset); 1705 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment, 1706 MachineMemOperand::MODereferenceable | 1707 MachineMemOperand::MOInvariant); 1708 1709 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg); 1710 return DAG.getMergeValues({ Val, Load.getValue(1) }, SL); 1711 } 1712 1713 SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA, 1714 const SDLoc &SL, SDValue Chain, 1715 const ISD::InputArg &Arg) const { 1716 MachineFunction &MF = DAG.getMachineFunction(); 1717 MachineFrameInfo &MFI = MF.getFrameInfo(); 1718 1719 if (Arg.Flags.isByVal()) { 1720 unsigned Size = Arg.Flags.getByValSize(); 1721 int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false); 1722 return DAG.getFrameIndex(FrameIdx, MVT::i32); 1723 } 1724 1725 unsigned ArgOffset = VA.getLocMemOffset(); 1726 unsigned ArgSize = VA.getValVT().getStoreSize(); 1727 1728 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true); 1729 1730 // Create load nodes to retrieve arguments from the stack. 1731 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); 1732 SDValue ArgValue; 1733 1734 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT) 1735 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; 1736 MVT MemVT = VA.getValVT(); 1737 1738 switch (VA.getLocInfo()) { 1739 default: 1740 break; 1741 case CCValAssign::BCvt: 1742 MemVT = VA.getLocVT(); 1743 break; 1744 case CCValAssign::SExt: 1745 ExtType = ISD::SEXTLOAD; 1746 break; 1747 case CCValAssign::ZExt: 1748 ExtType = ISD::ZEXTLOAD; 1749 break; 1750 case CCValAssign::AExt: 1751 ExtType = ISD::EXTLOAD; 1752 break; 1753 } 1754 1755 ArgValue = DAG.getExtLoad( 1756 ExtType, SL, VA.getLocVT(), Chain, FIN, 1757 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), 1758 MemVT); 1759 return ArgValue; 1760 } 1761 1762 SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG, 1763 const SIMachineFunctionInfo &MFI, 1764 EVT VT, 1765 AMDGPUFunctionArgInfo::PreloadedValue PVID) const { 1766 const ArgDescriptor *Reg; 1767 const TargetRegisterClass *RC; 1768 LLT Ty; 1769 1770 std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID); 1771 return CreateLiveInRegister(DAG, RC, Reg->getRegister(), VT); 1772 } 1773 1774 static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits, 1775 CallingConv::ID CallConv, 1776 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped, 1777 FunctionType *FType, 1778 SIMachineFunctionInfo *Info) { 1779 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) { 1780 const ISD::InputArg *Arg = &Ins[I]; 1781 1782 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) && 1783 "vector type argument should have been split"); 1784 1785 // First check if it's a PS input addr. 1786 if (CallConv == CallingConv::AMDGPU_PS && 1787 !Arg->Flags.isInReg() && PSInputNum <= 15) { 1788 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum); 1789 1790 // Inconveniently only the first part of the split is marked as isSplit, 1791 // so skip to the end. We only want to increment PSInputNum once for the 1792 // entire split argument. 1793 if (Arg->Flags.isSplit()) { 1794 while (!Arg->Flags.isSplitEnd()) { 1795 assert((!Arg->VT.isVector() || 1796 Arg->VT.getScalarSizeInBits() == 16) && 1797 "unexpected vector split in ps argument type"); 1798 if (!SkipArg) 1799 Splits.push_back(*Arg); 1800 Arg = &Ins[++I]; 1801 } 1802 } 1803 1804 if (SkipArg) { 1805 // We can safely skip PS inputs. 1806 Skipped.set(Arg->getOrigArgIndex()); 1807 ++PSInputNum; 1808 continue; 1809 } 1810 1811 Info->markPSInputAllocated(PSInputNum); 1812 if (Arg->Used) 1813 Info->markPSInputEnabled(PSInputNum); 1814 1815 ++PSInputNum; 1816 } 1817 1818 Splits.push_back(*Arg); 1819 } 1820 } 1821 1822 // Allocate special inputs passed in VGPRs. 1823 void SITargetLowering::allocateSpecialEntryInputVGPRs(CCState &CCInfo, 1824 MachineFunction &MF, 1825 const SIRegisterInfo &TRI, 1826 SIMachineFunctionInfo &Info) const { 1827 const LLT S32 = LLT::scalar(32); 1828 MachineRegisterInfo &MRI = MF.getRegInfo(); 1829 1830 if (Info.hasWorkItemIDX()) { 1831 Register Reg = AMDGPU::VGPR0; 1832 MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32); 1833 1834 CCInfo.AllocateReg(Reg); 1835 unsigned Mask = (Subtarget->hasPackedTID() && 1836 Info.hasWorkItemIDY()) ? 0x3ff : ~0u; 1837 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask)); 1838 } 1839 1840 if (Info.hasWorkItemIDY()) { 1841 assert(Info.hasWorkItemIDX()); 1842 if (Subtarget->hasPackedTID()) { 1843 Info.setWorkItemIDY(ArgDescriptor::createRegister(AMDGPU::VGPR0, 1844 0x3ff << 10)); 1845 } else { 1846 unsigned Reg = AMDGPU::VGPR1; 1847 MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32); 1848 1849 CCInfo.AllocateReg(Reg); 1850 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg)); 1851 } 1852 } 1853 1854 if (Info.hasWorkItemIDZ()) { 1855 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY()); 1856 if (Subtarget->hasPackedTID()) { 1857 Info.setWorkItemIDZ(ArgDescriptor::createRegister(AMDGPU::VGPR0, 1858 0x3ff << 20)); 1859 } else { 1860 unsigned Reg = AMDGPU::VGPR2; 1861 MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32); 1862 1863 CCInfo.AllocateReg(Reg); 1864 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg)); 1865 } 1866 } 1867 } 1868 1869 // Try to allocate a VGPR at the end of the argument list, or if no argument 1870 // VGPRs are left allocating a stack slot. 1871 // If \p Mask is is given it indicates bitfield position in the register. 1872 // If \p Arg is given use it with new ]p Mask instead of allocating new. 1873 static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u, 1874 ArgDescriptor Arg = ArgDescriptor()) { 1875 if (Arg.isSet()) 1876 return ArgDescriptor::createArg(Arg, Mask); 1877 1878 ArrayRef<MCPhysReg> ArgVGPRs 1879 = makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32); 1880 unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs); 1881 if (RegIdx == ArgVGPRs.size()) { 1882 // Spill to stack required. 1883 int64_t Offset = CCInfo.AllocateStack(4, Align(4)); 1884 1885 return ArgDescriptor::createStack(Offset, Mask); 1886 } 1887 1888 unsigned Reg = ArgVGPRs[RegIdx]; 1889 Reg = CCInfo.AllocateReg(Reg); 1890 assert(Reg != AMDGPU::NoRegister); 1891 1892 MachineFunction &MF = CCInfo.getMachineFunction(); 1893 Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass); 1894 MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32)); 1895 return ArgDescriptor::createRegister(Reg, Mask); 1896 } 1897 1898 static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo, 1899 const TargetRegisterClass *RC, 1900 unsigned NumArgRegs) { 1901 ArrayRef<MCPhysReg> ArgSGPRs = makeArrayRef(RC->begin(), 32); 1902 unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs); 1903 if (RegIdx == ArgSGPRs.size()) 1904 report_fatal_error("ran out of SGPRs for arguments"); 1905 1906 unsigned Reg = ArgSGPRs[RegIdx]; 1907 Reg = CCInfo.AllocateReg(Reg); 1908 assert(Reg != AMDGPU::NoRegister); 1909 1910 MachineFunction &MF = CCInfo.getMachineFunction(); 1911 MF.addLiveIn(Reg, RC); 1912 return ArgDescriptor::createRegister(Reg); 1913 } 1914 1915 // If this has a fixed position, we still should allocate the register in the 1916 // CCInfo state. Technically we could get away with this for values passed 1917 // outside of the normal argument range. 1918 static void allocateFixedSGPRInputImpl(CCState &CCInfo, 1919 const TargetRegisterClass *RC, 1920 MCRegister Reg) { 1921 Reg = CCInfo.AllocateReg(Reg); 1922 assert(Reg != AMDGPU::NoRegister); 1923 MachineFunction &MF = CCInfo.getMachineFunction(); 1924 MF.addLiveIn(Reg, RC); 1925 } 1926 1927 static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) { 1928 if (Arg) { 1929 allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 1930 Arg.getRegister()); 1931 } else 1932 Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32); 1933 } 1934 1935 static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) { 1936 if (Arg) { 1937 allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 1938 Arg.getRegister()); 1939 } else 1940 Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16); 1941 } 1942 1943 /// Allocate implicit function VGPR arguments at the end of allocated user 1944 /// arguments. 1945 void SITargetLowering::allocateSpecialInputVGPRs( 1946 CCState &CCInfo, MachineFunction &MF, 1947 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const { 1948 const unsigned Mask = 0x3ff; 1949 ArgDescriptor Arg; 1950 1951 if (Info.hasWorkItemIDX()) { 1952 Arg = allocateVGPR32Input(CCInfo, Mask); 1953 Info.setWorkItemIDX(Arg); 1954 } 1955 1956 if (Info.hasWorkItemIDY()) { 1957 Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg); 1958 Info.setWorkItemIDY(Arg); 1959 } 1960 1961 if (Info.hasWorkItemIDZ()) 1962 Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg)); 1963 } 1964 1965 /// Allocate implicit function VGPR arguments in fixed registers. 1966 void SITargetLowering::allocateSpecialInputVGPRsFixed( 1967 CCState &CCInfo, MachineFunction &MF, 1968 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const { 1969 Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31); 1970 if (!Reg) 1971 report_fatal_error("failed to allocated VGPR for implicit arguments"); 1972 1973 const unsigned Mask = 0x3ff; 1974 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask)); 1975 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10)); 1976 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20)); 1977 } 1978 1979 void SITargetLowering::allocateSpecialInputSGPRs( 1980 CCState &CCInfo, 1981 MachineFunction &MF, 1982 const SIRegisterInfo &TRI, 1983 SIMachineFunctionInfo &Info) const { 1984 auto &ArgInfo = Info.getArgInfo(); 1985 1986 // TODO: Unify handling with private memory pointers. 1987 1988 if (Info.hasDispatchPtr()) 1989 allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr); 1990 1991 if (Info.hasQueuePtr()) 1992 allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr); 1993 1994 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a 1995 // constant offset from the kernarg segment. 1996 if (Info.hasImplicitArgPtr()) 1997 allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr); 1998 1999 if (Info.hasDispatchID()) 2000 allocateSGPR64Input(CCInfo, ArgInfo.DispatchID); 2001 2002 // flat_scratch_init is not applicable for non-kernel functions. 2003 2004 if (Info.hasWorkGroupIDX()) 2005 allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX); 2006 2007 if (Info.hasWorkGroupIDY()) 2008 allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY); 2009 2010 if (Info.hasWorkGroupIDZ()) 2011 allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ); 2012 } 2013 2014 // Allocate special inputs passed in user SGPRs. 2015 void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo, 2016 MachineFunction &MF, 2017 const SIRegisterInfo &TRI, 2018 SIMachineFunctionInfo &Info) const { 2019 if (Info.hasImplicitBufferPtr()) { 2020 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI); 2021 MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass); 2022 CCInfo.AllocateReg(ImplicitBufferPtrReg); 2023 } 2024 2025 // FIXME: How should these inputs interact with inreg / custom SGPR inputs? 2026 if (Info.hasPrivateSegmentBuffer()) { 2027 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI); 2028 MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass); 2029 CCInfo.AllocateReg(PrivateSegmentBufferReg); 2030 } 2031 2032 if (Info.hasDispatchPtr()) { 2033 Register DispatchPtrReg = Info.addDispatchPtr(TRI); 2034 MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass); 2035 CCInfo.AllocateReg(DispatchPtrReg); 2036 } 2037 2038 if (Info.hasQueuePtr()) { 2039 Register QueuePtrReg = Info.addQueuePtr(TRI); 2040 MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass); 2041 CCInfo.AllocateReg(QueuePtrReg); 2042 } 2043 2044 if (Info.hasKernargSegmentPtr()) { 2045 MachineRegisterInfo &MRI = MF.getRegInfo(); 2046 Register InputPtrReg = Info.addKernargSegmentPtr(TRI); 2047 CCInfo.AllocateReg(InputPtrReg); 2048 2049 Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass); 2050 MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); 2051 } 2052 2053 if (Info.hasDispatchID()) { 2054 Register DispatchIDReg = Info.addDispatchID(TRI); 2055 MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass); 2056 CCInfo.AllocateReg(DispatchIDReg); 2057 } 2058 2059 if (Info.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) { 2060 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI); 2061 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass); 2062 CCInfo.AllocateReg(FlatScratchInitReg); 2063 } 2064 2065 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read 2066 // these from the dispatch pointer. 2067 } 2068 2069 // Allocate special input registers that are initialized per-wave. 2070 void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, 2071 MachineFunction &MF, 2072 SIMachineFunctionInfo &Info, 2073 CallingConv::ID CallConv, 2074 bool IsShader) const { 2075 if (Info.hasWorkGroupIDX()) { 2076 Register Reg = Info.addWorkGroupIDX(); 2077 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); 2078 CCInfo.AllocateReg(Reg); 2079 } 2080 2081 if (Info.hasWorkGroupIDY()) { 2082 Register Reg = Info.addWorkGroupIDY(); 2083 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); 2084 CCInfo.AllocateReg(Reg); 2085 } 2086 2087 if (Info.hasWorkGroupIDZ()) { 2088 Register Reg = Info.addWorkGroupIDZ(); 2089 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); 2090 CCInfo.AllocateReg(Reg); 2091 } 2092 2093 if (Info.hasWorkGroupInfo()) { 2094 Register Reg = Info.addWorkGroupInfo(); 2095 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); 2096 CCInfo.AllocateReg(Reg); 2097 } 2098 2099 if (Info.hasPrivateSegmentWaveByteOffset()) { 2100 // Scratch wave offset passed in system SGPR. 2101 unsigned PrivateSegmentWaveByteOffsetReg; 2102 2103 if (IsShader) { 2104 PrivateSegmentWaveByteOffsetReg = 2105 Info.getPrivateSegmentWaveByteOffsetSystemSGPR(); 2106 2107 // This is true if the scratch wave byte offset doesn't have a fixed 2108 // location. 2109 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) { 2110 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo); 2111 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg); 2112 } 2113 } else 2114 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset(); 2115 2116 MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass); 2117 CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg); 2118 } 2119 } 2120 2121 static void reservePrivateMemoryRegs(const TargetMachine &TM, 2122 MachineFunction &MF, 2123 const SIRegisterInfo &TRI, 2124 SIMachineFunctionInfo &Info) { 2125 // Now that we've figured out where the scratch register inputs are, see if 2126 // should reserve the arguments and use them directly. 2127 MachineFrameInfo &MFI = MF.getFrameInfo(); 2128 bool HasStackObjects = MFI.hasStackObjects(); 2129 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 2130 2131 // Record that we know we have non-spill stack objects so we don't need to 2132 // check all stack objects later. 2133 if (HasStackObjects) 2134 Info.setHasNonSpillStackObjects(true); 2135 2136 // Everything live out of a block is spilled with fast regalloc, so it's 2137 // almost certain that spilling will be required. 2138 if (TM.getOptLevel() == CodeGenOpt::None) 2139 HasStackObjects = true; 2140 2141 // For now assume stack access is needed in any callee functions, so we need 2142 // the scratch registers to pass in. 2143 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls(); 2144 2145 if (!ST.enableFlatScratch()) { 2146 if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) { 2147 // If we have stack objects, we unquestionably need the private buffer 2148 // resource. For the Code Object V2 ABI, this will be the first 4 user 2149 // SGPR inputs. We can reserve those and use them directly. 2150 2151 Register PrivateSegmentBufferReg = 2152 Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER); 2153 Info.setScratchRSrcReg(PrivateSegmentBufferReg); 2154 } else { 2155 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF); 2156 // We tentatively reserve the last registers (skipping the last registers 2157 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation, 2158 // we'll replace these with the ones immediately after those which were 2159 // really allocated. In the prologue copies will be inserted from the 2160 // argument to these reserved registers. 2161 2162 // Without HSA, relocations are used for the scratch pointer and the 2163 // buffer resource setup is always inserted in the prologue. Scratch wave 2164 // offset is still in an input SGPR. 2165 Info.setScratchRSrcReg(ReservedBufferReg); 2166 } 2167 } 2168 2169 MachineRegisterInfo &MRI = MF.getRegInfo(); 2170 2171 // For entry functions we have to set up the stack pointer if we use it, 2172 // whereas non-entry functions get this "for free". This means there is no 2173 // intrinsic advantage to using S32 over S34 in cases where we do not have 2174 // calls but do need a frame pointer (i.e. if we are requested to have one 2175 // because frame pointer elimination is disabled). To keep things simple we 2176 // only ever use S32 as the call ABI stack pointer, and so using it does not 2177 // imply we need a separate frame pointer. 2178 // 2179 // Try to use s32 as the SP, but move it if it would interfere with input 2180 // arguments. This won't work with calls though. 2181 // 2182 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input 2183 // registers. 2184 if (!MRI.isLiveIn(AMDGPU::SGPR32)) { 2185 Info.setStackPtrOffsetReg(AMDGPU::SGPR32); 2186 } else { 2187 assert(AMDGPU::isShader(MF.getFunction().getCallingConv())); 2188 2189 if (MFI.hasCalls()) 2190 report_fatal_error("call in graphics shader with too many input SGPRs"); 2191 2192 for (unsigned Reg : AMDGPU::SGPR_32RegClass) { 2193 if (!MRI.isLiveIn(Reg)) { 2194 Info.setStackPtrOffsetReg(Reg); 2195 break; 2196 } 2197 } 2198 2199 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG) 2200 report_fatal_error("failed to find register for SP"); 2201 } 2202 2203 // hasFP should be accurate for entry functions even before the frame is 2204 // finalized, because it does not rely on the known stack size, only 2205 // properties like whether variable sized objects are present. 2206 if (ST.getFrameLowering()->hasFP(MF)) { 2207 Info.setFrameOffsetReg(AMDGPU::SGPR33); 2208 } 2209 } 2210 2211 bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const { 2212 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 2213 return !Info->isEntryFunction(); 2214 } 2215 2216 void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { 2217 2218 } 2219 2220 void SITargetLowering::insertCopiesSplitCSR( 2221 MachineBasicBlock *Entry, 2222 const SmallVectorImpl<MachineBasicBlock *> &Exits) const { 2223 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); 2224 2225 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); 2226 if (!IStart) 2227 return; 2228 2229 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 2230 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); 2231 MachineBasicBlock::iterator MBBI = Entry->begin(); 2232 for (const MCPhysReg *I = IStart; *I; ++I) { 2233 const TargetRegisterClass *RC = nullptr; 2234 if (AMDGPU::SReg_64RegClass.contains(*I)) 2235 RC = &AMDGPU::SGPR_64RegClass; 2236 else if (AMDGPU::SReg_32RegClass.contains(*I)) 2237 RC = &AMDGPU::SGPR_32RegClass; 2238 else 2239 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 2240 2241 Register NewVR = MRI->createVirtualRegister(RC); 2242 // Create copy from CSR to a virtual register. 2243 Entry->addLiveIn(*I); 2244 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) 2245 .addReg(*I); 2246 2247 // Insert the copy-back instructions right before the terminator. 2248 for (auto *Exit : Exits) 2249 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), 2250 TII->get(TargetOpcode::COPY), *I) 2251 .addReg(NewVR); 2252 } 2253 } 2254 2255 SDValue SITargetLowering::LowerFormalArguments( 2256 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 2257 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 2258 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 2259 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); 2260 2261 MachineFunction &MF = DAG.getMachineFunction(); 2262 const Function &Fn = MF.getFunction(); 2263 FunctionType *FType = MF.getFunction().getFunctionType(); 2264 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 2265 2266 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) { 2267 DiagnosticInfoUnsupported NoGraphicsHSA( 2268 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()); 2269 DAG.getContext()->diagnose(NoGraphicsHSA); 2270 return DAG.getEntryNode(); 2271 } 2272 2273 Info->allocateModuleLDSGlobal(Fn.getParent()); 2274 2275 SmallVector<ISD::InputArg, 16> Splits; 2276 SmallVector<CCValAssign, 16> ArgLocs; 2277 BitVector Skipped(Ins.size()); 2278 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 2279 *DAG.getContext()); 2280 2281 bool IsGraphics = AMDGPU::isGraphics(CallConv); 2282 bool IsKernel = AMDGPU::isKernel(CallConv); 2283 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv); 2284 2285 if (IsGraphics) { 2286 assert(!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && 2287 (!Info->hasFlatScratchInit() || Subtarget->enableFlatScratch()) && 2288 !Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() && 2289 !Info->hasWorkGroupIDZ() && !Info->hasWorkGroupInfo() && 2290 !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() && 2291 !Info->hasWorkItemIDZ()); 2292 } 2293 2294 if (CallConv == CallingConv::AMDGPU_PS) { 2295 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info); 2296 2297 // At least one interpolation mode must be enabled or else the GPU will 2298 // hang. 2299 // 2300 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user 2301 // set PSInputAddr, the user wants to enable some bits after the compilation 2302 // based on run-time states. Since we can't know what the final PSInputEna 2303 // will look like, so we shouldn't do anything here and the user should take 2304 // responsibility for the correct programming. 2305 // 2306 // Otherwise, the following restrictions apply: 2307 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled. 2308 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be 2309 // enabled too. 2310 if ((Info->getPSInputAddr() & 0x7F) == 0 || 2311 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) { 2312 CCInfo.AllocateReg(AMDGPU::VGPR0); 2313 CCInfo.AllocateReg(AMDGPU::VGPR1); 2314 Info->markPSInputAllocated(0); 2315 Info->markPSInputEnabled(0); 2316 } 2317 if (Subtarget->isAmdPalOS()) { 2318 // For isAmdPalOS, the user does not enable some bits after compilation 2319 // based on run-time states; the register values being generated here are 2320 // the final ones set in hardware. Therefore we need to apply the 2321 // workaround to PSInputAddr and PSInputEnable together. (The case where 2322 // a bit is set in PSInputAddr but not PSInputEnable is where the 2323 // frontend set up an input arg for a particular interpolation mode, but 2324 // nothing uses that input arg. Really we should have an earlier pass 2325 // that removes such an arg.) 2326 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable(); 2327 if ((PsInputBits & 0x7F) == 0 || 2328 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1))) 2329 Info->markPSInputEnabled( 2330 countTrailingZeros(Info->getPSInputAddr(), ZB_Undefined)); 2331 } 2332 } else if (IsKernel) { 2333 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX()); 2334 } else { 2335 Splits.append(Ins.begin(), Ins.end()); 2336 } 2337 2338 if (IsEntryFunc) { 2339 allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info); 2340 allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info); 2341 } else { 2342 // For the fixed ABI, pass workitem IDs in the last argument register. 2343 if (AMDGPUTargetMachine::EnableFixedFunctionABI) 2344 allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); 2345 } 2346 2347 if (IsKernel) { 2348 analyzeFormalArgumentsCompute(CCInfo, Ins); 2349 } else { 2350 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg); 2351 CCInfo.AnalyzeFormalArguments(Splits, AssignFn); 2352 } 2353 2354 SmallVector<SDValue, 16> Chains; 2355 2356 // FIXME: This is the minimum kernel argument alignment. We should improve 2357 // this to the maximum alignment of the arguments. 2358 // 2359 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit 2360 // kern arg offset. 2361 const Align KernelArgBaseAlign = Align(16); 2362 2363 for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) { 2364 const ISD::InputArg &Arg = Ins[i]; 2365 if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) { 2366 InVals.push_back(DAG.getUNDEF(Arg.VT)); 2367 continue; 2368 } 2369 2370 CCValAssign &VA = ArgLocs[ArgIdx++]; 2371 MVT VT = VA.getLocVT(); 2372 2373 if (IsEntryFunc && VA.isMemLoc()) { 2374 VT = Ins[i].VT; 2375 EVT MemVT = VA.getLocVT(); 2376 2377 const uint64_t Offset = VA.getLocMemOffset(); 2378 Align Alignment = commonAlignment(KernelArgBaseAlign, Offset); 2379 2380 if (Arg.Flags.isByRef()) { 2381 SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset); 2382 2383 const GCNTargetMachine &TM = 2384 static_cast<const GCNTargetMachine &>(getTargetMachine()); 2385 if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS, 2386 Arg.Flags.getPointerAddrSpace())) { 2387 Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS, 2388 Arg.Flags.getPointerAddrSpace()); 2389 } 2390 2391 InVals.push_back(Ptr); 2392 continue; 2393 } 2394 2395 SDValue Arg = lowerKernargMemParameter( 2396 DAG, VT, MemVT, DL, Chain, Offset, Alignment, Ins[i].Flags.isSExt(), &Ins[i]); 2397 Chains.push_back(Arg.getValue(1)); 2398 2399 auto *ParamTy = 2400 dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex())); 2401 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS && 2402 ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || 2403 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) { 2404 // On SI local pointers are just offsets into LDS, so they are always 2405 // less than 16-bits. On CI and newer they could potentially be 2406 // real pointers, so we can't guarantee their size. 2407 Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg, 2408 DAG.getValueType(MVT::i16)); 2409 } 2410 2411 InVals.push_back(Arg); 2412 continue; 2413 } else if (!IsEntryFunc && VA.isMemLoc()) { 2414 SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg); 2415 InVals.push_back(Val); 2416 if (!Arg.Flags.isByVal()) 2417 Chains.push_back(Val.getValue(1)); 2418 continue; 2419 } 2420 2421 assert(VA.isRegLoc() && "Parameter must be in a register!"); 2422 2423 Register Reg = VA.getLocReg(); 2424 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); 2425 EVT ValVT = VA.getValVT(); 2426 2427 Reg = MF.addLiveIn(Reg, RC); 2428 SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT); 2429 2430 if (Arg.Flags.isSRet()) { 2431 // The return object should be reasonably addressable. 2432 2433 // FIXME: This helps when the return is a real sret. If it is a 2434 // automatically inserted sret (i.e. CanLowerReturn returns false), an 2435 // extra copy is inserted in SelectionDAGBuilder which obscures this. 2436 unsigned NumBits 2437 = 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex(); 2438 Val = DAG.getNode(ISD::AssertZext, DL, VT, Val, 2439 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits))); 2440 } 2441 2442 // If this is an 8 or 16-bit value, it is really passed promoted 2443 // to 32 bits. Insert an assert[sz]ext to capture this, then 2444 // truncate to the right size. 2445 switch (VA.getLocInfo()) { 2446 case CCValAssign::Full: 2447 break; 2448 case CCValAssign::BCvt: 2449 Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val); 2450 break; 2451 case CCValAssign::SExt: 2452 Val = DAG.getNode(ISD::AssertSext, DL, VT, Val, 2453 DAG.getValueType(ValVT)); 2454 Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val); 2455 break; 2456 case CCValAssign::ZExt: 2457 Val = DAG.getNode(ISD::AssertZext, DL, VT, Val, 2458 DAG.getValueType(ValVT)); 2459 Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val); 2460 break; 2461 case CCValAssign::AExt: 2462 Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val); 2463 break; 2464 default: 2465 llvm_unreachable("Unknown loc info!"); 2466 } 2467 2468 InVals.push_back(Val); 2469 } 2470 2471 if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) { 2472 // Special inputs come after user arguments. 2473 allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info); 2474 } 2475 2476 // Start adding system SGPRs. 2477 if (IsEntryFunc) { 2478 allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics); 2479 } else { 2480 CCInfo.AllocateReg(Info->getScratchRSrcReg()); 2481 allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); 2482 } 2483 2484 auto &ArgUsageInfo = 2485 DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>(); 2486 ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo()); 2487 2488 unsigned StackArgSize = CCInfo.getNextStackOffset(); 2489 Info->setBytesInStackArgArea(StackArgSize); 2490 2491 return Chains.empty() ? Chain : 2492 DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 2493 } 2494 2495 // TODO: If return values can't fit in registers, we should return as many as 2496 // possible in registers before passing on stack. 2497 bool SITargetLowering::CanLowerReturn( 2498 CallingConv::ID CallConv, 2499 MachineFunction &MF, bool IsVarArg, 2500 const SmallVectorImpl<ISD::OutputArg> &Outs, 2501 LLVMContext &Context) const { 2502 // Replacing returns with sret/stack usage doesn't make sense for shaders. 2503 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn 2504 // for shaders. Vector types should be explicitly handled by CC. 2505 if (AMDGPU::isEntryFunctionCC(CallConv)) 2506 return true; 2507 2508 SmallVector<CCValAssign, 16> RVLocs; 2509 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); 2510 return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg)); 2511 } 2512 2513 SDValue 2514 SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 2515 bool isVarArg, 2516 const SmallVectorImpl<ISD::OutputArg> &Outs, 2517 const SmallVectorImpl<SDValue> &OutVals, 2518 const SDLoc &DL, SelectionDAG &DAG) const { 2519 MachineFunction &MF = DAG.getMachineFunction(); 2520 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 2521 2522 if (AMDGPU::isKernel(CallConv)) { 2523 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs, 2524 OutVals, DL, DAG); 2525 } 2526 2527 bool IsShader = AMDGPU::isShader(CallConv); 2528 2529 Info->setIfReturnsVoid(Outs.empty()); 2530 bool IsWaveEnd = Info->returnsVoid() && IsShader; 2531 2532 // CCValAssign - represent the assignment of the return value to a location. 2533 SmallVector<CCValAssign, 48> RVLocs; 2534 SmallVector<ISD::OutputArg, 48> Splits; 2535 2536 // CCState - Info about the registers and stack slots. 2537 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 2538 *DAG.getContext()); 2539 2540 // Analyze outgoing return values. 2541 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg)); 2542 2543 SDValue Flag; 2544 SmallVector<SDValue, 48> RetOps; 2545 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 2546 2547 // Add return address for callable functions. 2548 if (!Info->isEntryFunction()) { 2549 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); 2550 SDValue ReturnAddrReg = CreateLiveInRegister( 2551 DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); 2552 2553 SDValue ReturnAddrVirtualReg = DAG.getRegister( 2554 MF.getRegInfo().createVirtualRegister(&AMDGPU::CCR_SGPR_64RegClass), 2555 MVT::i64); 2556 Chain = 2557 DAG.getCopyToReg(Chain, DL, ReturnAddrVirtualReg, ReturnAddrReg, Flag); 2558 Flag = Chain.getValue(1); 2559 RetOps.push_back(ReturnAddrVirtualReg); 2560 } 2561 2562 // Copy the result values into the output registers. 2563 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E; 2564 ++I, ++RealRVLocIdx) { 2565 CCValAssign &VA = RVLocs[I]; 2566 assert(VA.isRegLoc() && "Can only return in registers!"); 2567 // TODO: Partially return in registers if return values don't fit. 2568 SDValue Arg = OutVals[RealRVLocIdx]; 2569 2570 // Copied from other backends. 2571 switch (VA.getLocInfo()) { 2572 case CCValAssign::Full: 2573 break; 2574 case CCValAssign::BCvt: 2575 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); 2576 break; 2577 case CCValAssign::SExt: 2578 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); 2579 break; 2580 case CCValAssign::ZExt: 2581 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); 2582 break; 2583 case CCValAssign::AExt: 2584 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); 2585 break; 2586 default: 2587 llvm_unreachable("Unknown loc info!"); 2588 } 2589 2590 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag); 2591 Flag = Chain.getValue(1); 2592 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2593 } 2594 2595 // FIXME: Does sret work properly? 2596 if (!Info->isEntryFunction()) { 2597 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); 2598 const MCPhysReg *I = 2599 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 2600 if (I) { 2601 for (; *I; ++I) { 2602 if (AMDGPU::SReg_64RegClass.contains(*I)) 2603 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); 2604 else if (AMDGPU::SReg_32RegClass.contains(*I)) 2605 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); 2606 else 2607 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 2608 } 2609 } 2610 } 2611 2612 // Update chain and glue. 2613 RetOps[0] = Chain; 2614 if (Flag.getNode()) 2615 RetOps.push_back(Flag); 2616 2617 unsigned Opc = AMDGPUISD::ENDPGM; 2618 if (!IsWaveEnd) 2619 Opc = IsShader ? AMDGPUISD::RETURN_TO_EPILOG : AMDGPUISD::RET_FLAG; 2620 return DAG.getNode(Opc, DL, MVT::Other, RetOps); 2621 } 2622 2623 SDValue SITargetLowering::LowerCallResult( 2624 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg, 2625 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 2626 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn, 2627 SDValue ThisVal) const { 2628 CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg); 2629 2630 // Assign locations to each value returned by this call. 2631 SmallVector<CCValAssign, 16> RVLocs; 2632 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, 2633 *DAG.getContext()); 2634 CCInfo.AnalyzeCallResult(Ins, RetCC); 2635 2636 // Copy all of the result registers out of their specified physreg. 2637 for (unsigned i = 0; i != RVLocs.size(); ++i) { 2638 CCValAssign VA = RVLocs[i]; 2639 SDValue Val; 2640 2641 if (VA.isRegLoc()) { 2642 Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag); 2643 Chain = Val.getValue(1); 2644 InFlag = Val.getValue(2); 2645 } else if (VA.isMemLoc()) { 2646 report_fatal_error("TODO: return values in memory"); 2647 } else 2648 llvm_unreachable("unknown argument location type"); 2649 2650 switch (VA.getLocInfo()) { 2651 case CCValAssign::Full: 2652 break; 2653 case CCValAssign::BCvt: 2654 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); 2655 break; 2656 case CCValAssign::ZExt: 2657 Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val, 2658 DAG.getValueType(VA.getValVT())); 2659 Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val); 2660 break; 2661 case CCValAssign::SExt: 2662 Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val, 2663 DAG.getValueType(VA.getValVT())); 2664 Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val); 2665 break; 2666 case CCValAssign::AExt: 2667 Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val); 2668 break; 2669 default: 2670 llvm_unreachable("Unknown loc info!"); 2671 } 2672 2673 InVals.push_back(Val); 2674 } 2675 2676 return Chain; 2677 } 2678 2679 // Add code to pass special inputs required depending on used features separate 2680 // from the explicit user arguments present in the IR. 2681 void SITargetLowering::passSpecialInputs( 2682 CallLoweringInfo &CLI, 2683 CCState &CCInfo, 2684 const SIMachineFunctionInfo &Info, 2685 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, 2686 SmallVectorImpl<SDValue> &MemOpChains, 2687 SDValue Chain) const { 2688 // If we don't have a call site, this was a call inserted by 2689 // legalization. These can never use special inputs. 2690 if (!CLI.CB) 2691 return; 2692 2693 SelectionDAG &DAG = CLI.DAG; 2694 const SDLoc &DL = CLI.DL; 2695 2696 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); 2697 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo(); 2698 2699 const AMDGPUFunctionArgInfo *CalleeArgInfo 2700 = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo; 2701 if (const Function *CalleeFunc = CLI.CB->getCalledFunction()) { 2702 auto &ArgUsageInfo = 2703 DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>(); 2704 CalleeArgInfo = &ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc); 2705 } 2706 2707 // TODO: Unify with private memory register handling. This is complicated by 2708 // the fact that at least in kernels, the input argument is not necessarily 2709 // in the same location as the input. 2710 AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = { 2711 AMDGPUFunctionArgInfo::DISPATCH_PTR, 2712 AMDGPUFunctionArgInfo::QUEUE_PTR, 2713 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, 2714 AMDGPUFunctionArgInfo::DISPATCH_ID, 2715 AMDGPUFunctionArgInfo::WORKGROUP_ID_X, 2716 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, 2717 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z 2718 }; 2719 2720 for (auto InputID : InputRegs) { 2721 const ArgDescriptor *OutgoingArg; 2722 const TargetRegisterClass *ArgRC; 2723 LLT ArgTy; 2724 2725 std::tie(OutgoingArg, ArgRC, ArgTy) = 2726 CalleeArgInfo->getPreloadedValue(InputID); 2727 if (!OutgoingArg) 2728 continue; 2729 2730 const ArgDescriptor *IncomingArg; 2731 const TargetRegisterClass *IncomingArgRC; 2732 LLT Ty; 2733 std::tie(IncomingArg, IncomingArgRC, Ty) = 2734 CallerArgInfo.getPreloadedValue(InputID); 2735 assert(IncomingArgRC == ArgRC); 2736 2737 // All special arguments are ints for now. 2738 EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32; 2739 SDValue InputReg; 2740 2741 if (IncomingArg) { 2742 InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg); 2743 } else { 2744 // The implicit arg ptr is special because it doesn't have a corresponding 2745 // input for kernels, and is computed from the kernarg segment pointer. 2746 assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR); 2747 InputReg = getImplicitArgPtr(DAG, DL); 2748 } 2749 2750 if (OutgoingArg->isRegister()) { 2751 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg); 2752 if (!CCInfo.AllocateReg(OutgoingArg->getRegister())) 2753 report_fatal_error("failed to allocate implicit input argument"); 2754 } else { 2755 unsigned SpecialArgOffset = 2756 CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4)); 2757 SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg, 2758 SpecialArgOffset); 2759 MemOpChains.push_back(ArgStore); 2760 } 2761 } 2762 2763 // Pack workitem IDs into a single register or pass it as is if already 2764 // packed. 2765 const ArgDescriptor *OutgoingArg; 2766 const TargetRegisterClass *ArgRC; 2767 LLT Ty; 2768 2769 std::tie(OutgoingArg, ArgRC, Ty) = 2770 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X); 2771 if (!OutgoingArg) 2772 std::tie(OutgoingArg, ArgRC, Ty) = 2773 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y); 2774 if (!OutgoingArg) 2775 std::tie(OutgoingArg, ArgRC, Ty) = 2776 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z); 2777 if (!OutgoingArg) 2778 return; 2779 2780 const ArgDescriptor *IncomingArgX = std::get<0>( 2781 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X)); 2782 const ArgDescriptor *IncomingArgY = std::get<0>( 2783 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y)); 2784 const ArgDescriptor *IncomingArgZ = std::get<0>( 2785 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z)); 2786 2787 SDValue InputReg; 2788 SDLoc SL; 2789 2790 // If incoming ids are not packed we need to pack them. 2791 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX) 2792 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX); 2793 2794 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY) { 2795 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY); 2796 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, 2797 DAG.getShiftAmountConstant(10, MVT::i32, SL)); 2798 InputReg = InputReg.getNode() ? 2799 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; 2800 } 2801 2802 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ) { 2803 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ); 2804 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, 2805 DAG.getShiftAmountConstant(20, MVT::i32, SL)); 2806 InputReg = InputReg.getNode() ? 2807 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; 2808 } 2809 2810 if (!InputReg.getNode()) { 2811 // Workitem ids are already packed, any of present incoming arguments 2812 // will carry all required fields. 2813 ArgDescriptor IncomingArg = ArgDescriptor::createArg( 2814 IncomingArgX ? *IncomingArgX : 2815 IncomingArgY ? *IncomingArgY : 2816 *IncomingArgZ, ~0u); 2817 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg); 2818 } 2819 2820 if (OutgoingArg->isRegister()) { 2821 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg); 2822 CCInfo.AllocateReg(OutgoingArg->getRegister()); 2823 } else { 2824 unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4)); 2825 SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg, 2826 SpecialArgOffset); 2827 MemOpChains.push_back(ArgStore); 2828 } 2829 } 2830 2831 static bool canGuaranteeTCO(CallingConv::ID CC) { 2832 return CC == CallingConv::Fast; 2833 } 2834 2835 /// Return true if we might ever do TCO for calls with this calling convention. 2836 static bool mayTailCallThisCC(CallingConv::ID CC) { 2837 switch (CC) { 2838 case CallingConv::C: 2839 case CallingConv::AMDGPU_Gfx: 2840 return true; 2841 default: 2842 return canGuaranteeTCO(CC); 2843 } 2844 } 2845 2846 bool SITargetLowering::isEligibleForTailCallOptimization( 2847 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg, 2848 const SmallVectorImpl<ISD::OutputArg> &Outs, 2849 const SmallVectorImpl<SDValue> &OutVals, 2850 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const { 2851 if (!mayTailCallThisCC(CalleeCC)) 2852 return false; 2853 2854 // For a divergent call target, we need to do a waterfall loop over the 2855 // possible callees which precludes us from using a simple jump. 2856 if (Callee->isDivergent()) 2857 return false; 2858 2859 MachineFunction &MF = DAG.getMachineFunction(); 2860 const Function &CallerF = MF.getFunction(); 2861 CallingConv::ID CallerCC = CallerF.getCallingConv(); 2862 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); 2863 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); 2864 2865 // Kernels aren't callable, and don't have a live in return address so it 2866 // doesn't make sense to do a tail call with entry functions. 2867 if (!CallerPreserved) 2868 return false; 2869 2870 bool CCMatch = CallerCC == CalleeCC; 2871 2872 if (DAG.getTarget().Options.GuaranteedTailCallOpt) { 2873 if (canGuaranteeTCO(CalleeCC) && CCMatch) 2874 return true; 2875 return false; 2876 } 2877 2878 // TODO: Can we handle var args? 2879 if (IsVarArg) 2880 return false; 2881 2882 for (const Argument &Arg : CallerF.args()) { 2883 if (Arg.hasByValAttr()) 2884 return false; 2885 } 2886 2887 LLVMContext &Ctx = *DAG.getContext(); 2888 2889 // Check that the call results are passed in the same way. 2890 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins, 2891 CCAssignFnForCall(CalleeCC, IsVarArg), 2892 CCAssignFnForCall(CallerCC, IsVarArg))) 2893 return false; 2894 2895 // The callee has to preserve all registers the caller needs to preserve. 2896 if (!CCMatch) { 2897 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); 2898 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) 2899 return false; 2900 } 2901 2902 // Nothing more to check if the callee is taking no arguments. 2903 if (Outs.empty()) 2904 return true; 2905 2906 SmallVector<CCValAssign, 16> ArgLocs; 2907 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx); 2908 2909 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg)); 2910 2911 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 2912 // If the stack arguments for this call do not fit into our own save area then 2913 // the call cannot be made tail. 2914 // TODO: Is this really necessary? 2915 if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) 2916 return false; 2917 2918 const MachineRegisterInfo &MRI = MF.getRegInfo(); 2919 return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals); 2920 } 2921 2922 bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 2923 if (!CI->isTailCall()) 2924 return false; 2925 2926 const Function *ParentFn = CI->getParent()->getParent(); 2927 if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv())) 2928 return false; 2929 return true; 2930 } 2931 2932 // The wave scratch offset register is used as the global base pointer. 2933 SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, 2934 SmallVectorImpl<SDValue> &InVals) const { 2935 SelectionDAG &DAG = CLI.DAG; 2936 const SDLoc &DL = CLI.DL; 2937 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; 2938 SmallVector<SDValue, 32> &OutVals = CLI.OutVals; 2939 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; 2940 SDValue Chain = CLI.Chain; 2941 SDValue Callee = CLI.Callee; 2942 bool &IsTailCall = CLI.IsTailCall; 2943 CallingConv::ID CallConv = CLI.CallConv; 2944 bool IsVarArg = CLI.IsVarArg; 2945 bool IsSibCall = false; 2946 bool IsThisReturn = false; 2947 MachineFunction &MF = DAG.getMachineFunction(); 2948 2949 if (Callee.isUndef() || isNullConstant(Callee)) { 2950 if (!CLI.IsTailCall) { 2951 for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I) 2952 InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT)); 2953 } 2954 2955 return Chain; 2956 } 2957 2958 if (IsVarArg) { 2959 return lowerUnhandledCall(CLI, InVals, 2960 "unsupported call to variadic function "); 2961 } 2962 2963 if (!CLI.CB) 2964 report_fatal_error("unsupported libcall legalization"); 2965 2966 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) { 2967 return lowerUnhandledCall(CLI, InVals, 2968 "unsupported required tail call to function "); 2969 } 2970 2971 if (AMDGPU::isShader(CallConv)) { 2972 // Note the issue is with the CC of the called function, not of the call 2973 // itself. 2974 return lowerUnhandledCall(CLI, InVals, 2975 "unsupported call to a shader function "); 2976 } 2977 2978 if (AMDGPU::isShader(MF.getFunction().getCallingConv()) && 2979 CallConv != CallingConv::AMDGPU_Gfx) { 2980 // Only allow calls with specific calling conventions. 2981 return lowerUnhandledCall(CLI, InVals, 2982 "unsupported calling convention for call from " 2983 "graphics shader of function "); 2984 } 2985 2986 if (IsTailCall) { 2987 IsTailCall = isEligibleForTailCallOptimization( 2988 Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG); 2989 if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { 2990 report_fatal_error("failed to perform tail call elimination on a call " 2991 "site marked musttail"); 2992 } 2993 2994 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; 2995 2996 // A sibling call is one where we're under the usual C ABI and not planning 2997 // to change that but can still do a tail call: 2998 if (!TailCallOpt && IsTailCall) 2999 IsSibCall = true; 3000 3001 if (IsTailCall) 3002 ++NumTailCalls; 3003 } 3004 3005 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 3006 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 3007 SmallVector<SDValue, 8> MemOpChains; 3008 3009 // Analyze operands of the call, assigning locations to each operand. 3010 SmallVector<CCValAssign, 16> ArgLocs; 3011 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 3012 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg); 3013 3014 if (AMDGPUTargetMachine::EnableFixedFunctionABI && 3015 CallConv != CallingConv::AMDGPU_Gfx) { 3016 // With a fixed ABI, allocate fixed registers before user arguments. 3017 passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain); 3018 } 3019 3020 CCInfo.AnalyzeCallOperands(Outs, AssignFn); 3021 3022 // Get a count of how many bytes are to be pushed on the stack. 3023 unsigned NumBytes = CCInfo.getNextStackOffset(); 3024 3025 if (IsSibCall) { 3026 // Since we're not changing the ABI to make this a tail call, the memory 3027 // operands are already available in the caller's incoming argument space. 3028 NumBytes = 0; 3029 } 3030 3031 // FPDiff is the byte offset of the call's argument area from the callee's. 3032 // Stores to callee stack arguments will be placed in FixedStackSlots offset 3033 // by this amount for a tail call. In a sibling call it must be 0 because the 3034 // caller will deallocate the entire stack and the callee still expects its 3035 // arguments to begin at SP+0. Completely unused for non-tail calls. 3036 int32_t FPDiff = 0; 3037 MachineFrameInfo &MFI = MF.getFrameInfo(); 3038 3039 // Adjust the stack pointer for the new arguments... 3040 // These operations are automatically eliminated by the prolog/epilog pass 3041 if (!IsSibCall) { 3042 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL); 3043 3044 if (!Subtarget->enableFlatScratch()) { 3045 SmallVector<SDValue, 4> CopyFromChains; 3046 3047 // In the HSA case, this should be an identity copy. 3048 SDValue ScratchRSrcReg 3049 = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32); 3050 RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg); 3051 CopyFromChains.push_back(ScratchRSrcReg.getValue(1)); 3052 Chain = DAG.getTokenFactor(DL, CopyFromChains); 3053 } 3054 } 3055 3056 MVT PtrVT = MVT::i32; 3057 3058 // Walk the register/memloc assignments, inserting copies/loads. 3059 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3060 CCValAssign &VA = ArgLocs[i]; 3061 SDValue Arg = OutVals[i]; 3062 3063 // Promote the value if needed. 3064 switch (VA.getLocInfo()) { 3065 case CCValAssign::Full: 3066 break; 3067 case CCValAssign::BCvt: 3068 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); 3069 break; 3070 case CCValAssign::ZExt: 3071 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); 3072 break; 3073 case CCValAssign::SExt: 3074 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); 3075 break; 3076 case CCValAssign::AExt: 3077 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); 3078 break; 3079 case CCValAssign::FPExt: 3080 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); 3081 break; 3082 default: 3083 llvm_unreachable("Unknown loc info!"); 3084 } 3085 3086 if (VA.isRegLoc()) { 3087 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 3088 } else { 3089 assert(VA.isMemLoc()); 3090 3091 SDValue DstAddr; 3092 MachinePointerInfo DstInfo; 3093 3094 unsigned LocMemOffset = VA.getLocMemOffset(); 3095 int32_t Offset = LocMemOffset; 3096 3097 SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT); 3098 MaybeAlign Alignment; 3099 3100 if (IsTailCall) { 3101 ISD::ArgFlagsTy Flags = Outs[i].Flags; 3102 unsigned OpSize = Flags.isByVal() ? 3103 Flags.getByValSize() : VA.getValVT().getStoreSize(); 3104 3105 // FIXME: We can have better than the minimum byval required alignment. 3106 Alignment = 3107 Flags.isByVal() 3108 ? Flags.getNonZeroByValAlign() 3109 : commonAlignment(Subtarget->getStackAlignment(), Offset); 3110 3111 Offset = Offset + FPDiff; 3112 int FI = MFI.CreateFixedObject(OpSize, Offset, true); 3113 3114 DstAddr = DAG.getFrameIndex(FI, PtrVT); 3115 DstInfo = MachinePointerInfo::getFixedStack(MF, FI); 3116 3117 // Make sure any stack arguments overlapping with where we're storing 3118 // are loaded before this eventual operation. Otherwise they'll be 3119 // clobbered. 3120 3121 // FIXME: Why is this really necessary? This seems to just result in a 3122 // lot of code to copy the stack and write them back to the same 3123 // locations, which are supposed to be immutable? 3124 Chain = addTokenForArgument(Chain, DAG, MFI, FI); 3125 } else { 3126 DstAddr = PtrOff; 3127 DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset); 3128 Alignment = 3129 commonAlignment(Subtarget->getStackAlignment(), LocMemOffset); 3130 } 3131 3132 if (Outs[i].Flags.isByVal()) { 3133 SDValue SizeNode = 3134 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32); 3135 SDValue Cpy = 3136 DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode, 3137 Outs[i].Flags.getNonZeroByValAlign(), 3138 /*isVol = */ false, /*AlwaysInline = */ true, 3139 /*isTailCall = */ false, DstInfo, 3140 MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS)); 3141 3142 MemOpChains.push_back(Cpy); 3143 } else { 3144 SDValue Store = 3145 DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment); 3146 MemOpChains.push_back(Store); 3147 } 3148 } 3149 } 3150 3151 if (!AMDGPUTargetMachine::EnableFixedFunctionABI && 3152 CallConv != CallingConv::AMDGPU_Gfx) { 3153 // Copy special input registers after user input arguments. 3154 passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain); 3155 } 3156 3157 if (!MemOpChains.empty()) 3158 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 3159 3160 // Build a sequence of copy-to-reg nodes chained together with token chain 3161 // and flag operands which copy the outgoing args into the appropriate regs. 3162 SDValue InFlag; 3163 for (auto &RegToPass : RegsToPass) { 3164 Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first, 3165 RegToPass.second, InFlag); 3166 InFlag = Chain.getValue(1); 3167 } 3168 3169 3170 SDValue PhysReturnAddrReg; 3171 if (IsTailCall) { 3172 // Since the return is being combined with the call, we need to pass on the 3173 // return address. 3174 3175 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); 3176 SDValue ReturnAddrReg = CreateLiveInRegister( 3177 DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); 3178 3179 PhysReturnAddrReg = DAG.getRegister(TRI->getReturnAddressReg(MF), 3180 MVT::i64); 3181 Chain = DAG.getCopyToReg(Chain, DL, PhysReturnAddrReg, ReturnAddrReg, InFlag); 3182 InFlag = Chain.getValue(1); 3183 } 3184 3185 // We don't usually want to end the call-sequence here because we would tidy 3186 // the frame up *after* the call, however in the ABI-changing tail-call case 3187 // we've carefully laid out the parameters so that when sp is reset they'll be 3188 // in the correct location. 3189 if (IsTailCall && !IsSibCall) { 3190 Chain = DAG.getCALLSEQ_END(Chain, 3191 DAG.getTargetConstant(NumBytes, DL, MVT::i32), 3192 DAG.getTargetConstant(0, DL, MVT::i32), 3193 InFlag, DL); 3194 InFlag = Chain.getValue(1); 3195 } 3196 3197 std::vector<SDValue> Ops; 3198 Ops.push_back(Chain); 3199 Ops.push_back(Callee); 3200 // Add a redundant copy of the callee global which will not be legalized, as 3201 // we need direct access to the callee later. 3202 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) { 3203 const GlobalValue *GV = GSD->getGlobal(); 3204 Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64)); 3205 } else { 3206 Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); 3207 } 3208 3209 if (IsTailCall) { 3210 // Each tail call may have to adjust the stack by a different amount, so 3211 // this information must travel along with the operation for eventual 3212 // consumption by emitEpilogue. 3213 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); 3214 3215 Ops.push_back(PhysReturnAddrReg); 3216 } 3217 3218 // Add argument registers to the end of the list so that they are known live 3219 // into the call. 3220 for (auto &RegToPass : RegsToPass) { 3221 Ops.push_back(DAG.getRegister(RegToPass.first, 3222 RegToPass.second.getValueType())); 3223 } 3224 3225 // Add a register mask operand representing the call-preserved registers. 3226 3227 auto *TRI = static_cast<const SIRegisterInfo*>(Subtarget->getRegisterInfo()); 3228 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); 3229 assert(Mask && "Missing call preserved mask for calling convention"); 3230 Ops.push_back(DAG.getRegisterMask(Mask)); 3231 3232 if (InFlag.getNode()) 3233 Ops.push_back(InFlag); 3234 3235 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 3236 3237 // If we're doing a tall call, use a TC_RETURN here rather than an 3238 // actual call instruction. 3239 if (IsTailCall) { 3240 MFI.setHasTailCall(); 3241 return DAG.getNode(AMDGPUISD::TC_RETURN, DL, NodeTys, Ops); 3242 } 3243 3244 // Returns a chain and a flag for retval copy to use. 3245 SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops); 3246 Chain = Call.getValue(0); 3247 InFlag = Call.getValue(1); 3248 3249 uint64_t CalleePopBytes = NumBytes; 3250 Chain = DAG.getCALLSEQ_END(Chain, DAG.getTargetConstant(0, DL, MVT::i32), 3251 DAG.getTargetConstant(CalleePopBytes, DL, MVT::i32), 3252 InFlag, DL); 3253 if (!Ins.empty()) 3254 InFlag = Chain.getValue(1); 3255 3256 // Handle result values, copying them out of physregs into vregs that we 3257 // return. 3258 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG, 3259 InVals, IsThisReturn, 3260 IsThisReturn ? OutVals[0] : SDValue()); 3261 } 3262 3263 // This is identical to the default implementation in ExpandDYNAMIC_STACKALLOC, 3264 // except for applying the wave size scale to the increment amount. 3265 SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl( 3266 SDValue Op, SelectionDAG &DAG) const { 3267 const MachineFunction &MF = DAG.getMachineFunction(); 3268 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 3269 3270 SDLoc dl(Op); 3271 EVT VT = Op.getValueType(); 3272 SDValue Tmp1 = Op; 3273 SDValue Tmp2 = Op.getValue(1); 3274 SDValue Tmp3 = Op.getOperand(2); 3275 SDValue Chain = Tmp1.getOperand(0); 3276 3277 Register SPReg = Info->getStackPtrOffsetReg(); 3278 3279 // Chain the dynamic stack allocation so that it doesn't modify the stack 3280 // pointer when other instructions are using the stack. 3281 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); 3282 3283 SDValue Size = Tmp2.getOperand(1); 3284 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 3285 Chain = SP.getValue(1); 3286 MaybeAlign Alignment = cast<ConstantSDNode>(Tmp3)->getMaybeAlignValue(); 3287 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 3288 const TargetFrameLowering *TFL = ST.getFrameLowering(); 3289 unsigned Opc = 3290 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? 3291 ISD::ADD : ISD::SUB; 3292 3293 SDValue ScaledSize = DAG.getNode( 3294 ISD::SHL, dl, VT, Size, 3295 DAG.getConstant(ST.getWavefrontSizeLog2(), dl, MVT::i32)); 3296 3297 Align StackAlign = TFL->getStackAlign(); 3298 Tmp1 = DAG.getNode(Opc, dl, VT, SP, ScaledSize); // Value 3299 if (Alignment && *Alignment > StackAlign) { 3300 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, 3301 DAG.getConstant(-(uint64_t)Alignment->value() 3302 << ST.getWavefrontSizeLog2(), 3303 dl, VT)); 3304 } 3305 3306 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 3307 Tmp2 = DAG.getCALLSEQ_END( 3308 Chain, DAG.getIntPtrConstant(0, dl, true), 3309 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); 3310 3311 return DAG.getMergeValues({Tmp1, Tmp2}, dl); 3312 } 3313 3314 SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 3315 SelectionDAG &DAG) const { 3316 // We only handle constant sizes here to allow non-entry block, static sized 3317 // allocas. A truly dynamic value is more difficult to support because we 3318 // don't know if the size value is uniform or not. If the size isn't uniform, 3319 // we would need to do a wave reduction to get the maximum size to know how 3320 // much to increment the uniform stack pointer. 3321 SDValue Size = Op.getOperand(1); 3322 if (isa<ConstantSDNode>(Size)) 3323 return lowerDYNAMIC_STACKALLOCImpl(Op, DAG); // Use "generic" expansion. 3324 3325 return AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(Op, DAG); 3326 } 3327 3328 Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT, 3329 const MachineFunction &MF) const { 3330 Register Reg = StringSwitch<Register>(RegName) 3331 .Case("m0", AMDGPU::M0) 3332 .Case("exec", AMDGPU::EXEC) 3333 .Case("exec_lo", AMDGPU::EXEC_LO) 3334 .Case("exec_hi", AMDGPU::EXEC_HI) 3335 .Case("flat_scratch", AMDGPU::FLAT_SCR) 3336 .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO) 3337 .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI) 3338 .Default(Register()); 3339 3340 if (Reg == AMDGPU::NoRegister) { 3341 report_fatal_error(Twine("invalid register name \"" 3342 + StringRef(RegName) + "\".")); 3343 3344 } 3345 3346 if (!Subtarget->hasFlatScrRegister() && 3347 Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) { 3348 report_fatal_error(Twine("invalid register \"" 3349 + StringRef(RegName) + "\" for subtarget.")); 3350 } 3351 3352 switch (Reg) { 3353 case AMDGPU::M0: 3354 case AMDGPU::EXEC_LO: 3355 case AMDGPU::EXEC_HI: 3356 case AMDGPU::FLAT_SCR_LO: 3357 case AMDGPU::FLAT_SCR_HI: 3358 if (VT.getSizeInBits() == 32) 3359 return Reg; 3360 break; 3361 case AMDGPU::EXEC: 3362 case AMDGPU::FLAT_SCR: 3363 if (VT.getSizeInBits() == 64) 3364 return Reg; 3365 break; 3366 default: 3367 llvm_unreachable("missing register type checking"); 3368 } 3369 3370 report_fatal_error(Twine("invalid type for register \"" 3371 + StringRef(RegName) + "\".")); 3372 } 3373 3374 // If kill is not the last instruction, split the block so kill is always a 3375 // proper terminator. 3376 MachineBasicBlock * 3377 SITargetLowering::splitKillBlock(MachineInstr &MI, 3378 MachineBasicBlock *BB) const { 3379 MachineBasicBlock *SplitBB = BB->splitAt(MI, false /*UpdateLiveIns*/); 3380 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 3381 MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode())); 3382 return SplitBB; 3383 } 3384 3385 // Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true, 3386 // \p MI will be the only instruction in the loop body block. Otherwise, it will 3387 // be the first instruction in the remainder block. 3388 // 3389 /// \returns { LoopBody, Remainder } 3390 static std::pair<MachineBasicBlock *, MachineBasicBlock *> 3391 splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) { 3392 MachineFunction *MF = MBB.getParent(); 3393 MachineBasicBlock::iterator I(&MI); 3394 3395 // To insert the loop we need to split the block. Move everything after this 3396 // point to a new block, and insert a new empty block between the two. 3397 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock(); 3398 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock(); 3399 MachineFunction::iterator MBBI(MBB); 3400 ++MBBI; 3401 3402 MF->insert(MBBI, LoopBB); 3403 MF->insert(MBBI, RemainderBB); 3404 3405 LoopBB->addSuccessor(LoopBB); 3406 LoopBB->addSuccessor(RemainderBB); 3407 3408 // Move the rest of the block into a new block. 3409 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 3410 3411 if (InstInLoop) { 3412 auto Next = std::next(I); 3413 3414 // Move instruction to loop body. 3415 LoopBB->splice(LoopBB->begin(), &MBB, I, Next); 3416 3417 // Move the rest of the block. 3418 RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end()); 3419 } else { 3420 RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end()); 3421 } 3422 3423 MBB.addSuccessor(LoopBB); 3424 3425 return std::make_pair(LoopBB, RemainderBB); 3426 } 3427 3428 /// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it. 3429 void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const { 3430 MachineBasicBlock *MBB = MI.getParent(); 3431 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 3432 auto I = MI.getIterator(); 3433 auto E = std::next(I); 3434 3435 BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT)) 3436 .addImm(0); 3437 3438 MIBundleBuilder Bundler(*MBB, I, E); 3439 finalizeBundle(*MBB, Bundler.begin()); 3440 } 3441 3442 MachineBasicBlock * 3443 SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI, 3444 MachineBasicBlock *BB) const { 3445 const DebugLoc &DL = MI.getDebugLoc(); 3446 3447 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 3448 3449 MachineBasicBlock *LoopBB; 3450 MachineBasicBlock *RemainderBB; 3451 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 3452 3453 // Apparently kill flags are only valid if the def is in the same block? 3454 if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0)) 3455 Src->setIsKill(false); 3456 3457 std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true); 3458 3459 MachineBasicBlock::iterator I = LoopBB->end(); 3460 3461 const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg( 3462 AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1); 3463 3464 // Clear TRAP_STS.MEM_VIOL 3465 BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32)) 3466 .addImm(0) 3467 .addImm(EncodedReg); 3468 3469 bundleInstWithWaitcnt(MI); 3470 3471 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 3472 3473 // Load and check TRAP_STS.MEM_VIOL 3474 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg) 3475 .addImm(EncodedReg); 3476 3477 // FIXME: Do we need to use an isel pseudo that may clobber scc? 3478 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32)) 3479 .addReg(Reg, RegState::Kill) 3480 .addImm(0); 3481 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1)) 3482 .addMBB(LoopBB); 3483 3484 return RemainderBB; 3485 } 3486 3487 // Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the 3488 // wavefront. If the value is uniform and just happens to be in a VGPR, this 3489 // will only do one iteration. In the worst case, this will loop 64 times. 3490 // 3491 // TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value. 3492 static MachineBasicBlock::iterator 3493 emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, 3494 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, 3495 const DebugLoc &DL, const MachineOperand &Idx, 3496 unsigned InitReg, unsigned ResultReg, unsigned PhiReg, 3497 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, 3498 Register &SGPRIdxReg) { 3499 3500 MachineFunction *MF = OrigBB.getParent(); 3501 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 3502 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 3503 MachineBasicBlock::iterator I = LoopBB.begin(); 3504 3505 const TargetRegisterClass *BoolRC = TRI->getBoolRC(); 3506 Register PhiExec = MRI.createVirtualRegister(BoolRC); 3507 Register NewExec = MRI.createVirtualRegister(BoolRC); 3508 Register CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 3509 Register CondReg = MRI.createVirtualRegister(BoolRC); 3510 3511 BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg) 3512 .addReg(InitReg) 3513 .addMBB(&OrigBB) 3514 .addReg(ResultReg) 3515 .addMBB(&LoopBB); 3516 3517 BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec) 3518 .addReg(InitSaveExecReg) 3519 .addMBB(&OrigBB) 3520 .addReg(NewExec) 3521 .addMBB(&LoopBB); 3522 3523 // Read the next variant <- also loop target. 3524 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg) 3525 .addReg(Idx.getReg(), getUndefRegState(Idx.isUndef())); 3526 3527 // Compare the just read M0 value to all possible Idx values. 3528 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg) 3529 .addReg(CurrentIdxReg) 3530 .addReg(Idx.getReg(), 0, Idx.getSubReg()); 3531 3532 // Update EXEC, save the original EXEC value to VCC. 3533 BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 3534 : AMDGPU::S_AND_SAVEEXEC_B64), 3535 NewExec) 3536 .addReg(CondReg, RegState::Kill); 3537 3538 MRI.setSimpleHint(NewExec, CondReg); 3539 3540 if (UseGPRIdxMode) { 3541 if (Offset == 0) { 3542 SGPRIdxReg = CurrentIdxReg; 3543 } else { 3544 SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 3545 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg) 3546 .addReg(CurrentIdxReg, RegState::Kill) 3547 .addImm(Offset); 3548 } 3549 } else { 3550 // Move index from VCC into M0 3551 if (Offset == 0) { 3552 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) 3553 .addReg(CurrentIdxReg, RegState::Kill); 3554 } else { 3555 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0) 3556 .addReg(CurrentIdxReg, RegState::Kill) 3557 .addImm(Offset); 3558 } 3559 } 3560 3561 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 3562 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 3563 MachineInstr *InsertPt = 3564 BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_XOR_B32_term 3565 : AMDGPU::S_XOR_B64_term), Exec) 3566 .addReg(Exec) 3567 .addReg(NewExec); 3568 3569 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use 3570 // s_cbranch_scc0? 3571 3572 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover. 3573 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ)) 3574 .addMBB(&LoopBB); 3575 3576 return InsertPt->getIterator(); 3577 } 3578 3579 // This has slightly sub-optimal regalloc when the source vector is killed by 3580 // the read. The register allocator does not understand that the kill is 3581 // per-workitem, so is kept alive for the whole loop so we end up not re-using a 3582 // subregister from it, using 1 more VGPR than necessary. This was saved when 3583 // this was expanded after register allocation. 3584 static MachineBasicBlock::iterator 3585 loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, 3586 unsigned InitResultReg, unsigned PhiReg, int Offset, 3587 bool UseGPRIdxMode, Register &SGPRIdxReg) { 3588 MachineFunction *MF = MBB.getParent(); 3589 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 3590 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 3591 MachineRegisterInfo &MRI = MF->getRegInfo(); 3592 const DebugLoc &DL = MI.getDebugLoc(); 3593 MachineBasicBlock::iterator I(&MI); 3594 3595 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 3596 Register DstReg = MI.getOperand(0).getReg(); 3597 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 3598 Register TmpExec = MRI.createVirtualRegister(BoolXExecRC); 3599 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 3600 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 3601 3602 BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec); 3603 3604 // Save the EXEC mask 3605 BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec) 3606 .addReg(Exec); 3607 3608 MachineBasicBlock *LoopBB; 3609 MachineBasicBlock *RemainderBB; 3610 std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false); 3611 3612 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); 3613 3614 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx, 3615 InitResultReg, DstReg, PhiReg, TmpExec, 3616 Offset, UseGPRIdxMode, SGPRIdxReg); 3617 3618 MachineBasicBlock* LandingPad = MF->CreateMachineBasicBlock(); 3619 MachineFunction::iterator MBBI(LoopBB); 3620 ++MBBI; 3621 MF->insert(MBBI, LandingPad); 3622 LoopBB->removeSuccessor(RemainderBB); 3623 LandingPad->addSuccessor(RemainderBB); 3624 LoopBB->addSuccessor(LandingPad); 3625 MachineBasicBlock::iterator First = LandingPad->begin(); 3626 BuildMI(*LandingPad, First, DL, TII->get(MovExecOpc), Exec) 3627 .addReg(SaveExec); 3628 3629 return InsPt; 3630 } 3631 3632 // Returns subreg index, offset 3633 static std::pair<unsigned, int> 3634 computeIndirectRegAndOffset(const SIRegisterInfo &TRI, 3635 const TargetRegisterClass *SuperRC, 3636 unsigned VecReg, 3637 int Offset) { 3638 int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32; 3639 3640 // Skip out of bounds offsets, or else we would end up using an undefined 3641 // register. 3642 if (Offset >= NumElts || Offset < 0) 3643 return std::make_pair(AMDGPU::sub0, Offset); 3644 3645 return std::make_pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0); 3646 } 3647 3648 static void setM0ToIndexFromSGPR(const SIInstrInfo *TII, 3649 MachineRegisterInfo &MRI, MachineInstr &MI, 3650 int Offset) { 3651 MachineBasicBlock *MBB = MI.getParent(); 3652 const DebugLoc &DL = MI.getDebugLoc(); 3653 MachineBasicBlock::iterator I(&MI); 3654 3655 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); 3656 3657 assert(Idx->getReg() != AMDGPU::NoRegister); 3658 3659 if (Offset == 0) { 3660 BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0).add(*Idx); 3661 } else { 3662 BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0) 3663 .add(*Idx) 3664 .addImm(Offset); 3665 } 3666 } 3667 3668 static Register getIndirectSGPRIdx(const SIInstrInfo *TII, 3669 MachineRegisterInfo &MRI, MachineInstr &MI, 3670 int Offset) { 3671 MachineBasicBlock *MBB = MI.getParent(); 3672 const DebugLoc &DL = MI.getDebugLoc(); 3673 MachineBasicBlock::iterator I(&MI); 3674 3675 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); 3676 3677 if (Offset == 0) 3678 return Idx->getReg(); 3679 3680 Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 3681 BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp) 3682 .add(*Idx) 3683 .addImm(Offset); 3684 return Tmp; 3685 } 3686 3687 static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI, 3688 MachineBasicBlock &MBB, 3689 const GCNSubtarget &ST) { 3690 const SIInstrInfo *TII = ST.getInstrInfo(); 3691 const SIRegisterInfo &TRI = TII->getRegisterInfo(); 3692 MachineFunction *MF = MBB.getParent(); 3693 MachineRegisterInfo &MRI = MF->getRegInfo(); 3694 3695 Register Dst = MI.getOperand(0).getReg(); 3696 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); 3697 Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg(); 3698 int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm(); 3699 3700 const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg); 3701 const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg()); 3702 3703 unsigned SubReg; 3704 std::tie(SubReg, Offset) 3705 = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset); 3706 3707 const bool UseGPRIdxMode = ST.useVGPRIndexMode(); 3708 3709 // Check for a SGPR index. 3710 if (TII->getRegisterInfo().isSGPRClass(IdxRC)) { 3711 MachineBasicBlock::iterator I(&MI); 3712 const DebugLoc &DL = MI.getDebugLoc(); 3713 3714 if (UseGPRIdxMode) { 3715 // TODO: Look at the uses to avoid the copy. This may require rescheduling 3716 // to avoid interfering with other uses, so probably requires a new 3717 // optimization pass. 3718 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset); 3719 3720 const MCInstrDesc &GPRIDXDesc = 3721 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true); 3722 BuildMI(MBB, I, DL, GPRIDXDesc, Dst) 3723 .addReg(SrcReg) 3724 .addReg(Idx) 3725 .addImm(SubReg); 3726 } else { 3727 setM0ToIndexFromSGPR(TII, MRI, MI, Offset); 3728 3729 BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst) 3730 .addReg(SrcReg, 0, SubReg) 3731 .addReg(SrcReg, RegState::Implicit); 3732 } 3733 3734 MI.eraseFromParent(); 3735 3736 return &MBB; 3737 } 3738 3739 // Control flow needs to be inserted if indexing with a VGPR. 3740 const DebugLoc &DL = MI.getDebugLoc(); 3741 MachineBasicBlock::iterator I(&MI); 3742 3743 Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 3744 Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 3745 3746 BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg); 3747 3748 Register SGPRIdxReg; 3749 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset, 3750 UseGPRIdxMode, SGPRIdxReg); 3751 3752 MachineBasicBlock *LoopBB = InsPt->getParent(); 3753 3754 if (UseGPRIdxMode) { 3755 const MCInstrDesc &GPRIDXDesc = 3756 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true); 3757 3758 BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst) 3759 .addReg(SrcReg) 3760 .addReg(SGPRIdxReg) 3761 .addImm(SubReg); 3762 } else { 3763 BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst) 3764 .addReg(SrcReg, 0, SubReg) 3765 .addReg(SrcReg, RegState::Implicit); 3766 } 3767 3768 MI.eraseFromParent(); 3769 3770 return LoopBB; 3771 } 3772 3773 static MachineBasicBlock *emitIndirectDst(MachineInstr &MI, 3774 MachineBasicBlock &MBB, 3775 const GCNSubtarget &ST) { 3776 const SIInstrInfo *TII = ST.getInstrInfo(); 3777 const SIRegisterInfo &TRI = TII->getRegisterInfo(); 3778 MachineFunction *MF = MBB.getParent(); 3779 MachineRegisterInfo &MRI = MF->getRegInfo(); 3780 3781 Register Dst = MI.getOperand(0).getReg(); 3782 const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src); 3783 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); 3784 const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val); 3785 int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm(); 3786 const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg()); 3787 const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg()); 3788 3789 // This can be an immediate, but will be folded later. 3790 assert(Val->getReg()); 3791 3792 unsigned SubReg; 3793 std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC, 3794 SrcVec->getReg(), 3795 Offset); 3796 const bool UseGPRIdxMode = ST.useVGPRIndexMode(); 3797 3798 if (Idx->getReg() == AMDGPU::NoRegister) { 3799 MachineBasicBlock::iterator I(&MI); 3800 const DebugLoc &DL = MI.getDebugLoc(); 3801 3802 assert(Offset == 0); 3803 3804 BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst) 3805 .add(*SrcVec) 3806 .add(*Val) 3807 .addImm(SubReg); 3808 3809 MI.eraseFromParent(); 3810 return &MBB; 3811 } 3812 3813 // Check for a SGPR index. 3814 if (TII->getRegisterInfo().isSGPRClass(IdxRC)) { 3815 MachineBasicBlock::iterator I(&MI); 3816 const DebugLoc &DL = MI.getDebugLoc(); 3817 3818 if (UseGPRIdxMode) { 3819 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset); 3820 3821 const MCInstrDesc &GPRIDXDesc = 3822 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false); 3823 BuildMI(MBB, I, DL, GPRIDXDesc, Dst) 3824 .addReg(SrcVec->getReg()) 3825 .add(*Val) 3826 .addReg(Idx) 3827 .addImm(SubReg); 3828 } else { 3829 setM0ToIndexFromSGPR(TII, MRI, MI, Offset); 3830 3831 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo( 3832 TRI.getRegSizeInBits(*VecRC), 32, false); 3833 BuildMI(MBB, I, DL, MovRelDesc, Dst) 3834 .addReg(SrcVec->getReg()) 3835 .add(*Val) 3836 .addImm(SubReg); 3837 } 3838 MI.eraseFromParent(); 3839 return &MBB; 3840 } 3841 3842 // Control flow needs to be inserted if indexing with a VGPR. 3843 if (Val->isReg()) 3844 MRI.clearKillFlags(Val->getReg()); 3845 3846 const DebugLoc &DL = MI.getDebugLoc(); 3847 3848 Register PhiReg = MRI.createVirtualRegister(VecRC); 3849 3850 Register SGPRIdxReg; 3851 auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset, 3852 UseGPRIdxMode, SGPRIdxReg); 3853 MachineBasicBlock *LoopBB = InsPt->getParent(); 3854 3855 if (UseGPRIdxMode) { 3856 const MCInstrDesc &GPRIDXDesc = 3857 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false); 3858 3859 BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst) 3860 .addReg(PhiReg) 3861 .add(*Val) 3862 .addReg(SGPRIdxReg) 3863 .addImm(AMDGPU::sub0); 3864 } else { 3865 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo( 3866 TRI.getRegSizeInBits(*VecRC), 32, false); 3867 BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst) 3868 .addReg(PhiReg) 3869 .add(*Val) 3870 .addImm(AMDGPU::sub0); 3871 } 3872 3873 MI.eraseFromParent(); 3874 return LoopBB; 3875 } 3876 3877 MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( 3878 MachineInstr &MI, MachineBasicBlock *BB) const { 3879 3880 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 3881 MachineFunction *MF = BB->getParent(); 3882 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 3883 3884 switch (MI.getOpcode()) { 3885 case AMDGPU::S_UADDO_PSEUDO: 3886 case AMDGPU::S_USUBO_PSEUDO: { 3887 const DebugLoc &DL = MI.getDebugLoc(); 3888 MachineOperand &Dest0 = MI.getOperand(0); 3889 MachineOperand &Dest1 = MI.getOperand(1); 3890 MachineOperand &Src0 = MI.getOperand(2); 3891 MachineOperand &Src1 = MI.getOperand(3); 3892 3893 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 3894 ? AMDGPU::S_ADD_I32 3895 : AMDGPU::S_SUB_I32; 3896 BuildMI(*BB, MI, DL, TII->get(Opc), Dest0.getReg()).add(Src0).add(Src1); 3897 3898 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B64), Dest1.getReg()) 3899 .addImm(1) 3900 .addImm(0); 3901 3902 MI.eraseFromParent(); 3903 return BB; 3904 } 3905 case AMDGPU::S_ADD_U64_PSEUDO: 3906 case AMDGPU::S_SUB_U64_PSEUDO: { 3907 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 3908 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 3909 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 3910 const TargetRegisterClass *BoolRC = TRI->getBoolRC(); 3911 const DebugLoc &DL = MI.getDebugLoc(); 3912 3913 MachineOperand &Dest = MI.getOperand(0); 3914 MachineOperand &Src0 = MI.getOperand(1); 3915 MachineOperand &Src1 = MI.getOperand(2); 3916 3917 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 3918 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 3919 3920 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm( 3921 MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); 3922 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm( 3923 MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); 3924 3925 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm( 3926 MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); 3927 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm( 3928 MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); 3929 3930 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); 3931 3932 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; 3933 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; 3934 BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0); 3935 BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1).add(Src0Sub1).add(Src1Sub1); 3936 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg()) 3937 .addReg(DestSub0) 3938 .addImm(AMDGPU::sub0) 3939 .addReg(DestSub1) 3940 .addImm(AMDGPU::sub1); 3941 MI.eraseFromParent(); 3942 return BB; 3943 } 3944 case AMDGPU::V_ADD_U64_PSEUDO: 3945 case AMDGPU::V_SUB_U64_PSEUDO: { 3946 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 3947 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 3948 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 3949 const DebugLoc &DL = MI.getDebugLoc(); 3950 3951 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO); 3952 3953 const auto *CarryRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 3954 3955 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 3956 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 3957 3958 Register CarryReg = MRI.createVirtualRegister(CarryRC); 3959 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC); 3960 3961 MachineOperand &Dest = MI.getOperand(0); 3962 MachineOperand &Src0 = MI.getOperand(1); 3963 MachineOperand &Src1 = MI.getOperand(2); 3964 3965 const TargetRegisterClass *Src0RC = Src0.isReg() 3966 ? MRI.getRegClass(Src0.getReg()) 3967 : &AMDGPU::VReg_64RegClass; 3968 const TargetRegisterClass *Src1RC = Src1.isReg() 3969 ? MRI.getRegClass(Src1.getReg()) 3970 : &AMDGPU::VReg_64RegClass; 3971 3972 const TargetRegisterClass *Src0SubRC = 3973 TRI->getSubRegClass(Src0RC, AMDGPU::sub0); 3974 const TargetRegisterClass *Src1SubRC = 3975 TRI->getSubRegClass(Src1RC, AMDGPU::sub1); 3976 3977 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm( 3978 MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); 3979 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm( 3980 MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); 3981 3982 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm( 3983 MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC); 3984 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm( 3985 MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC); 3986 3987 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; 3988 MachineInstr *LoHalf = BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0) 3989 .addReg(CarryReg, RegState::Define) 3990 .add(SrcReg0Sub0) 3991 .add(SrcReg1Sub0) 3992 .addImm(0); // clamp bit 3993 3994 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; 3995 MachineInstr *HiHalf = 3996 BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1) 3997 .addReg(DeadCarryReg, RegState::Define | RegState::Dead) 3998 .add(SrcReg0Sub1) 3999 .add(SrcReg1Sub1) 4000 .addReg(CarryReg, RegState::Kill) 4001 .addImm(0); // clamp bit 4002 4003 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg()) 4004 .addReg(DestSub0) 4005 .addImm(AMDGPU::sub0) 4006 .addReg(DestSub1) 4007 .addImm(AMDGPU::sub1); 4008 TII->legalizeOperands(*LoHalf); 4009 TII->legalizeOperands(*HiHalf); 4010 MI.eraseFromParent(); 4011 return BB; 4012 } 4013 case AMDGPU::S_ADD_CO_PSEUDO: 4014 case AMDGPU::S_SUB_CO_PSEUDO: { 4015 // This pseudo has a chance to be selected 4016 // only from uniform add/subcarry node. All the VGPR operands 4017 // therefore assumed to be splat vectors. 4018 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 4019 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 4020 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 4021 MachineBasicBlock::iterator MII = MI; 4022 const DebugLoc &DL = MI.getDebugLoc(); 4023 MachineOperand &Dest = MI.getOperand(0); 4024 MachineOperand &CarryDest = MI.getOperand(1); 4025 MachineOperand &Src0 = MI.getOperand(2); 4026 MachineOperand &Src1 = MI.getOperand(3); 4027 MachineOperand &Src2 = MI.getOperand(4); 4028 unsigned Opc = (MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 4029 ? AMDGPU::S_ADDC_U32 4030 : AMDGPU::S_SUBB_U32; 4031 if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) { 4032 Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 4033 BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0) 4034 .addReg(Src0.getReg()); 4035 Src0.setReg(RegOp0); 4036 } 4037 if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) { 4038 Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 4039 BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1) 4040 .addReg(Src1.getReg()); 4041 Src1.setReg(RegOp1); 4042 } 4043 Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 4044 if (TRI->isVectorRegister(MRI, Src2.getReg())) { 4045 BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2) 4046 .addReg(Src2.getReg()); 4047 Src2.setReg(RegOp2); 4048 } 4049 4050 const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg()); 4051 if (TRI->getRegSizeInBits(*Src2RC) == 64) { 4052 if (ST.hasScalarCompareEq64()) { 4053 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64)) 4054 .addReg(Src2.getReg()) 4055 .addImm(0); 4056 } else { 4057 const TargetRegisterClass *SubRC = 4058 TRI->getSubRegClass(Src2RC, AMDGPU::sub0); 4059 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm( 4060 MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC); 4061 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm( 4062 MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC); 4063 Register Src2_32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 4064 4065 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_OR_B32), Src2_32) 4066 .add(Src2Sub0) 4067 .add(Src2Sub1); 4068 4069 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32)) 4070 .addReg(Src2_32, RegState::Kill) 4071 .addImm(0); 4072 } 4073 } else { 4074 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMPK_LG_U32)) 4075 .addReg(Src2.getReg()) 4076 .addImm(0); 4077 } 4078 4079 BuildMI(*BB, MII, DL, TII->get(Opc), Dest.getReg()).add(Src0).add(Src1); 4080 4081 BuildMI(*BB, MII, DL, TII->get(AMDGPU::COPY), CarryDest.getReg()) 4082 .addReg(AMDGPU::SCC); 4083 MI.eraseFromParent(); 4084 return BB; 4085 } 4086 case AMDGPU::SI_INIT_M0: { 4087 BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(), 4088 TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) 4089 .add(MI.getOperand(0)); 4090 MI.eraseFromParent(); 4091 return BB; 4092 } 4093 case AMDGPU::GET_GROUPSTATICSIZE: { 4094 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA || 4095 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL); 4096 DebugLoc DL = MI.getDebugLoc(); 4097 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32)) 4098 .add(MI.getOperand(0)) 4099 .addImm(MFI->getLDSSize()); 4100 MI.eraseFromParent(); 4101 return BB; 4102 } 4103 case AMDGPU::SI_INDIRECT_SRC_V1: 4104 case AMDGPU::SI_INDIRECT_SRC_V2: 4105 case AMDGPU::SI_INDIRECT_SRC_V4: 4106 case AMDGPU::SI_INDIRECT_SRC_V8: 4107 case AMDGPU::SI_INDIRECT_SRC_V16: 4108 case AMDGPU::SI_INDIRECT_SRC_V32: 4109 return emitIndirectSrc(MI, *BB, *getSubtarget()); 4110 case AMDGPU::SI_INDIRECT_DST_V1: 4111 case AMDGPU::SI_INDIRECT_DST_V2: 4112 case AMDGPU::SI_INDIRECT_DST_V4: 4113 case AMDGPU::SI_INDIRECT_DST_V8: 4114 case AMDGPU::SI_INDIRECT_DST_V16: 4115 case AMDGPU::SI_INDIRECT_DST_V32: 4116 return emitIndirectDst(MI, *BB, *getSubtarget()); 4117 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 4118 case AMDGPU::SI_KILL_I1_PSEUDO: 4119 return splitKillBlock(MI, BB); 4120 case AMDGPU::V_CNDMASK_B64_PSEUDO: { 4121 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 4122 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 4123 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 4124 4125 Register Dst = MI.getOperand(0).getReg(); 4126 Register Src0 = MI.getOperand(1).getReg(); 4127 Register Src1 = MI.getOperand(2).getReg(); 4128 const DebugLoc &DL = MI.getDebugLoc(); 4129 Register SrcCond = MI.getOperand(3).getReg(); 4130 4131 Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 4132 Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 4133 const auto *CondRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 4134 Register SrcCondCopy = MRI.createVirtualRegister(CondRC); 4135 4136 BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy) 4137 .addReg(SrcCond); 4138 BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo) 4139 .addImm(0) 4140 .addReg(Src0, 0, AMDGPU::sub0) 4141 .addImm(0) 4142 .addReg(Src1, 0, AMDGPU::sub0) 4143 .addReg(SrcCondCopy); 4144 BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi) 4145 .addImm(0) 4146 .addReg(Src0, 0, AMDGPU::sub1) 4147 .addImm(0) 4148 .addReg(Src1, 0, AMDGPU::sub1) 4149 .addReg(SrcCondCopy); 4150 4151 BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst) 4152 .addReg(DstLo) 4153 .addImm(AMDGPU::sub0) 4154 .addReg(DstHi) 4155 .addImm(AMDGPU::sub1); 4156 MI.eraseFromParent(); 4157 return BB; 4158 } 4159 case AMDGPU::SI_BR_UNDEF: { 4160 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 4161 const DebugLoc &DL = MI.getDebugLoc(); 4162 MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1)) 4163 .add(MI.getOperand(0)); 4164 Br->getOperand(1).setIsUndef(true); // read undef SCC 4165 MI.eraseFromParent(); 4166 return BB; 4167 } 4168 case AMDGPU::ADJCALLSTACKUP: 4169 case AMDGPU::ADJCALLSTACKDOWN: { 4170 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 4171 MachineInstrBuilder MIB(*MF, &MI); 4172 MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine) 4173 .addReg(Info->getStackPtrOffsetReg(), RegState::Implicit); 4174 return BB; 4175 } 4176 case AMDGPU::SI_CALL_ISEL: { 4177 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 4178 const DebugLoc &DL = MI.getDebugLoc(); 4179 4180 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF); 4181 4182 MachineInstrBuilder MIB; 4183 MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg); 4184 4185 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 4186 MIB.add(MI.getOperand(I)); 4187 4188 MIB.cloneMemRefs(MI); 4189 MI.eraseFromParent(); 4190 return BB; 4191 } 4192 case AMDGPU::V_ADD_CO_U32_e32: 4193 case AMDGPU::V_SUB_CO_U32_e32: 4194 case AMDGPU::V_SUBREV_CO_U32_e32: { 4195 // TODO: Define distinct V_*_I32_Pseudo instructions instead. 4196 const DebugLoc &DL = MI.getDebugLoc(); 4197 unsigned Opc = MI.getOpcode(); 4198 4199 bool NeedClampOperand = false; 4200 if (TII->pseudoToMCOpcode(Opc) == -1) { 4201 Opc = AMDGPU::getVOPe64(Opc); 4202 NeedClampOperand = true; 4203 } 4204 4205 auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg()); 4206 if (TII->isVOP3(*I)) { 4207 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 4208 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 4209 I.addReg(TRI->getVCC(), RegState::Define); 4210 } 4211 I.add(MI.getOperand(1)) 4212 .add(MI.getOperand(2)); 4213 if (NeedClampOperand) 4214 I.addImm(0); // clamp bit for e64 encoding 4215 4216 TII->legalizeOperands(*I); 4217 4218 MI.eraseFromParent(); 4219 return BB; 4220 } 4221 case AMDGPU::DS_GWS_INIT: 4222 case AMDGPU::DS_GWS_SEMA_V: 4223 case AMDGPU::DS_GWS_SEMA_BR: 4224 case AMDGPU::DS_GWS_SEMA_P: 4225 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL: 4226 case AMDGPU::DS_GWS_BARRIER: 4227 // A s_waitcnt 0 is required to be the instruction immediately following. 4228 if (getSubtarget()->hasGWSAutoReplay()) { 4229 bundleInstWithWaitcnt(MI); 4230 return BB; 4231 } 4232 4233 return emitGWSMemViolTestLoop(MI, BB); 4234 case AMDGPU::S_SETREG_B32: { 4235 // Try to optimize cases that only set the denormal mode or rounding mode. 4236 // 4237 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or 4238 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode 4239 // instead. 4240 // 4241 // FIXME: This could be predicates on the immediate, but tablegen doesn't 4242 // allow you to have a no side effect instruction in the output of a 4243 // sideeffecting pattern. 4244 unsigned ID, Offset, Width; 4245 AMDGPU::Hwreg::decodeHwreg(MI.getOperand(1).getImm(), ID, Offset, Width); 4246 if (ID != AMDGPU::Hwreg::ID_MODE) 4247 return BB; 4248 4249 const unsigned WidthMask = maskTrailingOnes<unsigned>(Width); 4250 const unsigned SetMask = WidthMask << Offset; 4251 4252 if (getSubtarget()->hasDenormModeInst()) { 4253 unsigned SetDenormOp = 0; 4254 unsigned SetRoundOp = 0; 4255 4256 // The dedicated instructions can only set the whole denorm or round mode 4257 // at once, not a subset of bits in either. 4258 if (SetMask == 4259 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) { 4260 // If this fully sets both the round and denorm mode, emit the two 4261 // dedicated instructions for these. 4262 SetRoundOp = AMDGPU::S_ROUND_MODE; 4263 SetDenormOp = AMDGPU::S_DENORM_MODE; 4264 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) { 4265 SetRoundOp = AMDGPU::S_ROUND_MODE; 4266 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) { 4267 SetDenormOp = AMDGPU::S_DENORM_MODE; 4268 } 4269 4270 if (SetRoundOp || SetDenormOp) { 4271 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 4272 MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg()); 4273 if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) { 4274 unsigned ImmVal = Def->getOperand(1).getImm(); 4275 if (SetRoundOp) { 4276 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetRoundOp)) 4277 .addImm(ImmVal & 0xf); 4278 4279 // If we also have the denorm mode, get just the denorm mode bits. 4280 ImmVal >>= 4; 4281 } 4282 4283 if (SetDenormOp) { 4284 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetDenormOp)) 4285 .addImm(ImmVal & 0xf); 4286 } 4287 4288 MI.eraseFromParent(); 4289 return BB; 4290 } 4291 } 4292 } 4293 4294 // If only FP bits are touched, used the no side effects pseudo. 4295 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK | 4296 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask) 4297 MI.setDesc(TII->get(AMDGPU::S_SETREG_B32_mode)); 4298 4299 return BB; 4300 } 4301 default: 4302 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 4303 } 4304 } 4305 4306 bool SITargetLowering::hasBitPreservingFPLogic(EVT VT) const { 4307 return isTypeLegal(VT.getScalarType()); 4308 } 4309 4310 bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const { 4311 // This currently forces unfolding various combinations of fsub into fma with 4312 // free fneg'd operands. As long as we have fast FMA (controlled by 4313 // isFMAFasterThanFMulAndFAdd), we should perform these. 4314 4315 // When fma is quarter rate, for f64 where add / sub are at best half rate, 4316 // most of these combines appear to be cycle neutral but save on instruction 4317 // count / code size. 4318 return true; 4319 } 4320 4321 EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx, 4322 EVT VT) const { 4323 if (!VT.isVector()) { 4324 return MVT::i1; 4325 } 4326 return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements()); 4327 } 4328 4329 MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { 4330 // TODO: Should i16 be used always if legal? For now it would force VALU 4331 // shifts. 4332 return (VT == MVT::i16) ? MVT::i16 : MVT::i32; 4333 } 4334 4335 LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const { 4336 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts()) 4337 ? Ty.changeElementSize(16) 4338 : Ty.changeElementSize(32); 4339 } 4340 4341 // Answering this is somewhat tricky and depends on the specific device which 4342 // have different rates for fma or all f64 operations. 4343 // 4344 // v_fma_f64 and v_mul_f64 always take the same number of cycles as each other 4345 // regardless of which device (although the number of cycles differs between 4346 // devices), so it is always profitable for f64. 4347 // 4348 // v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable 4349 // only on full rate devices. Normally, we should prefer selecting v_mad_f32 4350 // which we can always do even without fused FP ops since it returns the same 4351 // result as the separate operations and since it is always full 4352 // rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32 4353 // however does not support denormals, so we do report fma as faster if we have 4354 // a fast fma device and require denormals. 4355 // 4356 bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, 4357 EVT VT) const { 4358 VT = VT.getScalarType(); 4359 4360 switch (VT.getSimpleVT().SimpleTy) { 4361 case MVT::f32: { 4362 // If mad is not available this depends only on if f32 fma is full rate. 4363 if (!Subtarget->hasMadMacF32Insts()) 4364 return Subtarget->hasFastFMAF32(); 4365 4366 // Otherwise f32 mad is always full rate and returns the same result as 4367 // the separate operations so should be preferred over fma. 4368 // However does not support denomals. 4369 if (hasFP32Denormals(MF)) 4370 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts(); 4371 4372 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32. 4373 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts(); 4374 } 4375 case MVT::f64: 4376 return true; 4377 case MVT::f16: 4378 return Subtarget->has16BitInsts() && hasFP64FP16Denormals(MF); 4379 default: 4380 break; 4381 } 4382 4383 return false; 4384 } 4385 4386 bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG, 4387 const SDNode *N) const { 4388 // TODO: Check future ftz flag 4389 // v_mad_f32/v_mac_f32 do not support denormals. 4390 EVT VT = N->getValueType(0); 4391 if (VT == MVT::f32) 4392 return Subtarget->hasMadMacF32Insts() && 4393 !hasFP32Denormals(DAG.getMachineFunction()); 4394 if (VT == MVT::f16) { 4395 return Subtarget->hasMadF16() && 4396 !hasFP64FP16Denormals(DAG.getMachineFunction()); 4397 } 4398 4399 return false; 4400 } 4401 4402 //===----------------------------------------------------------------------===// 4403 // Custom DAG Lowering Operations 4404 //===----------------------------------------------------------------------===// 4405 4406 // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the 4407 // wider vector type is legal. 4408 SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op, 4409 SelectionDAG &DAG) const { 4410 unsigned Opc = Op.getOpcode(); 4411 EVT VT = Op.getValueType(); 4412 assert(VT == MVT::v4f16 || VT == MVT::v4i16); 4413 4414 SDValue Lo, Hi; 4415 std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0); 4416 4417 SDLoc SL(Op); 4418 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, 4419 Op->getFlags()); 4420 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi, 4421 Op->getFlags()); 4422 4423 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); 4424 } 4425 4426 // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the 4427 // wider vector type is legal. 4428 SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op, 4429 SelectionDAG &DAG) const { 4430 unsigned Opc = Op.getOpcode(); 4431 EVT VT = Op.getValueType(); 4432 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || 4433 VT == MVT::v8f32 || VT == MVT::v16f32 || VT == MVT::v32f32); 4434 4435 SDValue Lo0, Hi0; 4436 std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0); 4437 SDValue Lo1, Hi1; 4438 std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1); 4439 4440 SDLoc SL(Op); 4441 4442 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, 4443 Op->getFlags()); 4444 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, 4445 Op->getFlags()); 4446 4447 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); 4448 } 4449 4450 SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op, 4451 SelectionDAG &DAG) const { 4452 unsigned Opc = Op.getOpcode(); 4453 EVT VT = Op.getValueType(); 4454 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || 4455 VT == MVT::v8f32 || VT == MVT::v16f32 || VT == MVT::v32f32); 4456 4457 SDValue Lo0, Hi0; 4458 std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0); 4459 SDValue Lo1, Hi1; 4460 std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1); 4461 SDValue Lo2, Hi2; 4462 std::tie(Lo2, Hi2) = DAG.SplitVectorOperand(Op.getNode(), 2); 4463 4464 SDLoc SL(Op); 4465 4466 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Lo2, 4467 Op->getFlags()); 4468 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Hi2, 4469 Op->getFlags()); 4470 4471 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); 4472 } 4473 4474 4475 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 4476 switch (Op.getOpcode()) { 4477 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 4478 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 4479 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 4480 case ISD::LOAD: { 4481 SDValue Result = LowerLOAD(Op, DAG); 4482 assert((!Result.getNode() || 4483 Result.getNode()->getNumValues() == 2) && 4484 "Load should return a value and a chain"); 4485 return Result; 4486 } 4487 4488 case ISD::FSIN: 4489 case ISD::FCOS: 4490 return LowerTrig(Op, DAG); 4491 case ISD::SELECT: return LowerSELECT(Op, DAG); 4492 case ISD::FDIV: return LowerFDIV(Op, DAG); 4493 case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG); 4494 case ISD::STORE: return LowerSTORE(Op, DAG); 4495 case ISD::GlobalAddress: { 4496 MachineFunction &MF = DAG.getMachineFunction(); 4497 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 4498 return LowerGlobalAddress(MFI, Op, DAG); 4499 } 4500 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 4501 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG); 4502 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); 4503 case ISD::ADDRSPACECAST: return lowerADDRSPACECAST(Op, DAG); 4504 case ISD::INSERT_SUBVECTOR: 4505 return lowerINSERT_SUBVECTOR(Op, DAG); 4506 case ISD::INSERT_VECTOR_ELT: 4507 return lowerINSERT_VECTOR_ELT(Op, DAG); 4508 case ISD::EXTRACT_VECTOR_ELT: 4509 return lowerEXTRACT_VECTOR_ELT(Op, DAG); 4510 case ISD::VECTOR_SHUFFLE: 4511 return lowerVECTOR_SHUFFLE(Op, DAG); 4512 case ISD::BUILD_VECTOR: 4513 return lowerBUILD_VECTOR(Op, DAG); 4514 case ISD::FP_ROUND: 4515 return lowerFP_ROUND(Op, DAG); 4516 case ISD::TRAP: 4517 return lowerTRAP(Op, DAG); 4518 case ISD::DEBUGTRAP: 4519 return lowerDEBUGTRAP(Op, DAG); 4520 case ISD::FABS: 4521 case ISD::FNEG: 4522 case ISD::FCANONICALIZE: 4523 case ISD::BSWAP: 4524 return splitUnaryVectorOp(Op, DAG); 4525 case ISD::FMINNUM: 4526 case ISD::FMAXNUM: 4527 return lowerFMINNUM_FMAXNUM(Op, DAG); 4528 case ISD::FMA: 4529 return splitTernaryVectorOp(Op, DAG); 4530 case ISD::FP_TO_SINT: 4531 case ISD::FP_TO_UINT: 4532 return LowerFP_TO_INT(Op, DAG); 4533 case ISD::SHL: 4534 case ISD::SRA: 4535 case ISD::SRL: 4536 case ISD::ADD: 4537 case ISD::SUB: 4538 case ISD::MUL: 4539 case ISD::SMIN: 4540 case ISD::SMAX: 4541 case ISD::UMIN: 4542 case ISD::UMAX: 4543 case ISD::FADD: 4544 case ISD::FMUL: 4545 case ISD::FMINNUM_IEEE: 4546 case ISD::FMAXNUM_IEEE: 4547 case ISD::UADDSAT: 4548 case ISD::USUBSAT: 4549 case ISD::SADDSAT: 4550 case ISD::SSUBSAT: 4551 return splitBinaryVectorOp(Op, DAG); 4552 case ISD::SMULO: 4553 case ISD::UMULO: 4554 return lowerXMULO(Op, DAG); 4555 case ISD::DYNAMIC_STACKALLOC: 4556 return LowerDYNAMIC_STACKALLOC(Op, DAG); 4557 } 4558 return SDValue(); 4559 } 4560 4561 // Used for D16: Casts the result of an instruction into the right vector, 4562 // packs values if loads return unpacked values. 4563 static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT, 4564 const SDLoc &DL, 4565 SelectionDAG &DAG, bool Unpacked) { 4566 if (!LoadVT.isVector()) 4567 return Result; 4568 4569 // Cast back to the original packed type or to a larger type that is a 4570 // multiple of 32 bit for D16. Widening the return type is a required for 4571 // legalization. 4572 EVT FittingLoadVT = LoadVT; 4573 if ((LoadVT.getVectorNumElements() % 2) == 1) { 4574 FittingLoadVT = 4575 EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(), 4576 LoadVT.getVectorNumElements() + 1); 4577 } 4578 4579 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16. 4580 // Truncate to v2i16/v4i16. 4581 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger(); 4582 4583 // Workaround legalizer not scalarizing truncate after vector op 4584 // legalization but not creating intermediate vector trunc. 4585 SmallVector<SDValue, 4> Elts; 4586 DAG.ExtractVectorElements(Result, Elts); 4587 for (SDValue &Elt : Elts) 4588 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); 4589 4590 // Pad illegal v1i16/v3fi6 to v4i16 4591 if ((LoadVT.getVectorNumElements() % 2) == 1) 4592 Elts.push_back(DAG.getUNDEF(MVT::i16)); 4593 4594 Result = DAG.getBuildVector(IntLoadVT, DL, Elts); 4595 4596 // Bitcast to original type (v2f16/v4f16). 4597 return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result); 4598 } 4599 4600 // Cast back to the original packed type. 4601 return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result); 4602 } 4603 4604 SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, 4605 MemSDNode *M, 4606 SelectionDAG &DAG, 4607 ArrayRef<SDValue> Ops, 4608 bool IsIntrinsic) const { 4609 SDLoc DL(M); 4610 4611 bool Unpacked = Subtarget->hasUnpackedD16VMem(); 4612 EVT LoadVT = M->getValueType(0); 4613 4614 EVT EquivLoadVT = LoadVT; 4615 if (LoadVT.isVector()) { 4616 if (Unpacked) { 4617 EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, 4618 LoadVT.getVectorNumElements()); 4619 } else if ((LoadVT.getVectorNumElements() % 2) == 1) { 4620 // Widen v3f16 to legal type 4621 EquivLoadVT = 4622 EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(), 4623 LoadVT.getVectorNumElements() + 1); 4624 } 4625 } 4626 4627 // Change from v4f16/v2f16 to EquivLoadVT. 4628 SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other); 4629 4630 SDValue Load 4631 = DAG.getMemIntrinsicNode( 4632 IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, DL, 4633 VTList, Ops, M->getMemoryVT(), 4634 M->getMemOperand()); 4635 4636 SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked); 4637 4638 return DAG.getMergeValues({ Adjusted, Load.getValue(1) }, DL); 4639 } 4640 4641 SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, 4642 SelectionDAG &DAG, 4643 ArrayRef<SDValue> Ops) const { 4644 SDLoc DL(M); 4645 EVT LoadVT = M->getValueType(0); 4646 EVT EltType = LoadVT.getScalarType(); 4647 EVT IntVT = LoadVT.changeTypeToInteger(); 4648 4649 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16); 4650 4651 unsigned Opc = 4652 IsFormat ? AMDGPUISD::BUFFER_LOAD_FORMAT : AMDGPUISD::BUFFER_LOAD; 4653 4654 if (IsD16) { 4655 return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops); 4656 } 4657 4658 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics 4659 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32) 4660 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M); 4661 4662 if (isTypeLegal(LoadVT)) { 4663 return getMemIntrinsicNode(Opc, DL, M->getVTList(), Ops, IntVT, 4664 M->getMemOperand(), DAG); 4665 } 4666 4667 EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT); 4668 SDVTList VTList = DAG.getVTList(CastVT, MVT::Other); 4669 SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT, 4670 M->getMemOperand(), DAG); 4671 return DAG.getMergeValues( 4672 {DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)}, 4673 DL); 4674 } 4675 4676 static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, 4677 SDNode *N, SelectionDAG &DAG) { 4678 EVT VT = N->getValueType(0); 4679 const auto *CD = cast<ConstantSDNode>(N->getOperand(3)); 4680 unsigned CondCode = CD->getZExtValue(); 4681 if (!ICmpInst::isIntPredicate(static_cast<ICmpInst::Predicate>(CondCode))) 4682 return DAG.getUNDEF(VT); 4683 4684 ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode); 4685 4686 SDValue LHS = N->getOperand(1); 4687 SDValue RHS = N->getOperand(2); 4688 4689 SDLoc DL(N); 4690 4691 EVT CmpVT = LHS.getValueType(); 4692 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) { 4693 unsigned PromoteOp = ICmpInst::isSigned(IcInput) ? 4694 ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4695 LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS); 4696 RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS); 4697 } 4698 4699 ISD::CondCode CCOpcode = getICmpCondCode(IcInput); 4700 4701 unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize(); 4702 EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize); 4703 4704 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS, 4705 DAG.getCondCode(CCOpcode)); 4706 if (VT.bitsEq(CCVT)) 4707 return SetCC; 4708 return DAG.getZExtOrTrunc(SetCC, DL, VT); 4709 } 4710 4711 static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, 4712 SDNode *N, SelectionDAG &DAG) { 4713 EVT VT = N->getValueType(0); 4714 const auto *CD = cast<ConstantSDNode>(N->getOperand(3)); 4715 4716 unsigned CondCode = CD->getZExtValue(); 4717 if (!FCmpInst::isFPPredicate(static_cast<FCmpInst::Predicate>(CondCode))) 4718 return DAG.getUNDEF(VT); 4719 4720 SDValue Src0 = N->getOperand(1); 4721 SDValue Src1 = N->getOperand(2); 4722 EVT CmpVT = Src0.getValueType(); 4723 SDLoc SL(N); 4724 4725 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) { 4726 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); 4727 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); 4728 } 4729 4730 FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode); 4731 ISD::CondCode CCOpcode = getFCmpCondCode(IcInput); 4732 unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize(); 4733 EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize); 4734 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0, 4735 Src1, DAG.getCondCode(CCOpcode)); 4736 if (VT.bitsEq(CCVT)) 4737 return SetCC; 4738 return DAG.getZExtOrTrunc(SetCC, SL, VT); 4739 } 4740 4741 static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N, 4742 SelectionDAG &DAG) { 4743 EVT VT = N->getValueType(0); 4744 SDValue Src = N->getOperand(1); 4745 SDLoc SL(N); 4746 4747 if (Src.getOpcode() == ISD::SETCC) { 4748 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...) 4749 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0), 4750 Src.getOperand(1), Src.getOperand(2)); 4751 } 4752 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) { 4753 // (ballot 0) -> 0 4754 if (Arg->isNullValue()) 4755 return DAG.getConstant(0, SL, VT); 4756 4757 // (ballot 1) -> EXEC/EXEC_LO 4758 if (Arg->isOne()) { 4759 Register Exec; 4760 if (VT.getScalarSizeInBits() == 32) 4761 Exec = AMDGPU::EXEC_LO; 4762 else if (VT.getScalarSizeInBits() == 64) 4763 Exec = AMDGPU::EXEC; 4764 else 4765 return SDValue(); 4766 4767 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT); 4768 } 4769 } 4770 4771 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0) 4772 // ISD::SETNE) 4773 return DAG.getNode( 4774 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), 4775 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); 4776 } 4777 4778 void SITargetLowering::ReplaceNodeResults(SDNode *N, 4779 SmallVectorImpl<SDValue> &Results, 4780 SelectionDAG &DAG) const { 4781 switch (N->getOpcode()) { 4782 case ISD::INSERT_VECTOR_ELT: { 4783 if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG)) 4784 Results.push_back(Res); 4785 return; 4786 } 4787 case ISD::EXTRACT_VECTOR_ELT: { 4788 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG)) 4789 Results.push_back(Res); 4790 return; 4791 } 4792 case ISD::INTRINSIC_WO_CHAIN: { 4793 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 4794 switch (IID) { 4795 case Intrinsic::amdgcn_cvt_pkrtz: { 4796 SDValue Src0 = N->getOperand(1); 4797 SDValue Src1 = N->getOperand(2); 4798 SDLoc SL(N); 4799 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, 4800 Src0, Src1); 4801 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); 4802 return; 4803 } 4804 case Intrinsic::amdgcn_cvt_pknorm_i16: 4805 case Intrinsic::amdgcn_cvt_pknorm_u16: 4806 case Intrinsic::amdgcn_cvt_pk_i16: 4807 case Intrinsic::amdgcn_cvt_pk_u16: { 4808 SDValue Src0 = N->getOperand(1); 4809 SDValue Src1 = N->getOperand(2); 4810 SDLoc SL(N); 4811 unsigned Opcode; 4812 4813 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16) 4814 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32; 4815 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16) 4816 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32; 4817 else if (IID == Intrinsic::amdgcn_cvt_pk_i16) 4818 Opcode = AMDGPUISD::CVT_PK_I16_I32; 4819 else 4820 Opcode = AMDGPUISD::CVT_PK_U16_U32; 4821 4822 EVT VT = N->getValueType(0); 4823 if (isTypeLegal(VT)) 4824 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1)); 4825 else { 4826 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); 4827 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); 4828 } 4829 return; 4830 } 4831 } 4832 break; 4833 } 4834 case ISD::INTRINSIC_W_CHAIN: { 4835 if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) { 4836 if (Res.getOpcode() == ISD::MERGE_VALUES) { 4837 // FIXME: Hacky 4838 for (unsigned I = 0; I < Res.getNumOperands(); I++) { 4839 Results.push_back(Res.getOperand(I)); 4840 } 4841 } else { 4842 Results.push_back(Res); 4843 Results.push_back(Res.getValue(1)); 4844 } 4845 return; 4846 } 4847 4848 break; 4849 } 4850 case ISD::SELECT: { 4851 SDLoc SL(N); 4852 EVT VT = N->getValueType(0); 4853 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); 4854 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1)); 4855 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2)); 4856 4857 EVT SelectVT = NewVT; 4858 if (NewVT.bitsLT(MVT::i32)) { 4859 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); 4860 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); 4861 SelectVT = MVT::i32; 4862 } 4863 4864 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT, 4865 N->getOperand(0), LHS, RHS); 4866 4867 if (NewVT != SelectVT) 4868 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect); 4869 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect)); 4870 return; 4871 } 4872 case ISD::FNEG: { 4873 if (N->getValueType(0) != MVT::v2f16) 4874 break; 4875 4876 SDLoc SL(N); 4877 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); 4878 4879 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, 4880 BC, 4881 DAG.getConstant(0x80008000, SL, MVT::i32)); 4882 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); 4883 return; 4884 } 4885 case ISD::FABS: { 4886 if (N->getValueType(0) != MVT::v2f16) 4887 break; 4888 4889 SDLoc SL(N); 4890 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); 4891 4892 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, 4893 BC, 4894 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); 4895 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); 4896 return; 4897 } 4898 default: 4899 break; 4900 } 4901 } 4902 4903 /// Helper function for LowerBRCOND 4904 static SDNode *findUser(SDValue Value, unsigned Opcode) { 4905 4906 SDNode *Parent = Value.getNode(); 4907 for (SDNode::use_iterator I = Parent->use_begin(), E = Parent->use_end(); 4908 I != E; ++I) { 4909 4910 if (I.getUse().get() != Value) 4911 continue; 4912 4913 if (I->getOpcode() == Opcode) 4914 return *I; 4915 } 4916 return nullptr; 4917 } 4918 4919 unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const { 4920 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 4921 switch (cast<ConstantSDNode>(Intr->getOperand(1))->getZExtValue()) { 4922 case Intrinsic::amdgcn_if: 4923 return AMDGPUISD::IF; 4924 case Intrinsic::amdgcn_else: 4925 return AMDGPUISD::ELSE; 4926 case Intrinsic::amdgcn_loop: 4927 return AMDGPUISD::LOOP; 4928 case Intrinsic::amdgcn_end_cf: 4929 llvm_unreachable("should not occur"); 4930 default: 4931 return 0; 4932 } 4933 } 4934 4935 // break, if_break, else_break are all only used as inputs to loop, not 4936 // directly as branch conditions. 4937 return 0; 4938 } 4939 4940 bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const { 4941 const Triple &TT = getTargetMachine().getTargetTriple(); 4942 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 4943 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && 4944 AMDGPU::shouldEmitConstantsToTextSection(TT); 4945 } 4946 4947 bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const { 4948 // FIXME: Either avoid relying on address space here or change the default 4949 // address space for functions to avoid the explicit check. 4950 return (GV->getValueType()->isFunctionTy() || 4951 !isNonGlobalAddrSpace(GV->getAddressSpace())) && 4952 !shouldEmitFixup(GV) && 4953 !getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); 4954 } 4955 4956 bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const { 4957 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV); 4958 } 4959 4960 bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const { 4961 if (!GV->hasExternalLinkage()) 4962 return true; 4963 4964 const auto OS = getTargetMachine().getTargetTriple().getOS(); 4965 return OS == Triple::AMDHSA || OS == Triple::AMDPAL; 4966 } 4967 4968 /// This transforms the control flow intrinsics to get the branch destination as 4969 /// last parameter, also switches branch target with BR if the need arise 4970 SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, 4971 SelectionDAG &DAG) const { 4972 SDLoc DL(BRCOND); 4973 4974 SDNode *Intr = BRCOND.getOperand(1).getNode(); 4975 SDValue Target = BRCOND.getOperand(2); 4976 SDNode *BR = nullptr; 4977 SDNode *SetCC = nullptr; 4978 4979 if (Intr->getOpcode() == ISD::SETCC) { 4980 // As long as we negate the condition everything is fine 4981 SetCC = Intr; 4982 Intr = SetCC->getOperand(0).getNode(); 4983 4984 } else { 4985 // Get the target from BR if we don't negate the condition 4986 BR = findUser(BRCOND, ISD::BR); 4987 assert(BR && "brcond missing unconditional branch user"); 4988 Target = BR->getOperand(1); 4989 } 4990 4991 unsigned CFNode = isCFIntrinsic(Intr); 4992 if (CFNode == 0) { 4993 // This is a uniform branch so we don't need to legalize. 4994 return BRCOND; 4995 } 4996 4997 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID || 4998 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN; 4999 5000 assert(!SetCC || 5001 (SetCC->getConstantOperandVal(1) == 1 && 5002 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() == 5003 ISD::SETNE)); 5004 5005 // operands of the new intrinsic call 5006 SmallVector<SDValue, 4> Ops; 5007 if (HaveChain) 5008 Ops.push_back(BRCOND.getOperand(0)); 5009 5010 Ops.append(Intr->op_begin() + (HaveChain ? 2 : 1), Intr->op_end()); 5011 Ops.push_back(Target); 5012 5013 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end()); 5014 5015 // build the new intrinsic call 5016 SDNode *Result = DAG.getNode(CFNode, DL, DAG.getVTList(Res), Ops).getNode(); 5017 5018 if (!HaveChain) { 5019 SDValue Ops[] = { 5020 SDValue(Result, 0), 5021 BRCOND.getOperand(0) 5022 }; 5023 5024 Result = DAG.getMergeValues(Ops, DL).getNode(); 5025 } 5026 5027 if (BR) { 5028 // Give the branch instruction our target 5029 SDValue Ops[] = { 5030 BR->getOperand(0), 5031 BRCOND.getOperand(2) 5032 }; 5033 SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops); 5034 DAG.ReplaceAllUsesWith(BR, NewBR.getNode()); 5035 } 5036 5037 SDValue Chain = SDValue(Result, Result->getNumValues() - 1); 5038 5039 // Copy the intrinsic results to registers 5040 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) { 5041 SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg); 5042 if (!CopyToReg) 5043 continue; 5044 5045 Chain = DAG.getCopyToReg( 5046 Chain, DL, 5047 CopyToReg->getOperand(1), 5048 SDValue(Result, i - 1), 5049 SDValue()); 5050 5051 DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0)); 5052 } 5053 5054 // Remove the old intrinsic from the chain 5055 DAG.ReplaceAllUsesOfValueWith( 5056 SDValue(Intr, Intr->getNumValues() - 1), 5057 Intr->getOperand(0)); 5058 5059 return Chain; 5060 } 5061 5062 SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, 5063 SelectionDAG &DAG) const { 5064 MVT VT = Op.getSimpleValueType(); 5065 SDLoc DL(Op); 5066 // Checking the depth 5067 if (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() != 0) 5068 return DAG.getConstant(0, DL, VT); 5069 5070 MachineFunction &MF = DAG.getMachineFunction(); 5071 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 5072 // Check for kernel and shader functions 5073 if (Info->isEntryFunction()) 5074 return DAG.getConstant(0, DL, VT); 5075 5076 MachineFrameInfo &MFI = MF.getFrameInfo(); 5077 // There is a call to @llvm.returnaddress in this function 5078 MFI.setReturnAddressIsTaken(true); 5079 5080 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); 5081 // Get the return address reg and mark it as an implicit live-in 5082 Register Reg = MF.addLiveIn(TRI->getReturnAddressReg(MF), getRegClassFor(VT, Op.getNode()->isDivergent())); 5083 5084 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT); 5085 } 5086 5087 SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, 5088 SDValue Op, 5089 const SDLoc &DL, 5090 EVT VT) const { 5091 return Op.getValueType().bitsLE(VT) ? 5092 DAG.getNode(ISD::FP_EXTEND, DL, VT, Op) : 5093 DAG.getNode(ISD::FP_ROUND, DL, VT, Op, 5094 DAG.getTargetConstant(0, DL, MVT::i32)); 5095 } 5096 5097 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { 5098 assert(Op.getValueType() == MVT::f16 && 5099 "Do not know how to custom lower FP_ROUND for non-f16 type"); 5100 5101 SDValue Src = Op.getOperand(0); 5102 EVT SrcVT = Src.getValueType(); 5103 if (SrcVT != MVT::f64) 5104 return Op; 5105 5106 SDLoc DL(Op); 5107 5108 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); 5109 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); 5110 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); 5111 } 5112 5113 SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op, 5114 SelectionDAG &DAG) const { 5115 EVT VT = Op.getValueType(); 5116 const MachineFunction &MF = DAG.getMachineFunction(); 5117 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 5118 bool IsIEEEMode = Info->getMode().IEEE; 5119 5120 // FIXME: Assert during selection that this is only selected for 5121 // ieee_mode. Currently a combine can produce the ieee version for non-ieee 5122 // mode functions, but this happens to be OK since it's only done in cases 5123 // where there is known no sNaN. 5124 if (IsIEEEMode) 5125 return expandFMINNUM_FMAXNUM(Op.getNode(), DAG); 5126 5127 if (VT == MVT::v4f16) 5128 return splitBinaryVectorOp(Op, DAG); 5129 return Op; 5130 } 5131 5132 SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const { 5133 EVT VT = Op.getValueType(); 5134 SDLoc SL(Op); 5135 SDValue LHS = Op.getOperand(0); 5136 SDValue RHS = Op.getOperand(1); 5137 bool isSigned = Op.getOpcode() == ISD::SMULO; 5138 5139 if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) { 5140 const APInt &C = RHSC->getAPIntValue(); 5141 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X } 5142 if (C.isPowerOf2()) { 5143 // smulo(x, signed_min) is same as umulo(x, signed_min). 5144 bool UseArithShift = isSigned && !C.isMinSignedValue(); 5145 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); 5146 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt); 5147 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, 5148 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, 5149 SL, VT, Result, ShiftAmt), 5150 LHS, ISD::SETNE); 5151 return DAG.getMergeValues({ Result, Overflow }, SL); 5152 } 5153 } 5154 5155 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS); 5156 SDValue Top = DAG.getNode(isSigned ? ISD::MULHS : ISD::MULHU, 5157 SL, VT, LHS, RHS); 5158 5159 SDValue Sign = isSigned 5160 ? DAG.getNode(ISD::SRA, SL, VT, Result, 5161 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) 5162 : DAG.getConstant(0, SL, VT); 5163 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); 5164 5165 return DAG.getMergeValues({ Result, Overflow }, SL); 5166 } 5167 5168 SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const { 5169 if (!Subtarget->isTrapHandlerEnabled() || 5170 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) 5171 return lowerTrapEndpgm(Op, DAG); 5172 5173 if (Optional<uint8_t> HsaAbiVer = AMDGPU::getHsaAbiVersion(Subtarget)) { 5174 switch (*HsaAbiVer) { 5175 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 5176 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 5177 return lowerTrapHsaQueuePtr(Op, DAG); 5178 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 5179 return Subtarget->supportsGetDoorbellID() ? 5180 lowerTrapHsa(Op, DAG) : lowerTrapHsaQueuePtr(Op, DAG); 5181 } 5182 } 5183 5184 llvm_unreachable("Unknown trap handler"); 5185 } 5186 5187 SDValue SITargetLowering::lowerTrapEndpgm( 5188 SDValue Op, SelectionDAG &DAG) const { 5189 SDLoc SL(Op); 5190 SDValue Chain = Op.getOperand(0); 5191 return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain); 5192 } 5193 5194 SDValue SITargetLowering::lowerTrapHsaQueuePtr( 5195 SDValue Op, SelectionDAG &DAG) const { 5196 SDLoc SL(Op); 5197 SDValue Chain = Op.getOperand(0); 5198 5199 MachineFunction &MF = DAG.getMachineFunction(); 5200 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 5201 Register UserSGPR = Info->getQueuePtrUserSGPR(); 5202 assert(UserSGPR != AMDGPU::NoRegister); 5203 SDValue QueuePtr = CreateLiveInRegister( 5204 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); 5205 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); 5206 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, 5207 QueuePtr, SDValue()); 5208 5209 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap); 5210 SDValue Ops[] = { 5211 ToReg, 5212 DAG.getTargetConstant(TrapID, SL, MVT::i16), 5213 SGPR01, 5214 ToReg.getValue(1) 5215 }; 5216 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); 5217 } 5218 5219 SDValue SITargetLowering::lowerTrapHsa( 5220 SDValue Op, SelectionDAG &DAG) const { 5221 SDLoc SL(Op); 5222 SDValue Chain = Op.getOperand(0); 5223 5224 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap); 5225 SDValue Ops[] = { 5226 Chain, 5227 DAG.getTargetConstant(TrapID, SL, MVT::i16) 5228 }; 5229 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); 5230 } 5231 5232 SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const { 5233 SDLoc SL(Op); 5234 SDValue Chain = Op.getOperand(0); 5235 MachineFunction &MF = DAG.getMachineFunction(); 5236 5237 if (!Subtarget->isTrapHandlerEnabled() || 5238 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) { 5239 DiagnosticInfoUnsupported NoTrap(MF.getFunction(), 5240 "debugtrap handler not supported", 5241 Op.getDebugLoc(), 5242 DS_Warning); 5243 LLVMContext &Ctx = MF.getFunction().getContext(); 5244 Ctx.diagnose(NoTrap); 5245 return Chain; 5246 } 5247 5248 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap); 5249 SDValue Ops[] = { 5250 Chain, 5251 DAG.getTargetConstant(TrapID, SL, MVT::i16) 5252 }; 5253 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); 5254 } 5255 5256 SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL, 5257 SelectionDAG &DAG) const { 5258 // FIXME: Use inline constants (src_{shared, private}_base) instead. 5259 if (Subtarget->hasApertureRegs()) { 5260 unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ? 5261 AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE : 5262 AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE; 5263 unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ? 5264 AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE : 5265 AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE; 5266 unsigned Encoding = 5267 AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ | 5268 Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ | 5269 WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_; 5270 5271 SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16); 5272 SDValue ApertureReg = SDValue( 5273 DAG.getMachineNode(AMDGPU::S_GETREG_B32, DL, MVT::i32, EncodingImm), 0); 5274 SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32); 5275 return DAG.getNode(ISD::SHL, DL, MVT::i32, ApertureReg, ShiftAmount); 5276 } 5277 5278 MachineFunction &MF = DAG.getMachineFunction(); 5279 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 5280 Register UserSGPR = Info->getQueuePtrUserSGPR(); 5281 assert(UserSGPR != AMDGPU::NoRegister); 5282 5283 SDValue QueuePtr = CreateLiveInRegister( 5284 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); 5285 5286 // Offset into amd_queue_t for group_segment_aperture_base_hi / 5287 // private_segment_aperture_base_hi. 5288 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44; 5289 5290 SDValue Ptr = 5291 DAG.getObjectPtrOffset(DL, QueuePtr, TypeSize::Fixed(StructOffset)); 5292 5293 // TODO: Use custom target PseudoSourceValue. 5294 // TODO: We should use the value from the IR intrinsic call, but it might not 5295 // be available and how do we get it? 5296 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); 5297 return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, 5298 commonAlignment(Align(64), StructOffset), 5299 MachineMemOperand::MODereferenceable | 5300 MachineMemOperand::MOInvariant); 5301 } 5302 5303 SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op, 5304 SelectionDAG &DAG) const { 5305 SDLoc SL(Op); 5306 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Op); 5307 5308 SDValue Src = ASC->getOperand(0); 5309 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); 5310 5311 const AMDGPUTargetMachine &TM = 5312 static_cast<const AMDGPUTargetMachine &>(getTargetMachine()); 5313 5314 // flat -> local/private 5315 if (ASC->getSrcAddressSpace() == AMDGPUAS::FLAT_ADDRESS) { 5316 unsigned DestAS = ASC->getDestAddressSpace(); 5317 5318 if (DestAS == AMDGPUAS::LOCAL_ADDRESS || 5319 DestAS == AMDGPUAS::PRIVATE_ADDRESS) { 5320 unsigned NullVal = TM.getNullPointerValue(DestAS); 5321 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); 5322 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); 5323 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); 5324 5325 return DAG.getNode(ISD::SELECT, SL, MVT::i32, 5326 NonNull, Ptr, SegmentNullPtr); 5327 } 5328 } 5329 5330 // local/private -> flat 5331 if (ASC->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS) { 5332 unsigned SrcAS = ASC->getSrcAddressSpace(); 5333 5334 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS || 5335 SrcAS == AMDGPUAS::PRIVATE_ADDRESS) { 5336 unsigned NullVal = TM.getNullPointerValue(SrcAS); 5337 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); 5338 5339 SDValue NonNull 5340 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); 5341 5342 SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG); 5343 SDValue CvtPtr 5344 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); 5345 5346 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, 5347 DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr), 5348 FlatNullPtr); 5349 } 5350 } 5351 5352 if (ASC->getDestAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT && 5353 Src.getValueType() == MVT::i64) 5354 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); 5355 5356 // global <-> flat are no-ops and never emitted. 5357 5358 const MachineFunction &MF = DAG.getMachineFunction(); 5359 DiagnosticInfoUnsupported InvalidAddrSpaceCast( 5360 MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc()); 5361 DAG.getContext()->diagnose(InvalidAddrSpaceCast); 5362 5363 return DAG.getUNDEF(ASC->getValueType(0)); 5364 } 5365 5366 // This lowers an INSERT_SUBVECTOR by extracting the individual elements from 5367 // the small vector and inserting them into the big vector. That is better than 5368 // the default expansion of doing it via a stack slot. Even though the use of 5369 // the stack slot would be optimized away afterwards, the stack slot itself 5370 // remains. 5371 SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op, 5372 SelectionDAG &DAG) const { 5373 SDValue Vec = Op.getOperand(0); 5374 SDValue Ins = Op.getOperand(1); 5375 SDValue Idx = Op.getOperand(2); 5376 EVT VecVT = Vec.getValueType(); 5377 EVT InsVT = Ins.getValueType(); 5378 EVT EltVT = VecVT.getVectorElementType(); 5379 unsigned InsNumElts = InsVT.getVectorNumElements(); 5380 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 5381 SDLoc SL(Op); 5382 5383 for (unsigned I = 0; I != InsNumElts; ++I) { 5384 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins, 5385 DAG.getConstant(I, SL, MVT::i32)); 5386 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt, 5387 DAG.getConstant(IdxVal + I, SL, MVT::i32)); 5388 } 5389 return Vec; 5390 } 5391 5392 SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op, 5393 SelectionDAG &DAG) const { 5394 SDValue Vec = Op.getOperand(0); 5395 SDValue InsVal = Op.getOperand(1); 5396 SDValue Idx = Op.getOperand(2); 5397 EVT VecVT = Vec.getValueType(); 5398 EVT EltVT = VecVT.getVectorElementType(); 5399 unsigned VecSize = VecVT.getSizeInBits(); 5400 unsigned EltSize = EltVT.getSizeInBits(); 5401 5402 5403 assert(VecSize <= 64); 5404 5405 unsigned NumElts = VecVT.getVectorNumElements(); 5406 SDLoc SL(Op); 5407 auto KIdx = dyn_cast<ConstantSDNode>(Idx); 5408 5409 if (NumElts == 4 && EltSize == 16 && KIdx) { 5410 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); 5411 5412 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, 5413 DAG.getConstant(0, SL, MVT::i32)); 5414 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, 5415 DAG.getConstant(1, SL, MVT::i32)); 5416 5417 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); 5418 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); 5419 5420 unsigned Idx = KIdx->getZExtValue(); 5421 bool InsertLo = Idx < 2; 5422 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, 5423 InsertLo ? LoVec : HiVec, 5424 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), 5425 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); 5426 5427 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); 5428 5429 SDValue Concat = InsertLo ? 5430 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : 5431 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); 5432 5433 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat); 5434 } 5435 5436 if (isa<ConstantSDNode>(Idx)) 5437 return SDValue(); 5438 5439 MVT IntVT = MVT::getIntegerVT(VecSize); 5440 5441 // Avoid stack access for dynamic indexing. 5442 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec 5443 5444 // Create a congruent vector with the target value in each element so that 5445 // the required element can be masked and ORed into the target vector. 5446 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT, 5447 DAG.getSplatBuildVector(VecVT, SL, InsVal)); 5448 5449 assert(isPowerOf2_32(EltSize)); 5450 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); 5451 5452 // Convert vector index to bit-index. 5453 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); 5454 5455 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); 5456 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT, 5457 DAG.getConstant(0xffff, SL, IntVT), 5458 ScaledIdx); 5459 5460 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal); 5461 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT, 5462 DAG.getNOT(SL, BFM, IntVT), BCVec); 5463 5464 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS); 5465 return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI); 5466 } 5467 5468 SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op, 5469 SelectionDAG &DAG) const { 5470 SDLoc SL(Op); 5471 5472 EVT ResultVT = Op.getValueType(); 5473 SDValue Vec = Op.getOperand(0); 5474 SDValue Idx = Op.getOperand(1); 5475 EVT VecVT = Vec.getValueType(); 5476 unsigned VecSize = VecVT.getSizeInBits(); 5477 EVT EltVT = VecVT.getVectorElementType(); 5478 assert(VecSize <= 64); 5479 5480 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr); 5481 5482 // Make sure we do any optimizations that will make it easier to fold 5483 // source modifiers before obscuring it with bit operations. 5484 5485 // XXX - Why doesn't this get called when vector_shuffle is expanded? 5486 if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI)) 5487 return Combined; 5488 5489 unsigned EltSize = EltVT.getSizeInBits(); 5490 assert(isPowerOf2_32(EltSize)); 5491 5492 MVT IntVT = MVT::getIntegerVT(VecSize); 5493 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); 5494 5495 // Convert vector index to bit-index (* EltSize) 5496 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); 5497 5498 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); 5499 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx); 5500 5501 if (ResultVT == MVT::f16) { 5502 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); 5503 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result); 5504 } 5505 5506 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT); 5507 } 5508 5509 static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) { 5510 assert(Elt % 2 == 0); 5511 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0); 5512 } 5513 5514 SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op, 5515 SelectionDAG &DAG) const { 5516 SDLoc SL(Op); 5517 EVT ResultVT = Op.getValueType(); 5518 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 5519 5520 EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16; 5521 EVT EltVT = PackVT.getVectorElementType(); 5522 int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements(); 5523 5524 // vector_shuffle <0,1,6,7> lhs, rhs 5525 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2) 5526 // 5527 // vector_shuffle <6,7,2,3> lhs, rhs 5528 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2) 5529 // 5530 // vector_shuffle <6,7,0,1> lhs, rhs 5531 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0) 5532 5533 // Avoid scalarizing when both halves are reading from consecutive elements. 5534 SmallVector<SDValue, 4> Pieces; 5535 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) { 5536 if (elementPairIsContiguous(SVN->getMask(), I)) { 5537 const int Idx = SVN->getMaskElt(I); 5538 int VecIdx = Idx < SrcNumElts ? 0 : 1; 5539 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts; 5540 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, 5541 PackVT, SVN->getOperand(VecIdx), 5542 DAG.getConstant(EltIdx, SL, MVT::i32)); 5543 Pieces.push_back(SubVec); 5544 } else { 5545 const int Idx0 = SVN->getMaskElt(I); 5546 const int Idx1 = SVN->getMaskElt(I + 1); 5547 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1; 5548 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1; 5549 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts; 5550 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts; 5551 5552 SDValue Vec0 = SVN->getOperand(VecIdx0); 5553 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 5554 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); 5555 5556 SDValue Vec1 = SVN->getOperand(VecIdx1); 5557 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 5558 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); 5559 Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 })); 5560 } 5561 } 5562 5563 return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces); 5564 } 5565 5566 SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op, 5567 SelectionDAG &DAG) const { 5568 SDLoc SL(Op); 5569 EVT VT = Op.getValueType(); 5570 5571 if (VT == MVT::v4i16 || VT == MVT::v4f16) { 5572 EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), 2); 5573 5574 // Turn into pair of packed build_vectors. 5575 // TODO: Special case for constants that can be materialized with s_mov_b64. 5576 SDValue Lo = DAG.getBuildVector(HalfVT, SL, 5577 { Op.getOperand(0), Op.getOperand(1) }); 5578 SDValue Hi = DAG.getBuildVector(HalfVT, SL, 5579 { Op.getOperand(2), Op.getOperand(3) }); 5580 5581 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo); 5582 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi); 5583 5584 SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi }); 5585 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); 5586 } 5587 5588 assert(VT == MVT::v2f16 || VT == MVT::v2i16); 5589 assert(!Subtarget->hasVOP3PInsts() && "this should be legal"); 5590 5591 SDValue Lo = Op.getOperand(0); 5592 SDValue Hi = Op.getOperand(1); 5593 5594 // Avoid adding defined bits with the zero_extend. 5595 if (Hi.isUndef()) { 5596 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); 5597 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); 5598 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo); 5599 } 5600 5601 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); 5602 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); 5603 5604 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, 5605 DAG.getConstant(16, SL, MVT::i32)); 5606 if (Lo.isUndef()) 5607 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi); 5608 5609 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); 5610 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); 5611 5612 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); 5613 return DAG.getNode(ISD::BITCAST, SL, VT, Or); 5614 } 5615 5616 bool 5617 SITargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 5618 // We can fold offsets for anything that doesn't require a GOT relocation. 5619 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS || 5620 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 5621 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && 5622 !shouldEmitGOTReloc(GA->getGlobal()); 5623 } 5624 5625 static SDValue 5626 buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV, 5627 const SDLoc &DL, int64_t Offset, EVT PtrVT, 5628 unsigned GAFlags = SIInstrInfo::MO_NONE) { 5629 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!"); 5630 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is 5631 // lowered to the following code sequence: 5632 // 5633 // For constant address space: 5634 // s_getpc_b64 s[0:1] 5635 // s_add_u32 s0, s0, $symbol 5636 // s_addc_u32 s1, s1, 0 5637 // 5638 // s_getpc_b64 returns the address of the s_add_u32 instruction and then 5639 // a fixup or relocation is emitted to replace $symbol with a literal 5640 // constant, which is a pc-relative offset from the encoding of the $symbol 5641 // operand to the global variable. 5642 // 5643 // For global address space: 5644 // s_getpc_b64 s[0:1] 5645 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo 5646 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi 5647 // 5648 // s_getpc_b64 returns the address of the s_add_u32 instruction and then 5649 // fixups or relocations are emitted to replace $symbol@*@lo and 5650 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant, 5651 // which is a 64-bit pc-relative offset from the encoding of the $symbol 5652 // operand to the global variable. 5653 // 5654 // What we want here is an offset from the value returned by s_getpc 5655 // (which is the address of the s_add_u32 instruction) to the global 5656 // variable, but since the encoding of $symbol starts 4 bytes after the start 5657 // of the s_add_u32 instruction, we end up with an offset that is 4 bytes too 5658 // small. This requires us to add 4 to the global variable offset in order to 5659 // compute the correct address. Similarly for the s_addc_u32 instruction, the 5660 // encoding of $symbol starts 12 bytes after the start of the s_add_u32 5661 // instruction. 5662 SDValue PtrLo = 5663 DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, GAFlags); 5664 SDValue PtrHi; 5665 if (GAFlags == SIInstrInfo::MO_NONE) { 5666 PtrHi = DAG.getTargetConstant(0, DL, MVT::i32); 5667 } else { 5668 PtrHi = 5669 DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 12, GAFlags + 1); 5670 } 5671 return DAG.getNode(AMDGPUISD::PC_ADD_REL_OFFSET, DL, PtrVT, PtrLo, PtrHi); 5672 } 5673 5674 SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI, 5675 SDValue Op, 5676 SelectionDAG &DAG) const { 5677 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op); 5678 SDLoc DL(GSD); 5679 EVT PtrVT = Op.getValueType(); 5680 5681 const GlobalValue *GV = GSD->getGlobal(); 5682 if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && 5683 shouldUseLDSConstAddress(GV)) || 5684 GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS || 5685 GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) { 5686 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && 5687 GV->hasExternalLinkage()) { 5688 Type *Ty = GV->getValueType(); 5689 // HIP uses an unsized array `extern __shared__ T s[]` or similar 5690 // zero-sized type in other languages to declare the dynamic shared 5691 // memory which size is not known at the compile time. They will be 5692 // allocated by the runtime and placed directly after the static 5693 // allocated ones. They all share the same offset. 5694 if (DAG.getDataLayout().getTypeAllocSize(Ty).isZero()) { 5695 assert(PtrVT == MVT::i32 && "32-bit pointer is expected."); 5696 // Adjust alignment for that dynamic shared memory array. 5697 MFI->setDynLDSAlign(DAG.getDataLayout(), *cast<GlobalVariable>(GV)); 5698 return SDValue( 5699 DAG.getMachineNode(AMDGPU::GET_GROUPSTATICSIZE, DL, PtrVT), 0); 5700 } 5701 } 5702 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG); 5703 } 5704 5705 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) { 5706 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(), 5707 SIInstrInfo::MO_ABS32_LO); 5708 return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA); 5709 } 5710 5711 if (shouldEmitFixup(GV)) 5712 return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT); 5713 else if (shouldEmitPCReloc(GV)) 5714 return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT, 5715 SIInstrInfo::MO_REL32); 5716 5717 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT, 5718 SIInstrInfo::MO_GOTPCREL32); 5719 5720 Type *Ty = PtrVT.getTypeForEVT(*DAG.getContext()); 5721 PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS); 5722 const DataLayout &DataLayout = DAG.getDataLayout(); 5723 Align Alignment = DataLayout.getABITypeAlign(PtrTy); 5724 MachinePointerInfo PtrInfo 5725 = MachinePointerInfo::getGOT(DAG.getMachineFunction()); 5726 5727 return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Alignment, 5728 MachineMemOperand::MODereferenceable | 5729 MachineMemOperand::MOInvariant); 5730 } 5731 5732 SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain, 5733 const SDLoc &DL, SDValue V) const { 5734 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as 5735 // the destination register. 5736 // 5737 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions, 5738 // so we will end up with redundant moves to m0. 5739 // 5740 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result. 5741 5742 // A Null SDValue creates a glue result. 5743 SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue, 5744 V, Chain); 5745 return SDValue(M0, 0); 5746 } 5747 5748 SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, 5749 SDValue Op, 5750 MVT VT, 5751 unsigned Offset) const { 5752 SDLoc SL(Op); 5753 SDValue Param = lowerKernargMemParameter( 5754 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); 5755 // The local size values will have the hi 16-bits as zero. 5756 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, 5757 DAG.getValueType(VT)); 5758 } 5759 5760 static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, 5761 EVT VT) { 5762 DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(), 5763 "non-hsa intrinsic with hsa target", 5764 DL.getDebugLoc()); 5765 DAG.getContext()->diagnose(BadIntrin); 5766 return DAG.getUNDEF(VT); 5767 } 5768 5769 static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, 5770 EVT VT) { 5771 DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(), 5772 "intrinsic not supported on subtarget", 5773 DL.getDebugLoc()); 5774 DAG.getContext()->diagnose(BadIntrin); 5775 return DAG.getUNDEF(VT); 5776 } 5777 5778 static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, 5779 ArrayRef<SDValue> Elts) { 5780 assert(!Elts.empty()); 5781 MVT Type; 5782 unsigned NumElts; 5783 5784 if (Elts.size() == 1) { 5785 Type = MVT::f32; 5786 NumElts = 1; 5787 } else if (Elts.size() == 2) { 5788 Type = MVT::v2f32; 5789 NumElts = 2; 5790 } else if (Elts.size() == 3) { 5791 Type = MVT::v3f32; 5792 NumElts = 3; 5793 } else if (Elts.size() <= 4) { 5794 Type = MVT::v4f32; 5795 NumElts = 4; 5796 } else if (Elts.size() <= 8) { 5797 Type = MVT::v8f32; 5798 NumElts = 8; 5799 } else { 5800 assert(Elts.size() <= 16); 5801 Type = MVT::v16f32; 5802 NumElts = 16; 5803 } 5804 5805 SmallVector<SDValue, 16> VecElts(NumElts); 5806 for (unsigned i = 0; i < Elts.size(); ++i) { 5807 SDValue Elt = Elts[i]; 5808 if (Elt.getValueType() != MVT::f32) 5809 Elt = DAG.getBitcast(MVT::f32, Elt); 5810 VecElts[i] = Elt; 5811 } 5812 for (unsigned i = Elts.size(); i < NumElts; ++i) 5813 VecElts[i] = DAG.getUNDEF(MVT::f32); 5814 5815 if (NumElts == 1) 5816 return VecElts[0]; 5817 return DAG.getBuildVector(Type, DL, VecElts); 5818 } 5819 5820 static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, 5821 SDValue Src, int ExtraElts) { 5822 EVT SrcVT = Src.getValueType(); 5823 5824 SmallVector<SDValue, 8> Elts; 5825 5826 if (SrcVT.isVector()) 5827 DAG.ExtractVectorElements(Src, Elts); 5828 else 5829 Elts.push_back(Src); 5830 5831 SDValue Undef = DAG.getUNDEF(SrcVT.getScalarType()); 5832 while (ExtraElts--) 5833 Elts.push_back(Undef); 5834 5835 return DAG.getBuildVector(CastVT, DL, Elts); 5836 } 5837 5838 // Re-construct the required return value for a image load intrinsic. 5839 // This is more complicated due to the optional use TexFailCtrl which means the required 5840 // return type is an aggregate 5841 static SDValue constructRetValue(SelectionDAG &DAG, 5842 MachineSDNode *Result, 5843 ArrayRef<EVT> ResultTypes, 5844 bool IsTexFail, bool Unpacked, bool IsD16, 5845 int DMaskPop, int NumVDataDwords, 5846 const SDLoc &DL) { 5847 // Determine the required return type. This is the same regardless of IsTexFail flag 5848 EVT ReqRetVT = ResultTypes[0]; 5849 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1; 5850 int NumDataDwords = (!IsD16 || (IsD16 && Unpacked)) ? 5851 ReqRetNumElts : (ReqRetNumElts + 1) / 2; 5852 5853 int MaskPopDwords = (!IsD16 || (IsD16 && Unpacked)) ? 5854 DMaskPop : (DMaskPop + 1) / 2; 5855 5856 MVT DataDwordVT = NumDataDwords == 1 ? 5857 MVT::i32 : MVT::getVectorVT(MVT::i32, NumDataDwords); 5858 5859 MVT MaskPopVT = MaskPopDwords == 1 ? 5860 MVT::i32 : MVT::getVectorVT(MVT::i32, MaskPopDwords); 5861 5862 SDValue Data(Result, 0); 5863 SDValue TexFail; 5864 5865 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) { 5866 SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32); 5867 if (MaskPopVT.isVector()) { 5868 Data = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskPopVT, 5869 SDValue(Result, 0), ZeroIdx); 5870 } else { 5871 Data = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MaskPopVT, 5872 SDValue(Result, 0), ZeroIdx); 5873 } 5874 } 5875 5876 if (DataDwordVT.isVector()) 5877 Data = padEltsToUndef(DAG, DL, DataDwordVT, Data, 5878 NumDataDwords - MaskPopDwords); 5879 5880 if (IsD16) 5881 Data = adjustLoadValueTypeImpl(Data, ReqRetVT, DL, DAG, Unpacked); 5882 5883 EVT LegalReqRetVT = ReqRetVT; 5884 if (!ReqRetVT.isVector()) { 5885 Data = DAG.getNode(ISD::TRUNCATE, DL, ReqRetVT.changeTypeToInteger(), Data); 5886 } else { 5887 // We need to widen the return vector to a legal type 5888 if ((ReqRetVT.getVectorNumElements() % 2) == 1 && 5889 ReqRetVT.getVectorElementType().getSizeInBits() == 16) { 5890 LegalReqRetVT = 5891 EVT::getVectorVT(*DAG.getContext(), ReqRetVT.getVectorElementType(), 5892 ReqRetVT.getVectorNumElements() + 1); 5893 } 5894 } 5895 Data = DAG.getNode(ISD::BITCAST, DL, LegalReqRetVT, Data); 5896 5897 if (IsTexFail) { 5898 TexFail = 5899 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, SDValue(Result, 0), 5900 DAG.getConstant(MaskPopDwords, DL, MVT::i32)); 5901 5902 return DAG.getMergeValues({Data, TexFail, SDValue(Result, 1)}, DL); 5903 } 5904 5905 if (Result->getNumValues() == 1) 5906 return Data; 5907 5908 return DAG.getMergeValues({Data, SDValue(Result, 1)}, DL); 5909 } 5910 5911 static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, 5912 SDValue *LWE, bool &IsTexFail) { 5913 auto TexFailCtrlConst = cast<ConstantSDNode>(TexFailCtrl.getNode()); 5914 5915 uint64_t Value = TexFailCtrlConst->getZExtValue(); 5916 if (Value) { 5917 IsTexFail = true; 5918 } 5919 5920 SDLoc DL(TexFailCtrlConst); 5921 *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); 5922 Value &= ~(uint64_t)0x1; 5923 *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); 5924 Value &= ~(uint64_t)0x2; 5925 5926 return Value == 0; 5927 } 5928 5929 static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op, 5930 MVT PackVectorVT, 5931 SmallVectorImpl<SDValue> &PackedAddrs, 5932 unsigned DimIdx, unsigned EndIdx, 5933 unsigned NumGradients) { 5934 SDLoc DL(Op); 5935 for (unsigned I = DimIdx; I < EndIdx; I++) { 5936 SDValue Addr = Op.getOperand(I); 5937 5938 // Gradients are packed with undef for each coordinate. 5939 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this: 5940 // 1D: undef,dx/dh; undef,dx/dv 5941 // 2D: dy/dh,dx/dh; dy/dv,dx/dv 5942 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv 5943 if (((I + 1) >= EndIdx) || 5944 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 || 5945 I == DimIdx + NumGradients - 1))) { 5946 if (Addr.getValueType() != MVT::i16) 5947 Addr = DAG.getBitcast(MVT::i16, Addr); 5948 Addr = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Addr); 5949 } else { 5950 Addr = DAG.getBuildVector(PackVectorVT, DL, {Addr, Op.getOperand(I + 1)}); 5951 I++; 5952 } 5953 Addr = DAG.getBitcast(MVT::f32, Addr); 5954 PackedAddrs.push_back(Addr); 5955 } 5956 } 5957 5958 SDValue SITargetLowering::lowerImage(SDValue Op, 5959 const AMDGPU::ImageDimIntrinsicInfo *Intr, 5960 SelectionDAG &DAG, bool WithChain) const { 5961 SDLoc DL(Op); 5962 MachineFunction &MF = DAG.getMachineFunction(); 5963 const GCNSubtarget* ST = &MF.getSubtarget<GCNSubtarget>(); 5964 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 5965 AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); 5966 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); 5967 const AMDGPU::MIMGLZMappingInfo *LZMappingInfo = 5968 AMDGPU::getMIMGLZMappingInfo(Intr->BaseOpcode); 5969 const AMDGPU::MIMGMIPMappingInfo *MIPMappingInfo = 5970 AMDGPU::getMIMGMIPMappingInfo(Intr->BaseOpcode); 5971 unsigned IntrOpcode = Intr->BaseOpcode; 5972 bool IsGFX10Plus = AMDGPU::isGFX10Plus(*Subtarget); 5973 5974 SmallVector<EVT, 3> ResultTypes(Op->values()); 5975 SmallVector<EVT, 3> OrigResultTypes(Op->values()); 5976 bool IsD16 = false; 5977 bool IsG16 = false; 5978 bool IsA16 = false; 5979 SDValue VData; 5980 int NumVDataDwords; 5981 bool AdjustRetType = false; 5982 5983 // Offset of intrinsic arguments 5984 const unsigned ArgOffset = WithChain ? 2 : 1; 5985 5986 unsigned DMask; 5987 unsigned DMaskLanes = 0; 5988 5989 if (BaseOpcode->Atomic) { 5990 VData = Op.getOperand(2); 5991 5992 bool Is64Bit = VData.getValueType() == MVT::i64; 5993 if (BaseOpcode->AtomicX2) { 5994 SDValue VData2 = Op.getOperand(3); 5995 VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL, 5996 {VData, VData2}); 5997 if (Is64Bit) 5998 VData = DAG.getBitcast(MVT::v4i32, VData); 5999 6000 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32; 6001 DMask = Is64Bit ? 0xf : 0x3; 6002 NumVDataDwords = Is64Bit ? 4 : 2; 6003 } else { 6004 DMask = Is64Bit ? 0x3 : 0x1; 6005 NumVDataDwords = Is64Bit ? 2 : 1; 6006 } 6007 } else { 6008 auto *DMaskConst = 6009 cast<ConstantSDNode>(Op.getOperand(ArgOffset + Intr->DMaskIndex)); 6010 DMask = DMaskConst->getZExtValue(); 6011 DMaskLanes = BaseOpcode->Gather4 ? 4 : countPopulation(DMask); 6012 6013 if (BaseOpcode->Store) { 6014 VData = Op.getOperand(2); 6015 6016 MVT StoreVT = VData.getSimpleValueType(); 6017 if (StoreVT.getScalarType() == MVT::f16) { 6018 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16) 6019 return Op; // D16 is unsupported for this instruction 6020 6021 IsD16 = true; 6022 VData = handleD16VData(VData, DAG, true); 6023 } 6024 6025 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32; 6026 } else { 6027 // Work out the num dwords based on the dmask popcount and underlying type 6028 // and whether packing is supported. 6029 MVT LoadVT = ResultTypes[0].getSimpleVT(); 6030 if (LoadVT.getScalarType() == MVT::f16) { 6031 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16) 6032 return Op; // D16 is unsupported for this instruction 6033 6034 IsD16 = true; 6035 } 6036 6037 // Confirm that the return type is large enough for the dmask specified 6038 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) || 6039 (!LoadVT.isVector() && DMaskLanes > 1)) 6040 return Op; 6041 6042 // The sq block of gfx8 and gfx9 do not estimate register use correctly 6043 // for d16 image_gather4, image_gather4_l, and image_gather4_lz 6044 // instructions. 6045 if (IsD16 && !Subtarget->hasUnpackedD16VMem() && 6046 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug())) 6047 NumVDataDwords = (DMaskLanes + 1) / 2; 6048 else 6049 NumVDataDwords = DMaskLanes; 6050 6051 AdjustRetType = true; 6052 } 6053 } 6054 6055 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd; 6056 SmallVector<SDValue, 4> VAddrs; 6057 6058 // Optimize _L to _LZ when _L is zero 6059 if (LZMappingInfo) { 6060 if (auto *ConstantLod = dyn_cast<ConstantFPSDNode>( 6061 Op.getOperand(ArgOffset + Intr->LodIndex))) { 6062 if (ConstantLod->isZero() || ConstantLod->isNegative()) { 6063 IntrOpcode = LZMappingInfo->LZ; // set new opcode to _lz variant of _l 6064 VAddrEnd--; // remove 'lod' 6065 } 6066 } 6067 } 6068 6069 // Optimize _mip away, when 'lod' is zero 6070 if (MIPMappingInfo) { 6071 if (auto *ConstantLod = dyn_cast<ConstantSDNode>( 6072 Op.getOperand(ArgOffset + Intr->MipIndex))) { 6073 if (ConstantLod->isNullValue()) { 6074 IntrOpcode = MIPMappingInfo->NONMIP; // set new opcode to variant without _mip 6075 VAddrEnd--; // remove 'mip' 6076 } 6077 } 6078 } 6079 6080 // Push back extra arguments. 6081 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) 6082 VAddrs.push_back(Op.getOperand(ArgOffset + I)); 6083 6084 // Check for 16 bit addresses or derivatives and pack if true. 6085 MVT VAddrVT = 6086 Op.getOperand(ArgOffset + Intr->GradientStart).getSimpleValueType(); 6087 MVT VAddrScalarVT = VAddrVT.getScalarType(); 6088 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; 6089 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; 6090 6091 VAddrVT = Op.getOperand(ArgOffset + Intr->CoordStart).getSimpleValueType(); 6092 VAddrScalarVT = VAddrVT.getScalarType(); 6093 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; 6094 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; 6095 6096 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) { 6097 // 16 bit gradients are supported, but are tied to the A16 control 6098 // so both gradients and addresses must be 16 bit 6099 LLVM_DEBUG( 6100 dbgs() << "Failed to lower image intrinsic: 16 bit addresses " 6101 "require 16 bit args for both gradients and addresses"); 6102 return Op; 6103 } 6104 6105 if (IsA16) { 6106 if (!ST->hasA16()) { 6107 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not " 6108 "support 16 bit addresses\n"); 6109 return Op; 6110 } 6111 } 6112 6113 // We've dealt with incorrect input so we know that if IsA16, IsG16 6114 // are set then we have to compress/pack operands (either address, 6115 // gradient or both) 6116 // In the case where a16 and gradients are tied (no G16 support) then we 6117 // have already verified that both IsA16 and IsG16 are true 6118 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) { 6119 // Activate g16 6120 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo = 6121 AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode); 6122 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16 6123 } 6124 6125 // Add gradients (packed or unpacked) 6126 if (IsG16) { 6127 // Pack the gradients 6128 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart); 6129 packImage16bitOpsToDwords(DAG, Op, GradPackVectorVT, VAddrs, 6130 ArgOffset + Intr->GradientStart, 6131 ArgOffset + Intr->CoordStart, Intr->NumGradients); 6132 } else { 6133 for (unsigned I = ArgOffset + Intr->GradientStart; 6134 I < ArgOffset + Intr->CoordStart; I++) 6135 VAddrs.push_back(Op.getOperand(I)); 6136 } 6137 6138 // Add addresses (packed or unpacked) 6139 if (IsA16) { 6140 packImage16bitOpsToDwords(DAG, Op, AddrPackVectorVT, VAddrs, 6141 ArgOffset + Intr->CoordStart, VAddrEnd, 6142 0 /* No gradients */); 6143 } else { 6144 // Add uncompressed address 6145 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++) 6146 VAddrs.push_back(Op.getOperand(I)); 6147 } 6148 6149 // If the register allocator cannot place the address registers contiguously 6150 // without introducing moves, then using the non-sequential address encoding 6151 // is always preferable, since it saves VALU instructions and is usually a 6152 // wash in terms of code size or even better. 6153 // 6154 // However, we currently have no way of hinting to the register allocator that 6155 // MIMG addresses should be placed contiguously when it is possible to do so, 6156 // so force non-NSA for the common 2-address case as a heuristic. 6157 // 6158 // SIShrinkInstructions will convert NSA encodings to non-NSA after register 6159 // allocation when possible. 6160 bool UseNSA = 6161 ST->hasFeature(AMDGPU::FeatureNSAEncoding) && VAddrs.size() >= 3; 6162 SDValue VAddr; 6163 if (!UseNSA) 6164 VAddr = getBuildDwordsVector(DAG, DL, VAddrs); 6165 6166 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1); 6167 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1); 6168 SDValue Unorm; 6169 if (!BaseOpcode->Sampler) { 6170 Unorm = True; 6171 } else { 6172 auto UnormConst = 6173 cast<ConstantSDNode>(Op.getOperand(ArgOffset + Intr->UnormIndex)); 6174 6175 Unorm = UnormConst->getZExtValue() ? True : False; 6176 } 6177 6178 SDValue TFE; 6179 SDValue LWE; 6180 SDValue TexFail = Op.getOperand(ArgOffset + Intr->TexFailCtrlIndex); 6181 bool IsTexFail = false; 6182 if (!parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail)) 6183 return Op; 6184 6185 if (IsTexFail) { 6186 if (!DMaskLanes) { 6187 // Expecting to get an error flag since TFC is on - and dmask is 0 6188 // Force dmask to be at least 1 otherwise the instruction will fail 6189 DMask = 0x1; 6190 DMaskLanes = 1; 6191 NumVDataDwords = 1; 6192 } 6193 NumVDataDwords += 1; 6194 AdjustRetType = true; 6195 } 6196 6197 // Has something earlier tagged that the return type needs adjusting 6198 // This happens if the instruction is a load or has set TexFailCtrl flags 6199 if (AdjustRetType) { 6200 // NumVDataDwords reflects the true number of dwords required in the return type 6201 if (DMaskLanes == 0 && !BaseOpcode->Store) { 6202 // This is a no-op load. This can be eliminated 6203 SDValue Undef = DAG.getUNDEF(Op.getValueType()); 6204 if (isa<MemSDNode>(Op)) 6205 return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL); 6206 return Undef; 6207 } 6208 6209 EVT NewVT = NumVDataDwords > 1 ? 6210 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumVDataDwords) 6211 : MVT::i32; 6212 6213 ResultTypes[0] = NewVT; 6214 if (ResultTypes.size() == 3) { 6215 // Original result was aggregate type used for TexFailCtrl results 6216 // The actual instruction returns as a vector type which has now been 6217 // created. Remove the aggregate result. 6218 ResultTypes.erase(&ResultTypes[1]); 6219 } 6220 } 6221 6222 unsigned CPol = cast<ConstantSDNode>( 6223 Op.getOperand(ArgOffset + Intr->CachePolicyIndex))->getZExtValue(); 6224 if (BaseOpcode->Atomic) 6225 CPol |= AMDGPU::CPol::GLC; // TODO no-return optimization 6226 if (CPol & ~AMDGPU::CPol::ALL) 6227 return Op; 6228 6229 SmallVector<SDValue, 26> Ops; 6230 if (BaseOpcode->Store || BaseOpcode->Atomic) 6231 Ops.push_back(VData); // vdata 6232 if (UseNSA) 6233 append_range(Ops, VAddrs); 6234 else 6235 Ops.push_back(VAddr); 6236 Ops.push_back(Op.getOperand(ArgOffset + Intr->RsrcIndex)); 6237 if (BaseOpcode->Sampler) 6238 Ops.push_back(Op.getOperand(ArgOffset + Intr->SampIndex)); 6239 Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32)); 6240 if (IsGFX10Plus) 6241 Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32)); 6242 Ops.push_back(Unorm); 6243 Ops.push_back(DAG.getTargetConstant(CPol, DL, MVT::i32)); 6244 Ops.push_back(IsA16 && // r128, a16 for gfx9 6245 ST->hasFeature(AMDGPU::FeatureR128A16) ? True : False); 6246 if (IsGFX10Plus) 6247 Ops.push_back(IsA16 ? True : False); 6248 if (!Subtarget->hasGFX90AInsts()) { 6249 Ops.push_back(TFE); //tfe 6250 } else if (cast<ConstantSDNode>(TFE)->getZExtValue()) { 6251 report_fatal_error("TFE is not supported on this GPU"); 6252 } 6253 Ops.push_back(LWE); // lwe 6254 if (!IsGFX10Plus) 6255 Ops.push_back(DimInfo->DA ? True : False); 6256 if (BaseOpcode->HasD16) 6257 Ops.push_back(IsD16 ? True : False); 6258 if (isa<MemSDNode>(Op)) 6259 Ops.push_back(Op.getOperand(0)); // chain 6260 6261 int NumVAddrDwords = 6262 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32; 6263 int Opcode = -1; 6264 6265 if (IsGFX10Plus) { 6266 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, 6267 UseNSA ? AMDGPU::MIMGEncGfx10NSA 6268 : AMDGPU::MIMGEncGfx10Default, 6269 NumVDataDwords, NumVAddrDwords); 6270 } else { 6271 if (Subtarget->hasGFX90AInsts()) { 6272 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a, 6273 NumVDataDwords, NumVAddrDwords); 6274 if (Opcode == -1) 6275 report_fatal_error( 6276 "requested image instruction is not supported on this GPU"); 6277 } 6278 if (Opcode == -1 && 6279 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 6280 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8, 6281 NumVDataDwords, NumVAddrDwords); 6282 if (Opcode == -1) 6283 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6, 6284 NumVDataDwords, NumVAddrDwords); 6285 } 6286 assert(Opcode != -1); 6287 6288 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops); 6289 if (auto MemOp = dyn_cast<MemSDNode>(Op)) { 6290 MachineMemOperand *MemRef = MemOp->getMemOperand(); 6291 DAG.setNodeMemRefs(NewNode, {MemRef}); 6292 } 6293 6294 if (BaseOpcode->AtomicX2) { 6295 SmallVector<SDValue, 1> Elt; 6296 DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1); 6297 return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL); 6298 } 6299 if (BaseOpcode->Store) 6300 return SDValue(NewNode, 0); 6301 return constructRetValue(DAG, NewNode, 6302 OrigResultTypes, IsTexFail, 6303 Subtarget->hasUnpackedD16VMem(), IsD16, 6304 DMaskLanes, NumVDataDwords, DL); 6305 } 6306 6307 SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, 6308 SDValue Offset, SDValue CachePolicy, 6309 SelectionDAG &DAG) const { 6310 MachineFunction &MF = DAG.getMachineFunction(); 6311 6312 const DataLayout &DataLayout = DAG.getDataLayout(); 6313 Align Alignment = 6314 DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext())); 6315 6316 MachineMemOperand *MMO = MF.getMachineMemOperand( 6317 MachinePointerInfo(), 6318 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | 6319 MachineMemOperand::MOInvariant, 6320 VT.getStoreSize(), Alignment); 6321 6322 if (!Offset->isDivergent()) { 6323 SDValue Ops[] = { 6324 Rsrc, 6325 Offset, // Offset 6326 CachePolicy 6327 }; 6328 6329 // Widen vec3 load to vec4. 6330 if (VT.isVector() && VT.getVectorNumElements() == 3) { 6331 EVT WidenedVT = 6332 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4); 6333 auto WidenedOp = DAG.getMemIntrinsicNode( 6334 AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT), Ops, WidenedVT, 6335 MF.getMachineMemOperand(MMO, 0, WidenedVT.getStoreSize())); 6336 auto Subvector = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WidenedOp, 6337 DAG.getVectorIdxConstant(0, DL)); 6338 return Subvector; 6339 } 6340 6341 return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL, 6342 DAG.getVTList(VT), Ops, VT, MMO); 6343 } 6344 6345 // We have a divergent offset. Emit a MUBUF buffer load instead. We can 6346 // assume that the buffer is unswizzled. 6347 SmallVector<SDValue, 4> Loads; 6348 unsigned NumLoads = 1; 6349 MVT LoadVT = VT.getSimpleVT(); 6350 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1; 6351 assert((LoadVT.getScalarType() == MVT::i32 || 6352 LoadVT.getScalarType() == MVT::f32)); 6353 6354 if (NumElts == 8 || NumElts == 16) { 6355 NumLoads = NumElts / 4; 6356 LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4); 6357 } 6358 6359 SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); 6360 SDValue Ops[] = { 6361 DAG.getEntryNode(), // Chain 6362 Rsrc, // rsrc 6363 DAG.getConstant(0, DL, MVT::i32), // vindex 6364 {}, // voffset 6365 {}, // soffset 6366 {}, // offset 6367 CachePolicy, // cachepolicy 6368 DAG.getTargetConstant(0, DL, MVT::i1), // idxen 6369 }; 6370 6371 // Use the alignment to ensure that the required offsets will fit into the 6372 // immediate offsets. 6373 setBufferOffsets(Offset, DAG, &Ops[3], 6374 NumLoads > 1 ? Align(16 * NumLoads) : Align(4)); 6375 6376 uint64_t InstOffset = cast<ConstantSDNode>(Ops[5])->getZExtValue(); 6377 for (unsigned i = 0; i < NumLoads; ++i) { 6378 Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32); 6379 Loads.push_back(getMemIntrinsicNode(AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops, 6380 LoadVT, MMO, DAG)); 6381 } 6382 6383 if (NumElts == 8 || NumElts == 16) 6384 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Loads); 6385 6386 return Loads[0]; 6387 } 6388 6389 SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 6390 SelectionDAG &DAG) const { 6391 MachineFunction &MF = DAG.getMachineFunction(); 6392 auto MFI = MF.getInfo<SIMachineFunctionInfo>(); 6393 6394 EVT VT = Op.getValueType(); 6395 SDLoc DL(Op); 6396 unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 6397 6398 // TODO: Should this propagate fast-math-flags? 6399 6400 switch (IntrinsicID) { 6401 case Intrinsic::amdgcn_implicit_buffer_ptr: { 6402 if (getSubtarget()->isAmdHsaOrMesa(MF.getFunction())) 6403 return emitNonHSAIntrinsicError(DAG, DL, VT); 6404 return getPreloadedValue(DAG, *MFI, VT, 6405 AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR); 6406 } 6407 case Intrinsic::amdgcn_dispatch_ptr: 6408 case Intrinsic::amdgcn_queue_ptr: { 6409 if (!Subtarget->isAmdHsaOrMesa(MF.getFunction())) { 6410 DiagnosticInfoUnsupported BadIntrin( 6411 MF.getFunction(), "unsupported hsa intrinsic without hsa target", 6412 DL.getDebugLoc()); 6413 DAG.getContext()->diagnose(BadIntrin); 6414 return DAG.getUNDEF(VT); 6415 } 6416 6417 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr ? 6418 AMDGPUFunctionArgInfo::DISPATCH_PTR : AMDGPUFunctionArgInfo::QUEUE_PTR; 6419 return getPreloadedValue(DAG, *MFI, VT, RegID); 6420 } 6421 case Intrinsic::amdgcn_implicitarg_ptr: { 6422 if (MFI->isEntryFunction()) 6423 return getImplicitArgPtr(DAG, DL); 6424 return getPreloadedValue(DAG, *MFI, VT, 6425 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR); 6426 } 6427 case Intrinsic::amdgcn_kernarg_segment_ptr: { 6428 if (!AMDGPU::isKernel(MF.getFunction().getCallingConv())) { 6429 // This only makes sense to call in a kernel, so just lower to null. 6430 return DAG.getConstant(0, DL, VT); 6431 } 6432 6433 return getPreloadedValue(DAG, *MFI, VT, 6434 AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR); 6435 } 6436 case Intrinsic::amdgcn_dispatch_id: { 6437 return getPreloadedValue(DAG, *MFI, VT, AMDGPUFunctionArgInfo::DISPATCH_ID); 6438 } 6439 case Intrinsic::amdgcn_rcp: 6440 return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1)); 6441 case Intrinsic::amdgcn_rsq: 6442 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); 6443 case Intrinsic::amdgcn_rsq_legacy: 6444 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 6445 return emitRemovedIntrinsicError(DAG, DL, VT); 6446 return SDValue(); 6447 case Intrinsic::amdgcn_rcp_legacy: 6448 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 6449 return emitRemovedIntrinsicError(DAG, DL, VT); 6450 return DAG.getNode(AMDGPUISD::RCP_LEGACY, DL, VT, Op.getOperand(1)); 6451 case Intrinsic::amdgcn_rsq_clamp: { 6452 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS) 6453 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1)); 6454 6455 Type *Type = VT.getTypeForEVT(*DAG.getContext()); 6456 APFloat Max = APFloat::getLargest(Type->getFltSemantics()); 6457 APFloat Min = APFloat::getLargest(Type->getFltSemantics(), true); 6458 6459 SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); 6460 SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq, 6461 DAG.getConstantFP(Max, DL, VT)); 6462 return DAG.getNode(ISD::FMAXNUM, DL, VT, Tmp, 6463 DAG.getConstantFP(Min, DL, VT)); 6464 } 6465 case Intrinsic::r600_read_ngroups_x: 6466 if (Subtarget->isAmdHsaOS()) 6467 return emitNonHSAIntrinsicError(DAG, DL, VT); 6468 6469 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 6470 SI::KernelInputOffsets::NGROUPS_X, Align(4), 6471 false); 6472 case Intrinsic::r600_read_ngroups_y: 6473 if (Subtarget->isAmdHsaOS()) 6474 return emitNonHSAIntrinsicError(DAG, DL, VT); 6475 6476 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 6477 SI::KernelInputOffsets::NGROUPS_Y, Align(4), 6478 false); 6479 case Intrinsic::r600_read_ngroups_z: 6480 if (Subtarget->isAmdHsaOS()) 6481 return emitNonHSAIntrinsicError(DAG, DL, VT); 6482 6483 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 6484 SI::KernelInputOffsets::NGROUPS_Z, Align(4), 6485 false); 6486 case Intrinsic::r600_read_global_size_x: 6487 if (Subtarget->isAmdHsaOS()) 6488 return emitNonHSAIntrinsicError(DAG, DL, VT); 6489 6490 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 6491 SI::KernelInputOffsets::GLOBAL_SIZE_X, 6492 Align(4), false); 6493 case Intrinsic::r600_read_global_size_y: 6494 if (Subtarget->isAmdHsaOS()) 6495 return emitNonHSAIntrinsicError(DAG, DL, VT); 6496 6497 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 6498 SI::KernelInputOffsets::GLOBAL_SIZE_Y, 6499 Align(4), false); 6500 case Intrinsic::r600_read_global_size_z: 6501 if (Subtarget->isAmdHsaOS()) 6502 return emitNonHSAIntrinsicError(DAG, DL, VT); 6503 6504 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), 6505 SI::KernelInputOffsets::GLOBAL_SIZE_Z, 6506 Align(4), false); 6507 case Intrinsic::r600_read_local_size_x: 6508 if (Subtarget->isAmdHsaOS()) 6509 return emitNonHSAIntrinsicError(DAG, DL, VT); 6510 6511 return lowerImplicitZextParam(DAG, Op, MVT::i16, 6512 SI::KernelInputOffsets::LOCAL_SIZE_X); 6513 case Intrinsic::r600_read_local_size_y: 6514 if (Subtarget->isAmdHsaOS()) 6515 return emitNonHSAIntrinsicError(DAG, DL, VT); 6516 6517 return lowerImplicitZextParam(DAG, Op, MVT::i16, 6518 SI::KernelInputOffsets::LOCAL_SIZE_Y); 6519 case Intrinsic::r600_read_local_size_z: 6520 if (Subtarget->isAmdHsaOS()) 6521 return emitNonHSAIntrinsicError(DAG, DL, VT); 6522 6523 return lowerImplicitZextParam(DAG, Op, MVT::i16, 6524 SI::KernelInputOffsets::LOCAL_SIZE_Z); 6525 case Intrinsic::amdgcn_workgroup_id_x: 6526 return getPreloadedValue(DAG, *MFI, VT, 6527 AMDGPUFunctionArgInfo::WORKGROUP_ID_X); 6528 case Intrinsic::amdgcn_workgroup_id_y: 6529 return getPreloadedValue(DAG, *MFI, VT, 6530 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y); 6531 case Intrinsic::amdgcn_workgroup_id_z: 6532 return getPreloadedValue(DAG, *MFI, VT, 6533 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z); 6534 case Intrinsic::amdgcn_workitem_id_x: 6535 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, 6536 SDLoc(DAG.getEntryNode()), 6537 MFI->getArgInfo().WorkItemIDX); 6538 case Intrinsic::amdgcn_workitem_id_y: 6539 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, 6540 SDLoc(DAG.getEntryNode()), 6541 MFI->getArgInfo().WorkItemIDY); 6542 case Intrinsic::amdgcn_workitem_id_z: 6543 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, 6544 SDLoc(DAG.getEntryNode()), 6545 MFI->getArgInfo().WorkItemIDZ); 6546 case Intrinsic::amdgcn_wavefrontsize: 6547 return DAG.getConstant(MF.getSubtarget<GCNSubtarget>().getWavefrontSize(), 6548 SDLoc(Op), MVT::i32); 6549 case Intrinsic::amdgcn_s_buffer_load: { 6550 unsigned CPol = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); 6551 if (CPol & ~AMDGPU::CPol::ALL) 6552 return Op; 6553 return lowerSBuffer(VT, DL, Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), 6554 DAG); 6555 } 6556 case Intrinsic::amdgcn_fdiv_fast: 6557 return lowerFDIV_FAST(Op, DAG); 6558 case Intrinsic::amdgcn_sin: 6559 return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1)); 6560 6561 case Intrinsic::amdgcn_cos: 6562 return DAG.getNode(AMDGPUISD::COS_HW, DL, VT, Op.getOperand(1)); 6563 6564 case Intrinsic::amdgcn_mul_u24: 6565 return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT, Op.getOperand(1), Op.getOperand(2)); 6566 case Intrinsic::amdgcn_mul_i24: 6567 return DAG.getNode(AMDGPUISD::MUL_I24, DL, VT, Op.getOperand(1), Op.getOperand(2)); 6568 6569 case Intrinsic::amdgcn_log_clamp: { 6570 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS) 6571 return SDValue(); 6572 6573 return emitRemovedIntrinsicError(DAG, DL, VT); 6574 } 6575 case Intrinsic::amdgcn_ldexp: 6576 return DAG.getNode(AMDGPUISD::LDEXP, DL, VT, 6577 Op.getOperand(1), Op.getOperand(2)); 6578 6579 case Intrinsic::amdgcn_fract: 6580 return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); 6581 6582 case Intrinsic::amdgcn_class: 6583 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, 6584 Op.getOperand(1), Op.getOperand(2)); 6585 case Intrinsic::amdgcn_div_fmas: 6586 return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, 6587 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), 6588 Op.getOperand(4)); 6589 6590 case Intrinsic::amdgcn_div_fixup: 6591 return DAG.getNode(AMDGPUISD::DIV_FIXUP, DL, VT, 6592 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 6593 6594 case Intrinsic::amdgcn_div_scale: { 6595 const ConstantSDNode *Param = cast<ConstantSDNode>(Op.getOperand(3)); 6596 6597 // Translate to the operands expected by the machine instruction. The 6598 // first parameter must be the same as the first instruction. 6599 SDValue Numerator = Op.getOperand(1); 6600 SDValue Denominator = Op.getOperand(2); 6601 6602 // Note this order is opposite of the machine instruction's operations, 6603 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The 6604 // intrinsic has the numerator as the first operand to match a normal 6605 // division operation. 6606 6607 SDValue Src0 = Param->isAllOnesValue() ? Numerator : Denominator; 6608 6609 return DAG.getNode(AMDGPUISD::DIV_SCALE, DL, Op->getVTList(), Src0, 6610 Denominator, Numerator); 6611 } 6612 case Intrinsic::amdgcn_icmp: { 6613 // There is a Pat that handles this variant, so return it as-is. 6614 if (Op.getOperand(1).getValueType() == MVT::i1 && 6615 Op.getConstantOperandVal(2) == 0 && 6616 Op.getConstantOperandVal(3) == ICmpInst::Predicate::ICMP_NE) 6617 return Op; 6618 return lowerICMPIntrinsic(*this, Op.getNode(), DAG); 6619 } 6620 case Intrinsic::amdgcn_fcmp: { 6621 return lowerFCMPIntrinsic(*this, Op.getNode(), DAG); 6622 } 6623 case Intrinsic::amdgcn_ballot: 6624 return lowerBALLOTIntrinsic(*this, Op.getNode(), DAG); 6625 case Intrinsic::amdgcn_fmed3: 6626 return DAG.getNode(AMDGPUISD::FMED3, DL, VT, 6627 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 6628 case Intrinsic::amdgcn_fdot2: 6629 return DAG.getNode(AMDGPUISD::FDOT2, DL, VT, 6630 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), 6631 Op.getOperand(4)); 6632 case Intrinsic::amdgcn_fmul_legacy: 6633 return DAG.getNode(AMDGPUISD::FMUL_LEGACY, DL, VT, 6634 Op.getOperand(1), Op.getOperand(2)); 6635 case Intrinsic::amdgcn_sffbh: 6636 return DAG.getNode(AMDGPUISD::FFBH_I32, DL, VT, Op.getOperand(1)); 6637 case Intrinsic::amdgcn_sbfe: 6638 return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT, 6639 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 6640 case Intrinsic::amdgcn_ubfe: 6641 return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT, 6642 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 6643 case Intrinsic::amdgcn_cvt_pkrtz: 6644 case Intrinsic::amdgcn_cvt_pknorm_i16: 6645 case Intrinsic::amdgcn_cvt_pknorm_u16: 6646 case Intrinsic::amdgcn_cvt_pk_i16: 6647 case Intrinsic::amdgcn_cvt_pk_u16: { 6648 // FIXME: Stop adding cast if v2f16/v2i16 are legal. 6649 EVT VT = Op.getValueType(); 6650 unsigned Opcode; 6651 6652 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz) 6653 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32; 6654 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16) 6655 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32; 6656 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16) 6657 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32; 6658 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16) 6659 Opcode = AMDGPUISD::CVT_PK_I16_I32; 6660 else 6661 Opcode = AMDGPUISD::CVT_PK_U16_U32; 6662 6663 if (isTypeLegal(VT)) 6664 return DAG.getNode(Opcode, DL, VT, Op.getOperand(1), Op.getOperand(2)); 6665 6666 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32, 6667 Op.getOperand(1), Op.getOperand(2)); 6668 return DAG.getNode(ISD::BITCAST, DL, VT, Node); 6669 } 6670 case Intrinsic::amdgcn_fmad_ftz: 6671 return DAG.getNode(AMDGPUISD::FMAD_FTZ, DL, VT, Op.getOperand(1), 6672 Op.getOperand(2), Op.getOperand(3)); 6673 6674 case Intrinsic::amdgcn_if_break: 6675 return SDValue(DAG.getMachineNode(AMDGPU::SI_IF_BREAK, DL, VT, 6676 Op->getOperand(1), Op->getOperand(2)), 0); 6677 6678 case Intrinsic::amdgcn_groupstaticsize: { 6679 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS(); 6680 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) 6681 return Op; 6682 6683 const Module *M = MF.getFunction().getParent(); 6684 const GlobalValue *GV = 6685 M->getNamedValue(Intrinsic::getName(Intrinsic::amdgcn_groupstaticsize)); 6686 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0, 6687 SIInstrInfo::MO_ABS32_LO); 6688 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; 6689 } 6690 case Intrinsic::amdgcn_is_shared: 6691 case Intrinsic::amdgcn_is_private: { 6692 SDLoc SL(Op); 6693 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared) ? 6694 AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS; 6695 SDValue Aperture = getSegmentAperture(AS, SL, DAG); 6696 SDValue SrcVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, 6697 Op.getOperand(1)); 6698 6699 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec, 6700 DAG.getConstant(1, SL, MVT::i32)); 6701 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ); 6702 } 6703 case Intrinsic::amdgcn_alignbit: 6704 return DAG.getNode(ISD::FSHR, DL, VT, 6705 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 6706 case Intrinsic::amdgcn_perm: 6707 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op.getOperand(1), 6708 Op.getOperand(2), Op.getOperand(3)); 6709 case Intrinsic::amdgcn_reloc_constant: { 6710 Module *M = const_cast<Module *>(MF.getFunction().getParent()); 6711 const MDNode *Metadata = cast<MDNodeSDNode>(Op.getOperand(1))->getMD(); 6712 auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString(); 6713 auto RelocSymbol = cast<GlobalVariable>( 6714 M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext()))); 6715 SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0, 6716 SIInstrInfo::MO_ABS32_LO); 6717 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; 6718 } 6719 default: 6720 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr = 6721 AMDGPU::getImageDimIntrinsicInfo(IntrinsicID)) 6722 return lowerImage(Op, ImageDimIntr, DAG, false); 6723 6724 return Op; 6725 } 6726 } 6727 6728 // This function computes an appropriate offset to pass to 6729 // MachineMemOperand::setOffset() based on the offset inputs to 6730 // an intrinsic. If any of the offsets are non-contstant or 6731 // if VIndex is non-zero then this function returns 0. Otherwise, 6732 // it returns the sum of VOffset, SOffset, and Offset. 6733 static unsigned getBufferOffsetForMMO(SDValue VOffset, 6734 SDValue SOffset, 6735 SDValue Offset, 6736 SDValue VIndex = SDValue()) { 6737 6738 if (!isa<ConstantSDNode>(VOffset) || !isa<ConstantSDNode>(SOffset) || 6739 !isa<ConstantSDNode>(Offset)) 6740 return 0; 6741 6742 if (VIndex) { 6743 if (!isa<ConstantSDNode>(VIndex) || !cast<ConstantSDNode>(VIndex)->isNullValue()) 6744 return 0; 6745 } 6746 6747 return cast<ConstantSDNode>(VOffset)->getSExtValue() + 6748 cast<ConstantSDNode>(SOffset)->getSExtValue() + 6749 cast<ConstantSDNode>(Offset)->getSExtValue(); 6750 } 6751 6752 SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op, 6753 SelectionDAG &DAG, 6754 unsigned NewOpcode) const { 6755 SDLoc DL(Op); 6756 6757 SDValue VData = Op.getOperand(2); 6758 auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); 6759 SDValue Ops[] = { 6760 Op.getOperand(0), // Chain 6761 VData, // vdata 6762 Op.getOperand(3), // rsrc 6763 DAG.getConstant(0, DL, MVT::i32), // vindex 6764 Offsets.first, // voffset 6765 Op.getOperand(5), // soffset 6766 Offsets.second, // offset 6767 Op.getOperand(6), // cachepolicy 6768 DAG.getTargetConstant(0, DL, MVT::i1), // idxen 6769 }; 6770 6771 auto *M = cast<MemSDNode>(Op); 6772 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6])); 6773 6774 EVT MemVT = VData.getValueType(); 6775 return DAG.getMemIntrinsicNode(NewOpcode, DL, Op->getVTList(), Ops, MemVT, 6776 M->getMemOperand()); 6777 } 6778 6779 SDValue 6780 SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG, 6781 unsigned NewOpcode) const { 6782 SDLoc DL(Op); 6783 6784 SDValue VData = Op.getOperand(2); 6785 auto Offsets = splitBufferOffsets(Op.getOperand(5), DAG); 6786 SDValue Ops[] = { 6787 Op.getOperand(0), // Chain 6788 VData, // vdata 6789 Op.getOperand(3), // rsrc 6790 Op.getOperand(4), // vindex 6791 Offsets.first, // voffset 6792 Op.getOperand(6), // soffset 6793 Offsets.second, // offset 6794 Op.getOperand(7), // cachepolicy 6795 DAG.getTargetConstant(1, DL, MVT::i1), // idxen 6796 }; 6797 6798 auto *M = cast<MemSDNode>(Op); 6799 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6], 6800 Ops[3])); 6801 6802 EVT MemVT = VData.getValueType(); 6803 return DAG.getMemIntrinsicNode(NewOpcode, DL, Op->getVTList(), Ops, MemVT, 6804 M->getMemOperand()); 6805 } 6806 6807 SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, 6808 SelectionDAG &DAG) const { 6809 unsigned IntrID = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6810 SDLoc DL(Op); 6811 6812 switch (IntrID) { 6813 case Intrinsic::amdgcn_ds_ordered_add: 6814 case Intrinsic::amdgcn_ds_ordered_swap: { 6815 MemSDNode *M = cast<MemSDNode>(Op); 6816 SDValue Chain = M->getOperand(0); 6817 SDValue M0 = M->getOperand(2); 6818 SDValue Value = M->getOperand(3); 6819 unsigned IndexOperand = M->getConstantOperandVal(7); 6820 unsigned WaveRelease = M->getConstantOperandVal(8); 6821 unsigned WaveDone = M->getConstantOperandVal(9); 6822 6823 unsigned OrderedCountIndex = IndexOperand & 0x3f; 6824 IndexOperand &= ~0x3f; 6825 unsigned CountDw = 0; 6826 6827 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) { 6828 CountDw = (IndexOperand >> 24) & 0xf; 6829 IndexOperand &= ~(0xf << 24); 6830 6831 if (CountDw < 1 || CountDw > 4) { 6832 report_fatal_error( 6833 "ds_ordered_count: dword count must be between 1 and 4"); 6834 } 6835 } 6836 6837 if (IndexOperand) 6838 report_fatal_error("ds_ordered_count: bad index operand"); 6839 6840 if (WaveDone && !WaveRelease) 6841 report_fatal_error("ds_ordered_count: wave_done requires wave_release"); 6842 6843 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1; 6844 unsigned ShaderType = 6845 SIInstrInfo::getDSShaderTypeValue(DAG.getMachineFunction()); 6846 unsigned Offset0 = OrderedCountIndex << 2; 6847 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (ShaderType << 2) | 6848 (Instruction << 4); 6849 6850 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) 6851 Offset1 |= (CountDw - 1) << 6; 6852 6853 unsigned Offset = Offset0 | (Offset1 << 8); 6854 6855 SDValue Ops[] = { 6856 Chain, 6857 Value, 6858 DAG.getTargetConstant(Offset, DL, MVT::i16), 6859 copyToM0(DAG, Chain, DL, M0).getValue(1), // Glue 6860 }; 6861 return DAG.getMemIntrinsicNode(AMDGPUISD::DS_ORDERED_COUNT, DL, 6862 M->getVTList(), Ops, M->getMemoryVT(), 6863 M->getMemOperand()); 6864 } 6865 case Intrinsic::amdgcn_ds_fadd: { 6866 MemSDNode *M = cast<MemSDNode>(Op); 6867 unsigned Opc; 6868 switch (IntrID) { 6869 case Intrinsic::amdgcn_ds_fadd: 6870 Opc = ISD::ATOMIC_LOAD_FADD; 6871 break; 6872 } 6873 6874 return DAG.getAtomic(Opc, SDLoc(Op), M->getMemoryVT(), 6875 M->getOperand(0), M->getOperand(2), M->getOperand(3), 6876 M->getMemOperand()); 6877 } 6878 case Intrinsic::amdgcn_atomic_inc: 6879 case Intrinsic::amdgcn_atomic_dec: 6880 case Intrinsic::amdgcn_ds_fmin: 6881 case Intrinsic::amdgcn_ds_fmax: { 6882 MemSDNode *M = cast<MemSDNode>(Op); 6883 unsigned Opc; 6884 switch (IntrID) { 6885 case Intrinsic::amdgcn_atomic_inc: 6886 Opc = AMDGPUISD::ATOMIC_INC; 6887 break; 6888 case Intrinsic::amdgcn_atomic_dec: 6889 Opc = AMDGPUISD::ATOMIC_DEC; 6890 break; 6891 case Intrinsic::amdgcn_ds_fmin: 6892 Opc = AMDGPUISD::ATOMIC_LOAD_FMIN; 6893 break; 6894 case Intrinsic::amdgcn_ds_fmax: 6895 Opc = AMDGPUISD::ATOMIC_LOAD_FMAX; 6896 break; 6897 default: 6898 llvm_unreachable("Unknown intrinsic!"); 6899 } 6900 SDValue Ops[] = { 6901 M->getOperand(0), // Chain 6902 M->getOperand(2), // Ptr 6903 M->getOperand(3) // Value 6904 }; 6905 6906 return DAG.getMemIntrinsicNode(Opc, SDLoc(Op), M->getVTList(), Ops, 6907 M->getMemoryVT(), M->getMemOperand()); 6908 } 6909 case Intrinsic::amdgcn_buffer_load: 6910 case Intrinsic::amdgcn_buffer_load_format: { 6911 unsigned Glc = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue(); 6912 unsigned Slc = cast<ConstantSDNode>(Op.getOperand(6))->getZExtValue(); 6913 unsigned IdxEn = 1; 6914 if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 6915 IdxEn = Idx->getZExtValue() != 0; 6916 SDValue Ops[] = { 6917 Op.getOperand(0), // Chain 6918 Op.getOperand(2), // rsrc 6919 Op.getOperand(3), // vindex 6920 SDValue(), // voffset -- will be set by setBufferOffsets 6921 SDValue(), // soffset -- will be set by setBufferOffsets 6922 SDValue(), // offset -- will be set by setBufferOffsets 6923 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy 6924 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen 6925 }; 6926 6927 unsigned Offset = setBufferOffsets(Op.getOperand(4), DAG, &Ops[3]); 6928 // We don't know the offset if vindex is non-zero, so clear it. 6929 if (IdxEn) 6930 Offset = 0; 6931 6932 unsigned Opc = (IntrID == Intrinsic::amdgcn_buffer_load) ? 6933 AMDGPUISD::BUFFER_LOAD : AMDGPUISD::BUFFER_LOAD_FORMAT; 6934 6935 EVT VT = Op.getValueType(); 6936 EVT IntVT = VT.changeTypeToInteger(); 6937 auto *M = cast<MemSDNode>(Op); 6938 M->getMemOperand()->setOffset(Offset); 6939 EVT LoadVT = Op.getValueType(); 6940 6941 if (LoadVT.getScalarType() == MVT::f16) 6942 return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, 6943 M, DAG, Ops); 6944 6945 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics 6946 if (LoadVT.getScalarType() == MVT::i8 || 6947 LoadVT.getScalarType() == MVT::i16) 6948 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M); 6949 6950 return getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, IntVT, 6951 M->getMemOperand(), DAG); 6952 } 6953 case Intrinsic::amdgcn_raw_buffer_load: 6954 case Intrinsic::amdgcn_raw_buffer_load_format: { 6955 const bool IsFormat = IntrID == Intrinsic::amdgcn_raw_buffer_load_format; 6956 6957 auto Offsets = splitBufferOffsets(Op.getOperand(3), DAG); 6958 SDValue Ops[] = { 6959 Op.getOperand(0), // Chain 6960 Op.getOperand(2), // rsrc 6961 DAG.getConstant(0, DL, MVT::i32), // vindex 6962 Offsets.first, // voffset 6963 Op.getOperand(4), // soffset 6964 Offsets.second, // offset 6965 Op.getOperand(5), // cachepolicy, swizzled buffer 6966 DAG.getTargetConstant(0, DL, MVT::i1), // idxen 6967 }; 6968 6969 auto *M = cast<MemSDNode>(Op); 6970 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[3], Ops[4], Ops[5])); 6971 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops); 6972 } 6973 case Intrinsic::amdgcn_struct_buffer_load: 6974 case Intrinsic::amdgcn_struct_buffer_load_format: { 6975 const bool IsFormat = IntrID == Intrinsic::amdgcn_struct_buffer_load_format; 6976 6977 auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); 6978 SDValue Ops[] = { 6979 Op.getOperand(0), // Chain 6980 Op.getOperand(2), // rsrc 6981 Op.getOperand(3), // vindex 6982 Offsets.first, // voffset 6983 Op.getOperand(5), // soffset 6984 Offsets.second, // offset 6985 Op.getOperand(6), // cachepolicy, swizzled buffer 6986 DAG.getTargetConstant(1, DL, MVT::i1), // idxen 6987 }; 6988 6989 auto *M = cast<MemSDNode>(Op); 6990 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[3], Ops[4], Ops[5], 6991 Ops[2])); 6992 return lowerIntrinsicLoad(cast<MemSDNode>(Op), IsFormat, DAG, Ops); 6993 } 6994 case Intrinsic::amdgcn_tbuffer_load: { 6995 MemSDNode *M = cast<MemSDNode>(Op); 6996 EVT LoadVT = Op.getValueType(); 6997 6998 unsigned Dfmt = cast<ConstantSDNode>(Op.getOperand(7))->getZExtValue(); 6999 unsigned Nfmt = cast<ConstantSDNode>(Op.getOperand(8))->getZExtValue(); 7000 unsigned Glc = cast<ConstantSDNode>(Op.getOperand(9))->getZExtValue(); 7001 unsigned Slc = cast<ConstantSDNode>(Op.getOperand(10))->getZExtValue(); 7002 unsigned IdxEn = 1; 7003 if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 7004 IdxEn = Idx->getZExtValue() != 0; 7005 SDValue Ops[] = { 7006 Op.getOperand(0), // Chain 7007 Op.getOperand(2), // rsrc 7008 Op.getOperand(3), // vindex 7009 Op.getOperand(4), // voffset 7010 Op.getOperand(5), // soffset 7011 Op.getOperand(6), // offset 7012 DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format 7013 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy 7014 DAG.getTargetConstant(IdxEn, DL, MVT::i1) // idxen 7015 }; 7016 7017 if (LoadVT.getScalarType() == MVT::f16) 7018 return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, 7019 M, DAG, Ops); 7020 return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL, 7021 Op->getVTList(), Ops, LoadVT, M->getMemOperand(), 7022 DAG); 7023 } 7024 case Intrinsic::amdgcn_raw_tbuffer_load: { 7025 MemSDNode *M = cast<MemSDNode>(Op); 7026 EVT LoadVT = Op.getValueType(); 7027 auto Offsets = splitBufferOffsets(Op.getOperand(3), DAG); 7028 7029 SDValue Ops[] = { 7030 Op.getOperand(0), // Chain 7031 Op.getOperand(2), // rsrc 7032 DAG.getConstant(0, DL, MVT::i32), // vindex 7033 Offsets.first, // voffset 7034 Op.getOperand(4), // soffset 7035 Offsets.second, // offset 7036 Op.getOperand(5), // format 7037 Op.getOperand(6), // cachepolicy, swizzled buffer 7038 DAG.getTargetConstant(0, DL, MVT::i1), // idxen 7039 }; 7040 7041 if (LoadVT.getScalarType() == MVT::f16) 7042 return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, 7043 M, DAG, Ops); 7044 return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL, 7045 Op->getVTList(), Ops, LoadVT, M->getMemOperand(), 7046 DAG); 7047 } 7048 case Intrinsic::amdgcn_struct_tbuffer_load: { 7049 MemSDNode *M = cast<MemSDNode>(Op); 7050 EVT LoadVT = Op.getValueType(); 7051 auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); 7052 7053 SDValue Ops[] = { 7054 Op.getOperand(0), // Chain 7055 Op.getOperand(2), // rsrc 7056 Op.getOperand(3), // vindex 7057 Offsets.first, // voffset 7058 Op.getOperand(5), // soffset 7059 Offsets.second, // offset 7060 Op.getOperand(6), // format 7061 Op.getOperand(7), // cachepolicy, swizzled buffer 7062 DAG.getTargetConstant(1, DL, MVT::i1), // idxen 7063 }; 7064 7065 if (LoadVT.getScalarType() == MVT::f16) 7066 return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, 7067 M, DAG, Ops); 7068 return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL, 7069 Op->getVTList(), Ops, LoadVT, M->getMemOperand(), 7070 DAG); 7071 } 7072 case Intrinsic::amdgcn_buffer_atomic_swap: 7073 case Intrinsic::amdgcn_buffer_atomic_add: 7074 case Intrinsic::amdgcn_buffer_atomic_sub: 7075 case Intrinsic::amdgcn_buffer_atomic_csub: 7076 case Intrinsic::amdgcn_buffer_atomic_smin: 7077 case Intrinsic::amdgcn_buffer_atomic_umin: 7078 case Intrinsic::amdgcn_buffer_atomic_smax: 7079 case Intrinsic::amdgcn_buffer_atomic_umax: 7080 case Intrinsic::amdgcn_buffer_atomic_and: 7081 case Intrinsic::amdgcn_buffer_atomic_or: 7082 case Intrinsic::amdgcn_buffer_atomic_xor: 7083 case Intrinsic::amdgcn_buffer_atomic_fadd: { 7084 unsigned Slc = cast<ConstantSDNode>(Op.getOperand(6))->getZExtValue(); 7085 unsigned IdxEn = 1; 7086 if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(4))) 7087 IdxEn = Idx->getZExtValue() != 0; 7088 SDValue Ops[] = { 7089 Op.getOperand(0), // Chain 7090 Op.getOperand(2), // vdata 7091 Op.getOperand(3), // rsrc 7092 Op.getOperand(4), // vindex 7093 SDValue(), // voffset -- will be set by setBufferOffsets 7094 SDValue(), // soffset -- will be set by setBufferOffsets 7095 SDValue(), // offset -- will be set by setBufferOffsets 7096 DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy 7097 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen 7098 }; 7099 unsigned Offset = setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]); 7100 // We don't know the offset if vindex is non-zero, so clear it. 7101 if (IdxEn) 7102 Offset = 0; 7103 EVT VT = Op.getValueType(); 7104 7105 auto *M = cast<MemSDNode>(Op); 7106 M->getMemOperand()->setOffset(Offset); 7107 unsigned Opcode = 0; 7108 7109 switch (IntrID) { 7110 case Intrinsic::amdgcn_buffer_atomic_swap: 7111 Opcode = AMDGPUISD::BUFFER_ATOMIC_SWAP; 7112 break; 7113 case Intrinsic::amdgcn_buffer_atomic_add: 7114 Opcode = AMDGPUISD::BUFFER_ATOMIC_ADD; 7115 break; 7116 case Intrinsic::amdgcn_buffer_atomic_sub: 7117 Opcode = AMDGPUISD::BUFFER_ATOMIC_SUB; 7118 break; 7119 case Intrinsic::amdgcn_buffer_atomic_csub: 7120 Opcode = AMDGPUISD::BUFFER_ATOMIC_CSUB; 7121 break; 7122 case Intrinsic::amdgcn_buffer_atomic_smin: 7123 Opcode = AMDGPUISD::BUFFER_ATOMIC_SMIN; 7124 break; 7125 case Intrinsic::amdgcn_buffer_atomic_umin: 7126 Opcode = AMDGPUISD::BUFFER_ATOMIC_UMIN; 7127 break; 7128 case Intrinsic::amdgcn_buffer_atomic_smax: 7129 Opcode = AMDGPUISD::BUFFER_ATOMIC_SMAX; 7130 break; 7131 case Intrinsic::amdgcn_buffer_atomic_umax: 7132 Opcode = AMDGPUISD::BUFFER_ATOMIC_UMAX; 7133 break; 7134 case Intrinsic::amdgcn_buffer_atomic_and: 7135 Opcode = AMDGPUISD::BUFFER_ATOMIC_AND; 7136 break; 7137 case Intrinsic::amdgcn_buffer_atomic_or: 7138 Opcode = AMDGPUISD::BUFFER_ATOMIC_OR; 7139 break; 7140 case Intrinsic::amdgcn_buffer_atomic_xor: 7141 Opcode = AMDGPUISD::BUFFER_ATOMIC_XOR; 7142 break; 7143 case Intrinsic::amdgcn_buffer_atomic_fadd: 7144 if (!Op.getValue(0).use_empty() && !Subtarget->hasGFX90AInsts()) { 7145 DiagnosticInfoUnsupported 7146 NoFpRet(DAG.getMachineFunction().getFunction(), 7147 "return versions of fp atomics not supported", 7148 DL.getDebugLoc(), DS_Error); 7149 DAG.getContext()->diagnose(NoFpRet); 7150 return SDValue(); 7151 } 7152 Opcode = AMDGPUISD::BUFFER_ATOMIC_FADD; 7153 break; 7154 default: 7155 llvm_unreachable("unhandled atomic opcode"); 7156 } 7157 7158 return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT, 7159 M->getMemOperand()); 7160 } 7161 case Intrinsic::amdgcn_raw_buffer_atomic_fadd: 7162 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FADD); 7163 case Intrinsic::amdgcn_struct_buffer_atomic_fadd: 7164 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FADD); 7165 case Intrinsic::amdgcn_raw_buffer_atomic_fmin: 7166 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMIN); 7167 case Intrinsic::amdgcn_struct_buffer_atomic_fmin: 7168 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMIN); 7169 case Intrinsic::amdgcn_raw_buffer_atomic_fmax: 7170 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMAX); 7171 case Intrinsic::amdgcn_struct_buffer_atomic_fmax: 7172 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMAX); 7173 case Intrinsic::amdgcn_raw_buffer_atomic_swap: 7174 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SWAP); 7175 case Intrinsic::amdgcn_raw_buffer_atomic_add: 7176 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_ADD); 7177 case Intrinsic::amdgcn_raw_buffer_atomic_sub: 7178 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SUB); 7179 case Intrinsic::amdgcn_raw_buffer_atomic_smin: 7180 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMIN); 7181 case Intrinsic::amdgcn_raw_buffer_atomic_umin: 7182 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMIN); 7183 case Intrinsic::amdgcn_raw_buffer_atomic_smax: 7184 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMAX); 7185 case Intrinsic::amdgcn_raw_buffer_atomic_umax: 7186 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMAX); 7187 case Intrinsic::amdgcn_raw_buffer_atomic_and: 7188 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_AND); 7189 case Intrinsic::amdgcn_raw_buffer_atomic_or: 7190 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_OR); 7191 case Intrinsic::amdgcn_raw_buffer_atomic_xor: 7192 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_XOR); 7193 case Intrinsic::amdgcn_raw_buffer_atomic_inc: 7194 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_INC); 7195 case Intrinsic::amdgcn_raw_buffer_atomic_dec: 7196 return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_DEC); 7197 case Intrinsic::amdgcn_struct_buffer_atomic_swap: 7198 return lowerStructBufferAtomicIntrin(Op, DAG, 7199 AMDGPUISD::BUFFER_ATOMIC_SWAP); 7200 case Intrinsic::amdgcn_struct_buffer_atomic_add: 7201 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_ADD); 7202 case Intrinsic::amdgcn_struct_buffer_atomic_sub: 7203 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SUB); 7204 case Intrinsic::amdgcn_struct_buffer_atomic_smin: 7205 return lowerStructBufferAtomicIntrin(Op, DAG, 7206 AMDGPUISD::BUFFER_ATOMIC_SMIN); 7207 case Intrinsic::amdgcn_struct_buffer_atomic_umin: 7208 return lowerStructBufferAtomicIntrin(Op, DAG, 7209 AMDGPUISD::BUFFER_ATOMIC_UMIN); 7210 case Intrinsic::amdgcn_struct_buffer_atomic_smax: 7211 return lowerStructBufferAtomicIntrin(Op, DAG, 7212 AMDGPUISD::BUFFER_ATOMIC_SMAX); 7213 case Intrinsic::amdgcn_struct_buffer_atomic_umax: 7214 return lowerStructBufferAtomicIntrin(Op, DAG, 7215 AMDGPUISD::BUFFER_ATOMIC_UMAX); 7216 case Intrinsic::amdgcn_struct_buffer_atomic_and: 7217 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_AND); 7218 case Intrinsic::amdgcn_struct_buffer_atomic_or: 7219 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_OR); 7220 case Intrinsic::amdgcn_struct_buffer_atomic_xor: 7221 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_XOR); 7222 case Intrinsic::amdgcn_struct_buffer_atomic_inc: 7223 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_INC); 7224 case Intrinsic::amdgcn_struct_buffer_atomic_dec: 7225 return lowerStructBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_DEC); 7226 7227 case Intrinsic::amdgcn_buffer_atomic_cmpswap: { 7228 unsigned Slc = cast<ConstantSDNode>(Op.getOperand(7))->getZExtValue(); 7229 unsigned IdxEn = 1; 7230 if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(5))) 7231 IdxEn = Idx->getZExtValue() != 0; 7232 SDValue Ops[] = { 7233 Op.getOperand(0), // Chain 7234 Op.getOperand(2), // src 7235 Op.getOperand(3), // cmp 7236 Op.getOperand(4), // rsrc 7237 Op.getOperand(5), // vindex 7238 SDValue(), // voffset -- will be set by setBufferOffsets 7239 SDValue(), // soffset -- will be set by setBufferOffsets 7240 SDValue(), // offset -- will be set by setBufferOffsets 7241 DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy 7242 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen 7243 }; 7244 unsigned Offset = setBufferOffsets(Op.getOperand(6), DAG, &Ops[5]); 7245 // We don't know the offset if vindex is non-zero, so clear it. 7246 if (IdxEn) 7247 Offset = 0; 7248 EVT VT = Op.getValueType(); 7249 auto *M = cast<MemSDNode>(Op); 7250 M->getMemOperand()->setOffset(Offset); 7251 7252 return DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, DL, 7253 Op->getVTList(), Ops, VT, M->getMemOperand()); 7254 } 7255 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap: { 7256 auto Offsets = splitBufferOffsets(Op.getOperand(5), DAG); 7257 SDValue Ops[] = { 7258 Op.getOperand(0), // Chain 7259 Op.getOperand(2), // src 7260 Op.getOperand(3), // cmp 7261 Op.getOperand(4), // rsrc 7262 DAG.getConstant(0, DL, MVT::i32), // vindex 7263 Offsets.first, // voffset 7264 Op.getOperand(6), // soffset 7265 Offsets.second, // offset 7266 Op.getOperand(7), // cachepolicy 7267 DAG.getTargetConstant(0, DL, MVT::i1), // idxen 7268 }; 7269 EVT VT = Op.getValueType(); 7270 auto *M = cast<MemSDNode>(Op); 7271 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[5], Ops[6], Ops[7])); 7272 7273 return DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, DL, 7274 Op->getVTList(), Ops, VT, M->getMemOperand()); 7275 } 7276 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap: { 7277 auto Offsets = splitBufferOffsets(Op.getOperand(6), DAG); 7278 SDValue Ops[] = { 7279 Op.getOperand(0), // Chain 7280 Op.getOperand(2), // src 7281 Op.getOperand(3), // cmp 7282 Op.getOperand(4), // rsrc 7283 Op.getOperand(5), // vindex 7284 Offsets.first, // voffset 7285 Op.getOperand(7), // soffset 7286 Offsets.second, // offset 7287 Op.getOperand(8), // cachepolicy 7288 DAG.getTargetConstant(1, DL, MVT::i1), // idxen 7289 }; 7290 EVT VT = Op.getValueType(); 7291 auto *M = cast<MemSDNode>(Op); 7292 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[5], Ops[6], Ops[7], 7293 Ops[4])); 7294 7295 return DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, DL, 7296 Op->getVTList(), Ops, VT, M->getMemOperand()); 7297 } 7298 case Intrinsic::amdgcn_image_bvh_intersect_ray: { 7299 SDLoc DL(Op); 7300 MemSDNode *M = cast<MemSDNode>(Op); 7301 SDValue NodePtr = M->getOperand(2); 7302 SDValue RayExtent = M->getOperand(3); 7303 SDValue RayOrigin = M->getOperand(4); 7304 SDValue RayDir = M->getOperand(5); 7305 SDValue RayInvDir = M->getOperand(6); 7306 SDValue TDescr = M->getOperand(7); 7307 7308 assert(NodePtr.getValueType() == MVT::i32 || 7309 NodePtr.getValueType() == MVT::i64); 7310 assert(RayDir.getValueType() == MVT::v4f16 || 7311 RayDir.getValueType() == MVT::v4f32); 7312 7313 bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16; 7314 bool Is64 = NodePtr.getValueType() == MVT::i64; 7315 unsigned Opcode = IsA16 ? Is64 ? AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16_nsa 7316 : AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16_nsa 7317 : Is64 ? AMDGPU::IMAGE_BVH64_INTERSECT_RAY_nsa 7318 : AMDGPU::IMAGE_BVH_INTERSECT_RAY_nsa; 7319 7320 SmallVector<SDValue, 16> Ops; 7321 7322 auto packLanes = [&DAG, &Ops, &DL] (SDValue Op, bool IsAligned) { 7323 SmallVector<SDValue, 3> Lanes; 7324 DAG.ExtractVectorElements(Op, Lanes, 0, 3); 7325 if (Lanes[0].getValueSizeInBits() == 32) { 7326 for (unsigned I = 0; I < 3; ++I) 7327 Ops.push_back(DAG.getBitcast(MVT::i32, Lanes[I])); 7328 } else { 7329 if (IsAligned) { 7330 Ops.push_back( 7331 DAG.getBitcast(MVT::i32, 7332 DAG.getBuildVector(MVT::v2f16, DL, 7333 { Lanes[0], Lanes[1] }))); 7334 Ops.push_back(Lanes[2]); 7335 } else { 7336 SDValue Elt0 = Ops.pop_back_val(); 7337 Ops.push_back( 7338 DAG.getBitcast(MVT::i32, 7339 DAG.getBuildVector(MVT::v2f16, DL, 7340 { Elt0, Lanes[0] }))); 7341 Ops.push_back( 7342 DAG.getBitcast(MVT::i32, 7343 DAG.getBuildVector(MVT::v2f16, DL, 7344 { Lanes[1], Lanes[2] }))); 7345 } 7346 } 7347 }; 7348 7349 if (Is64) 7350 DAG.ExtractVectorElements(DAG.getBitcast(MVT::v2i32, NodePtr), Ops, 0, 2); 7351 else 7352 Ops.push_back(NodePtr); 7353 7354 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent)); 7355 packLanes(RayOrigin, true); 7356 packLanes(RayDir, true); 7357 packLanes(RayInvDir, false); 7358 Ops.push_back(TDescr); 7359 if (IsA16) 7360 Ops.push_back(DAG.getTargetConstant(1, DL, MVT::i1)); 7361 Ops.push_back(M->getChain()); 7362 7363 auto *NewNode = DAG.getMachineNode(Opcode, DL, M->getVTList(), Ops); 7364 MachineMemOperand *MemRef = M->getMemOperand(); 7365 DAG.setNodeMemRefs(NewNode, {MemRef}); 7366 return SDValue(NewNode, 0); 7367 } 7368 case Intrinsic::amdgcn_global_atomic_fadd: 7369 if (!Op.getValue(0).use_empty() && !Subtarget->hasGFX90AInsts()) { 7370 DiagnosticInfoUnsupported 7371 NoFpRet(DAG.getMachineFunction().getFunction(), 7372 "return versions of fp atomics not supported", 7373 DL.getDebugLoc(), DS_Error); 7374 DAG.getContext()->diagnose(NoFpRet); 7375 return SDValue(); 7376 } 7377 LLVM_FALLTHROUGH; 7378 case Intrinsic::amdgcn_global_atomic_fmin: 7379 case Intrinsic::amdgcn_global_atomic_fmax: 7380 case Intrinsic::amdgcn_flat_atomic_fadd: 7381 case Intrinsic::amdgcn_flat_atomic_fmin: 7382 case Intrinsic::amdgcn_flat_atomic_fmax: { 7383 MemSDNode *M = cast<MemSDNode>(Op); 7384 SDValue Ops[] = { 7385 M->getOperand(0), // Chain 7386 M->getOperand(2), // Ptr 7387 M->getOperand(3) // Value 7388 }; 7389 unsigned Opcode = 0; 7390 switch (IntrID) { 7391 case Intrinsic::amdgcn_global_atomic_fadd: 7392 case Intrinsic::amdgcn_flat_atomic_fadd: { 7393 EVT VT = Op.getOperand(3).getValueType(); 7394 return DAG.getAtomic(ISD::ATOMIC_LOAD_FADD, DL, VT, 7395 DAG.getVTList(VT, MVT::Other), Ops, 7396 M->getMemOperand()); 7397 } 7398 case Intrinsic::amdgcn_global_atomic_fmin: 7399 case Intrinsic::amdgcn_flat_atomic_fmin: { 7400 Opcode = AMDGPUISD::ATOMIC_LOAD_FMIN; 7401 break; 7402 } 7403 case Intrinsic::amdgcn_global_atomic_fmax: 7404 case Intrinsic::amdgcn_flat_atomic_fmax: { 7405 Opcode = AMDGPUISD::ATOMIC_LOAD_FMAX; 7406 break; 7407 } 7408 default: 7409 llvm_unreachable("unhandled atomic opcode"); 7410 } 7411 return DAG.getMemIntrinsicNode(Opcode, SDLoc(Op), 7412 M->getVTList(), Ops, M->getMemoryVT(), 7413 M->getMemOperand()); 7414 } 7415 default: 7416 7417 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr = 7418 AMDGPU::getImageDimIntrinsicInfo(IntrID)) 7419 return lowerImage(Op, ImageDimIntr, DAG, true); 7420 7421 return SDValue(); 7422 } 7423 } 7424 7425 // Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to 7426 // dwordx4 if on SI. 7427 SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL, 7428 SDVTList VTList, 7429 ArrayRef<SDValue> Ops, EVT MemVT, 7430 MachineMemOperand *MMO, 7431 SelectionDAG &DAG) const { 7432 EVT VT = VTList.VTs[0]; 7433 EVT WidenedVT = VT; 7434 EVT WidenedMemVT = MemVT; 7435 if (!Subtarget->hasDwordx3LoadStores() && 7436 (WidenedVT == MVT::v3i32 || WidenedVT == MVT::v3f32)) { 7437 WidenedVT = EVT::getVectorVT(*DAG.getContext(), 7438 WidenedVT.getVectorElementType(), 4); 7439 WidenedMemVT = EVT::getVectorVT(*DAG.getContext(), 7440 WidenedMemVT.getVectorElementType(), 4); 7441 MMO = DAG.getMachineFunction().getMachineMemOperand(MMO, 0, 16); 7442 } 7443 7444 assert(VTList.NumVTs == 2); 7445 SDVTList WidenedVTList = DAG.getVTList(WidenedVT, VTList.VTs[1]); 7446 7447 auto NewOp = DAG.getMemIntrinsicNode(Opcode, DL, WidenedVTList, Ops, 7448 WidenedMemVT, MMO); 7449 if (WidenedVT != VT) { 7450 auto Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, NewOp, 7451 DAG.getVectorIdxConstant(0, DL)); 7452 NewOp = DAG.getMergeValues({ Extract, SDValue(NewOp.getNode(), 1) }, DL); 7453 } 7454 return NewOp; 7455 } 7456 7457 SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG, 7458 bool ImageStore) const { 7459 EVT StoreVT = VData.getValueType(); 7460 7461 // No change for f16 and legal vector D16 types. 7462 if (!StoreVT.isVector()) 7463 return VData; 7464 7465 SDLoc DL(VData); 7466 unsigned NumElements = StoreVT.getVectorNumElements(); 7467 7468 if (Subtarget->hasUnpackedD16VMem()) { 7469 // We need to unpack the packed data to store. 7470 EVT IntStoreVT = StoreVT.changeTypeToInteger(); 7471 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData); 7472 7473 EVT EquivStoreVT = 7474 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElements); 7475 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, EquivStoreVT, IntVData); 7476 return DAG.UnrollVectorOp(ZExt.getNode()); 7477 } 7478 7479 // The sq block of gfx8.1 does not estimate register use correctly for d16 7480 // image store instructions. The data operand is computed as if it were not a 7481 // d16 image instruction. 7482 if (ImageStore && Subtarget->hasImageStoreD16Bug()) { 7483 // Bitcast to i16 7484 EVT IntStoreVT = StoreVT.changeTypeToInteger(); 7485 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData); 7486 7487 // Decompose into scalars 7488 SmallVector<SDValue, 4> Elts; 7489 DAG.ExtractVectorElements(IntVData, Elts); 7490 7491 // Group pairs of i16 into v2i16 and bitcast to i32 7492 SmallVector<SDValue, 4> PackedElts; 7493 for (unsigned I = 0; I < Elts.size() / 2; I += 1) { 7494 SDValue Pair = 7495 DAG.getBuildVector(MVT::v2i16, DL, {Elts[I * 2], Elts[I * 2 + 1]}); 7496 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); 7497 PackedElts.push_back(IntPair); 7498 } 7499 if ((NumElements % 2) == 1) { 7500 // Handle v3i16 7501 unsigned I = Elts.size() / 2; 7502 SDValue Pair = DAG.getBuildVector(MVT::v2i16, DL, 7503 {Elts[I * 2], DAG.getUNDEF(MVT::i16)}); 7504 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); 7505 PackedElts.push_back(IntPair); 7506 } 7507 7508 // Pad using UNDEF 7509 PackedElts.resize(Elts.size(), DAG.getUNDEF(MVT::i32)); 7510 7511 // Build final vector 7512 EVT VecVT = 7513 EVT::getVectorVT(*DAG.getContext(), MVT::i32, PackedElts.size()); 7514 return DAG.getBuildVector(VecVT, DL, PackedElts); 7515 } 7516 7517 if (NumElements == 3) { 7518 EVT IntStoreVT = 7519 EVT::getIntegerVT(*DAG.getContext(), StoreVT.getStoreSizeInBits()); 7520 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData); 7521 7522 EVT WidenedStoreVT = EVT::getVectorVT( 7523 *DAG.getContext(), StoreVT.getVectorElementType(), NumElements + 1); 7524 EVT WidenedIntVT = EVT::getIntegerVT(*DAG.getContext(), 7525 WidenedStoreVT.getStoreSizeInBits()); 7526 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenedIntVT, IntVData); 7527 return DAG.getNode(ISD::BITCAST, DL, WidenedStoreVT, ZExt); 7528 } 7529 7530 assert(isTypeLegal(StoreVT)); 7531 return VData; 7532 } 7533 7534 SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, 7535 SelectionDAG &DAG) const { 7536 SDLoc DL(Op); 7537 SDValue Chain = Op.getOperand(0); 7538 unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 7539 MachineFunction &MF = DAG.getMachineFunction(); 7540 7541 switch (IntrinsicID) { 7542 case Intrinsic::amdgcn_exp_compr: { 7543 SDValue Src0 = Op.getOperand(4); 7544 SDValue Src1 = Op.getOperand(5); 7545 // Hack around illegal type on SI by directly selecting it. 7546 if (isTypeLegal(Src0.getValueType())) 7547 return SDValue(); 7548 7549 const ConstantSDNode *Done = cast<ConstantSDNode>(Op.getOperand(6)); 7550 SDValue Undef = DAG.getUNDEF(MVT::f32); 7551 const SDValue Ops[] = { 7552 Op.getOperand(2), // tgt 7553 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), // src0 7554 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src1), // src1 7555 Undef, // src2 7556 Undef, // src3 7557 Op.getOperand(7), // vm 7558 DAG.getTargetConstant(1, DL, MVT::i1), // compr 7559 Op.getOperand(3), // en 7560 Op.getOperand(0) // Chain 7561 }; 7562 7563 unsigned Opc = Done->isNullValue() ? AMDGPU::EXP : AMDGPU::EXP_DONE; 7564 return SDValue(DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops), 0); 7565 } 7566 case Intrinsic::amdgcn_s_barrier: { 7567 if (getTargetMachine().getOptLevel() > CodeGenOpt::None) { 7568 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 7569 unsigned WGSize = ST.getFlatWorkGroupSizes(MF.getFunction()).second; 7570 if (WGSize <= ST.getWavefrontSize()) 7571 return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other, 7572 Op.getOperand(0)), 0); 7573 } 7574 return SDValue(); 7575 }; 7576 case Intrinsic::amdgcn_tbuffer_store: { 7577 SDValue VData = Op.getOperand(2); 7578 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); 7579 if (IsD16) 7580 VData = handleD16VData(VData, DAG); 7581 unsigned Dfmt = cast<ConstantSDNode>(Op.getOperand(8))->getZExtValue(); 7582 unsigned Nfmt = cast<ConstantSDNode>(Op.getOperand(9))->getZExtValue(); 7583 unsigned Glc = cast<ConstantSDNode>(Op.getOperand(10))->getZExtValue(); 7584 unsigned Slc = cast<ConstantSDNode>(Op.getOperand(11))->getZExtValue(); 7585 unsigned IdxEn = 1; 7586 if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(4))) 7587 IdxEn = Idx->getZExtValue() != 0; 7588 SDValue Ops[] = { 7589 Chain, 7590 VData, // vdata 7591 Op.getOperand(3), // rsrc 7592 Op.getOperand(4), // vindex 7593 Op.getOperand(5), // voffset 7594 Op.getOperand(6), // soffset 7595 Op.getOperand(7), // offset 7596 DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format 7597 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy 7598 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idexen 7599 }; 7600 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 : 7601 AMDGPUISD::TBUFFER_STORE_FORMAT; 7602 MemSDNode *M = cast<MemSDNode>(Op); 7603 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, 7604 M->getMemoryVT(), M->getMemOperand()); 7605 } 7606 7607 case Intrinsic::amdgcn_struct_tbuffer_store: { 7608 SDValue VData = Op.getOperand(2); 7609 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); 7610 if (IsD16) 7611 VData = handleD16VData(VData, DAG); 7612 auto Offsets = splitBufferOffsets(Op.getOperand(5), DAG); 7613 SDValue Ops[] = { 7614 Chain, 7615 VData, // vdata 7616 Op.getOperand(3), // rsrc 7617 Op.getOperand(4), // vindex 7618 Offsets.first, // voffset 7619 Op.getOperand(6), // soffset 7620 Offsets.second, // offset 7621 Op.getOperand(7), // format 7622 Op.getOperand(8), // cachepolicy, swizzled buffer 7623 DAG.getTargetConstant(1, DL, MVT::i1), // idexen 7624 }; 7625 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 : 7626 AMDGPUISD::TBUFFER_STORE_FORMAT; 7627 MemSDNode *M = cast<MemSDNode>(Op); 7628 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, 7629 M->getMemoryVT(), M->getMemOperand()); 7630 } 7631 7632 case Intrinsic::amdgcn_raw_tbuffer_store: { 7633 SDValue VData = Op.getOperand(2); 7634 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); 7635 if (IsD16) 7636 VData = handleD16VData(VData, DAG); 7637 auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); 7638 SDValue Ops[] = { 7639 Chain, 7640 VData, // vdata 7641 Op.getOperand(3), // rsrc 7642 DAG.getConstant(0, DL, MVT::i32), // vindex 7643 Offsets.first, // voffset 7644 Op.getOperand(5), // soffset 7645 Offsets.second, // offset 7646 Op.getOperand(6), // format 7647 Op.getOperand(7), // cachepolicy, swizzled buffer 7648 DAG.getTargetConstant(0, DL, MVT::i1), // idexen 7649 }; 7650 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 : 7651 AMDGPUISD::TBUFFER_STORE_FORMAT; 7652 MemSDNode *M = cast<MemSDNode>(Op); 7653 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, 7654 M->getMemoryVT(), M->getMemOperand()); 7655 } 7656 7657 case Intrinsic::amdgcn_buffer_store: 7658 case Intrinsic::amdgcn_buffer_store_format: { 7659 SDValue VData = Op.getOperand(2); 7660 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); 7661 if (IsD16) 7662 VData = handleD16VData(VData, DAG); 7663 unsigned Glc = cast<ConstantSDNode>(Op.getOperand(6))->getZExtValue(); 7664 unsigned Slc = cast<ConstantSDNode>(Op.getOperand(7))->getZExtValue(); 7665 unsigned IdxEn = 1; 7666 if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(4))) 7667 IdxEn = Idx->getZExtValue() != 0; 7668 SDValue Ops[] = { 7669 Chain, 7670 VData, 7671 Op.getOperand(3), // rsrc 7672 Op.getOperand(4), // vindex 7673 SDValue(), // voffset -- will be set by setBufferOffsets 7674 SDValue(), // soffset -- will be set by setBufferOffsets 7675 SDValue(), // offset -- will be set by setBufferOffsets 7676 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy 7677 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen 7678 }; 7679 unsigned Offset = setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]); 7680 // We don't know the offset if vindex is non-zero, so clear it. 7681 if (IdxEn) 7682 Offset = 0; 7683 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_buffer_store ? 7684 AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT; 7685 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc; 7686 MemSDNode *M = cast<MemSDNode>(Op); 7687 M->getMemOperand()->setOffset(Offset); 7688 7689 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics 7690 EVT VDataType = VData.getValueType().getScalarType(); 7691 if (VDataType == MVT::i8 || VDataType == MVT::i16) 7692 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M); 7693 7694 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, 7695 M->getMemoryVT(), M->getMemOperand()); 7696 } 7697 7698 case Intrinsic::amdgcn_raw_buffer_store: 7699 case Intrinsic::amdgcn_raw_buffer_store_format: { 7700 const bool IsFormat = 7701 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format; 7702 7703 SDValue VData = Op.getOperand(2); 7704 EVT VDataVT = VData.getValueType(); 7705 EVT EltType = VDataVT.getScalarType(); 7706 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16); 7707 if (IsD16) { 7708 VData = handleD16VData(VData, DAG); 7709 VDataVT = VData.getValueType(); 7710 } 7711 7712 if (!isTypeLegal(VDataVT)) { 7713 VData = 7714 DAG.getNode(ISD::BITCAST, DL, 7715 getEquivalentMemType(*DAG.getContext(), VDataVT), VData); 7716 } 7717 7718 auto Offsets = splitBufferOffsets(Op.getOperand(4), DAG); 7719 SDValue Ops[] = { 7720 Chain, 7721 VData, 7722 Op.getOperand(3), // rsrc 7723 DAG.getConstant(0, DL, MVT::i32), // vindex 7724 Offsets.first, // voffset 7725 Op.getOperand(5), // soffset 7726 Offsets.second, // offset 7727 Op.getOperand(6), // cachepolicy, swizzled buffer 7728 DAG.getTargetConstant(0, DL, MVT::i1), // idxen 7729 }; 7730 unsigned Opc = 7731 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE; 7732 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc; 7733 MemSDNode *M = cast<MemSDNode>(Op); 7734 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6])); 7735 7736 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics 7737 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32) 7738 return handleByteShortBufferStores(DAG, VDataVT, DL, Ops, M); 7739 7740 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, 7741 M->getMemoryVT(), M->getMemOperand()); 7742 } 7743 7744 case Intrinsic::amdgcn_struct_buffer_store: 7745 case Intrinsic::amdgcn_struct_buffer_store_format: { 7746 const bool IsFormat = 7747 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format; 7748 7749 SDValue VData = Op.getOperand(2); 7750 EVT VDataVT = VData.getValueType(); 7751 EVT EltType = VDataVT.getScalarType(); 7752 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16); 7753 7754 if (IsD16) { 7755 VData = handleD16VData(VData, DAG); 7756 VDataVT = VData.getValueType(); 7757 } 7758 7759 if (!isTypeLegal(VDataVT)) { 7760 VData = 7761 DAG.getNode(ISD::BITCAST, DL, 7762 getEquivalentMemType(*DAG.getContext(), VDataVT), VData); 7763 } 7764 7765 auto Offsets = splitBufferOffsets(Op.getOperand(5), DAG); 7766 SDValue Ops[] = { 7767 Chain, 7768 VData, 7769 Op.getOperand(3), // rsrc 7770 Op.getOperand(4), // vindex 7771 Offsets.first, // voffset 7772 Op.getOperand(6), // soffset 7773 Offsets.second, // offset 7774 Op.getOperand(7), // cachepolicy, swizzled buffer 7775 DAG.getTargetConstant(1, DL, MVT::i1), // idxen 7776 }; 7777 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_struct_buffer_store ? 7778 AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT; 7779 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc; 7780 MemSDNode *M = cast<MemSDNode>(Op); 7781 M->getMemOperand()->setOffset(getBufferOffsetForMMO(Ops[4], Ops[5], Ops[6], 7782 Ops[3])); 7783 7784 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics 7785 EVT VDataType = VData.getValueType().getScalarType(); 7786 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32) 7787 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M); 7788 7789 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, 7790 M->getMemoryVT(), M->getMemOperand()); 7791 } 7792 case Intrinsic::amdgcn_end_cf: 7793 return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, 7794 Op->getOperand(2), Chain), 0); 7795 7796 default: { 7797 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr = 7798 AMDGPU::getImageDimIntrinsicInfo(IntrinsicID)) 7799 return lowerImage(Op, ImageDimIntr, DAG, true); 7800 7801 return Op; 7802 } 7803 } 7804 } 7805 7806 // The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args: 7807 // offset (the offset that is included in bounds checking and swizzling, to be 7808 // split between the instruction's voffset and immoffset fields) and soffset 7809 // (the offset that is excluded from bounds checking and swizzling, to go in 7810 // the instruction's soffset field). This function takes the first kind of 7811 // offset and figures out how to split it between voffset and immoffset. 7812 std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets( 7813 SDValue Offset, SelectionDAG &DAG) const { 7814 SDLoc DL(Offset); 7815 const unsigned MaxImm = 4095; 7816 SDValue N0 = Offset; 7817 ConstantSDNode *C1 = nullptr; 7818 7819 if ((C1 = dyn_cast<ConstantSDNode>(N0))) 7820 N0 = SDValue(); 7821 else if (DAG.isBaseWithConstantOffset(N0)) { 7822 C1 = cast<ConstantSDNode>(N0.getOperand(1)); 7823 N0 = N0.getOperand(0); 7824 } 7825 7826 if (C1) { 7827 unsigned ImmOffset = C1->getZExtValue(); 7828 // If the immediate value is too big for the immoffset field, put the value 7829 // and -4096 into the immoffset field so that the value that is copied/added 7830 // for the voffset field is a multiple of 4096, and it stands more chance 7831 // of being CSEd with the copy/add for another similar load/store. 7832 // However, do not do that rounding down to a multiple of 4096 if that is a 7833 // negative number, as it appears to be illegal to have a negative offset 7834 // in the vgpr, even if adding the immediate offset makes it positive. 7835 unsigned Overflow = ImmOffset & ~MaxImm; 7836 ImmOffset -= Overflow; 7837 if ((int32_t)Overflow < 0) { 7838 Overflow += ImmOffset; 7839 ImmOffset = 0; 7840 } 7841 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32)); 7842 if (Overflow) { 7843 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32); 7844 if (!N0) 7845 N0 = OverflowVal; 7846 else { 7847 SDValue Ops[] = { N0, OverflowVal }; 7848 N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops); 7849 } 7850 } 7851 } 7852 if (!N0) 7853 N0 = DAG.getConstant(0, DL, MVT::i32); 7854 if (!C1) 7855 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32)); 7856 return {N0, SDValue(C1, 0)}; 7857 } 7858 7859 // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the 7860 // three offsets (voffset, soffset and instoffset) into the SDValue[3] array 7861 // pointed to by Offsets. 7862 unsigned SITargetLowering::setBufferOffsets(SDValue CombinedOffset, 7863 SelectionDAG &DAG, SDValue *Offsets, 7864 Align Alignment) const { 7865 SDLoc DL(CombinedOffset); 7866 if (auto C = dyn_cast<ConstantSDNode>(CombinedOffset)) { 7867 uint32_t Imm = C->getZExtValue(); 7868 uint32_t SOffset, ImmOffset; 7869 if (AMDGPU::splitMUBUFOffset(Imm, SOffset, ImmOffset, Subtarget, 7870 Alignment)) { 7871 Offsets[0] = DAG.getConstant(0, DL, MVT::i32); 7872 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); 7873 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); 7874 return SOffset + ImmOffset; 7875 } 7876 } 7877 if (DAG.isBaseWithConstantOffset(CombinedOffset)) { 7878 SDValue N0 = CombinedOffset.getOperand(0); 7879 SDValue N1 = CombinedOffset.getOperand(1); 7880 uint32_t SOffset, ImmOffset; 7881 int Offset = cast<ConstantSDNode>(N1)->getSExtValue(); 7882 if (Offset >= 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset, 7883 Subtarget, Alignment)) { 7884 Offsets[0] = N0; 7885 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); 7886 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); 7887 return 0; 7888 } 7889 } 7890 Offsets[0] = CombinedOffset; 7891 Offsets[1] = DAG.getConstant(0, DL, MVT::i32); 7892 Offsets[2] = DAG.getTargetConstant(0, DL, MVT::i32); 7893 return 0; 7894 } 7895 7896 // Handle 8 bit and 16 bit buffer loads 7897 SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG, 7898 EVT LoadVT, SDLoc DL, 7899 ArrayRef<SDValue> Ops, 7900 MemSDNode *M) const { 7901 EVT IntVT = LoadVT.changeTypeToInteger(); 7902 unsigned Opc = (LoadVT.getScalarType() == MVT::i8) ? 7903 AMDGPUISD::BUFFER_LOAD_UBYTE : AMDGPUISD::BUFFER_LOAD_USHORT; 7904 7905 SDVTList ResList = DAG.getVTList(MVT::i32, MVT::Other); 7906 SDValue BufferLoad = DAG.getMemIntrinsicNode(Opc, DL, ResList, 7907 Ops, IntVT, 7908 M->getMemOperand()); 7909 SDValue LoadVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, BufferLoad); 7910 LoadVal = DAG.getNode(ISD::BITCAST, DL, LoadVT, LoadVal); 7911 7912 return DAG.getMergeValues({LoadVal, BufferLoad.getValue(1)}, DL); 7913 } 7914 7915 // Handle 8 bit and 16 bit buffer stores 7916 SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG, 7917 EVT VDataType, SDLoc DL, 7918 SDValue Ops[], 7919 MemSDNode *M) const { 7920 if (VDataType == MVT::f16) 7921 Ops[1] = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Ops[1]); 7922 7923 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]); 7924 Ops[1] = BufferStoreExt; 7925 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE : 7926 AMDGPUISD::BUFFER_STORE_SHORT; 7927 ArrayRef<SDValue> OpsRef = makeArrayRef(&Ops[0], 9); 7928 return DAG.getMemIntrinsicNode(Opc, DL, M->getVTList(), OpsRef, VDataType, 7929 M->getMemOperand()); 7930 } 7931 7932 static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, 7933 ISD::LoadExtType ExtType, SDValue Op, 7934 const SDLoc &SL, EVT VT) { 7935 if (VT.bitsLT(Op.getValueType())) 7936 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op); 7937 7938 switch (ExtType) { 7939 case ISD::SEXTLOAD: 7940 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op); 7941 case ISD::ZEXTLOAD: 7942 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op); 7943 case ISD::EXTLOAD: 7944 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op); 7945 case ISD::NON_EXTLOAD: 7946 return Op; 7947 } 7948 7949 llvm_unreachable("invalid ext type"); 7950 } 7951 7952 SDValue SITargetLowering::widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const { 7953 SelectionDAG &DAG = DCI.DAG; 7954 if (Ld->getAlignment() < 4 || Ld->isDivergent()) 7955 return SDValue(); 7956 7957 // FIXME: Constant loads should all be marked invariant. 7958 unsigned AS = Ld->getAddressSpace(); 7959 if (AS != AMDGPUAS::CONSTANT_ADDRESS && 7960 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT && 7961 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant())) 7962 return SDValue(); 7963 7964 // Don't do this early, since it may interfere with adjacent load merging for 7965 // illegal types. We can avoid losing alignment information for exotic types 7966 // pre-legalize. 7967 EVT MemVT = Ld->getMemoryVT(); 7968 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) || 7969 MemVT.getSizeInBits() >= 32) 7970 return SDValue(); 7971 7972 SDLoc SL(Ld); 7973 7974 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) && 7975 "unexpected vector extload"); 7976 7977 // TODO: Drop only high part of range. 7978 SDValue Ptr = Ld->getBasePtr(); 7979 SDValue NewLoad = DAG.getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, 7980 MVT::i32, SL, Ld->getChain(), Ptr, 7981 Ld->getOffset(), 7982 Ld->getPointerInfo(), MVT::i32, 7983 Ld->getAlignment(), 7984 Ld->getMemOperand()->getFlags(), 7985 Ld->getAAInfo(), 7986 nullptr); // Drop ranges 7987 7988 EVT TruncVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()); 7989 if (MemVT.isFloatingPoint()) { 7990 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD && 7991 "unexpected fp extload"); 7992 TruncVT = MemVT.changeTypeToInteger(); 7993 } 7994 7995 SDValue Cvt = NewLoad; 7996 if (Ld->getExtensionType() == ISD::SEXTLOAD) { 7997 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, 7998 DAG.getValueType(TruncVT)); 7999 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD || 8000 Ld->getExtensionType() == ISD::NON_EXTLOAD) { 8001 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT); 8002 } else { 8003 assert(Ld->getExtensionType() == ISD::EXTLOAD); 8004 } 8005 8006 EVT VT = Ld->getValueType(0); 8007 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 8008 8009 DCI.AddToWorklist(Cvt.getNode()); 8010 8011 // We may need to handle exotic cases, such as i16->i64 extloads, so insert 8012 // the appropriate extension from the 32-bit load. 8013 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT); 8014 DCI.AddToWorklist(Cvt.getNode()); 8015 8016 // Handle conversion back to floating point if necessary. 8017 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt); 8018 8019 return DAG.getMergeValues({ Cvt, NewLoad.getValue(1) }, SL); 8020 } 8021 8022 SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 8023 SDLoc DL(Op); 8024 LoadSDNode *Load = cast<LoadSDNode>(Op); 8025 ISD::LoadExtType ExtType = Load->getExtensionType(); 8026 EVT MemVT = Load->getMemoryVT(); 8027 8028 if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) { 8029 if (MemVT == MVT::i16 && isTypeLegal(MVT::i16)) 8030 return SDValue(); 8031 8032 // FIXME: Copied from PPC 8033 // First, load into 32 bits, then truncate to 1 bit. 8034 8035 SDValue Chain = Load->getChain(); 8036 SDValue BasePtr = Load->getBasePtr(); 8037 MachineMemOperand *MMO = Load->getMemOperand(); 8038 8039 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16; 8040 8041 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, 8042 BasePtr, RealMemVT, MMO); 8043 8044 if (!MemVT.isVector()) { 8045 SDValue Ops[] = { 8046 DAG.getNode(ISD::TRUNCATE, DL, MemVT, NewLD), 8047 NewLD.getValue(1) 8048 }; 8049 8050 return DAG.getMergeValues(Ops, DL); 8051 } 8052 8053 SmallVector<SDValue, 3> Elts; 8054 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) { 8055 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD, 8056 DAG.getConstant(I, DL, MVT::i32)); 8057 8058 Elts.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Elt)); 8059 } 8060 8061 SDValue Ops[] = { 8062 DAG.getBuildVector(MemVT, DL, Elts), 8063 NewLD.getValue(1) 8064 }; 8065 8066 return DAG.getMergeValues(Ops, DL); 8067 } 8068 8069 if (!MemVT.isVector()) 8070 return SDValue(); 8071 8072 assert(Op.getValueType().getVectorElementType() == MVT::i32 && 8073 "Custom lowering for non-i32 vectors hasn't been implemented."); 8074 8075 unsigned Alignment = Load->getAlignment(); 8076 unsigned AS = Load->getAddressSpace(); 8077 if (Subtarget->hasLDSMisalignedBug() && 8078 AS == AMDGPUAS::FLAT_ADDRESS && 8079 Alignment < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) { 8080 return SplitVectorLoad(Op, DAG); 8081 } 8082 8083 MachineFunction &MF = DAG.getMachineFunction(); 8084 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 8085 // If there is a possibilty that flat instruction access scratch memory 8086 // then we need to use the same legalization rules we use for private. 8087 if (AS == AMDGPUAS::FLAT_ADDRESS && 8088 !Subtarget->hasMultiDwordFlatScratchAddressing()) 8089 AS = MFI->hasFlatScratchInit() ? 8090 AMDGPUAS::PRIVATE_ADDRESS : AMDGPUAS::GLOBAL_ADDRESS; 8091 8092 unsigned NumElements = MemVT.getVectorNumElements(); 8093 8094 if (AS == AMDGPUAS::CONSTANT_ADDRESS || 8095 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) { 8096 if (!Op->isDivergent() && Alignment >= 4 && NumElements < 32) { 8097 if (MemVT.isPow2VectorType()) 8098 return SDValue(); 8099 return WidenOrSplitVectorLoad(Op, DAG); 8100 } 8101 // Non-uniform loads will be selected to MUBUF instructions, so they 8102 // have the same legalization requirements as global and private 8103 // loads. 8104 // 8105 } 8106 8107 if (AS == AMDGPUAS::CONSTANT_ADDRESS || 8108 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT || 8109 AS == AMDGPUAS::GLOBAL_ADDRESS) { 8110 if (Subtarget->getScalarizeGlobalBehavior() && !Op->isDivergent() && 8111 Load->isSimple() && isMemOpHasNoClobberedMemOperand(Load) && 8112 Alignment >= 4 && NumElements < 32) { 8113 if (MemVT.isPow2VectorType()) 8114 return SDValue(); 8115 return WidenOrSplitVectorLoad(Op, DAG); 8116 } 8117 // Non-uniform loads will be selected to MUBUF instructions, so they 8118 // have the same legalization requirements as global and private 8119 // loads. 8120 // 8121 } 8122 if (AS == AMDGPUAS::CONSTANT_ADDRESS || 8123 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT || 8124 AS == AMDGPUAS::GLOBAL_ADDRESS || 8125 AS == AMDGPUAS::FLAT_ADDRESS) { 8126 if (NumElements > 4) 8127 return SplitVectorLoad(Op, DAG); 8128 // v3 loads not supported on SI. 8129 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores()) 8130 return WidenOrSplitVectorLoad(Op, DAG); 8131 8132 // v3 and v4 loads are supported for private and global memory. 8133 return SDValue(); 8134 } 8135 if (AS == AMDGPUAS::PRIVATE_ADDRESS) { 8136 // Depending on the setting of the private_element_size field in the 8137 // resource descriptor, we can only make private accesses up to a certain 8138 // size. 8139 switch (Subtarget->getMaxPrivateElementSize()) { 8140 case 4: { 8141 SDValue Ops[2]; 8142 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG); 8143 return DAG.getMergeValues(Ops, DL); 8144 } 8145 case 8: 8146 if (NumElements > 2) 8147 return SplitVectorLoad(Op, DAG); 8148 return SDValue(); 8149 case 16: 8150 // Same as global/flat 8151 if (NumElements > 4) 8152 return SplitVectorLoad(Op, DAG); 8153 // v3 loads not supported on SI. 8154 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores()) 8155 return WidenOrSplitVectorLoad(Op, DAG); 8156 8157 return SDValue(); 8158 default: 8159 llvm_unreachable("unsupported private_element_size"); 8160 } 8161 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) { 8162 // Use ds_read_b128 or ds_read_b96 when possible. 8163 if (Subtarget->hasDS96AndDS128() && 8164 ((Subtarget->useDS128() && MemVT.getStoreSize() == 16) || 8165 MemVT.getStoreSize() == 12) && 8166 allowsMisalignedMemoryAccessesImpl(MemVT.getSizeInBits(), AS, 8167 Load->getAlign())) 8168 return SDValue(); 8169 8170 if (NumElements > 2) 8171 return SplitVectorLoad(Op, DAG); 8172 8173 // SI has a hardware bug in the LDS / GDS boounds checking: if the base 8174 // address is negative, then the instruction is incorrectly treated as 8175 // out-of-bounds even if base + offsets is in bounds. Split vectorized 8176 // loads here to avoid emitting ds_read2_b32. We may re-combine the 8177 // load later in the SILoadStoreOptimizer. 8178 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS && 8179 NumElements == 2 && MemVT.getStoreSize() == 8 && 8180 Load->getAlignment() < 8) { 8181 return SplitVectorLoad(Op, DAG); 8182 } 8183 } 8184 8185 if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(), 8186 MemVT, *Load->getMemOperand())) { 8187 SDValue Ops[2]; 8188 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG); 8189 return DAG.getMergeValues(Ops, DL); 8190 } 8191 8192 return SDValue(); 8193 } 8194 8195 SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { 8196 EVT VT = Op.getValueType(); 8197 assert(VT.getSizeInBits() == 64); 8198 8199 SDLoc DL(Op); 8200 SDValue Cond = Op.getOperand(0); 8201 8202 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); 8203 SDValue One = DAG.getConstant(1, DL, MVT::i32); 8204 8205 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1)); 8206 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2)); 8207 8208 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero); 8209 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero); 8210 8211 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1); 8212 8213 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One); 8214 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One); 8215 8216 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1); 8217 8218 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi}); 8219 return DAG.getNode(ISD::BITCAST, DL, VT, Res); 8220 } 8221 8222 // Catch division cases where we can use shortcuts with rcp and rsq 8223 // instructions. 8224 SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op, 8225 SelectionDAG &DAG) const { 8226 SDLoc SL(Op); 8227 SDValue LHS = Op.getOperand(0); 8228 SDValue RHS = Op.getOperand(1); 8229 EVT VT = Op.getValueType(); 8230 const SDNodeFlags Flags = Op->getFlags(); 8231 8232 bool AllowInaccurateRcp = Flags.hasApproximateFuncs(); 8233 8234 // Without !fpmath accuracy information, we can't do more because we don't 8235 // know exactly whether rcp is accurate enough to meet !fpmath requirement. 8236 if (!AllowInaccurateRcp) 8237 return SDValue(); 8238 8239 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) { 8240 if (CLHS->isExactlyValue(1.0)) { 8241 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to 8242 // the CI documentation has a worst case error of 1 ulp. 8243 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to 8244 // use it as long as we aren't trying to use denormals. 8245 // 8246 // v_rcp_f16 and v_rsq_f16 DO support denormals. 8247 8248 // 1.0 / sqrt(x) -> rsq(x) 8249 8250 // XXX - Is UnsafeFPMath sufficient to do this for f64? The maximum ULP 8251 // error seems really high at 2^29 ULP. 8252 if (RHS.getOpcode() == ISD::FSQRT) 8253 return DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0)); 8254 8255 // 1.0 / x -> rcp(x) 8256 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); 8257 } 8258 8259 // Same as for 1.0, but expand the sign out of the constant. 8260 if (CLHS->isExactlyValue(-1.0)) { 8261 // -1.0 / x -> rcp (fneg x) 8262 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 8263 return DAG.getNode(AMDGPUISD::RCP, SL, VT, FNegRHS); 8264 } 8265 } 8266 8267 // Turn into multiply by the reciprocal. 8268 // x / y -> x * (1.0 / y) 8269 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); 8270 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags); 8271 } 8272 8273 SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op, 8274 SelectionDAG &DAG) const { 8275 SDLoc SL(Op); 8276 SDValue X = Op.getOperand(0); 8277 SDValue Y = Op.getOperand(1); 8278 EVT VT = Op.getValueType(); 8279 const SDNodeFlags Flags = Op->getFlags(); 8280 8281 bool AllowInaccurateDiv = Flags.hasApproximateFuncs() || 8282 DAG.getTarget().Options.UnsafeFPMath; 8283 if (!AllowInaccurateDiv) 8284 return SDValue(); 8285 8286 SDValue NegY = DAG.getNode(ISD::FNEG, SL, VT, Y); 8287 SDValue One = DAG.getConstantFP(1.0, SL, VT); 8288 8289 SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y); 8290 SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One); 8291 8292 R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R); 8293 SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One); 8294 R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R); 8295 SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R); 8296 SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X); 8297 return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret); 8298 } 8299 8300 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, 8301 EVT VT, SDValue A, SDValue B, SDValue GlueChain, 8302 SDNodeFlags Flags) { 8303 if (GlueChain->getNumValues() <= 1) { 8304 return DAG.getNode(Opcode, SL, VT, A, B, Flags); 8305 } 8306 8307 assert(GlueChain->getNumValues() == 3); 8308 8309 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); 8310 switch (Opcode) { 8311 default: llvm_unreachable("no chain equivalent for opcode"); 8312 case ISD::FMUL: 8313 Opcode = AMDGPUISD::FMUL_W_CHAIN; 8314 break; 8315 } 8316 8317 return DAG.getNode(Opcode, SL, VTList, 8318 {GlueChain.getValue(1), A, B, GlueChain.getValue(2)}, 8319 Flags); 8320 } 8321 8322 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, 8323 EVT VT, SDValue A, SDValue B, SDValue C, 8324 SDValue GlueChain, SDNodeFlags Flags) { 8325 if (GlueChain->getNumValues() <= 1) { 8326 return DAG.getNode(Opcode, SL, VT, {A, B, C}, Flags); 8327 } 8328 8329 assert(GlueChain->getNumValues() == 3); 8330 8331 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); 8332 switch (Opcode) { 8333 default: llvm_unreachable("no chain equivalent for opcode"); 8334 case ISD::FMA: 8335 Opcode = AMDGPUISD::FMA_W_CHAIN; 8336 break; 8337 } 8338 8339 return DAG.getNode(Opcode, SL, VTList, 8340 {GlueChain.getValue(1), A, B, C, GlueChain.getValue(2)}, 8341 Flags); 8342 } 8343 8344 SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const { 8345 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG)) 8346 return FastLowered; 8347 8348 SDLoc SL(Op); 8349 SDValue Src0 = Op.getOperand(0); 8350 SDValue Src1 = Op.getOperand(1); 8351 8352 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); 8353 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); 8354 8355 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); 8356 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); 8357 8358 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); 8359 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); 8360 8361 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); 8362 } 8363 8364 // Faster 2.5 ULP division that does not support denormals. 8365 SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const { 8366 SDLoc SL(Op); 8367 SDValue LHS = Op.getOperand(1); 8368 SDValue RHS = Op.getOperand(2); 8369 8370 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS); 8371 8372 const APFloat K0Val(BitsToFloat(0x6f800000)); 8373 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); 8374 8375 const APFloat K1Val(BitsToFloat(0x2f800000)); 8376 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); 8377 8378 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); 8379 8380 EVT SetCCVT = 8381 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); 8382 8383 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT); 8384 8385 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One); 8386 8387 // TODO: Should this propagate fast-math-flags? 8388 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3); 8389 8390 // rcp does not support denormals. 8391 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1); 8392 8393 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0); 8394 8395 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul); 8396 } 8397 8398 // Returns immediate value for setting the F32 denorm mode when using the 8399 // S_DENORM_MODE instruction. 8400 static SDValue getSPDenormModeValue(int SPDenormMode, SelectionDAG &DAG, 8401 const SDLoc &SL, const GCNSubtarget *ST) { 8402 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE"); 8403 int DPDenormModeDefault = hasFP64FP16Denormals(DAG.getMachineFunction()) 8404 ? FP_DENORM_FLUSH_NONE 8405 : FP_DENORM_FLUSH_IN_FLUSH_OUT; 8406 8407 int Mode = SPDenormMode | (DPDenormModeDefault << 2); 8408 return DAG.getTargetConstant(Mode, SL, MVT::i32); 8409 } 8410 8411 SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const { 8412 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG)) 8413 return FastLowered; 8414 8415 // The selection matcher assumes anything with a chain selecting to a 8416 // mayRaiseFPException machine instruction. Since we're introducing a chain 8417 // here, we need to explicitly report nofpexcept for the regular fdiv 8418 // lowering. 8419 SDNodeFlags Flags = Op->getFlags(); 8420 Flags.setNoFPExcept(true); 8421 8422 SDLoc SL(Op); 8423 SDValue LHS = Op.getOperand(0); 8424 SDValue RHS = Op.getOperand(1); 8425 8426 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); 8427 8428 SDVTList ScaleVT = DAG.getVTList(MVT::f32, MVT::i1); 8429 8430 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, 8431 {RHS, RHS, LHS}, Flags); 8432 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, 8433 {LHS, RHS, LHS}, Flags); 8434 8435 // Denominator is scaled to not be denormal, so using rcp is ok. 8436 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, 8437 DenominatorScaled, Flags); 8438 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, 8439 DenominatorScaled, Flags); 8440 8441 const unsigned Denorm32Reg = AMDGPU::Hwreg::ID_MODE | 8442 (4 << AMDGPU::Hwreg::OFFSET_SHIFT_) | 8443 (1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_); 8444 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32); 8445 8446 const bool HasFP32Denormals = hasFP32Denormals(DAG.getMachineFunction()); 8447 8448 if (!HasFP32Denormals) { 8449 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV 8450 // lowering. The chain dependence is insufficient, and we need glue. We do 8451 // not need the glue variants in a strictfp function. 8452 8453 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); 8454 8455 SDNode *EnableDenorm; 8456 if (Subtarget->hasDenormModeInst()) { 8457 const SDValue EnableDenormValue = 8458 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, SL, Subtarget); 8459 8460 EnableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, 8461 DAG.getEntryNode(), EnableDenormValue).getNode(); 8462 } else { 8463 const SDValue EnableDenormValue = DAG.getConstant(FP_DENORM_FLUSH_NONE, 8464 SL, MVT::i32); 8465 EnableDenorm = 8466 DAG.getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs, 8467 {EnableDenormValue, BitField, DAG.getEntryNode()}); 8468 } 8469 8470 SDValue Ops[3] = { 8471 NegDivScale0, 8472 SDValue(EnableDenorm, 0), 8473 SDValue(EnableDenorm, 1) 8474 }; 8475 8476 NegDivScale0 = DAG.getMergeValues(Ops, SL); 8477 } 8478 8479 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, 8480 ApproxRcp, One, NegDivScale0, Flags); 8481 8482 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, 8483 ApproxRcp, Fma0, Flags); 8484 8485 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, 8486 Fma1, Fma1, Flags); 8487 8488 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, 8489 NumeratorScaled, Mul, Flags); 8490 8491 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, 8492 Fma2, Fma1, Mul, Fma2, Flags); 8493 8494 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, 8495 NumeratorScaled, Fma3, Flags); 8496 8497 if (!HasFP32Denormals) { 8498 SDNode *DisableDenorm; 8499 if (Subtarget->hasDenormModeInst()) { 8500 const SDValue DisableDenormValue = 8501 getSPDenormModeValue(FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, SL, Subtarget); 8502 8503 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, 8504 Fma4.getValue(1), DisableDenormValue, 8505 Fma4.getValue(2)).getNode(); 8506 } else { 8507 const SDValue DisableDenormValue = 8508 DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); 8509 8510 DisableDenorm = DAG.getMachineNode( 8511 AMDGPU::S_SETREG_B32, SL, MVT::Other, 8512 {DisableDenormValue, BitField, Fma4.getValue(1), Fma4.getValue(2)}); 8513 } 8514 8515 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, 8516 SDValue(DisableDenorm, 0), DAG.getRoot()); 8517 DAG.setRoot(OutputChain); 8518 } 8519 8520 SDValue Scale = NumeratorScaled.getValue(1); 8521 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, 8522 {Fma4, Fma1, Fma3, Scale}, Flags); 8523 8524 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags); 8525 } 8526 8527 SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const { 8528 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG)) 8529 return FastLowered; 8530 8531 SDLoc SL(Op); 8532 SDValue X = Op.getOperand(0); 8533 SDValue Y = Op.getOperand(1); 8534 8535 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); 8536 8537 SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1); 8538 8539 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X); 8540 8541 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); 8542 8543 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); 8544 8545 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); 8546 8547 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); 8548 8549 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); 8550 8551 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X); 8552 8553 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); 8554 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); 8555 8556 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, 8557 NegDivScale0, Mul, DivScale1); 8558 8559 SDValue Scale; 8560 8561 if (!Subtarget->hasUsableDivScaleConditionOutput()) { 8562 // Workaround a hardware bug on SI where the condition output from div_scale 8563 // is not usable. 8564 8565 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); 8566 8567 // Figure out if the scale to use for div_fmas. 8568 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); 8569 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); 8570 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); 8571 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); 8572 8573 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); 8574 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); 8575 8576 SDValue Scale0Hi 8577 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); 8578 SDValue Scale1Hi 8579 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); 8580 8581 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); 8582 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); 8583 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); 8584 } else { 8585 Scale = DivScale1.getValue(1); 8586 } 8587 8588 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, 8589 Fma4, Fma3, Mul, Scale); 8590 8591 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); 8592 } 8593 8594 SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const { 8595 EVT VT = Op.getValueType(); 8596 8597 if (VT == MVT::f32) 8598 return LowerFDIV32(Op, DAG); 8599 8600 if (VT == MVT::f64) 8601 return LowerFDIV64(Op, DAG); 8602 8603 if (VT == MVT::f16) 8604 return LowerFDIV16(Op, DAG); 8605 8606 llvm_unreachable("Unexpected type for fdiv"); 8607 } 8608 8609 SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 8610 SDLoc DL(Op); 8611 StoreSDNode *Store = cast<StoreSDNode>(Op); 8612 EVT VT = Store->getMemoryVT(); 8613 8614 if (VT == MVT::i1) { 8615 return DAG.getTruncStore(Store->getChain(), DL, 8616 DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32), 8617 Store->getBasePtr(), MVT::i1, Store->getMemOperand()); 8618 } 8619 8620 assert(VT.isVector() && 8621 Store->getValue().getValueType().getScalarType() == MVT::i32); 8622 8623 unsigned AS = Store->getAddressSpace(); 8624 if (Subtarget->hasLDSMisalignedBug() && 8625 AS == AMDGPUAS::FLAT_ADDRESS && 8626 Store->getAlignment() < VT.getStoreSize() && VT.getSizeInBits() > 32) { 8627 return SplitVectorStore(Op, DAG); 8628 } 8629 8630 MachineFunction &MF = DAG.getMachineFunction(); 8631 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 8632 // If there is a possibilty that flat instruction access scratch memory 8633 // then we need to use the same legalization rules we use for private. 8634 if (AS == AMDGPUAS::FLAT_ADDRESS && 8635 !Subtarget->hasMultiDwordFlatScratchAddressing()) 8636 AS = MFI->hasFlatScratchInit() ? 8637 AMDGPUAS::PRIVATE_ADDRESS : AMDGPUAS::GLOBAL_ADDRESS; 8638 8639 unsigned NumElements = VT.getVectorNumElements(); 8640 if (AS == AMDGPUAS::GLOBAL_ADDRESS || 8641 AS == AMDGPUAS::FLAT_ADDRESS) { 8642 if (NumElements > 4) 8643 return SplitVectorStore(Op, DAG); 8644 // v3 stores not supported on SI. 8645 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores()) 8646 return SplitVectorStore(Op, DAG); 8647 8648 if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(), 8649 VT, *Store->getMemOperand())) 8650 return expandUnalignedStore(Store, DAG); 8651 8652 return SDValue(); 8653 } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) { 8654 switch (Subtarget->getMaxPrivateElementSize()) { 8655 case 4: 8656 return scalarizeVectorStore(Store, DAG); 8657 case 8: 8658 if (NumElements > 2) 8659 return SplitVectorStore(Op, DAG); 8660 return SDValue(); 8661 case 16: 8662 if (NumElements > 4 || 8663 (NumElements == 3 && !Subtarget->enableFlatScratch())) 8664 return SplitVectorStore(Op, DAG); 8665 return SDValue(); 8666 default: 8667 llvm_unreachable("unsupported private_element_size"); 8668 } 8669 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) { 8670 // Use ds_write_b128 or ds_write_b96 when possible. 8671 if (Subtarget->hasDS96AndDS128() && 8672 ((Subtarget->useDS128() && VT.getStoreSize() == 16) || 8673 (VT.getStoreSize() == 12)) && 8674 allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AS, 8675 Store->getAlign())) 8676 return SDValue(); 8677 8678 if (NumElements > 2) 8679 return SplitVectorStore(Op, DAG); 8680 8681 // SI has a hardware bug in the LDS / GDS boounds checking: if the base 8682 // address is negative, then the instruction is incorrectly treated as 8683 // out-of-bounds even if base + offsets is in bounds. Split vectorized 8684 // stores here to avoid emitting ds_write2_b32. We may re-combine the 8685 // store later in the SILoadStoreOptimizer. 8686 if (!Subtarget->hasUsableDSOffset() && 8687 NumElements == 2 && VT.getStoreSize() == 8 && 8688 Store->getAlignment() < 8) { 8689 return SplitVectorStore(Op, DAG); 8690 } 8691 8692 if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(), 8693 VT, *Store->getMemOperand())) { 8694 if (VT.isVector()) 8695 return SplitVectorStore(Op, DAG); 8696 return expandUnalignedStore(Store, DAG); 8697 } 8698 8699 return SDValue(); 8700 } else { 8701 llvm_unreachable("unhandled address space"); 8702 } 8703 } 8704 8705 SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const { 8706 SDLoc DL(Op); 8707 EVT VT = Op.getValueType(); 8708 SDValue Arg = Op.getOperand(0); 8709 SDValue TrigVal; 8710 8711 // Propagate fast-math flags so that the multiply we introduce can be folded 8712 // if Arg is already the result of a multiply by constant. 8713 auto Flags = Op->getFlags(); 8714 8715 SDValue OneOver2Pi = DAG.getConstantFP(0.5 * numbers::inv_pi, DL, VT); 8716 8717 if (Subtarget->hasTrigReducedRange()) { 8718 SDValue MulVal = DAG.getNode(ISD::FMUL, DL, VT, Arg, OneOver2Pi, Flags); 8719 TrigVal = DAG.getNode(AMDGPUISD::FRACT, DL, VT, MulVal, Flags); 8720 } else { 8721 TrigVal = DAG.getNode(ISD::FMUL, DL, VT, Arg, OneOver2Pi, Flags); 8722 } 8723 8724 switch (Op.getOpcode()) { 8725 case ISD::FCOS: 8726 return DAG.getNode(AMDGPUISD::COS_HW, SDLoc(Op), VT, TrigVal, Flags); 8727 case ISD::FSIN: 8728 return DAG.getNode(AMDGPUISD::SIN_HW, SDLoc(Op), VT, TrigVal, Flags); 8729 default: 8730 llvm_unreachable("Wrong trig opcode"); 8731 } 8732 } 8733 8734 SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const { 8735 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Op); 8736 assert(AtomicNode->isCompareAndSwap()); 8737 unsigned AS = AtomicNode->getAddressSpace(); 8738 8739 // No custom lowering required for local address space 8740 if (!AMDGPU::isFlatGlobalAddrSpace(AS)) 8741 return Op; 8742 8743 // Non-local address space requires custom lowering for atomic compare 8744 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2 8745 SDLoc DL(Op); 8746 SDValue ChainIn = Op.getOperand(0); 8747 SDValue Addr = Op.getOperand(1); 8748 SDValue Old = Op.getOperand(2); 8749 SDValue New = Op.getOperand(3); 8750 EVT VT = Op.getValueType(); 8751 MVT SimpleVT = VT.getSimpleVT(); 8752 MVT VecType = MVT::getVectorVT(SimpleVT, 2); 8753 8754 SDValue NewOld = DAG.getBuildVector(VecType, DL, {New, Old}); 8755 SDValue Ops[] = { ChainIn, Addr, NewOld }; 8756 8757 return DAG.getMemIntrinsicNode(AMDGPUISD::ATOMIC_CMP_SWAP, DL, Op->getVTList(), 8758 Ops, VT, AtomicNode->getMemOperand()); 8759 } 8760 8761 //===----------------------------------------------------------------------===// 8762 // Custom DAG optimizations 8763 //===----------------------------------------------------------------------===// 8764 8765 SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N, 8766 DAGCombinerInfo &DCI) const { 8767 EVT VT = N->getValueType(0); 8768 EVT ScalarVT = VT.getScalarType(); 8769 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16) 8770 return SDValue(); 8771 8772 SelectionDAG &DAG = DCI.DAG; 8773 SDLoc DL(N); 8774 8775 SDValue Src = N->getOperand(0); 8776 EVT SrcVT = Src.getValueType(); 8777 8778 // TODO: We could try to match extracting the higher bytes, which would be 8779 // easier if i8 vectors weren't promoted to i32 vectors, particularly after 8780 // types are legalized. v4i8 -> v4f32 is probably the only case to worry 8781 // about in practice. 8782 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) { 8783 if (DAG.MaskedValueIsZero(Src, APInt::getHighBitsSet(32, 24))) { 8784 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src); 8785 DCI.AddToWorklist(Cvt.getNode()); 8786 8787 // For the f16 case, fold to a cast to f32 and then cast back to f16. 8788 if (ScalarVT != MVT::f32) { 8789 Cvt = DAG.getNode(ISD::FP_ROUND, DL, VT, Cvt, 8790 DAG.getTargetConstant(0, DL, MVT::i32)); 8791 } 8792 return Cvt; 8793 } 8794 } 8795 8796 return SDValue(); 8797 } 8798 8799 // (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2) 8800 8801 // This is a variant of 8802 // (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2), 8803 // 8804 // The normal DAG combiner will do this, but only if the add has one use since 8805 // that would increase the number of instructions. 8806 // 8807 // This prevents us from seeing a constant offset that can be folded into a 8808 // memory instruction's addressing mode. If we know the resulting add offset of 8809 // a pointer can be folded into an addressing offset, we can replace the pointer 8810 // operand with the add of new constant offset. This eliminates one of the uses, 8811 // and may allow the remaining use to also be simplified. 8812 // 8813 SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, 8814 unsigned AddrSpace, 8815 EVT MemVT, 8816 DAGCombinerInfo &DCI) const { 8817 SDValue N0 = N->getOperand(0); 8818 SDValue N1 = N->getOperand(1); 8819 8820 // We only do this to handle cases where it's profitable when there are 8821 // multiple uses of the add, so defer to the standard combine. 8822 if ((N0.getOpcode() != ISD::ADD && N0.getOpcode() != ISD::OR) || 8823 N0->hasOneUse()) 8824 return SDValue(); 8825 8826 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N1); 8827 if (!CN1) 8828 return SDValue(); 8829 8830 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 8831 if (!CAdd) 8832 return SDValue(); 8833 8834 // If the resulting offset is too large, we can't fold it into the addressing 8835 // mode offset. 8836 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue(); 8837 Type *Ty = MemVT.getTypeForEVT(*DCI.DAG.getContext()); 8838 8839 AddrMode AM; 8840 AM.HasBaseReg = true; 8841 AM.BaseOffs = Offset.getSExtValue(); 8842 if (!isLegalAddressingMode(DCI.DAG.getDataLayout(), AM, Ty, AddrSpace)) 8843 return SDValue(); 8844 8845 SelectionDAG &DAG = DCI.DAG; 8846 SDLoc SL(N); 8847 EVT VT = N->getValueType(0); 8848 8849 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1); 8850 SDValue COffset = DAG.getConstant(Offset, SL, VT); 8851 8852 SDNodeFlags Flags; 8853 Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap() && 8854 (N0.getOpcode() == ISD::OR || 8855 N0->getFlags().hasNoUnsignedWrap())); 8856 8857 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags); 8858 } 8859 8860 /// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset 8861 /// by the chain and intrinsic ID. Theoretically we would also need to check the 8862 /// specific intrinsic, but they all place the pointer operand first. 8863 static unsigned getBasePtrIndex(const MemSDNode *N) { 8864 switch (N->getOpcode()) { 8865 case ISD::STORE: 8866 case ISD::INTRINSIC_W_CHAIN: 8867 case ISD::INTRINSIC_VOID: 8868 return 2; 8869 default: 8870 return 1; 8871 } 8872 } 8873 8874 SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N, 8875 DAGCombinerInfo &DCI) const { 8876 SelectionDAG &DAG = DCI.DAG; 8877 SDLoc SL(N); 8878 8879 unsigned PtrIdx = getBasePtrIndex(N); 8880 SDValue Ptr = N->getOperand(PtrIdx); 8881 8882 // TODO: We could also do this for multiplies. 8883 if (Ptr.getOpcode() == ISD::SHL) { 8884 SDValue NewPtr = performSHLPtrCombine(Ptr.getNode(), N->getAddressSpace(), 8885 N->getMemoryVT(), DCI); 8886 if (NewPtr) { 8887 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 8888 8889 NewOps[PtrIdx] = NewPtr; 8890 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 8891 } 8892 } 8893 8894 return SDValue(); 8895 } 8896 8897 static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) { 8898 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) || 8899 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) || 8900 (Opc == ISD::XOR && Val == 0); 8901 } 8902 8903 // Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This 8904 // will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit 8905 // integer combine opportunities since most 64-bit operations are decomposed 8906 // this way. TODO: We won't want this for SALU especially if it is an inline 8907 // immediate. 8908 SDValue SITargetLowering::splitBinaryBitConstantOp( 8909 DAGCombinerInfo &DCI, 8910 const SDLoc &SL, 8911 unsigned Opc, SDValue LHS, 8912 const ConstantSDNode *CRHS) const { 8913 uint64_t Val = CRHS->getZExtValue(); 8914 uint32_t ValLo = Lo_32(Val); 8915 uint32_t ValHi = Hi_32(Val); 8916 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 8917 8918 if ((bitOpWithConstantIsReducible(Opc, ValLo) || 8919 bitOpWithConstantIsReducible(Opc, ValHi)) || 8920 (CRHS->hasOneUse() && !TII->isInlineConstant(CRHS->getAPIntValue()))) { 8921 // If we need to materialize a 64-bit immediate, it will be split up later 8922 // anyway. Avoid creating the harder to understand 64-bit immediate 8923 // materialization. 8924 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi); 8925 } 8926 8927 return SDValue(); 8928 } 8929 8930 // Returns true if argument is a boolean value which is not serialized into 8931 // memory or argument and does not require v_cndmask_b32 to be deserialized. 8932 static bool isBoolSGPR(SDValue V) { 8933 if (V.getValueType() != MVT::i1) 8934 return false; 8935 switch (V.getOpcode()) { 8936 default: 8937 break; 8938 case ISD::SETCC: 8939 case AMDGPUISD::FP_CLASS: 8940 return true; 8941 case ISD::AND: 8942 case ISD::OR: 8943 case ISD::XOR: 8944 return isBoolSGPR(V.getOperand(0)) && isBoolSGPR(V.getOperand(1)); 8945 } 8946 return false; 8947 } 8948 8949 // If a constant has all zeroes or all ones within each byte return it. 8950 // Otherwise return 0. 8951 static uint32_t getConstantPermuteMask(uint32_t C) { 8952 // 0xff for any zero byte in the mask 8953 uint32_t ZeroByteMask = 0; 8954 if (!(C & 0x000000ff)) ZeroByteMask |= 0x000000ff; 8955 if (!(C & 0x0000ff00)) ZeroByteMask |= 0x0000ff00; 8956 if (!(C & 0x00ff0000)) ZeroByteMask |= 0x00ff0000; 8957 if (!(C & 0xff000000)) ZeroByteMask |= 0xff000000; 8958 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte 8959 if ((NonZeroByteMask & C) != NonZeroByteMask) 8960 return 0; // Partial bytes selected. 8961 return C; 8962 } 8963 8964 // Check if a node selects whole bytes from its operand 0 starting at a byte 8965 // boundary while masking the rest. Returns select mask as in the v_perm_b32 8966 // or -1 if not succeeded. 8967 // Note byte select encoding: 8968 // value 0-3 selects corresponding source byte; 8969 // value 0xc selects zero; 8970 // value 0xff selects 0xff. 8971 static uint32_t getPermuteMask(SelectionDAG &DAG, SDValue V) { 8972 assert(V.getValueSizeInBits() == 32); 8973 8974 if (V.getNumOperands() != 2) 8975 return ~0; 8976 8977 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(V.getOperand(1)); 8978 if (!N1) 8979 return ~0; 8980 8981 uint32_t C = N1->getZExtValue(); 8982 8983 switch (V.getOpcode()) { 8984 default: 8985 break; 8986 case ISD::AND: 8987 if (uint32_t ConstMask = getConstantPermuteMask(C)) { 8988 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask); 8989 } 8990 break; 8991 8992 case ISD::OR: 8993 if (uint32_t ConstMask = getConstantPermuteMask(C)) { 8994 return (0x03020100 & ~ConstMask) | ConstMask; 8995 } 8996 break; 8997 8998 case ISD::SHL: 8999 if (C % 8) 9000 return ~0; 9001 9002 return uint32_t((0x030201000c0c0c0cull << C) >> 32); 9003 9004 case ISD::SRL: 9005 if (C % 8) 9006 return ~0; 9007 9008 return uint32_t(0x0c0c0c0c03020100ull >> C); 9009 } 9010 9011 return ~0; 9012 } 9013 9014 SDValue SITargetLowering::performAndCombine(SDNode *N, 9015 DAGCombinerInfo &DCI) const { 9016 if (DCI.isBeforeLegalize()) 9017 return SDValue(); 9018 9019 SelectionDAG &DAG = DCI.DAG; 9020 EVT VT = N->getValueType(0); 9021 SDValue LHS = N->getOperand(0); 9022 SDValue RHS = N->getOperand(1); 9023 9024 9025 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS); 9026 if (VT == MVT::i64 && CRHS) { 9027 if (SDValue Split 9028 = splitBinaryBitConstantOp(DCI, SDLoc(N), ISD::AND, LHS, CRHS)) 9029 return Split; 9030 } 9031 9032 if (CRHS && VT == MVT::i32) { 9033 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb 9034 // nb = number of trailing zeroes in mask 9035 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass, 9036 // given that we are selecting 8 or 16 bit fields starting at byte boundary. 9037 uint64_t Mask = CRHS->getZExtValue(); 9038 unsigned Bits = countPopulation(Mask); 9039 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL && 9040 (Bits == 8 || Bits == 16) && isShiftedMask_64(Mask) && !(Mask & 1)) { 9041 if (auto *CShift = dyn_cast<ConstantSDNode>(LHS->getOperand(1))) { 9042 unsigned Shift = CShift->getZExtValue(); 9043 unsigned NB = CRHS->getAPIntValue().countTrailingZeros(); 9044 unsigned Offset = NB + Shift; 9045 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary. 9046 SDLoc SL(N); 9047 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, 9048 LHS->getOperand(0), 9049 DAG.getConstant(Offset, SL, MVT::i32), 9050 DAG.getConstant(Bits, SL, MVT::i32)); 9051 EVT NarrowVT = EVT::getIntegerVT(*DAG.getContext(), Bits); 9052 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE, 9053 DAG.getValueType(NarrowVT)); 9054 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(LHS), VT, Ext, 9055 DAG.getConstant(NB, SDLoc(CRHS), MVT::i32)); 9056 return Shl; 9057 } 9058 } 9059 } 9060 9061 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2) 9062 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM && 9063 isa<ConstantSDNode>(LHS.getOperand(2))) { 9064 uint32_t Sel = getConstantPermuteMask(Mask); 9065 if (!Sel) 9066 return SDValue(); 9067 9068 // Select 0xc for all zero bytes 9069 Sel = (LHS.getConstantOperandVal(2) & Sel) | (~Sel & 0x0c0c0c0c); 9070 SDLoc DL(N); 9071 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), 9072 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); 9073 } 9074 } 9075 9076 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) -> 9077 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity) 9078 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) { 9079 ISD::CondCode LCC = cast<CondCodeSDNode>(LHS.getOperand(2))->get(); 9080 ISD::CondCode RCC = cast<CondCodeSDNode>(RHS.getOperand(2))->get(); 9081 9082 SDValue X = LHS.getOperand(0); 9083 SDValue Y = RHS.getOperand(0); 9084 if (Y.getOpcode() != ISD::FABS || Y.getOperand(0) != X) 9085 return SDValue(); 9086 9087 if (LCC == ISD::SETO) { 9088 if (X != LHS.getOperand(1)) 9089 return SDValue(); 9090 9091 if (RCC == ISD::SETUNE) { 9092 const ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(RHS.getOperand(1)); 9093 if (!C1 || !C1->isInfinity() || C1->isNegative()) 9094 return SDValue(); 9095 9096 const uint32_t Mask = SIInstrFlags::N_NORMAL | 9097 SIInstrFlags::N_SUBNORMAL | 9098 SIInstrFlags::N_ZERO | 9099 SIInstrFlags::P_ZERO | 9100 SIInstrFlags::P_SUBNORMAL | 9101 SIInstrFlags::P_NORMAL; 9102 9103 static_assert(((~(SIInstrFlags::S_NAN | 9104 SIInstrFlags::Q_NAN | 9105 SIInstrFlags::N_INFINITY | 9106 SIInstrFlags::P_INFINITY)) & 0x3ff) == Mask, 9107 "mask not equal"); 9108 9109 SDLoc DL(N); 9110 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, 9111 X, DAG.getConstant(Mask, DL, MVT::i32)); 9112 } 9113 } 9114 } 9115 9116 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS) 9117 std::swap(LHS, RHS); 9118 9119 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS && 9120 RHS.hasOneUse()) { 9121 ISD::CondCode LCC = cast<CondCodeSDNode>(LHS.getOperand(2))->get(); 9122 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan | n_nan) 9123 // and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan | n_nan) 9124 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS.getOperand(1)); 9125 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask && 9126 (RHS.getOperand(0) == LHS.getOperand(0) && 9127 LHS.getOperand(0) == LHS.getOperand(1))) { 9128 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN; 9129 unsigned NewMask = LCC == ISD::SETO ? 9130 Mask->getZExtValue() & ~OrdMask : 9131 Mask->getZExtValue() & OrdMask; 9132 9133 SDLoc DL(N); 9134 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, RHS.getOperand(0), 9135 DAG.getConstant(NewMask, DL, MVT::i32)); 9136 } 9137 } 9138 9139 if (VT == MVT::i32 && 9140 (RHS.getOpcode() == ISD::SIGN_EXTEND || LHS.getOpcode() == ISD::SIGN_EXTEND)) { 9141 // and x, (sext cc from i1) => select cc, x, 0 9142 if (RHS.getOpcode() != ISD::SIGN_EXTEND) 9143 std::swap(LHS, RHS); 9144 if (isBoolSGPR(RHS.getOperand(0))) 9145 return DAG.getSelect(SDLoc(N), MVT::i32, RHS.getOperand(0), 9146 LHS, DAG.getConstant(0, SDLoc(N), MVT::i32)); 9147 } 9148 9149 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2) 9150 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 9151 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && 9152 N->isDivergent() && TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) { 9153 uint32_t LHSMask = getPermuteMask(DAG, LHS); 9154 uint32_t RHSMask = getPermuteMask(DAG, RHS); 9155 if (LHSMask != ~0u && RHSMask != ~0u) { 9156 // Canonicalize the expression in an attempt to have fewer unique masks 9157 // and therefore fewer registers used to hold the masks. 9158 if (LHSMask > RHSMask) { 9159 std::swap(LHSMask, RHSMask); 9160 std::swap(LHS, RHS); 9161 } 9162 9163 // Select 0xc for each lane used from source operand. Zero has 0xc mask 9164 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range. 9165 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c; 9166 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c; 9167 9168 // Check of we need to combine values from two sources within a byte. 9169 if (!(LHSUsedLanes & RHSUsedLanes) && 9170 // If we select high and lower word keep it for SDWA. 9171 // TODO: teach SDWA to work with v_perm_b32 and remove the check. 9172 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) { 9173 // Each byte in each mask is either selector mask 0-3, or has higher 9174 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for 9175 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise 9176 // mask which is not 0xff wins. By anding both masks we have a correct 9177 // result except that 0x0c shall be corrected to give 0x0c only. 9178 uint32_t Mask = LHSMask & RHSMask; 9179 for (unsigned I = 0; I < 32; I += 8) { 9180 uint32_t ByteSel = 0xff << I; 9181 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c) 9182 Mask &= (0x0c << I) & 0xffffffff; 9183 } 9184 9185 // Add 4 to each active LHS lane. It will not affect any existing 0xff 9186 // or 0x0c. 9187 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404); 9188 SDLoc DL(N); 9189 9190 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, 9191 LHS.getOperand(0), RHS.getOperand(0), 9192 DAG.getConstant(Sel, DL, MVT::i32)); 9193 } 9194 } 9195 } 9196 9197 return SDValue(); 9198 } 9199 9200 SDValue SITargetLowering::performOrCombine(SDNode *N, 9201 DAGCombinerInfo &DCI) const { 9202 SelectionDAG &DAG = DCI.DAG; 9203 SDValue LHS = N->getOperand(0); 9204 SDValue RHS = N->getOperand(1); 9205 9206 EVT VT = N->getValueType(0); 9207 if (VT == MVT::i1) { 9208 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2) 9209 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS && 9210 RHS.getOpcode() == AMDGPUISD::FP_CLASS) { 9211 SDValue Src = LHS.getOperand(0); 9212 if (Src != RHS.getOperand(0)) 9213 return SDValue(); 9214 9215 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(LHS.getOperand(1)); 9216 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS.getOperand(1)); 9217 if (!CLHS || !CRHS) 9218 return SDValue(); 9219 9220 // Only 10 bits are used. 9221 static const uint32_t MaxMask = 0x3ff; 9222 9223 uint32_t NewMask = (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask; 9224 SDLoc DL(N); 9225 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, 9226 Src, DAG.getConstant(NewMask, DL, MVT::i32)); 9227 } 9228 9229 return SDValue(); 9230 } 9231 9232 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2) 9233 if (isa<ConstantSDNode>(RHS) && LHS.hasOneUse() && 9234 LHS.getOpcode() == AMDGPUISD::PERM && 9235 isa<ConstantSDNode>(LHS.getOperand(2))) { 9236 uint32_t Sel = getConstantPermuteMask(N->getConstantOperandVal(1)); 9237 if (!Sel) 9238 return SDValue(); 9239 9240 Sel |= LHS.getConstantOperandVal(2); 9241 SDLoc DL(N); 9242 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), 9243 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); 9244 } 9245 9246 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2) 9247 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 9248 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && 9249 N->isDivergent() && TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) { 9250 uint32_t LHSMask = getPermuteMask(DAG, LHS); 9251 uint32_t RHSMask = getPermuteMask(DAG, RHS); 9252 if (LHSMask != ~0u && RHSMask != ~0u) { 9253 // Canonicalize the expression in an attempt to have fewer unique masks 9254 // and therefore fewer registers used to hold the masks. 9255 if (LHSMask > RHSMask) { 9256 std::swap(LHSMask, RHSMask); 9257 std::swap(LHS, RHS); 9258 } 9259 9260 // Select 0xc for each lane used from source operand. Zero has 0xc mask 9261 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range. 9262 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c; 9263 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c; 9264 9265 // Check of we need to combine values from two sources within a byte. 9266 if (!(LHSUsedLanes & RHSUsedLanes) && 9267 // If we select high and lower word keep it for SDWA. 9268 // TODO: teach SDWA to work with v_perm_b32 and remove the check. 9269 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) { 9270 // Kill zero bytes selected by other mask. Zero value is 0xc. 9271 LHSMask &= ~RHSUsedLanes; 9272 RHSMask &= ~LHSUsedLanes; 9273 // Add 4 to each active LHS lane 9274 LHSMask |= LHSUsedLanes & 0x04040404; 9275 // Combine masks 9276 uint32_t Sel = LHSMask | RHSMask; 9277 SDLoc DL(N); 9278 9279 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, 9280 LHS.getOperand(0), RHS.getOperand(0), 9281 DAG.getConstant(Sel, DL, MVT::i32)); 9282 } 9283 } 9284 } 9285 9286 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps()) 9287 return SDValue(); 9288 9289 // TODO: This could be a generic combine with a predicate for extracting the 9290 // high half of an integer being free. 9291 9292 // (or i64:x, (zero_extend i32:y)) -> 9293 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x))) 9294 if (LHS.getOpcode() == ISD::ZERO_EXTEND && 9295 RHS.getOpcode() != ISD::ZERO_EXTEND) 9296 std::swap(LHS, RHS); 9297 9298 if (RHS.getOpcode() == ISD::ZERO_EXTEND) { 9299 SDValue ExtSrc = RHS.getOperand(0); 9300 EVT SrcVT = ExtSrc.getValueType(); 9301 if (SrcVT == MVT::i32) { 9302 SDLoc SL(N); 9303 SDValue LowLHS, HiBits; 9304 std::tie(LowLHS, HiBits) = split64BitValue(LHS, DAG); 9305 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); 9306 9307 DCI.AddToWorklist(LowOr.getNode()); 9308 DCI.AddToWorklist(HiBits.getNode()); 9309 9310 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, 9311 LowOr, HiBits); 9312 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); 9313 } 9314 } 9315 9316 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(N->getOperand(1)); 9317 if (CRHS) { 9318 if (SDValue Split 9319 = splitBinaryBitConstantOp(DCI, SDLoc(N), ISD::OR, LHS, CRHS)) 9320 return Split; 9321 } 9322 9323 return SDValue(); 9324 } 9325 9326 SDValue SITargetLowering::performXorCombine(SDNode *N, 9327 DAGCombinerInfo &DCI) const { 9328 EVT VT = N->getValueType(0); 9329 if (VT != MVT::i64) 9330 return SDValue(); 9331 9332 SDValue LHS = N->getOperand(0); 9333 SDValue RHS = N->getOperand(1); 9334 9335 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS); 9336 if (CRHS) { 9337 if (SDValue Split 9338 = splitBinaryBitConstantOp(DCI, SDLoc(N), ISD::XOR, LHS, CRHS)) 9339 return Split; 9340 } 9341 9342 return SDValue(); 9343 } 9344 9345 // Instructions that will be lowered with a final instruction that zeros the 9346 // high result bits. 9347 // XXX - probably only need to list legal operations. 9348 static bool fp16SrcZerosHighBits(unsigned Opc) { 9349 switch (Opc) { 9350 case ISD::FADD: 9351 case ISD::FSUB: 9352 case ISD::FMUL: 9353 case ISD::FDIV: 9354 case ISD::FREM: 9355 case ISD::FMA: 9356 case ISD::FMAD: 9357 case ISD::FCANONICALIZE: 9358 case ISD::FP_ROUND: 9359 case ISD::UINT_TO_FP: 9360 case ISD::SINT_TO_FP: 9361 case ISD::FABS: 9362 // Fabs is lowered to a bit operation, but it's an and which will clear the 9363 // high bits anyway. 9364 case ISD::FSQRT: 9365 case ISD::FSIN: 9366 case ISD::FCOS: 9367 case ISD::FPOWI: 9368 case ISD::FPOW: 9369 case ISD::FLOG: 9370 case ISD::FLOG2: 9371 case ISD::FLOG10: 9372 case ISD::FEXP: 9373 case ISD::FEXP2: 9374 case ISD::FCEIL: 9375 case ISD::FTRUNC: 9376 case ISD::FRINT: 9377 case ISD::FNEARBYINT: 9378 case ISD::FROUND: 9379 case ISD::FFLOOR: 9380 case ISD::FMINNUM: 9381 case ISD::FMAXNUM: 9382 case AMDGPUISD::FRACT: 9383 case AMDGPUISD::CLAMP: 9384 case AMDGPUISD::COS_HW: 9385 case AMDGPUISD::SIN_HW: 9386 case AMDGPUISD::FMIN3: 9387 case AMDGPUISD::FMAX3: 9388 case AMDGPUISD::FMED3: 9389 case AMDGPUISD::FMAD_FTZ: 9390 case AMDGPUISD::RCP: 9391 case AMDGPUISD::RSQ: 9392 case AMDGPUISD::RCP_IFLAG: 9393 case AMDGPUISD::LDEXP: 9394 return true; 9395 default: 9396 // fcopysign, select and others may be lowered to 32-bit bit operations 9397 // which don't zero the high bits. 9398 return false; 9399 } 9400 } 9401 9402 SDValue SITargetLowering::performZeroExtendCombine(SDNode *N, 9403 DAGCombinerInfo &DCI) const { 9404 if (!Subtarget->has16BitInsts() || 9405 DCI.getDAGCombineLevel() < AfterLegalizeDAG) 9406 return SDValue(); 9407 9408 EVT VT = N->getValueType(0); 9409 if (VT != MVT::i32) 9410 return SDValue(); 9411 9412 SDValue Src = N->getOperand(0); 9413 if (Src.getValueType() != MVT::i16) 9414 return SDValue(); 9415 9416 // (i32 zext (i16 (bitcast f16:$src))) -> fp16_zext $src 9417 // FIXME: It is not universally true that the high bits are zeroed on gfx9. 9418 if (Src.getOpcode() == ISD::BITCAST) { 9419 SDValue BCSrc = Src.getOperand(0); 9420 if (BCSrc.getValueType() == MVT::f16 && 9421 fp16SrcZerosHighBits(BCSrc.getOpcode())) 9422 return DCI.DAG.getNode(AMDGPUISD::FP16_ZEXT, SDLoc(N), VT, BCSrc); 9423 } 9424 9425 return SDValue(); 9426 } 9427 9428 SDValue SITargetLowering::performSignExtendInRegCombine(SDNode *N, 9429 DAGCombinerInfo &DCI) 9430 const { 9431 SDValue Src = N->getOperand(0); 9432 auto *VTSign = cast<VTSDNode>(N->getOperand(1)); 9433 9434 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE && 9435 VTSign->getVT() == MVT::i8) || 9436 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT && 9437 VTSign->getVT() == MVT::i16)) && 9438 Src.hasOneUse()) { 9439 auto *M = cast<MemSDNode>(Src); 9440 SDValue Ops[] = { 9441 Src.getOperand(0), // Chain 9442 Src.getOperand(1), // rsrc 9443 Src.getOperand(2), // vindex 9444 Src.getOperand(3), // voffset 9445 Src.getOperand(4), // soffset 9446 Src.getOperand(5), // offset 9447 Src.getOperand(6), 9448 Src.getOperand(7) 9449 }; 9450 // replace with BUFFER_LOAD_BYTE/SHORT 9451 SDVTList ResList = DCI.DAG.getVTList(MVT::i32, 9452 Src.getOperand(0).getValueType()); 9453 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE) ? 9454 AMDGPUISD::BUFFER_LOAD_BYTE : AMDGPUISD::BUFFER_LOAD_SHORT; 9455 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(Opc, SDLoc(N), 9456 ResList, 9457 Ops, M->getMemoryVT(), 9458 M->getMemOperand()); 9459 return DCI.DAG.getMergeValues({BufferLoadSignExt, 9460 BufferLoadSignExt.getValue(1)}, SDLoc(N)); 9461 } 9462 return SDValue(); 9463 } 9464 9465 SDValue SITargetLowering::performClassCombine(SDNode *N, 9466 DAGCombinerInfo &DCI) const { 9467 SelectionDAG &DAG = DCI.DAG; 9468 SDValue Mask = N->getOperand(1); 9469 9470 // fp_class x, 0 -> false 9471 if (const ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Mask)) { 9472 if (CMask->isNullValue()) 9473 return DAG.getConstant(0, SDLoc(N), MVT::i1); 9474 } 9475 9476 if (N->getOperand(0).isUndef()) 9477 return DAG.getUNDEF(MVT::i1); 9478 9479 return SDValue(); 9480 } 9481 9482 SDValue SITargetLowering::performRcpCombine(SDNode *N, 9483 DAGCombinerInfo &DCI) const { 9484 EVT VT = N->getValueType(0); 9485 SDValue N0 = N->getOperand(0); 9486 9487 if (N0.isUndef()) 9488 return N0; 9489 9490 if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP || 9491 N0.getOpcode() == ISD::SINT_TO_FP)) { 9492 return DCI.DAG.getNode(AMDGPUISD::RCP_IFLAG, SDLoc(N), VT, N0, 9493 N->getFlags()); 9494 } 9495 9496 if ((VT == MVT::f32 || VT == MVT::f16) && N0.getOpcode() == ISD::FSQRT) { 9497 return DCI.DAG.getNode(AMDGPUISD::RSQ, SDLoc(N), VT, 9498 N0.getOperand(0), N->getFlags()); 9499 } 9500 9501 return AMDGPUTargetLowering::performRcpCombine(N, DCI); 9502 } 9503 9504 bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, 9505 unsigned MaxDepth) const { 9506 unsigned Opcode = Op.getOpcode(); 9507 if (Opcode == ISD::FCANONICALIZE) 9508 return true; 9509 9510 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Op)) { 9511 auto F = CFP->getValueAPF(); 9512 if (F.isNaN() && F.isSignaling()) 9513 return false; 9514 return !F.isDenormal() || denormalsEnabledForType(DAG, Op.getValueType()); 9515 } 9516 9517 // If source is a result of another standard FP operation it is already in 9518 // canonical form. 9519 if (MaxDepth == 0) 9520 return false; 9521 9522 switch (Opcode) { 9523 // These will flush denorms if required. 9524 case ISD::FADD: 9525 case ISD::FSUB: 9526 case ISD::FMUL: 9527 case ISD::FCEIL: 9528 case ISD::FFLOOR: 9529 case ISD::FMA: 9530 case ISD::FMAD: 9531 case ISD::FSQRT: 9532 case ISD::FDIV: 9533 case ISD::FREM: 9534 case ISD::FP_ROUND: 9535 case ISD::FP_EXTEND: 9536 case AMDGPUISD::FMUL_LEGACY: 9537 case AMDGPUISD::FMAD_FTZ: 9538 case AMDGPUISD::RCP: 9539 case AMDGPUISD::RSQ: 9540 case AMDGPUISD::RSQ_CLAMP: 9541 case AMDGPUISD::RCP_LEGACY: 9542 case AMDGPUISD::RCP_IFLAG: 9543 case AMDGPUISD::DIV_SCALE: 9544 case AMDGPUISD::DIV_FMAS: 9545 case AMDGPUISD::DIV_FIXUP: 9546 case AMDGPUISD::FRACT: 9547 case AMDGPUISD::LDEXP: 9548 case AMDGPUISD::CVT_PKRTZ_F16_F32: 9549 case AMDGPUISD::CVT_F32_UBYTE0: 9550 case AMDGPUISD::CVT_F32_UBYTE1: 9551 case AMDGPUISD::CVT_F32_UBYTE2: 9552 case AMDGPUISD::CVT_F32_UBYTE3: 9553 return true; 9554 9555 // It can/will be lowered or combined as a bit operation. 9556 // Need to check their input recursively to handle. 9557 case ISD::FNEG: 9558 case ISD::FABS: 9559 case ISD::FCOPYSIGN: 9560 return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1); 9561 9562 case ISD::FSIN: 9563 case ISD::FCOS: 9564 case ISD::FSINCOS: 9565 return Op.getValueType().getScalarType() != MVT::f16; 9566 9567 case ISD::FMINNUM: 9568 case ISD::FMAXNUM: 9569 case ISD::FMINNUM_IEEE: 9570 case ISD::FMAXNUM_IEEE: 9571 case AMDGPUISD::CLAMP: 9572 case AMDGPUISD::FMED3: 9573 case AMDGPUISD::FMAX3: 9574 case AMDGPUISD::FMIN3: { 9575 // FIXME: Shouldn't treat the generic operations different based these. 9576 // However, we aren't really required to flush the result from 9577 // minnum/maxnum.. 9578 9579 // snans will be quieted, so we only need to worry about denormals. 9580 if (Subtarget->supportsMinMaxDenormModes() || 9581 denormalsEnabledForType(DAG, Op.getValueType())) 9582 return true; 9583 9584 // Flushing may be required. 9585 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such 9586 // targets need to check their input recursively. 9587 9588 // FIXME: Does this apply with clamp? It's implemented with max. 9589 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) { 9590 if (!isCanonicalized(DAG, Op.getOperand(I), MaxDepth - 1)) 9591 return false; 9592 } 9593 9594 return true; 9595 } 9596 case ISD::SELECT: { 9597 return isCanonicalized(DAG, Op.getOperand(1), MaxDepth - 1) && 9598 isCanonicalized(DAG, Op.getOperand(2), MaxDepth - 1); 9599 } 9600 case ISD::BUILD_VECTOR: { 9601 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 9602 SDValue SrcOp = Op.getOperand(i); 9603 if (!isCanonicalized(DAG, SrcOp, MaxDepth - 1)) 9604 return false; 9605 } 9606 9607 return true; 9608 } 9609 case ISD::EXTRACT_VECTOR_ELT: 9610 case ISD::EXTRACT_SUBVECTOR: { 9611 return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1); 9612 } 9613 case ISD::INSERT_VECTOR_ELT: { 9614 return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1) && 9615 isCanonicalized(DAG, Op.getOperand(1), MaxDepth - 1); 9616 } 9617 case ISD::UNDEF: 9618 // Could be anything. 9619 return false; 9620 9621 case ISD::BITCAST: 9622 return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1); 9623 case ISD::TRUNCATE: { 9624 // Hack round the mess we make when legalizing extract_vector_elt 9625 if (Op.getValueType() == MVT::i16) { 9626 SDValue TruncSrc = Op.getOperand(0); 9627 if (TruncSrc.getValueType() == MVT::i32 && 9628 TruncSrc.getOpcode() == ISD::BITCAST && 9629 TruncSrc.getOperand(0).getValueType() == MVT::v2f16) { 9630 return isCanonicalized(DAG, TruncSrc.getOperand(0), MaxDepth - 1); 9631 } 9632 } 9633 return false; 9634 } 9635 case ISD::INTRINSIC_WO_CHAIN: { 9636 unsigned IntrinsicID 9637 = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 9638 // TODO: Handle more intrinsics 9639 switch (IntrinsicID) { 9640 case Intrinsic::amdgcn_cvt_pkrtz: 9641 case Intrinsic::amdgcn_cubeid: 9642 case Intrinsic::amdgcn_frexp_mant: 9643 case Intrinsic::amdgcn_fdot2: 9644 case Intrinsic::amdgcn_rcp: 9645 case Intrinsic::amdgcn_rsq: 9646 case Intrinsic::amdgcn_rsq_clamp: 9647 case Intrinsic::amdgcn_rcp_legacy: 9648 case Intrinsic::amdgcn_rsq_legacy: 9649 case Intrinsic::amdgcn_trig_preop: 9650 return true; 9651 default: 9652 break; 9653 } 9654 9655 LLVM_FALLTHROUGH; 9656 } 9657 default: 9658 return denormalsEnabledForType(DAG, Op.getValueType()) && 9659 DAG.isKnownNeverSNaN(Op); 9660 } 9661 9662 llvm_unreachable("invalid operation"); 9663 } 9664 9665 bool SITargetLowering::isCanonicalized(Register Reg, MachineFunction &MF, 9666 unsigned MaxDepth) const { 9667 MachineRegisterInfo &MRI = MF.getRegInfo(); 9668 MachineInstr *MI = MRI.getVRegDef(Reg); 9669 unsigned Opcode = MI->getOpcode(); 9670 9671 if (Opcode == AMDGPU::G_FCANONICALIZE) 9672 return true; 9673 9674 if (Opcode == AMDGPU::G_FCONSTANT) { 9675 auto F = MI->getOperand(1).getFPImm()->getValueAPF(); 9676 if (F.isNaN() && F.isSignaling()) 9677 return false; 9678 return !F.isDenormal() || denormalsEnabledForType(MRI.getType(Reg), MF); 9679 } 9680 9681 if (MaxDepth == 0) 9682 return false; 9683 9684 switch (Opcode) { 9685 case AMDGPU::G_FMINNUM_IEEE: 9686 case AMDGPU::G_FMAXNUM_IEEE: { 9687 if (Subtarget->supportsMinMaxDenormModes() || 9688 denormalsEnabledForType(MRI.getType(Reg), MF)) 9689 return true; 9690 for (unsigned I = 1, E = MI->getNumOperands(); I != E; ++I) { 9691 if (!isCanonicalized(MI->getOperand(I).getReg(), MF, MaxDepth - 1)) 9692 return false; 9693 } 9694 return true; 9695 } 9696 default: 9697 return denormalsEnabledForType(MRI.getType(Reg), MF) && 9698 isKnownNeverSNaN(Reg, MRI); 9699 } 9700 9701 llvm_unreachable("invalid operation"); 9702 } 9703 9704 // Constant fold canonicalize. 9705 SDValue SITargetLowering::getCanonicalConstantFP( 9706 SelectionDAG &DAG, const SDLoc &SL, EVT VT, const APFloat &C) const { 9707 // Flush denormals to 0 if not enabled. 9708 if (C.isDenormal() && !denormalsEnabledForType(DAG, VT)) 9709 return DAG.getConstantFP(0.0, SL, VT); 9710 9711 if (C.isNaN()) { 9712 APFloat CanonicalQNaN = APFloat::getQNaN(C.getSemantics()); 9713 if (C.isSignaling()) { 9714 // Quiet a signaling NaN. 9715 // FIXME: Is this supposed to preserve payload bits? 9716 return DAG.getConstantFP(CanonicalQNaN, SL, VT); 9717 } 9718 9719 // Make sure it is the canonical NaN bitpattern. 9720 // 9721 // TODO: Can we use -1 as the canonical NaN value since it's an inline 9722 // immediate? 9723 if (C.bitcastToAPInt() != CanonicalQNaN.bitcastToAPInt()) 9724 return DAG.getConstantFP(CanonicalQNaN, SL, VT); 9725 } 9726 9727 // Already canonical. 9728 return DAG.getConstantFP(C, SL, VT); 9729 } 9730 9731 static bool vectorEltWillFoldAway(SDValue Op) { 9732 return Op.isUndef() || isa<ConstantFPSDNode>(Op); 9733 } 9734 9735 SDValue SITargetLowering::performFCanonicalizeCombine( 9736 SDNode *N, 9737 DAGCombinerInfo &DCI) const { 9738 SelectionDAG &DAG = DCI.DAG; 9739 SDValue N0 = N->getOperand(0); 9740 EVT VT = N->getValueType(0); 9741 9742 // fcanonicalize undef -> qnan 9743 if (N0.isUndef()) { 9744 APFloat QNaN = APFloat::getQNaN(SelectionDAG::EVTToAPFloatSemantics(VT)); 9745 return DAG.getConstantFP(QNaN, SDLoc(N), VT); 9746 } 9747 9748 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N0)) { 9749 EVT VT = N->getValueType(0); 9750 return getCanonicalConstantFP(DAG, SDLoc(N), VT, CFP->getValueAPF()); 9751 } 9752 9753 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x), 9754 // (fcanonicalize k) 9755 // 9756 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0 9757 9758 // TODO: This could be better with wider vectors that will be split to v2f16, 9759 // and to consider uses since there aren't that many packed operations. 9760 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 && 9761 isTypeLegal(MVT::v2f16)) { 9762 SDLoc SL(N); 9763 SDValue NewElts[2]; 9764 SDValue Lo = N0.getOperand(0); 9765 SDValue Hi = N0.getOperand(1); 9766 EVT EltVT = Lo.getValueType(); 9767 9768 if (vectorEltWillFoldAway(Lo) || vectorEltWillFoldAway(Hi)) { 9769 for (unsigned I = 0; I != 2; ++I) { 9770 SDValue Op = N0.getOperand(I); 9771 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) { 9772 NewElts[I] = getCanonicalConstantFP(DAG, SL, EltVT, 9773 CFP->getValueAPF()); 9774 } else if (Op.isUndef()) { 9775 // Handled below based on what the other operand is. 9776 NewElts[I] = Op; 9777 } else { 9778 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op); 9779 } 9780 } 9781 9782 // If one half is undef, and one is constant, perfer a splat vector rather 9783 // than the normal qNaN. If it's a register, prefer 0.0 since that's 9784 // cheaper to use and may be free with a packed operation. 9785 if (NewElts[0].isUndef()) { 9786 if (isa<ConstantFPSDNode>(NewElts[1])) 9787 NewElts[0] = isa<ConstantFPSDNode>(NewElts[1]) ? 9788 NewElts[1]: DAG.getConstantFP(0.0f, SL, EltVT); 9789 } 9790 9791 if (NewElts[1].isUndef()) { 9792 NewElts[1] = isa<ConstantFPSDNode>(NewElts[0]) ? 9793 NewElts[0] : DAG.getConstantFP(0.0f, SL, EltVT); 9794 } 9795 9796 return DAG.getBuildVector(VT, SL, NewElts); 9797 } 9798 } 9799 9800 unsigned SrcOpc = N0.getOpcode(); 9801 9802 // If it's free to do so, push canonicalizes further up the source, which may 9803 // find a canonical source. 9804 // 9805 // TODO: More opcodes. Note this is unsafe for the the _ieee minnum/maxnum for 9806 // sNaNs. 9807 if (SrcOpc == ISD::FMINNUM || SrcOpc == ISD::FMAXNUM) { 9808 auto *CRHS = dyn_cast<ConstantFPSDNode>(N0.getOperand(1)); 9809 if (CRHS && N0.hasOneUse()) { 9810 SDLoc SL(N); 9811 SDValue Canon0 = DAG.getNode(ISD::FCANONICALIZE, SL, VT, 9812 N0.getOperand(0)); 9813 SDValue Canon1 = getCanonicalConstantFP(DAG, SL, VT, CRHS->getValueAPF()); 9814 DCI.AddToWorklist(Canon0.getNode()); 9815 9816 return DAG.getNode(N0.getOpcode(), SL, VT, Canon0, Canon1); 9817 } 9818 } 9819 9820 return isCanonicalized(DAG, N0) ? N0 : SDValue(); 9821 } 9822 9823 static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) { 9824 switch (Opc) { 9825 case ISD::FMAXNUM: 9826 case ISD::FMAXNUM_IEEE: 9827 return AMDGPUISD::FMAX3; 9828 case ISD::SMAX: 9829 return AMDGPUISD::SMAX3; 9830 case ISD::UMAX: 9831 return AMDGPUISD::UMAX3; 9832 case ISD::FMINNUM: 9833 case ISD::FMINNUM_IEEE: 9834 return AMDGPUISD::FMIN3; 9835 case ISD::SMIN: 9836 return AMDGPUISD::SMIN3; 9837 case ISD::UMIN: 9838 return AMDGPUISD::UMIN3; 9839 default: 9840 llvm_unreachable("Not a min/max opcode"); 9841 } 9842 } 9843 9844 SDValue SITargetLowering::performIntMed3ImmCombine( 9845 SelectionDAG &DAG, const SDLoc &SL, 9846 SDValue Op0, SDValue Op1, bool Signed) const { 9847 ConstantSDNode *K1 = dyn_cast<ConstantSDNode>(Op1); 9848 if (!K1) 9849 return SDValue(); 9850 9851 ConstantSDNode *K0 = dyn_cast<ConstantSDNode>(Op0.getOperand(1)); 9852 if (!K0) 9853 return SDValue(); 9854 9855 if (Signed) { 9856 if (K0->getAPIntValue().sge(K1->getAPIntValue())) 9857 return SDValue(); 9858 } else { 9859 if (K0->getAPIntValue().uge(K1->getAPIntValue())) 9860 return SDValue(); 9861 } 9862 9863 EVT VT = K0->getValueType(0); 9864 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3; 9865 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16())) { 9866 return DAG.getNode(Med3Opc, SL, VT, 9867 Op0.getOperand(0), SDValue(K0, 0), SDValue(K1, 0)); 9868 } 9869 9870 // If there isn't a 16-bit med3 operation, convert to 32-bit. 9871 if (VT == MVT::i16) { 9872 MVT NVT = MVT::i32; 9873 unsigned ExtOp = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 9874 9875 SDValue Tmp1 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(0)); 9876 SDValue Tmp2 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(1)); 9877 SDValue Tmp3 = DAG.getNode(ExtOp, SL, NVT, Op1); 9878 9879 SDValue Med3 = DAG.getNode(Med3Opc, SL, NVT, Tmp1, Tmp2, Tmp3); 9880 return DAG.getNode(ISD::TRUNCATE, SL, VT, Med3); 9881 } 9882 9883 return SDValue(); 9884 } 9885 9886 static ConstantFPSDNode *getSplatConstantFP(SDValue Op) { 9887 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 9888 return C; 9889 9890 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op)) { 9891 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode()) 9892 return C; 9893 } 9894 9895 return nullptr; 9896 } 9897 9898 SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG, 9899 const SDLoc &SL, 9900 SDValue Op0, 9901 SDValue Op1) const { 9902 ConstantFPSDNode *K1 = getSplatConstantFP(Op1); 9903 if (!K1) 9904 return SDValue(); 9905 9906 ConstantFPSDNode *K0 = getSplatConstantFP(Op0.getOperand(1)); 9907 if (!K0) 9908 return SDValue(); 9909 9910 // Ordered >= (although NaN inputs should have folded away by now). 9911 if (K0->getValueAPF() > K1->getValueAPF()) 9912 return SDValue(); 9913 9914 const MachineFunction &MF = DAG.getMachineFunction(); 9915 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 9916 9917 // TODO: Check IEEE bit enabled? 9918 EVT VT = Op0.getValueType(); 9919 if (Info->getMode().DX10Clamp) { 9920 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the 9921 // hardware fmed3 behavior converting to a min. 9922 // FIXME: Should this be allowing -0.0? 9923 if (K1->isExactlyValue(1.0) && K0->isExactlyValue(0.0)) 9924 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0)); 9925 } 9926 9927 // med3 for f16 is only available on gfx9+, and not available for v2f16. 9928 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) { 9929 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a 9930 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would 9931 // then give the other result, which is different from med3 with a NaN 9932 // input. 9933 SDValue Var = Op0.getOperand(0); 9934 if (!DAG.isKnownNeverSNaN(Var)) 9935 return SDValue(); 9936 9937 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 9938 9939 if ((!K0->hasOneUse() || 9940 TII->isInlineConstant(K0->getValueAPF().bitcastToAPInt())) && 9941 (!K1->hasOneUse() || 9942 TII->isInlineConstant(K1->getValueAPF().bitcastToAPInt()))) { 9943 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0), 9944 Var, SDValue(K0, 0), SDValue(K1, 0)); 9945 } 9946 } 9947 9948 return SDValue(); 9949 } 9950 9951 SDValue SITargetLowering::performMinMaxCombine(SDNode *N, 9952 DAGCombinerInfo &DCI) const { 9953 SelectionDAG &DAG = DCI.DAG; 9954 9955 EVT VT = N->getValueType(0); 9956 unsigned Opc = N->getOpcode(); 9957 SDValue Op0 = N->getOperand(0); 9958 SDValue Op1 = N->getOperand(1); 9959 9960 // Only do this if the inner op has one use since this will just increases 9961 // register pressure for no benefit. 9962 9963 if (Opc != AMDGPUISD::FMIN_LEGACY && Opc != AMDGPUISD::FMAX_LEGACY && 9964 !VT.isVector() && 9965 (VT == MVT::i32 || VT == MVT::f32 || 9966 ((VT == MVT::f16 || VT == MVT::i16) && Subtarget->hasMin3Max3_16()))) { 9967 // max(max(a, b), c) -> max3(a, b, c) 9968 // min(min(a, b), c) -> min3(a, b, c) 9969 if (Op0.getOpcode() == Opc && Op0.hasOneUse()) { 9970 SDLoc DL(N); 9971 return DAG.getNode(minMaxOpcToMin3Max3Opc(Opc), 9972 DL, 9973 N->getValueType(0), 9974 Op0.getOperand(0), 9975 Op0.getOperand(1), 9976 Op1); 9977 } 9978 9979 // Try commuted. 9980 // max(a, max(b, c)) -> max3(a, b, c) 9981 // min(a, min(b, c)) -> min3(a, b, c) 9982 if (Op1.getOpcode() == Opc && Op1.hasOneUse()) { 9983 SDLoc DL(N); 9984 return DAG.getNode(minMaxOpcToMin3Max3Opc(Opc), 9985 DL, 9986 N->getValueType(0), 9987 Op0, 9988 Op1.getOperand(0), 9989 Op1.getOperand(1)); 9990 } 9991 } 9992 9993 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1) 9994 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) { 9995 if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, true)) 9996 return Med3; 9997 } 9998 9999 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) { 10000 if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, false)) 10001 return Med3; 10002 } 10003 10004 // fminnum(fmaxnum(x, K0), K1), K0 < K1 && !is_snan(x) -> fmed3(x, K0, K1) 10005 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) || 10006 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) || 10007 (Opc == AMDGPUISD::FMIN_LEGACY && 10008 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) && 10009 (VT == MVT::f32 || VT == MVT::f64 || 10010 (VT == MVT::f16 && Subtarget->has16BitInsts()) || 10011 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) && 10012 Op0.hasOneUse()) { 10013 if (SDValue Res = performFPMed3ImmCombine(DAG, SDLoc(N), Op0, Op1)) 10014 return Res; 10015 } 10016 10017 return SDValue(); 10018 } 10019 10020 static bool isClampZeroToOne(SDValue A, SDValue B) { 10021 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) { 10022 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) { 10023 // FIXME: Should this be allowing -0.0? 10024 return (CA->isExactlyValue(0.0) && CB->isExactlyValue(1.0)) || 10025 (CA->isExactlyValue(1.0) && CB->isExactlyValue(0.0)); 10026 } 10027 } 10028 10029 return false; 10030 } 10031 10032 // FIXME: Should only worry about snans for version with chain. 10033 SDValue SITargetLowering::performFMed3Combine(SDNode *N, 10034 DAGCombinerInfo &DCI) const { 10035 EVT VT = N->getValueType(0); 10036 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and 10037 // NaNs. With a NaN input, the order of the operands may change the result. 10038 10039 SelectionDAG &DAG = DCI.DAG; 10040 SDLoc SL(N); 10041 10042 SDValue Src0 = N->getOperand(0); 10043 SDValue Src1 = N->getOperand(1); 10044 SDValue Src2 = N->getOperand(2); 10045 10046 if (isClampZeroToOne(Src0, Src1)) { 10047 // const_a, const_b, x -> clamp is safe in all cases including signaling 10048 // nans. 10049 // FIXME: Should this be allowing -0.0? 10050 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2); 10051 } 10052 10053 const MachineFunction &MF = DAG.getMachineFunction(); 10054 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 10055 10056 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother 10057 // handling no dx10-clamp? 10058 if (Info->getMode().DX10Clamp) { 10059 // If NaNs is clamped to 0, we are free to reorder the inputs. 10060 10061 if (isa<ConstantFPSDNode>(Src0) && !isa<ConstantFPSDNode>(Src1)) 10062 std::swap(Src0, Src1); 10063 10064 if (isa<ConstantFPSDNode>(Src1) && !isa<ConstantFPSDNode>(Src2)) 10065 std::swap(Src1, Src2); 10066 10067 if (isa<ConstantFPSDNode>(Src0) && !isa<ConstantFPSDNode>(Src1)) 10068 std::swap(Src0, Src1); 10069 10070 if (isClampZeroToOne(Src1, Src2)) 10071 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0); 10072 } 10073 10074 return SDValue(); 10075 } 10076 10077 SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N, 10078 DAGCombinerInfo &DCI) const { 10079 SDValue Src0 = N->getOperand(0); 10080 SDValue Src1 = N->getOperand(1); 10081 if (Src0.isUndef() && Src1.isUndef()) 10082 return DCI.DAG.getUNDEF(N->getValueType(0)); 10083 return SDValue(); 10084 } 10085 10086 // Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be 10087 // expanded into a set of cmp/select instructions. 10088 bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize, 10089 unsigned NumElem, 10090 bool IsDivergentIdx) { 10091 if (UseDivergentRegisterIndexing) 10092 return false; 10093 10094 unsigned VecSize = EltSize * NumElem; 10095 10096 // Sub-dword vectors of size 2 dword or less have better implementation. 10097 if (VecSize <= 64 && EltSize < 32) 10098 return false; 10099 10100 // Always expand the rest of sub-dword instructions, otherwise it will be 10101 // lowered via memory. 10102 if (EltSize < 32) 10103 return true; 10104 10105 // Always do this if var-idx is divergent, otherwise it will become a loop. 10106 if (IsDivergentIdx) 10107 return true; 10108 10109 // Large vectors would yield too many compares and v_cndmask_b32 instructions. 10110 unsigned NumInsts = NumElem /* Number of compares */ + 10111 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */; 10112 return NumInsts <= 16; 10113 } 10114 10115 static bool shouldExpandVectorDynExt(SDNode *N) { 10116 SDValue Idx = N->getOperand(N->getNumOperands() - 1); 10117 if (isa<ConstantSDNode>(Idx)) 10118 return false; 10119 10120 SDValue Vec = N->getOperand(0); 10121 EVT VecVT = Vec.getValueType(); 10122 EVT EltVT = VecVT.getVectorElementType(); 10123 unsigned EltSize = EltVT.getSizeInBits(); 10124 unsigned NumElem = VecVT.getVectorNumElements(); 10125 10126 return SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem, 10127 Idx->isDivergent()); 10128 } 10129 10130 SDValue SITargetLowering::performExtractVectorEltCombine( 10131 SDNode *N, DAGCombinerInfo &DCI) const { 10132 SDValue Vec = N->getOperand(0); 10133 SelectionDAG &DAG = DCI.DAG; 10134 10135 EVT VecVT = Vec.getValueType(); 10136 EVT EltVT = VecVT.getVectorElementType(); 10137 10138 if ((Vec.getOpcode() == ISD::FNEG || 10139 Vec.getOpcode() == ISD::FABS) && allUsesHaveSourceMods(N)) { 10140 SDLoc SL(N); 10141 EVT EltVT = N->getValueType(0); 10142 SDValue Idx = N->getOperand(1); 10143 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 10144 Vec.getOperand(0), Idx); 10145 return DAG.getNode(Vec.getOpcode(), SL, EltVT, Elt); 10146 } 10147 10148 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx) 10149 // => 10150 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx) 10151 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx) 10152 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt 10153 if (Vec.hasOneUse() && DCI.isBeforeLegalize()) { 10154 SDLoc SL(N); 10155 EVT EltVT = N->getValueType(0); 10156 SDValue Idx = N->getOperand(1); 10157 unsigned Opc = Vec.getOpcode(); 10158 10159 switch(Opc) { 10160 default: 10161 break; 10162 // TODO: Support other binary operations. 10163 case ISD::FADD: 10164 case ISD::FSUB: 10165 case ISD::FMUL: 10166 case ISD::ADD: 10167 case ISD::UMIN: 10168 case ISD::UMAX: 10169 case ISD::SMIN: 10170 case ISD::SMAX: 10171 case ISD::FMAXNUM: 10172 case ISD::FMINNUM: 10173 case ISD::FMAXNUM_IEEE: 10174 case ISD::FMINNUM_IEEE: { 10175 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 10176 Vec.getOperand(0), Idx); 10177 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 10178 Vec.getOperand(1), Idx); 10179 10180 DCI.AddToWorklist(Elt0.getNode()); 10181 DCI.AddToWorklist(Elt1.getNode()); 10182 return DAG.getNode(Opc, SL, EltVT, Elt0, Elt1, Vec->getFlags()); 10183 } 10184 } 10185 } 10186 10187 unsigned VecSize = VecVT.getSizeInBits(); 10188 unsigned EltSize = EltVT.getSizeInBits(); 10189 10190 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx) 10191 if (::shouldExpandVectorDynExt(N)) { 10192 SDLoc SL(N); 10193 SDValue Idx = N->getOperand(1); 10194 SDValue V; 10195 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) { 10196 SDValue IC = DAG.getVectorIdxConstant(I, SL); 10197 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC); 10198 if (I == 0) 10199 V = Elt; 10200 else 10201 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ); 10202 } 10203 return V; 10204 } 10205 10206 if (!DCI.isBeforeLegalize()) 10207 return SDValue(); 10208 10209 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit 10210 // elements. This exposes more load reduction opportunities by replacing 10211 // multiple small extract_vector_elements with a single 32-bit extract. 10212 auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10213 if (isa<MemSDNode>(Vec) && 10214 EltSize <= 16 && 10215 EltVT.isByteSized() && 10216 VecSize > 32 && 10217 VecSize % 32 == 0 && 10218 Idx) { 10219 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT); 10220 10221 unsigned BitIndex = Idx->getZExtValue() * EltSize; 10222 unsigned EltIdx = BitIndex / 32; 10223 unsigned LeftoverBitIdx = BitIndex % 32; 10224 SDLoc SL(N); 10225 10226 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec); 10227 DCI.AddToWorklist(Cast.getNode()); 10228 10229 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, 10230 DAG.getConstant(EltIdx, SL, MVT::i32)); 10231 DCI.AddToWorklist(Elt.getNode()); 10232 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, 10233 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); 10234 DCI.AddToWorklist(Srl.getNode()); 10235 10236 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, EltVT.changeTypeToInteger(), Srl); 10237 DCI.AddToWorklist(Trunc.getNode()); 10238 return DAG.getNode(ISD::BITCAST, SL, EltVT, Trunc); 10239 } 10240 10241 return SDValue(); 10242 } 10243 10244 SDValue 10245 SITargetLowering::performInsertVectorEltCombine(SDNode *N, 10246 DAGCombinerInfo &DCI) const { 10247 SDValue Vec = N->getOperand(0); 10248 SDValue Idx = N->getOperand(2); 10249 EVT VecVT = Vec.getValueType(); 10250 EVT EltVT = VecVT.getVectorElementType(); 10251 10252 // INSERT_VECTOR_ELT (<n x e>, var-idx) 10253 // => BUILD_VECTOR n x select (e, const-idx) 10254 if (!::shouldExpandVectorDynExt(N)) 10255 return SDValue(); 10256 10257 SelectionDAG &DAG = DCI.DAG; 10258 SDLoc SL(N); 10259 SDValue Ins = N->getOperand(1); 10260 EVT IdxVT = Idx.getValueType(); 10261 10262 SmallVector<SDValue, 16> Ops; 10263 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) { 10264 SDValue IC = DAG.getConstant(I, SL, IdxVT); 10265 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC); 10266 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ); 10267 Ops.push_back(V); 10268 } 10269 10270 return DAG.getBuildVector(VecVT, SL, Ops); 10271 } 10272 10273 unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG, 10274 const SDNode *N0, 10275 const SDNode *N1) const { 10276 EVT VT = N0->getValueType(0); 10277 10278 // Only do this if we are not trying to support denormals. v_mad_f32 does not 10279 // support denormals ever. 10280 if (((VT == MVT::f32 && !hasFP32Denormals(DAG.getMachineFunction())) || 10281 (VT == MVT::f16 && !hasFP64FP16Denormals(DAG.getMachineFunction()) && 10282 getSubtarget()->hasMadF16())) && 10283 isOperationLegal(ISD::FMAD, VT)) 10284 return ISD::FMAD; 10285 10286 const TargetOptions &Options = DAG.getTarget().Options; 10287 if ((Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath || 10288 (N0->getFlags().hasAllowContract() && 10289 N1->getFlags().hasAllowContract())) && 10290 isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { 10291 return ISD::FMA; 10292 } 10293 10294 return 0; 10295 } 10296 10297 // For a reassociatable opcode perform: 10298 // op x, (op y, z) -> op (op x, z), y, if x and z are uniform 10299 SDValue SITargetLowering::reassociateScalarOps(SDNode *N, 10300 SelectionDAG &DAG) const { 10301 EVT VT = N->getValueType(0); 10302 if (VT != MVT::i32 && VT != MVT::i64) 10303 return SDValue(); 10304 10305 unsigned Opc = N->getOpcode(); 10306 SDValue Op0 = N->getOperand(0); 10307 SDValue Op1 = N->getOperand(1); 10308 10309 if (!(Op0->isDivergent() ^ Op1->isDivergent())) 10310 return SDValue(); 10311 10312 if (Op0->isDivergent()) 10313 std::swap(Op0, Op1); 10314 10315 if (Op1.getOpcode() != Opc || !Op1.hasOneUse()) 10316 return SDValue(); 10317 10318 SDValue Op2 = Op1.getOperand(1); 10319 Op1 = Op1.getOperand(0); 10320 if (!(Op1->isDivergent() ^ Op2->isDivergent())) 10321 return SDValue(); 10322 10323 if (Op1->isDivergent()) 10324 std::swap(Op1, Op2); 10325 10326 // If either operand is constant this will conflict with 10327 // DAGCombiner::ReassociateOps(). 10328 if (DAG.isConstantIntBuildVectorOrConstantInt(Op0) || 10329 DAG.isConstantIntBuildVectorOrConstantInt(Op1)) 10330 return SDValue(); 10331 10332 SDLoc SL(N); 10333 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1); 10334 return DAG.getNode(Opc, SL, VT, Add1, Op2); 10335 } 10336 10337 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, 10338 EVT VT, 10339 SDValue N0, SDValue N1, SDValue N2, 10340 bool Signed) { 10341 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32; 10342 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i1); 10343 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2); 10344 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad); 10345 } 10346 10347 SDValue SITargetLowering::performAddCombine(SDNode *N, 10348 DAGCombinerInfo &DCI) const { 10349 SelectionDAG &DAG = DCI.DAG; 10350 EVT VT = N->getValueType(0); 10351 SDLoc SL(N); 10352 SDValue LHS = N->getOperand(0); 10353 SDValue RHS = N->getOperand(1); 10354 10355 if ((LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) 10356 && Subtarget->hasMad64_32() && 10357 !VT.isVector() && VT.getScalarSizeInBits() > 32 && 10358 VT.getScalarSizeInBits() <= 64) { 10359 if (LHS.getOpcode() != ISD::MUL) 10360 std::swap(LHS, RHS); 10361 10362 SDValue MulLHS = LHS.getOperand(0); 10363 SDValue MulRHS = LHS.getOperand(1); 10364 SDValue AddRHS = RHS; 10365 10366 // TODO: Maybe restrict if SGPR inputs. 10367 if (numBitsUnsigned(MulLHS, DAG) <= 32 && 10368 numBitsUnsigned(MulRHS, DAG) <= 32) { 10369 MulLHS = DAG.getZExtOrTrunc(MulLHS, SL, MVT::i32); 10370 MulRHS = DAG.getZExtOrTrunc(MulRHS, SL, MVT::i32); 10371 AddRHS = DAG.getZExtOrTrunc(AddRHS, SL, MVT::i64); 10372 return getMad64_32(DAG, SL, VT, MulLHS, MulRHS, AddRHS, false); 10373 } 10374 10375 if (numBitsSigned(MulLHS, DAG) < 32 && numBitsSigned(MulRHS, DAG) < 32) { 10376 MulLHS = DAG.getSExtOrTrunc(MulLHS, SL, MVT::i32); 10377 MulRHS = DAG.getSExtOrTrunc(MulRHS, SL, MVT::i32); 10378 AddRHS = DAG.getSExtOrTrunc(AddRHS, SL, MVT::i64); 10379 return getMad64_32(DAG, SL, VT, MulLHS, MulRHS, AddRHS, true); 10380 } 10381 10382 return SDValue(); 10383 } 10384 10385 if (SDValue V = reassociateScalarOps(N, DAG)) { 10386 return V; 10387 } 10388 10389 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG()) 10390 return SDValue(); 10391 10392 // add x, zext (setcc) => addcarry x, 0, setcc 10393 // add x, sext (setcc) => subcarry x, 0, setcc 10394 unsigned Opc = LHS.getOpcode(); 10395 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND || 10396 Opc == ISD::ANY_EXTEND || Opc == ISD::ADDCARRY) 10397 std::swap(RHS, LHS); 10398 10399 Opc = RHS.getOpcode(); 10400 switch (Opc) { 10401 default: break; 10402 case ISD::ZERO_EXTEND: 10403 case ISD::SIGN_EXTEND: 10404 case ISD::ANY_EXTEND: { 10405 auto Cond = RHS.getOperand(0); 10406 // If this won't be a real VOPC output, we would still need to insert an 10407 // extra instruction anyway. 10408 if (!isBoolSGPR(Cond)) 10409 break; 10410 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); 10411 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; 10412 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::SUBCARRY : ISD::ADDCARRY; 10413 return DAG.getNode(Opc, SL, VTList, Args); 10414 } 10415 case ISD::ADDCARRY: { 10416 // add x, (addcarry y, 0, cc) => addcarry x, y, cc 10417 auto C = dyn_cast<ConstantSDNode>(RHS.getOperand(1)); 10418 if (!C || C->getZExtValue() != 0) break; 10419 SDValue Args[] = { LHS, RHS.getOperand(0), RHS.getOperand(2) }; 10420 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), RHS->getVTList(), Args); 10421 } 10422 } 10423 return SDValue(); 10424 } 10425 10426 SDValue SITargetLowering::performSubCombine(SDNode *N, 10427 DAGCombinerInfo &DCI) const { 10428 SelectionDAG &DAG = DCI.DAG; 10429 EVT VT = N->getValueType(0); 10430 10431 if (VT != MVT::i32) 10432 return SDValue(); 10433 10434 SDLoc SL(N); 10435 SDValue LHS = N->getOperand(0); 10436 SDValue RHS = N->getOperand(1); 10437 10438 // sub x, zext (setcc) => subcarry x, 0, setcc 10439 // sub x, sext (setcc) => addcarry x, 0, setcc 10440 unsigned Opc = RHS.getOpcode(); 10441 switch (Opc) { 10442 default: break; 10443 case ISD::ZERO_EXTEND: 10444 case ISD::SIGN_EXTEND: 10445 case ISD::ANY_EXTEND: { 10446 auto Cond = RHS.getOperand(0); 10447 // If this won't be a real VOPC output, we would still need to insert an 10448 // extra instruction anyway. 10449 if (!isBoolSGPR(Cond)) 10450 break; 10451 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); 10452 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; 10453 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::ADDCARRY : ISD::SUBCARRY; 10454 return DAG.getNode(Opc, SL, VTList, Args); 10455 } 10456 } 10457 10458 if (LHS.getOpcode() == ISD::SUBCARRY) { 10459 // sub (subcarry x, 0, cc), y => subcarry x, y, cc 10460 auto C = dyn_cast<ConstantSDNode>(LHS.getOperand(1)); 10461 if (!C || !C->isNullValue()) 10462 return SDValue(); 10463 SDValue Args[] = { LHS.getOperand(0), RHS, LHS.getOperand(2) }; 10464 return DAG.getNode(ISD::SUBCARRY, SDLoc(N), LHS->getVTList(), Args); 10465 } 10466 return SDValue(); 10467 } 10468 10469 SDValue SITargetLowering::performAddCarrySubCarryCombine(SDNode *N, 10470 DAGCombinerInfo &DCI) const { 10471 10472 if (N->getValueType(0) != MVT::i32) 10473 return SDValue(); 10474 10475 auto C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 10476 if (!C || C->getZExtValue() != 0) 10477 return SDValue(); 10478 10479 SelectionDAG &DAG = DCI.DAG; 10480 SDValue LHS = N->getOperand(0); 10481 10482 // addcarry (add x, y), 0, cc => addcarry x, y, cc 10483 // subcarry (sub x, y), 0, cc => subcarry x, y, cc 10484 unsigned LHSOpc = LHS.getOpcode(); 10485 unsigned Opc = N->getOpcode(); 10486 if ((LHSOpc == ISD::ADD && Opc == ISD::ADDCARRY) || 10487 (LHSOpc == ISD::SUB && Opc == ISD::SUBCARRY)) { 10488 SDValue Args[] = { LHS.getOperand(0), LHS.getOperand(1), N->getOperand(2) }; 10489 return DAG.getNode(Opc, SDLoc(N), N->getVTList(), Args); 10490 } 10491 return SDValue(); 10492 } 10493 10494 SDValue SITargetLowering::performFAddCombine(SDNode *N, 10495 DAGCombinerInfo &DCI) const { 10496 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) 10497 return SDValue(); 10498 10499 SelectionDAG &DAG = DCI.DAG; 10500 EVT VT = N->getValueType(0); 10501 10502 SDLoc SL(N); 10503 SDValue LHS = N->getOperand(0); 10504 SDValue RHS = N->getOperand(1); 10505 10506 // These should really be instruction patterns, but writing patterns with 10507 // source modiifiers is a pain. 10508 10509 // fadd (fadd (a, a), b) -> mad 2.0, a, b 10510 if (LHS.getOpcode() == ISD::FADD) { 10511 SDValue A = LHS.getOperand(0); 10512 if (A == LHS.getOperand(1)) { 10513 unsigned FusedOp = getFusedOpcode(DAG, N, LHS.getNode()); 10514 if (FusedOp != 0) { 10515 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); 10516 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS); 10517 } 10518 } 10519 } 10520 10521 // fadd (b, fadd (a, a)) -> mad 2.0, a, b 10522 if (RHS.getOpcode() == ISD::FADD) { 10523 SDValue A = RHS.getOperand(0); 10524 if (A == RHS.getOperand(1)) { 10525 unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode()); 10526 if (FusedOp != 0) { 10527 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); 10528 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS); 10529 } 10530 } 10531 } 10532 10533 return SDValue(); 10534 } 10535 10536 SDValue SITargetLowering::performFSubCombine(SDNode *N, 10537 DAGCombinerInfo &DCI) const { 10538 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) 10539 return SDValue(); 10540 10541 SelectionDAG &DAG = DCI.DAG; 10542 SDLoc SL(N); 10543 EVT VT = N->getValueType(0); 10544 assert(!VT.isVector()); 10545 10546 // Try to get the fneg to fold into the source modifier. This undoes generic 10547 // DAG combines and folds them into the mad. 10548 // 10549 // Only do this if we are not trying to support denormals. v_mad_f32 does 10550 // not support denormals ever. 10551 SDValue LHS = N->getOperand(0); 10552 SDValue RHS = N->getOperand(1); 10553 if (LHS.getOpcode() == ISD::FADD) { 10554 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c) 10555 SDValue A = LHS.getOperand(0); 10556 if (A == LHS.getOperand(1)) { 10557 unsigned FusedOp = getFusedOpcode(DAG, N, LHS.getNode()); 10558 if (FusedOp != 0){ 10559 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); 10560 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 10561 10562 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS); 10563 } 10564 } 10565 } 10566 10567 if (RHS.getOpcode() == ISD::FADD) { 10568 // (fsub c, (fadd a, a)) -> mad -2.0, a, c 10569 10570 SDValue A = RHS.getOperand(0); 10571 if (A == RHS.getOperand(1)) { 10572 unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode()); 10573 if (FusedOp != 0){ 10574 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT); 10575 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS); 10576 } 10577 } 10578 } 10579 10580 return SDValue(); 10581 } 10582 10583 SDValue SITargetLowering::performFMACombine(SDNode *N, 10584 DAGCombinerInfo &DCI) const { 10585 SelectionDAG &DAG = DCI.DAG; 10586 EVT VT = N->getValueType(0); 10587 SDLoc SL(N); 10588 10589 if (!Subtarget->hasDot7Insts() || VT != MVT::f32) 10590 return SDValue(); 10591 10592 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) -> 10593 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z)) 10594 SDValue Op1 = N->getOperand(0); 10595 SDValue Op2 = N->getOperand(1); 10596 SDValue FMA = N->getOperand(2); 10597 10598 if (FMA.getOpcode() != ISD::FMA || 10599 Op1.getOpcode() != ISD::FP_EXTEND || 10600 Op2.getOpcode() != ISD::FP_EXTEND) 10601 return SDValue(); 10602 10603 // fdot2_f32_f16 always flushes fp32 denormal operand and output to zero, 10604 // regardless of the denorm mode setting. Therefore, unsafe-fp-math/fp-contract 10605 // is sufficient to allow generaing fdot2. 10606 const TargetOptions &Options = DAG.getTarget().Options; 10607 if (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath || 10608 (N->getFlags().hasAllowContract() && 10609 FMA->getFlags().hasAllowContract())) { 10610 Op1 = Op1.getOperand(0); 10611 Op2 = Op2.getOperand(0); 10612 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 10613 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 10614 return SDValue(); 10615 10616 SDValue Vec1 = Op1.getOperand(0); 10617 SDValue Idx1 = Op1.getOperand(1); 10618 SDValue Vec2 = Op2.getOperand(0); 10619 10620 SDValue FMAOp1 = FMA.getOperand(0); 10621 SDValue FMAOp2 = FMA.getOperand(1); 10622 SDValue FMAAcc = FMA.getOperand(2); 10623 10624 if (FMAOp1.getOpcode() != ISD::FP_EXTEND || 10625 FMAOp2.getOpcode() != ISD::FP_EXTEND) 10626 return SDValue(); 10627 10628 FMAOp1 = FMAOp1.getOperand(0); 10629 FMAOp2 = FMAOp2.getOperand(0); 10630 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 10631 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 10632 return SDValue(); 10633 10634 SDValue Vec3 = FMAOp1.getOperand(0); 10635 SDValue Vec4 = FMAOp2.getOperand(0); 10636 SDValue Idx2 = FMAOp1.getOperand(1); 10637 10638 if (Idx1 != Op2.getOperand(1) || Idx2 != FMAOp2.getOperand(1) || 10639 // Idx1 and Idx2 cannot be the same. 10640 Idx1 == Idx2) 10641 return SDValue(); 10642 10643 if (Vec1 == Vec2 || Vec3 == Vec4) 10644 return SDValue(); 10645 10646 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16) 10647 return SDValue(); 10648 10649 if ((Vec1 == Vec3 && Vec2 == Vec4) || 10650 (Vec1 == Vec4 && Vec2 == Vec3)) { 10651 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, 10652 DAG.getTargetConstant(0, SL, MVT::i1)); 10653 } 10654 } 10655 return SDValue(); 10656 } 10657 10658 SDValue SITargetLowering::performSetCCCombine(SDNode *N, 10659 DAGCombinerInfo &DCI) const { 10660 SelectionDAG &DAG = DCI.DAG; 10661 SDLoc SL(N); 10662 10663 SDValue LHS = N->getOperand(0); 10664 SDValue RHS = N->getOperand(1); 10665 EVT VT = LHS.getValueType(); 10666 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 10667 10668 auto CRHS = dyn_cast<ConstantSDNode>(RHS); 10669 if (!CRHS) { 10670 CRHS = dyn_cast<ConstantSDNode>(LHS); 10671 if (CRHS) { 10672 std::swap(LHS, RHS); 10673 CC = getSetCCSwappedOperands(CC); 10674 } 10675 } 10676 10677 if (CRHS) { 10678 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND && 10679 isBoolSGPR(LHS.getOperand(0))) { 10680 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1 10681 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc 10682 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1 10683 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc 10684 if ((CRHS->isAllOnesValue() && 10685 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) || 10686 (CRHS->isNullValue() && 10687 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE))) 10688 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), 10689 DAG.getConstant(-1, SL, MVT::i1)); 10690 if ((CRHS->isAllOnesValue() && 10691 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) || 10692 (CRHS->isNullValue() && 10693 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT))) 10694 return LHS.getOperand(0); 10695 } 10696 10697 uint64_t CRHSVal = CRHS->getZExtValue(); 10698 if ((CC == ISD::SETEQ || CC == ISD::SETNE) && 10699 LHS.getOpcode() == ISD::SELECT && 10700 isa<ConstantSDNode>(LHS.getOperand(1)) && 10701 isa<ConstantSDNode>(LHS.getOperand(2)) && 10702 LHS.getConstantOperandVal(1) != LHS.getConstantOperandVal(2) && 10703 isBoolSGPR(LHS.getOperand(0))) { 10704 // Given CT != FT: 10705 // setcc (select cc, CT, CF), CF, eq => xor cc, -1 10706 // setcc (select cc, CT, CF), CF, ne => cc 10707 // setcc (select cc, CT, CF), CT, ne => xor cc, -1 10708 // setcc (select cc, CT, CF), CT, eq => cc 10709 uint64_t CT = LHS.getConstantOperandVal(1); 10710 uint64_t CF = LHS.getConstantOperandVal(2); 10711 10712 if ((CF == CRHSVal && CC == ISD::SETEQ) || 10713 (CT == CRHSVal && CC == ISD::SETNE)) 10714 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), 10715 DAG.getConstant(-1, SL, MVT::i1)); 10716 if ((CF == CRHSVal && CC == ISD::SETNE) || 10717 (CT == CRHSVal && CC == ISD::SETEQ)) 10718 return LHS.getOperand(0); 10719 } 10720 } 10721 10722 if (VT != MVT::f32 && VT != MVT::f64 && (Subtarget->has16BitInsts() && 10723 VT != MVT::f16)) 10724 return SDValue(); 10725 10726 // Match isinf/isfinite pattern 10727 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity)) 10728 // (fcmp one (fabs x), inf) -> (fp_class x, 10729 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero) 10730 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) && LHS.getOpcode() == ISD::FABS) { 10731 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS); 10732 if (!CRHS) 10733 return SDValue(); 10734 10735 const APFloat &APF = CRHS->getValueAPF(); 10736 if (APF.isInfinity() && !APF.isNegative()) { 10737 const unsigned IsInfMask = SIInstrFlags::P_INFINITY | 10738 SIInstrFlags::N_INFINITY; 10739 const unsigned IsFiniteMask = SIInstrFlags::N_ZERO | 10740 SIInstrFlags::P_ZERO | 10741 SIInstrFlags::N_NORMAL | 10742 SIInstrFlags::P_NORMAL | 10743 SIInstrFlags::N_SUBNORMAL | 10744 SIInstrFlags::P_SUBNORMAL; 10745 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask; 10746 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), 10747 DAG.getConstant(Mask, SL, MVT::i32)); 10748 } 10749 } 10750 10751 return SDValue(); 10752 } 10753 10754 SDValue SITargetLowering::performCvtF32UByteNCombine(SDNode *N, 10755 DAGCombinerInfo &DCI) const { 10756 SelectionDAG &DAG = DCI.DAG; 10757 SDLoc SL(N); 10758 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0; 10759 10760 SDValue Src = N->getOperand(0); 10761 SDValue Shift = N->getOperand(0); 10762 10763 // TODO: Extend type shouldn't matter (assuming legal types). 10764 if (Shift.getOpcode() == ISD::ZERO_EXTEND) 10765 Shift = Shift.getOperand(0); 10766 10767 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) { 10768 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x 10769 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x 10770 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x 10771 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x 10772 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x 10773 if (auto *C = dyn_cast<ConstantSDNode>(Shift.getOperand(1))) { 10774 Shift = DAG.getZExtOrTrunc(Shift.getOperand(0), 10775 SDLoc(Shift.getOperand(0)), MVT::i32); 10776 10777 unsigned ShiftOffset = 8 * Offset; 10778 if (Shift.getOpcode() == ISD::SHL) 10779 ShiftOffset -= C->getZExtValue(); 10780 else 10781 ShiftOffset += C->getZExtValue(); 10782 10783 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) { 10784 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL, 10785 MVT::f32, Shift); 10786 } 10787 } 10788 } 10789 10790 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 10791 APInt DemandedBits = APInt::getBitsSet(32, 8 * Offset, 8 * Offset + 8); 10792 if (TLI.SimplifyDemandedBits(Src, DemandedBits, DCI)) { 10793 // We simplified Src. If this node is not dead, visit it again so it is 10794 // folded properly. 10795 if (N->getOpcode() != ISD::DELETED_NODE) 10796 DCI.AddToWorklist(N); 10797 return SDValue(N, 0); 10798 } 10799 10800 // Handle (or x, (srl y, 8)) pattern when known bits are zero. 10801 if (SDValue DemandedSrc = 10802 TLI.SimplifyMultipleUseDemandedBits(Src, DemandedBits, DAG)) 10803 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc); 10804 10805 return SDValue(); 10806 } 10807 10808 SDValue SITargetLowering::performClampCombine(SDNode *N, 10809 DAGCombinerInfo &DCI) const { 10810 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(N->getOperand(0)); 10811 if (!CSrc) 10812 return SDValue(); 10813 10814 const MachineFunction &MF = DCI.DAG.getMachineFunction(); 10815 const APFloat &F = CSrc->getValueAPF(); 10816 APFloat Zero = APFloat::getZero(F.getSemantics()); 10817 if (F < Zero || 10818 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) { 10819 return DCI.DAG.getConstantFP(Zero, SDLoc(N), N->getValueType(0)); 10820 } 10821 10822 APFloat One(F.getSemantics(), "1.0"); 10823 if (F > One) 10824 return DCI.DAG.getConstantFP(One, SDLoc(N), N->getValueType(0)); 10825 10826 return SDValue(CSrc, 0); 10827 } 10828 10829 10830 SDValue SITargetLowering::PerformDAGCombine(SDNode *N, 10831 DAGCombinerInfo &DCI) const { 10832 if (getTargetMachine().getOptLevel() == CodeGenOpt::None) 10833 return SDValue(); 10834 switch (N->getOpcode()) { 10835 case ISD::ADD: 10836 return performAddCombine(N, DCI); 10837 case ISD::SUB: 10838 return performSubCombine(N, DCI); 10839 case ISD::ADDCARRY: 10840 case ISD::SUBCARRY: 10841 return performAddCarrySubCarryCombine(N, DCI); 10842 case ISD::FADD: 10843 return performFAddCombine(N, DCI); 10844 case ISD::FSUB: 10845 return performFSubCombine(N, DCI); 10846 case ISD::SETCC: 10847 return performSetCCCombine(N, DCI); 10848 case ISD::FMAXNUM: 10849 case ISD::FMINNUM: 10850 case ISD::FMAXNUM_IEEE: 10851 case ISD::FMINNUM_IEEE: 10852 case ISD::SMAX: 10853 case ISD::SMIN: 10854 case ISD::UMAX: 10855 case ISD::UMIN: 10856 case AMDGPUISD::FMIN_LEGACY: 10857 case AMDGPUISD::FMAX_LEGACY: 10858 return performMinMaxCombine(N, DCI); 10859 case ISD::FMA: 10860 return performFMACombine(N, DCI); 10861 case ISD::AND: 10862 return performAndCombine(N, DCI); 10863 case ISD::OR: 10864 return performOrCombine(N, DCI); 10865 case ISD::XOR: 10866 return performXorCombine(N, DCI); 10867 case ISD::ZERO_EXTEND: 10868 return performZeroExtendCombine(N, DCI); 10869 case ISD::SIGN_EXTEND_INREG: 10870 return performSignExtendInRegCombine(N , DCI); 10871 case AMDGPUISD::FP_CLASS: 10872 return performClassCombine(N, DCI); 10873 case ISD::FCANONICALIZE: 10874 return performFCanonicalizeCombine(N, DCI); 10875 case AMDGPUISD::RCP: 10876 return performRcpCombine(N, DCI); 10877 case AMDGPUISD::FRACT: 10878 case AMDGPUISD::RSQ: 10879 case AMDGPUISD::RCP_LEGACY: 10880 case AMDGPUISD::RCP_IFLAG: 10881 case AMDGPUISD::RSQ_CLAMP: 10882 case AMDGPUISD::LDEXP: { 10883 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted 10884 SDValue Src = N->getOperand(0); 10885 if (Src.isUndef()) 10886 return Src; 10887 break; 10888 } 10889 case ISD::SINT_TO_FP: 10890 case ISD::UINT_TO_FP: 10891 return performUCharToFloatCombine(N, DCI); 10892 case AMDGPUISD::CVT_F32_UBYTE0: 10893 case AMDGPUISD::CVT_F32_UBYTE1: 10894 case AMDGPUISD::CVT_F32_UBYTE2: 10895 case AMDGPUISD::CVT_F32_UBYTE3: 10896 return performCvtF32UByteNCombine(N, DCI); 10897 case AMDGPUISD::FMED3: 10898 return performFMed3Combine(N, DCI); 10899 case AMDGPUISD::CVT_PKRTZ_F16_F32: 10900 return performCvtPkRTZCombine(N, DCI); 10901 case AMDGPUISD::CLAMP: 10902 return performClampCombine(N, DCI); 10903 case ISD::SCALAR_TO_VECTOR: { 10904 SelectionDAG &DAG = DCI.DAG; 10905 EVT VT = N->getValueType(0); 10906 10907 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x)) 10908 if (VT == MVT::v2i16 || VT == MVT::v2f16) { 10909 SDLoc SL(N); 10910 SDValue Src = N->getOperand(0); 10911 EVT EltVT = Src.getValueType(); 10912 if (EltVT == MVT::f16) 10913 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); 10914 10915 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); 10916 return DAG.getNode(ISD::BITCAST, SL, VT, Ext); 10917 } 10918 10919 break; 10920 } 10921 case ISD::EXTRACT_VECTOR_ELT: 10922 return performExtractVectorEltCombine(N, DCI); 10923 case ISD::INSERT_VECTOR_ELT: 10924 return performInsertVectorEltCombine(N, DCI); 10925 case ISD::LOAD: { 10926 if (SDValue Widended = widenLoad(cast<LoadSDNode>(N), DCI)) 10927 return Widended; 10928 LLVM_FALLTHROUGH; 10929 } 10930 default: { 10931 if (!DCI.isBeforeLegalize()) { 10932 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(N)) 10933 return performMemSDNodeCombine(MemNode, DCI); 10934 } 10935 10936 break; 10937 } 10938 } 10939 10940 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 10941 } 10942 10943 /// Helper function for adjustWritemask 10944 static unsigned SubIdx2Lane(unsigned Idx) { 10945 switch (Idx) { 10946 default: return ~0u; 10947 case AMDGPU::sub0: return 0; 10948 case AMDGPU::sub1: return 1; 10949 case AMDGPU::sub2: return 2; 10950 case AMDGPU::sub3: return 3; 10951 case AMDGPU::sub4: return 4; // Possible with TFE/LWE 10952 } 10953 } 10954 10955 /// Adjust the writemask of MIMG instructions 10956 SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node, 10957 SelectionDAG &DAG) const { 10958 unsigned Opcode = Node->getMachineOpcode(); 10959 10960 // Subtract 1 because the vdata output is not a MachineSDNode operand. 10961 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::d16) - 1; 10962 if (D16Idx >= 0 && Node->getConstantOperandVal(D16Idx)) 10963 return Node; // not implemented for D16 10964 10965 SDNode *Users[5] = { nullptr }; 10966 unsigned Lane = 0; 10967 unsigned DmaskIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::dmask) - 1; 10968 unsigned OldDmask = Node->getConstantOperandVal(DmaskIdx); 10969 unsigned NewDmask = 0; 10970 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::tfe) - 1; 10971 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::lwe) - 1; 10972 bool UsesTFC = ((int(TFEIdx) >= 0 && Node->getConstantOperandVal(TFEIdx)) || 10973 Node->getConstantOperandVal(LWEIdx)) ? 1 : 0; 10974 unsigned TFCLane = 0; 10975 bool HasChain = Node->getNumValues() > 1; 10976 10977 if (OldDmask == 0) { 10978 // These are folded out, but on the chance it happens don't assert. 10979 return Node; 10980 } 10981 10982 unsigned OldBitsSet = countPopulation(OldDmask); 10983 // Work out which is the TFE/LWE lane if that is enabled. 10984 if (UsesTFC) { 10985 TFCLane = OldBitsSet; 10986 } 10987 10988 // Try to figure out the used register components 10989 for (SDNode::use_iterator I = Node->use_begin(), E = Node->use_end(); 10990 I != E; ++I) { 10991 10992 // Don't look at users of the chain. 10993 if (I.getUse().getResNo() != 0) 10994 continue; 10995 10996 // Abort if we can't understand the usage 10997 if (!I->isMachineOpcode() || 10998 I->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG) 10999 return Node; 11000 11001 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used. 11002 // Note that subregs are packed, i.e. Lane==0 is the first bit set 11003 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit 11004 // set, etc. 11005 Lane = SubIdx2Lane(I->getConstantOperandVal(1)); 11006 if (Lane == ~0u) 11007 return Node; 11008 11009 // Check if the use is for the TFE/LWE generated result at VGPRn+1. 11010 if (UsesTFC && Lane == TFCLane) { 11011 Users[Lane] = *I; 11012 } else { 11013 // Set which texture component corresponds to the lane. 11014 unsigned Comp; 11015 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) { 11016 Comp = countTrailingZeros(Dmask); 11017 Dmask &= ~(1 << Comp); 11018 } 11019 11020 // Abort if we have more than one user per component. 11021 if (Users[Lane]) 11022 return Node; 11023 11024 Users[Lane] = *I; 11025 NewDmask |= 1 << Comp; 11026 } 11027 } 11028 11029 // Don't allow 0 dmask, as hardware assumes one channel enabled. 11030 bool NoChannels = !NewDmask; 11031 if (NoChannels) { 11032 if (!UsesTFC) { 11033 // No uses of the result and not using TFC. Then do nothing. 11034 return Node; 11035 } 11036 // If the original dmask has one channel - then nothing to do 11037 if (OldBitsSet == 1) 11038 return Node; 11039 // Use an arbitrary dmask - required for the instruction to work 11040 NewDmask = 1; 11041 } 11042 // Abort if there's no change 11043 if (NewDmask == OldDmask) 11044 return Node; 11045 11046 unsigned BitsSet = countPopulation(NewDmask); 11047 11048 // Check for TFE or LWE - increase the number of channels by one to account 11049 // for the extra return value 11050 // This will need adjustment for D16 if this is also included in 11051 // adjustWriteMask (this function) but at present D16 are excluded. 11052 unsigned NewChannels = BitsSet + UsesTFC; 11053 11054 int NewOpcode = 11055 AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), NewChannels); 11056 assert(NewOpcode != -1 && 11057 NewOpcode != static_cast<int>(Node->getMachineOpcode()) && 11058 "failed to find equivalent MIMG op"); 11059 11060 // Adjust the writemask in the node 11061 SmallVector<SDValue, 12> Ops; 11062 Ops.insert(Ops.end(), Node->op_begin(), Node->op_begin() + DmaskIdx); 11063 Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32)); 11064 Ops.insert(Ops.end(), Node->op_begin() + DmaskIdx + 1, Node->op_end()); 11065 11066 MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); 11067 11068 MVT ResultVT = NewChannels == 1 ? 11069 SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 : 11070 NewChannels == 5 ? 8 : NewChannels); 11071 SDVTList NewVTList = HasChain ? 11072 DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT); 11073 11074 11075 MachineSDNode *NewNode = DAG.getMachineNode(NewOpcode, SDLoc(Node), 11076 NewVTList, Ops); 11077 11078 if (HasChain) { 11079 // Update chain. 11080 DAG.setNodeMemRefs(NewNode, Node->memoperands()); 11081 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(NewNode, 1)); 11082 } 11083 11084 if (NewChannels == 1) { 11085 assert(Node->hasNUsesOfValue(1, 0)); 11086 SDNode *Copy = DAG.getMachineNode(TargetOpcode::COPY, 11087 SDLoc(Node), Users[Lane]->getValueType(0), 11088 SDValue(NewNode, 0)); 11089 DAG.ReplaceAllUsesWith(Users[Lane], Copy); 11090 return nullptr; 11091 } 11092 11093 // Update the users of the node with the new indices 11094 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) { 11095 SDNode *User = Users[i]; 11096 if (!User) { 11097 // Handle the special case of NoChannels. We set NewDmask to 1 above, but 11098 // Users[0] is still nullptr because channel 0 doesn't really have a use. 11099 if (i || !NoChannels) 11100 continue; 11101 } else { 11102 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); 11103 DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op); 11104 } 11105 11106 switch (Idx) { 11107 default: break; 11108 case AMDGPU::sub0: Idx = AMDGPU::sub1; break; 11109 case AMDGPU::sub1: Idx = AMDGPU::sub2; break; 11110 case AMDGPU::sub2: Idx = AMDGPU::sub3; break; 11111 case AMDGPU::sub3: Idx = AMDGPU::sub4; break; 11112 } 11113 } 11114 11115 DAG.RemoveDeadNode(Node); 11116 return nullptr; 11117 } 11118 11119 static bool isFrameIndexOp(SDValue Op) { 11120 if (Op.getOpcode() == ISD::AssertZext) 11121 Op = Op.getOperand(0); 11122 11123 return isa<FrameIndexSDNode>(Op); 11124 } 11125 11126 /// Legalize target independent instructions (e.g. INSERT_SUBREG) 11127 /// with frame index operands. 11128 /// LLVM assumes that inputs are to these instructions are registers. 11129 SDNode *SITargetLowering::legalizeTargetIndependentNode(SDNode *Node, 11130 SelectionDAG &DAG) const { 11131 if (Node->getOpcode() == ISD::CopyToReg) { 11132 RegisterSDNode *DestReg = cast<RegisterSDNode>(Node->getOperand(1)); 11133 SDValue SrcVal = Node->getOperand(2); 11134 11135 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have 11136 // to try understanding copies to physical registers. 11137 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) { 11138 SDLoc SL(Node); 11139 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 11140 SDValue VReg = DAG.getRegister( 11141 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1); 11142 11143 SDNode *Glued = Node->getGluedNode(); 11144 SDValue ToVReg 11145 = DAG.getCopyToReg(Node->getOperand(0), SL, VReg, SrcVal, 11146 SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0)); 11147 SDValue ToResultReg 11148 = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0), 11149 VReg, ToVReg.getValue(1)); 11150 DAG.ReplaceAllUsesWith(Node, ToResultReg.getNode()); 11151 DAG.RemoveDeadNode(Node); 11152 return ToResultReg.getNode(); 11153 } 11154 } 11155 11156 SmallVector<SDValue, 8> Ops; 11157 for (unsigned i = 0; i < Node->getNumOperands(); ++i) { 11158 if (!isFrameIndexOp(Node->getOperand(i))) { 11159 Ops.push_back(Node->getOperand(i)); 11160 continue; 11161 } 11162 11163 SDLoc DL(Node); 11164 Ops.push_back(SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, 11165 Node->getOperand(i).getValueType(), 11166 Node->getOperand(i)), 0)); 11167 } 11168 11169 return DAG.UpdateNodeOperands(Node, Ops); 11170 } 11171 11172 /// Fold the instructions after selecting them. 11173 /// Returns null if users were already updated. 11174 SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node, 11175 SelectionDAG &DAG) const { 11176 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 11177 unsigned Opcode = Node->getMachineOpcode(); 11178 11179 if (TII->isMIMG(Opcode) && !TII->get(Opcode).mayStore() && 11180 !TII->isGather4(Opcode) && 11181 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::dmask) != -1) { 11182 return adjustWritemask(Node, DAG); 11183 } 11184 11185 if (Opcode == AMDGPU::INSERT_SUBREG || 11186 Opcode == AMDGPU::REG_SEQUENCE) { 11187 legalizeTargetIndependentNode(Node, DAG); 11188 return Node; 11189 } 11190 11191 switch (Opcode) { 11192 case AMDGPU::V_DIV_SCALE_F32_e64: 11193 case AMDGPU::V_DIV_SCALE_F64_e64: { 11194 // Satisfy the operand register constraint when one of the inputs is 11195 // undefined. Ordinarily each undef value will have its own implicit_def of 11196 // a vreg, so force these to use a single register. 11197 SDValue Src0 = Node->getOperand(1); 11198 SDValue Src1 = Node->getOperand(3); 11199 SDValue Src2 = Node->getOperand(5); 11200 11201 if ((Src0.isMachineOpcode() && 11202 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) && 11203 (Src0 == Src1 || Src0 == Src2)) 11204 break; 11205 11206 MVT VT = Src0.getValueType().getSimpleVT(); 11207 const TargetRegisterClass *RC = 11208 getRegClassFor(VT, Src0.getNode()->isDivergent()); 11209 11210 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 11211 SDValue UndefReg = DAG.getRegister(MRI.createVirtualRegister(RC), VT); 11212 11213 SDValue ImpDef = DAG.getCopyToReg(DAG.getEntryNode(), SDLoc(Node), 11214 UndefReg, Src0, SDValue()); 11215 11216 // src0 must be the same register as src1 or src2, even if the value is 11217 // undefined, so make sure we don't violate this constraint. 11218 if (Src0.isMachineOpcode() && 11219 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) { 11220 if (Src1.isMachineOpcode() && 11221 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) 11222 Src0 = Src1; 11223 else if (Src2.isMachineOpcode() && 11224 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) 11225 Src0 = Src2; 11226 else { 11227 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF); 11228 Src0 = UndefReg; 11229 Src1 = UndefReg; 11230 } 11231 } else 11232 break; 11233 11234 SmallVector<SDValue, 9> Ops(Node->op_begin(), Node->op_end()); 11235 Ops[1] = Src0; 11236 Ops[3] = Src1; 11237 Ops[5] = Src2; 11238 Ops.push_back(ImpDef.getValue(1)); 11239 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 11240 } 11241 default: 11242 break; 11243 } 11244 11245 return Node; 11246 } 11247 11248 // Any MIMG instructions that use tfe or lwe require an initialization of the 11249 // result register that will be written in the case of a memory access failure. 11250 // The required code is also added to tie this init code to the result of the 11251 // img instruction. 11252 void SITargetLowering::AddIMGInit(MachineInstr &MI) const { 11253 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 11254 const SIRegisterInfo &TRI = TII->getRegisterInfo(); 11255 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); 11256 MachineBasicBlock &MBB = *MI.getParent(); 11257 11258 MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); 11259 MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); 11260 MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); 11261 11262 if (!TFE && !LWE) // intersect_ray 11263 return; 11264 11265 unsigned TFEVal = TFE ? TFE->getImm() : 0; 11266 unsigned LWEVal = LWE->getImm(); 11267 unsigned D16Val = D16 ? D16->getImm() : 0; 11268 11269 if (!TFEVal && !LWEVal) 11270 return; 11271 11272 // At least one of TFE or LWE are non-zero 11273 // We have to insert a suitable initialization of the result value and 11274 // tie this to the dest of the image instruction. 11275 11276 const DebugLoc &DL = MI.getDebugLoc(); 11277 11278 int DstIdx = 11279 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 11280 11281 // Calculate which dword we have to initialize to 0. 11282 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); 11283 11284 // check that dmask operand is found. 11285 assert(MO_Dmask && "Expected dmask operand in instruction"); 11286 11287 unsigned dmask = MO_Dmask->getImm(); 11288 // Determine the number of active lanes taking into account the 11289 // Gather4 special case 11290 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : countPopulation(dmask); 11291 11292 bool Packed = !Subtarget->hasUnpackedD16VMem(); 11293 11294 unsigned InitIdx = 11295 D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; 11296 11297 // Abandon attempt if the dst size isn't large enough 11298 // - this is in fact an error but this is picked up elsewhere and 11299 // reported correctly. 11300 uint32_t DstSize = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; 11301 if (DstSize < InitIdx) 11302 return; 11303 11304 // Create a register for the intialization value. 11305 Register PrevDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); 11306 unsigned NewDst = 0; // Final initialized value will be in here 11307 11308 // If PRTStrictNull feature is enabled (the default) then initialize 11309 // all the result registers to 0, otherwise just the error indication 11310 // register (VGPRn+1) 11311 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1; 11312 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1); 11313 11314 BuildMI(MBB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), PrevDst); 11315 for (; SizeLeft; SizeLeft--, CurrIdx++) { 11316 NewDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); 11317 // Initialize dword 11318 Register SubReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 11319 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), SubReg) 11320 .addImm(0); 11321 // Insert into the super-reg 11322 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewDst) 11323 .addReg(PrevDst) 11324 .addReg(SubReg) 11325 .addImm(SIRegisterInfo::getSubRegFromChannel(CurrIdx)); 11326 11327 PrevDst = NewDst; 11328 } 11329 11330 // Add as an implicit operand 11331 MI.addOperand(MachineOperand::CreateReg(NewDst, false, true)); 11332 11333 // Tie the just added implicit operand to the dst 11334 MI.tieOperands(DstIdx, MI.getNumOperands() - 1); 11335 } 11336 11337 /// Assign the register class depending on the number of 11338 /// bits set in the writemask 11339 void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, 11340 SDNode *Node) const { 11341 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 11342 11343 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 11344 11345 if (TII->isVOP3(MI.getOpcode())) { 11346 // Make sure constant bus requirements are respected. 11347 TII->legalizeOperandsVOP3(MRI, MI); 11348 11349 // Prefer VGPRs over AGPRs in mAI instructions where possible. 11350 // This saves a chain-copy of registers and better ballance register 11351 // use between vgpr and agpr as agpr tuples tend to be big. 11352 if (const MCOperandInfo *OpInfo = MI.getDesc().OpInfo) { 11353 unsigned Opc = MI.getOpcode(); 11354 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); 11355 for (auto I : { AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 11356 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) }) { 11357 if (I == -1) 11358 break; 11359 MachineOperand &Op = MI.getOperand(I); 11360 if ((OpInfo[I].RegClass != llvm::AMDGPU::AV_64RegClassID && 11361 OpInfo[I].RegClass != llvm::AMDGPU::AV_32RegClassID) || 11362 !Op.getReg().isVirtual() || !TRI->isAGPR(MRI, Op.getReg())) 11363 continue; 11364 auto *Src = MRI.getUniqueVRegDef(Op.getReg()); 11365 if (!Src || !Src->isCopy() || 11366 !TRI->isSGPRReg(MRI, Src->getOperand(1).getReg())) 11367 continue; 11368 auto *RC = TRI->getRegClassForReg(MRI, Op.getReg()); 11369 auto *NewRC = TRI->getEquivalentVGPRClass(RC); 11370 // All uses of agpr64 and agpr32 can also accept vgpr except for 11371 // v_accvgpr_read, but we do not produce agpr reads during selection, 11372 // so no use checks are needed. 11373 MRI.setRegClass(Op.getReg(), NewRC); 11374 } 11375 } 11376 11377 return; 11378 } 11379 11380 // Replace unused atomics with the no return version. 11381 int NoRetAtomicOp = AMDGPU::getAtomicNoRetOp(MI.getOpcode()); 11382 if (NoRetAtomicOp != -1) { 11383 if (!Node->hasAnyUseOfValue(0)) { 11384 int CPolIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 11385 AMDGPU::OpName::cpol); 11386 if (CPolIdx != -1) { 11387 MachineOperand &CPol = MI.getOperand(CPolIdx); 11388 CPol.setImm(CPol.getImm() & ~AMDGPU::CPol::GLC); 11389 } 11390 MI.RemoveOperand(0); 11391 MI.setDesc(TII->get(NoRetAtomicOp)); 11392 return; 11393 } 11394 11395 // For mubuf_atomic_cmpswap, we need to have tablegen use an extract_subreg 11396 // instruction, because the return type of these instructions is a vec2 of 11397 // the memory type, so it can be tied to the input operand. 11398 // This means these instructions always have a use, so we need to add a 11399 // special case to check if the atomic has only one extract_subreg use, 11400 // which itself has no uses. 11401 if ((Node->hasNUsesOfValue(1, 0) && 11402 Node->use_begin()->isMachineOpcode() && 11403 Node->use_begin()->getMachineOpcode() == AMDGPU::EXTRACT_SUBREG && 11404 !Node->use_begin()->hasAnyUseOfValue(0))) { 11405 Register Def = MI.getOperand(0).getReg(); 11406 11407 // Change this into a noret atomic. 11408 MI.setDesc(TII->get(NoRetAtomicOp)); 11409 MI.RemoveOperand(0); 11410 11411 // If we only remove the def operand from the atomic instruction, the 11412 // extract_subreg will be left with a use of a vreg without a def. 11413 // So we need to insert an implicit_def to avoid machine verifier 11414 // errors. 11415 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), 11416 TII->get(AMDGPU::IMPLICIT_DEF), Def); 11417 } 11418 return; 11419 } 11420 11421 if (TII->isMIMG(MI) && !MI.mayStore()) 11422 AddIMGInit(MI); 11423 } 11424 11425 static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, 11426 uint64_t Val) { 11427 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32); 11428 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0); 11429 } 11430 11431 MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG, 11432 const SDLoc &DL, 11433 SDValue Ptr) const { 11434 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 11435 11436 // Build the half of the subregister with the constants before building the 11437 // full 128-bit register. If we are building multiple resource descriptors, 11438 // this will allow CSEing of the 2-component register. 11439 const SDValue Ops0[] = { 11440 DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32), 11441 buildSMovImm32(DAG, DL, 0), 11442 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 11443 buildSMovImm32(DAG, DL, TII->getDefaultRsrcDataFormat() >> 32), 11444 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32) 11445 }; 11446 11447 SDValue SubRegHi = SDValue(DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, 11448 MVT::v2i32, Ops0), 0); 11449 11450 // Combine the constants and the pointer. 11451 const SDValue Ops1[] = { 11452 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), 11453 Ptr, 11454 DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32), 11455 SubRegHi, 11456 DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32) 11457 }; 11458 11459 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1); 11460 } 11461 11462 /// Return a resource descriptor with the 'Add TID' bit enabled 11463 /// The TID (Thread ID) is multiplied by the stride value (bits [61:48] 11464 /// of the resource descriptor) to create an offset, which is added to 11465 /// the resource pointer. 11466 MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL, 11467 SDValue Ptr, uint32_t RsrcDword1, 11468 uint64_t RsrcDword2And3) const { 11469 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr); 11470 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr); 11471 if (RsrcDword1) { 11472 PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi, 11473 DAG.getConstant(RsrcDword1, DL, MVT::i32)), 11474 0); 11475 } 11476 11477 SDValue DataLo = buildSMovImm32(DAG, DL, 11478 RsrcDword2And3 & UINT64_C(0xFFFFFFFF)); 11479 SDValue DataHi = buildSMovImm32(DAG, DL, RsrcDword2And3 >> 32); 11480 11481 const SDValue Ops[] = { 11482 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), 11483 PtrLo, 11484 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 11485 PtrHi, 11486 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32), 11487 DataLo, 11488 DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32), 11489 DataHi, 11490 DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32) 11491 }; 11492 11493 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops); 11494 } 11495 11496 //===----------------------------------------------------------------------===// 11497 // SI Inline Assembly Support 11498 //===----------------------------------------------------------------------===// 11499 11500 std::pair<unsigned, const TargetRegisterClass *> 11501 SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_, 11502 StringRef Constraint, 11503 MVT VT) const { 11504 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_); 11505 11506 const TargetRegisterClass *RC = nullptr; 11507 if (Constraint.size() == 1) { 11508 const unsigned BitWidth = VT.getSizeInBits(); 11509 switch (Constraint[0]) { 11510 default: 11511 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 11512 case 's': 11513 case 'r': 11514 switch (BitWidth) { 11515 case 16: 11516 RC = &AMDGPU::SReg_32RegClass; 11517 break; 11518 case 64: 11519 RC = &AMDGPU::SGPR_64RegClass; 11520 break; 11521 default: 11522 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth); 11523 if (!RC) 11524 return std::make_pair(0U, nullptr); 11525 break; 11526 } 11527 break; 11528 case 'v': 11529 switch (BitWidth) { 11530 case 16: 11531 RC = &AMDGPU::VGPR_32RegClass; 11532 break; 11533 default: 11534 RC = TRI->getVGPRClassForBitWidth(BitWidth); 11535 if (!RC) 11536 return std::make_pair(0U, nullptr); 11537 break; 11538 } 11539 break; 11540 case 'a': 11541 if (!Subtarget->hasMAIInsts()) 11542 break; 11543 switch (BitWidth) { 11544 case 16: 11545 RC = &AMDGPU::AGPR_32RegClass; 11546 break; 11547 default: 11548 RC = TRI->getAGPRClassForBitWidth(BitWidth); 11549 if (!RC) 11550 return std::make_pair(0U, nullptr); 11551 break; 11552 } 11553 break; 11554 } 11555 // We actually support i128, i16 and f16 as inline parameters 11556 // even if they are not reported as legal 11557 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 || 11558 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16)) 11559 return std::make_pair(0U, RC); 11560 } 11561 11562 if (Constraint.size() > 1) { 11563 if (Constraint[1] == 'v') { 11564 RC = &AMDGPU::VGPR_32RegClass; 11565 } else if (Constraint[1] == 's') { 11566 RC = &AMDGPU::SGPR_32RegClass; 11567 } else if (Constraint[1] == 'a') { 11568 RC = &AMDGPU::AGPR_32RegClass; 11569 } 11570 11571 if (RC) { 11572 uint32_t Idx; 11573 bool Failed = Constraint.substr(2).getAsInteger(10, Idx); 11574 if (!Failed && Idx < RC->getNumRegs()) 11575 return std::make_pair(RC->getRegister(Idx), RC); 11576 } 11577 } 11578 11579 // FIXME: Returns VS_32 for physical SGPR constraints 11580 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 11581 } 11582 11583 static bool isImmConstraint(StringRef Constraint) { 11584 if (Constraint.size() == 1) { 11585 switch (Constraint[0]) { 11586 default: break; 11587 case 'I': 11588 case 'J': 11589 case 'A': 11590 case 'B': 11591 case 'C': 11592 return true; 11593 } 11594 } else if (Constraint == "DA" || 11595 Constraint == "DB") { 11596 return true; 11597 } 11598 return false; 11599 } 11600 11601 SITargetLowering::ConstraintType 11602 SITargetLowering::getConstraintType(StringRef Constraint) const { 11603 if (Constraint.size() == 1) { 11604 switch (Constraint[0]) { 11605 default: break; 11606 case 's': 11607 case 'v': 11608 case 'a': 11609 return C_RegisterClass; 11610 } 11611 } 11612 if (isImmConstraint(Constraint)) { 11613 return C_Other; 11614 } 11615 return TargetLowering::getConstraintType(Constraint); 11616 } 11617 11618 static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) { 11619 if (!AMDGPU::isInlinableIntLiteral(Val)) { 11620 Val = Val & maskTrailingOnes<uint64_t>(Size); 11621 } 11622 return Val; 11623 } 11624 11625 void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op, 11626 std::string &Constraint, 11627 std::vector<SDValue> &Ops, 11628 SelectionDAG &DAG) const { 11629 if (isImmConstraint(Constraint)) { 11630 uint64_t Val; 11631 if (getAsmOperandConstVal(Op, Val) && 11632 checkAsmConstraintVal(Op, Constraint, Val)) { 11633 Val = clearUnusedBits(Val, Op.getScalarValueSizeInBits()); 11634 Ops.push_back(DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64)); 11635 } 11636 } else { 11637 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 11638 } 11639 } 11640 11641 bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const { 11642 unsigned Size = Op.getScalarValueSizeInBits(); 11643 if (Size > 64) 11644 return false; 11645 11646 if (Size == 16 && !Subtarget->has16BitInsts()) 11647 return false; 11648 11649 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 11650 Val = C->getSExtValue(); 11651 return true; 11652 } 11653 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) { 11654 Val = C->getValueAPF().bitcastToAPInt().getSExtValue(); 11655 return true; 11656 } 11657 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Op)) { 11658 if (Size != 16 || Op.getNumOperands() != 2) 11659 return false; 11660 if (Op.getOperand(0).isUndef() || Op.getOperand(1).isUndef()) 11661 return false; 11662 if (ConstantSDNode *C = V->getConstantSplatNode()) { 11663 Val = C->getSExtValue(); 11664 return true; 11665 } 11666 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) { 11667 Val = C->getValueAPF().bitcastToAPInt().getSExtValue(); 11668 return true; 11669 } 11670 } 11671 11672 return false; 11673 } 11674 11675 bool SITargetLowering::checkAsmConstraintVal(SDValue Op, 11676 const std::string &Constraint, 11677 uint64_t Val) const { 11678 if (Constraint.size() == 1) { 11679 switch (Constraint[0]) { 11680 case 'I': 11681 return AMDGPU::isInlinableIntLiteral(Val); 11682 case 'J': 11683 return isInt<16>(Val); 11684 case 'A': 11685 return checkAsmConstraintValA(Op, Val); 11686 case 'B': 11687 return isInt<32>(Val); 11688 case 'C': 11689 return isUInt<32>(clearUnusedBits(Val, Op.getScalarValueSizeInBits())) || 11690 AMDGPU::isInlinableIntLiteral(Val); 11691 default: 11692 break; 11693 } 11694 } else if (Constraint.size() == 2) { 11695 if (Constraint == "DA") { 11696 int64_t HiBits = static_cast<int32_t>(Val >> 32); 11697 int64_t LoBits = static_cast<int32_t>(Val); 11698 return checkAsmConstraintValA(Op, HiBits, 32) && 11699 checkAsmConstraintValA(Op, LoBits, 32); 11700 } 11701 if (Constraint == "DB") { 11702 return true; 11703 } 11704 } 11705 llvm_unreachable("Invalid asm constraint"); 11706 } 11707 11708 bool SITargetLowering::checkAsmConstraintValA(SDValue Op, 11709 uint64_t Val, 11710 unsigned MaxSize) const { 11711 unsigned Size = std::min<unsigned>(Op.getScalarValueSizeInBits(), MaxSize); 11712 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm(); 11713 if ((Size == 16 && AMDGPU::isInlinableLiteral16(Val, HasInv2Pi)) || 11714 (Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) || 11715 (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) { 11716 return true; 11717 } 11718 return false; 11719 } 11720 11721 static int getAlignedAGPRClassID(unsigned UnalignedClassID) { 11722 switch (UnalignedClassID) { 11723 case AMDGPU::VReg_64RegClassID: 11724 return AMDGPU::VReg_64_Align2RegClassID; 11725 case AMDGPU::VReg_96RegClassID: 11726 return AMDGPU::VReg_96_Align2RegClassID; 11727 case AMDGPU::VReg_128RegClassID: 11728 return AMDGPU::VReg_128_Align2RegClassID; 11729 case AMDGPU::VReg_160RegClassID: 11730 return AMDGPU::VReg_160_Align2RegClassID; 11731 case AMDGPU::VReg_192RegClassID: 11732 return AMDGPU::VReg_192_Align2RegClassID; 11733 case AMDGPU::VReg_256RegClassID: 11734 return AMDGPU::VReg_256_Align2RegClassID; 11735 case AMDGPU::VReg_512RegClassID: 11736 return AMDGPU::VReg_512_Align2RegClassID; 11737 case AMDGPU::VReg_1024RegClassID: 11738 return AMDGPU::VReg_1024_Align2RegClassID; 11739 case AMDGPU::AReg_64RegClassID: 11740 return AMDGPU::AReg_64_Align2RegClassID; 11741 case AMDGPU::AReg_96RegClassID: 11742 return AMDGPU::AReg_96_Align2RegClassID; 11743 case AMDGPU::AReg_128RegClassID: 11744 return AMDGPU::AReg_128_Align2RegClassID; 11745 case AMDGPU::AReg_160RegClassID: 11746 return AMDGPU::AReg_160_Align2RegClassID; 11747 case AMDGPU::AReg_192RegClassID: 11748 return AMDGPU::AReg_192_Align2RegClassID; 11749 case AMDGPU::AReg_256RegClassID: 11750 return AMDGPU::AReg_256_Align2RegClassID; 11751 case AMDGPU::AReg_512RegClassID: 11752 return AMDGPU::AReg_512_Align2RegClassID; 11753 case AMDGPU::AReg_1024RegClassID: 11754 return AMDGPU::AReg_1024_Align2RegClassID; 11755 default: 11756 return -1; 11757 } 11758 } 11759 11760 // Figure out which registers should be reserved for stack access. Only after 11761 // the function is legalized do we know all of the non-spill stack objects or if 11762 // calls are present. 11763 void SITargetLowering::finalizeLowering(MachineFunction &MF) const { 11764 MachineRegisterInfo &MRI = MF.getRegInfo(); 11765 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 11766 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 11767 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); 11768 const SIInstrInfo *TII = ST.getInstrInfo(); 11769 11770 if (Info->isEntryFunction()) { 11771 // Callable functions have fixed registers used for stack access. 11772 reservePrivateMemoryRegs(getTargetMachine(), MF, *TRI, *Info); 11773 } 11774 11775 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(), 11776 Info->getStackPtrOffsetReg())); 11777 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG) 11778 MRI.replaceRegWith(AMDGPU::SP_REG, Info->getStackPtrOffsetReg()); 11779 11780 // We need to worry about replacing the default register with itself in case 11781 // of MIR testcases missing the MFI. 11782 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG) 11783 MRI.replaceRegWith(AMDGPU::PRIVATE_RSRC_REG, Info->getScratchRSrcReg()); 11784 11785 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG) 11786 MRI.replaceRegWith(AMDGPU::FP_REG, Info->getFrameOffsetReg()); 11787 11788 Info->limitOccupancy(MF); 11789 11790 if (ST.isWave32() && !MF.empty()) { 11791 for (auto &MBB : MF) { 11792 for (auto &MI : MBB) { 11793 TII->fixImplicitOperands(MI); 11794 } 11795 } 11796 } 11797 11798 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned 11799 // classes if required. Ideally the register class constraints would differ 11800 // per-subtarget, but there's no easy way to achieve that right now. This is 11801 // not a problem for VGPRs because the correctly aligned VGPR class is implied 11802 // from using them as the register class for legal types. 11803 if (ST.needsAlignedVGPRs()) { 11804 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 11805 const Register Reg = Register::index2VirtReg(I); 11806 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg); 11807 if (!RC) 11808 continue; 11809 int NewClassID = getAlignedAGPRClassID(RC->getID()); 11810 if (NewClassID != -1) 11811 MRI.setRegClass(Reg, TRI->getRegClass(NewClassID)); 11812 } 11813 } 11814 11815 TargetLoweringBase::finalizeLowering(MF); 11816 11817 // Allocate a VGPR for future SGPR Spill if 11818 // "amdgpu-reserve-vgpr-for-sgpr-spill" option is used 11819 // FIXME: We won't need this hack if we split SGPR allocation from VGPR 11820 if (VGPRReserveforSGPRSpill && TRI->spillSGPRToVGPR() && 11821 !Info->VGPRReservedForSGPRSpill && !Info->isEntryFunction()) 11822 Info->reserveVGPRforSGPRSpills(MF); 11823 } 11824 11825 void SITargetLowering::computeKnownBitsForFrameIndex( 11826 const int FI, KnownBits &Known, const MachineFunction &MF) const { 11827 TargetLowering::computeKnownBitsForFrameIndex(FI, Known, MF); 11828 11829 // Set the high bits to zero based on the maximum allowed scratch size per 11830 // wave. We can't use vaddr in MUBUF instructions if we don't know the address 11831 // calculation won't overflow, so assume the sign bit is never set. 11832 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex()); 11833 } 11834 11835 static void knownBitsForWorkitemID(const GCNSubtarget &ST, GISelKnownBits &KB, 11836 KnownBits &Known, unsigned Dim) { 11837 unsigned MaxValue = 11838 ST.getMaxWorkitemID(KB.getMachineFunction().getFunction(), Dim); 11839 Known.Zero.setHighBits(countLeadingZeros(MaxValue)); 11840 } 11841 11842 void SITargetLowering::computeKnownBitsForTargetInstr( 11843 GISelKnownBits &KB, Register R, KnownBits &Known, const APInt &DemandedElts, 11844 const MachineRegisterInfo &MRI, unsigned Depth) const { 11845 const MachineInstr *MI = MRI.getVRegDef(R); 11846 switch (MI->getOpcode()) { 11847 case AMDGPU::G_INTRINSIC: { 11848 switch (MI->getIntrinsicID()) { 11849 case Intrinsic::amdgcn_workitem_id_x: 11850 knownBitsForWorkitemID(*getSubtarget(), KB, Known, 0); 11851 break; 11852 case Intrinsic::amdgcn_workitem_id_y: 11853 knownBitsForWorkitemID(*getSubtarget(), KB, Known, 1); 11854 break; 11855 case Intrinsic::amdgcn_workitem_id_z: 11856 knownBitsForWorkitemID(*getSubtarget(), KB, Known, 2); 11857 break; 11858 case Intrinsic::amdgcn_mbcnt_lo: 11859 case Intrinsic::amdgcn_mbcnt_hi: { 11860 // These return at most the wavefront size - 1. 11861 unsigned Size = MRI.getType(R).getSizeInBits(); 11862 Known.Zero.setHighBits(Size - getSubtarget()->getWavefrontSizeLog2()); 11863 break; 11864 } 11865 case Intrinsic::amdgcn_groupstaticsize: { 11866 // We can report everything over the maximum size as 0. We can't report 11867 // based on the actual size because we don't know if it's accurate or not 11868 // at any given point. 11869 Known.Zero.setHighBits(countLeadingZeros(getSubtarget()->getLocalMemorySize())); 11870 break; 11871 } 11872 } 11873 break; 11874 } 11875 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: 11876 Known.Zero.setHighBits(24); 11877 break; 11878 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: 11879 Known.Zero.setHighBits(16); 11880 break; 11881 } 11882 } 11883 11884 Align SITargetLowering::computeKnownAlignForTargetInstr( 11885 GISelKnownBits &KB, Register R, const MachineRegisterInfo &MRI, 11886 unsigned Depth) const { 11887 const MachineInstr *MI = MRI.getVRegDef(R); 11888 switch (MI->getOpcode()) { 11889 case AMDGPU::G_INTRINSIC: 11890 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: { 11891 // FIXME: Can this move to generic code? What about the case where the call 11892 // site specifies a lower alignment? 11893 Intrinsic::ID IID = MI->getIntrinsicID(); 11894 LLVMContext &Ctx = KB.getMachineFunction().getFunction().getContext(); 11895 AttributeList Attrs = Intrinsic::getAttributes(Ctx, IID); 11896 if (MaybeAlign RetAlign = Attrs.getRetAlignment()) 11897 return *RetAlign; 11898 return Align(1); 11899 } 11900 default: 11901 return Align(1); 11902 } 11903 } 11904 11905 Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { 11906 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML); 11907 const Align CacheLineAlign = Align(64); 11908 11909 // Pre-GFX10 target did not benefit from loop alignment 11910 if (!ML || DisableLoopAlignment || 11911 (getSubtarget()->getGeneration() < AMDGPUSubtarget::GFX10) || 11912 getSubtarget()->hasInstFwdPrefetchBug()) 11913 return PrefAlign; 11914 11915 // On GFX10 I$ is 4 x 64 bytes cache lines. 11916 // By default prefetcher keeps one cache line behind and reads two ahead. 11917 // We can modify it with S_INST_PREFETCH for larger loops to have two lines 11918 // behind and one ahead. 11919 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes. 11920 // If loop fits 64 bytes it always spans no more than two cache lines and 11921 // does not need an alignment. 11922 // Else if loop is less or equal 128 bytes we do not need to modify prefetch, 11923 // Else if loop is less or equal 192 bytes we need two lines behind. 11924 11925 const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); 11926 const MachineBasicBlock *Header = ML->getHeader(); 11927 if (Header->getAlignment() != PrefAlign) 11928 return Header->getAlignment(); // Already processed. 11929 11930 unsigned LoopSize = 0; 11931 for (const MachineBasicBlock *MBB : ML->blocks()) { 11932 // If inner loop block is aligned assume in average half of the alignment 11933 // size to be added as nops. 11934 if (MBB != Header) 11935 LoopSize += MBB->getAlignment().value() / 2; 11936 11937 for (const MachineInstr &MI : *MBB) { 11938 LoopSize += TII->getInstSizeInBytes(MI); 11939 if (LoopSize > 192) 11940 return PrefAlign; 11941 } 11942 } 11943 11944 if (LoopSize <= 64) 11945 return PrefAlign; 11946 11947 if (LoopSize <= 128) 11948 return CacheLineAlign; 11949 11950 // If any of parent loops is surrounded by prefetch instructions do not 11951 // insert new for inner loop, which would reset parent's settings. 11952 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) { 11953 if (MachineBasicBlock *Exit = P->getExitBlock()) { 11954 auto I = Exit->getFirstNonDebugInstr(); 11955 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH) 11956 return CacheLineAlign; 11957 } 11958 } 11959 11960 MachineBasicBlock *Pre = ML->getLoopPreheader(); 11961 MachineBasicBlock *Exit = ML->getExitBlock(); 11962 11963 if (Pre && Exit) { 11964 BuildMI(*Pre, Pre->getFirstTerminator(), DebugLoc(), 11965 TII->get(AMDGPU::S_INST_PREFETCH)) 11966 .addImm(1); // prefetch 2 lines behind PC 11967 11968 BuildMI(*Exit, Exit->getFirstNonDebugInstr(), DebugLoc(), 11969 TII->get(AMDGPU::S_INST_PREFETCH)) 11970 .addImm(2); // prefetch 1 line behind PC 11971 } 11972 11973 return CacheLineAlign; 11974 } 11975 11976 LLVM_ATTRIBUTE_UNUSED 11977 static bool isCopyFromRegOfInlineAsm(const SDNode *N) { 11978 assert(N->getOpcode() == ISD::CopyFromReg); 11979 do { 11980 // Follow the chain until we find an INLINEASM node. 11981 N = N->getOperand(0).getNode(); 11982 if (N->getOpcode() == ISD::INLINEASM || 11983 N->getOpcode() == ISD::INLINEASM_BR) 11984 return true; 11985 } while (N->getOpcode() == ISD::CopyFromReg); 11986 return false; 11987 } 11988 11989 bool SITargetLowering::isSDNodeSourceOfDivergence( 11990 const SDNode *N, FunctionLoweringInfo *FLI, 11991 LegacyDivergenceAnalysis *KDA) const { 11992 switch (N->getOpcode()) { 11993 case ISD::CopyFromReg: { 11994 const RegisterSDNode *R = cast<RegisterSDNode>(N->getOperand(1)); 11995 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo(); 11996 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); 11997 Register Reg = R->getReg(); 11998 11999 // FIXME: Why does this need to consider isLiveIn? 12000 if (Reg.isPhysical() || MRI.isLiveIn(Reg)) 12001 return !TRI->isSGPRReg(MRI, Reg); 12002 12003 if (const Value *V = FLI->getValueFromVirtualReg(R->getReg())) 12004 return KDA->isDivergent(V); 12005 12006 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N)); 12007 return !TRI->isSGPRReg(MRI, Reg); 12008 } 12009 case ISD::LOAD: { 12010 const LoadSDNode *L = cast<LoadSDNode>(N); 12011 unsigned AS = L->getAddressSpace(); 12012 // A flat load may access private memory. 12013 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS; 12014 } 12015 case ISD::CALLSEQ_END: 12016 return true; 12017 case ISD::INTRINSIC_WO_CHAIN: 12018 return AMDGPU::isIntrinsicSourceOfDivergence( 12019 cast<ConstantSDNode>(N->getOperand(0))->getZExtValue()); 12020 case ISD::INTRINSIC_W_CHAIN: 12021 return AMDGPU::isIntrinsicSourceOfDivergence( 12022 cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()); 12023 case AMDGPUISD::ATOMIC_CMP_SWAP: 12024 case AMDGPUISD::ATOMIC_INC: 12025 case AMDGPUISD::ATOMIC_DEC: 12026 case AMDGPUISD::ATOMIC_LOAD_FMIN: 12027 case AMDGPUISD::ATOMIC_LOAD_FMAX: 12028 case AMDGPUISD::BUFFER_ATOMIC_SWAP: 12029 case AMDGPUISD::BUFFER_ATOMIC_ADD: 12030 case AMDGPUISD::BUFFER_ATOMIC_SUB: 12031 case AMDGPUISD::BUFFER_ATOMIC_SMIN: 12032 case AMDGPUISD::BUFFER_ATOMIC_UMIN: 12033 case AMDGPUISD::BUFFER_ATOMIC_SMAX: 12034 case AMDGPUISD::BUFFER_ATOMIC_UMAX: 12035 case AMDGPUISD::BUFFER_ATOMIC_AND: 12036 case AMDGPUISD::BUFFER_ATOMIC_OR: 12037 case AMDGPUISD::BUFFER_ATOMIC_XOR: 12038 case AMDGPUISD::BUFFER_ATOMIC_INC: 12039 case AMDGPUISD::BUFFER_ATOMIC_DEC: 12040 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP: 12041 case AMDGPUISD::BUFFER_ATOMIC_CSUB: 12042 case AMDGPUISD::BUFFER_ATOMIC_FADD: 12043 case AMDGPUISD::BUFFER_ATOMIC_FMIN: 12044 case AMDGPUISD::BUFFER_ATOMIC_FMAX: 12045 // Target-specific read-modify-write atomics are sources of divergence. 12046 return true; 12047 default: 12048 if (auto *A = dyn_cast<AtomicSDNode>(N)) { 12049 // Generic read-modify-write atomics are sources of divergence. 12050 return A->readMem() && A->writeMem(); 12051 } 12052 return false; 12053 } 12054 } 12055 12056 bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG, 12057 EVT VT) const { 12058 switch (VT.getScalarType().getSimpleVT().SimpleTy) { 12059 case MVT::f32: 12060 return hasFP32Denormals(DAG.getMachineFunction()); 12061 case MVT::f64: 12062 case MVT::f16: 12063 return hasFP64FP16Denormals(DAG.getMachineFunction()); 12064 default: 12065 return false; 12066 } 12067 } 12068 12069 bool SITargetLowering::denormalsEnabledForType(LLT Ty, 12070 MachineFunction &MF) const { 12071 switch (Ty.getScalarSizeInBits()) { 12072 case 32: 12073 return hasFP32Denormals(MF); 12074 case 64: 12075 case 16: 12076 return hasFP64FP16Denormals(MF); 12077 default: 12078 return false; 12079 } 12080 } 12081 12082 bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, 12083 const SelectionDAG &DAG, 12084 bool SNaN, 12085 unsigned Depth) const { 12086 if (Op.getOpcode() == AMDGPUISD::CLAMP) { 12087 const MachineFunction &MF = DAG.getMachineFunction(); 12088 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 12089 12090 if (Info->getMode().DX10Clamp) 12091 return true; // Clamped to 0. 12092 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 12093 } 12094 12095 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DAG, 12096 SNaN, Depth); 12097 } 12098 12099 // Global FP atomic instructions have a hardcoded FP mode and do not support 12100 // FP32 denormals, and only support v2f16 denormals. 12101 static bool fpModeMatchesGlobalFPAtomicMode(const AtomicRMWInst *RMW) { 12102 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics(); 12103 auto DenormMode = RMW->getParent()->getParent()->getDenormalMode(Flt); 12104 if (&Flt == &APFloat::IEEEsingle()) 12105 return DenormMode == DenormalMode::getPreserveSign(); 12106 return DenormMode == DenormalMode::getIEEE(); 12107 } 12108 12109 TargetLowering::AtomicExpansionKind 12110 SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { 12111 switch (RMW->getOperation()) { 12112 case AtomicRMWInst::FAdd: { 12113 Type *Ty = RMW->getType(); 12114 12115 // We don't have a way to support 16-bit atomics now, so just leave them 12116 // as-is. 12117 if (Ty->isHalfTy()) 12118 return AtomicExpansionKind::None; 12119 12120 if (!Ty->isFloatTy() && (!Subtarget->hasGFX90AInsts() || !Ty->isDoubleTy())) 12121 return AtomicExpansionKind::CmpXChg; 12122 12123 unsigned AS = RMW->getPointerAddressSpace(); 12124 12125 if ((AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) && 12126 Subtarget->hasAtomicFaddInsts()) { 12127 // The amdgpu-unsafe-fp-atomics attribute enables generation of unsafe 12128 // floating point atomic instructions. May generate more efficient code, 12129 // but may not respect rounding and denormal modes, and may give incorrect 12130 // results for certain memory destinations. 12131 if (RMW->getFunction() 12132 ->getFnAttribute("amdgpu-unsafe-fp-atomics") 12133 .getValueAsString() != "true") 12134 return AtomicExpansionKind::CmpXChg; 12135 12136 if (Subtarget->hasGFX90AInsts()) { 12137 if (Ty->isFloatTy() && AS == AMDGPUAS::FLAT_ADDRESS) 12138 return AtomicExpansionKind::CmpXChg; 12139 12140 auto SSID = RMW->getSyncScopeID(); 12141 if (SSID == SyncScope::System || 12142 SSID == RMW->getContext().getOrInsertSyncScopeID("one-as")) 12143 return AtomicExpansionKind::CmpXChg; 12144 12145 return AtomicExpansionKind::None; 12146 } 12147 12148 if (AS == AMDGPUAS::FLAT_ADDRESS) 12149 return AtomicExpansionKind::CmpXChg; 12150 12151 return RMW->use_empty() ? AtomicExpansionKind::None 12152 : AtomicExpansionKind::CmpXChg; 12153 } 12154 12155 // DS FP atomics do repect the denormal mode, but the rounding mode is fixed 12156 // to round-to-nearest-even. 12157 // The only exception is DS_ADD_F64 which never flushes regardless of mode. 12158 if (AS == AMDGPUAS::LOCAL_ADDRESS && Subtarget->hasLDSFPAtomics()) { 12159 if (!Ty->isDoubleTy()) 12160 return AtomicExpansionKind::None; 12161 12162 return (fpModeMatchesGlobalFPAtomicMode(RMW) || 12163 RMW->getFunction() 12164 ->getFnAttribute("amdgpu-unsafe-fp-atomics") 12165 .getValueAsString() == "true") 12166 ? AtomicExpansionKind::None 12167 : AtomicExpansionKind::CmpXChg; 12168 } 12169 12170 return AtomicExpansionKind::CmpXChg; 12171 } 12172 default: 12173 break; 12174 } 12175 12176 return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW); 12177 } 12178 12179 const TargetRegisterClass * 12180 SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const { 12181 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, false); 12182 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); 12183 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent) 12184 return Subtarget->getWavefrontSize() == 64 ? &AMDGPU::SReg_64RegClass 12185 : &AMDGPU::SReg_32RegClass; 12186 if (!TRI->isSGPRClass(RC) && !isDivergent) 12187 return TRI->getEquivalentSGPRClass(RC); 12188 else if (TRI->isSGPRClass(RC) && isDivergent) 12189 return TRI->getEquivalentVGPRClass(RC); 12190 12191 return RC; 12192 } 12193 12194 // FIXME: This is a workaround for DivergenceAnalysis not understanding always 12195 // uniform values (as produced by the mask results of control flow intrinsics) 12196 // used outside of divergent blocks. The phi users need to also be treated as 12197 // always uniform. 12198 static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited, 12199 unsigned WaveSize) { 12200 // FIXME: We asssume we never cast the mask results of a control flow 12201 // intrinsic. 12202 // Early exit if the type won't be consistent as a compile time hack. 12203 IntegerType *IT = dyn_cast<IntegerType>(V->getType()); 12204 if (!IT || IT->getBitWidth() != WaveSize) 12205 return false; 12206 12207 if (!isa<Instruction>(V)) 12208 return false; 12209 if (!Visited.insert(V).second) 12210 return false; 12211 bool Result = false; 12212 for (auto U : V->users()) { 12213 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(U)) { 12214 if (V == U->getOperand(1)) { 12215 switch (Intrinsic->getIntrinsicID()) { 12216 default: 12217 Result = false; 12218 break; 12219 case Intrinsic::amdgcn_if_break: 12220 case Intrinsic::amdgcn_if: 12221 case Intrinsic::amdgcn_else: 12222 Result = true; 12223 break; 12224 } 12225 } 12226 if (V == U->getOperand(0)) { 12227 switch (Intrinsic->getIntrinsicID()) { 12228 default: 12229 Result = false; 12230 break; 12231 case Intrinsic::amdgcn_end_cf: 12232 case Intrinsic::amdgcn_loop: 12233 Result = true; 12234 break; 12235 } 12236 } 12237 } else { 12238 Result = hasCFUser(U, Visited, WaveSize); 12239 } 12240 if (Result) 12241 break; 12242 } 12243 return Result; 12244 } 12245 12246 bool SITargetLowering::requiresUniformRegister(MachineFunction &MF, 12247 const Value *V) const { 12248 if (const CallInst *CI = dyn_cast<CallInst>(V)) { 12249 if (CI->isInlineAsm()) { 12250 // FIXME: This cannot give a correct answer. This should only trigger in 12251 // the case where inline asm returns mixed SGPR and VGPR results, used 12252 // outside the defining block. We don't have a specific result to 12253 // consider, so this assumes if any value is SGPR, the overall register 12254 // also needs to be SGPR. 12255 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo(); 12256 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints( 12257 MF.getDataLayout(), Subtarget->getRegisterInfo(), *CI); 12258 for (auto &TC : TargetConstraints) { 12259 if (TC.Type == InlineAsm::isOutput) { 12260 ComputeConstraintToUse(TC, SDValue()); 12261 unsigned AssignedReg; 12262 const TargetRegisterClass *RC; 12263 std::tie(AssignedReg, RC) = getRegForInlineAsmConstraint( 12264 SIRI, TC.ConstraintCode, TC.ConstraintVT); 12265 if (RC) { 12266 MachineRegisterInfo &MRI = MF.getRegInfo(); 12267 if (AssignedReg != 0 && SIRI->isSGPRReg(MRI, AssignedReg)) 12268 return true; 12269 else if (SIRI->isSGPRClass(RC)) 12270 return true; 12271 } 12272 } 12273 } 12274 } 12275 } 12276 SmallPtrSet<const Value *, 16> Visited; 12277 return hasCFUser(V, Visited, Subtarget->getWavefrontSize()); 12278 } 12279 12280 std::pair<InstructionCost, MVT> 12281 SITargetLowering::getTypeLegalizationCost(const DataLayout &DL, 12282 Type *Ty) const { 12283 std::pair<InstructionCost, MVT> Cost = 12284 TargetLoweringBase::getTypeLegalizationCost(DL, Ty); 12285 auto Size = DL.getTypeSizeInBits(Ty); 12286 // Maximum load or store can handle 8 dwords for scalar and 4 for 12287 // vector ALU. Let's assume anything above 8 dwords is expensive 12288 // even if legal. 12289 if (Size <= 256) 12290 return Cost; 12291 12292 Cost.first = (Size + 255) / 256; 12293 return Cost; 12294 } 12295
Indexes created Sun Jun 07 00:24:08 UTC 2026