Lines Matching defs:Bld
537 CGBuilderTy &Bld = CGF.Builder;
541 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id");
548 CGBuilderTy &Bld = CGF.Builder;
552 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask),
563 CGBuilderTy &Bld = CGF.Builder;
571 ThreadLimit = Bld.CreateNUWSub(GPUNumThreads, GPUWarpSize, "thread_limit");
585 CGBuilderTy &Bld = CGF.Builder;
589 llvm::Value *Mask = Bld.CreateNUWSub(RT.getGPUWarpSize(CGF), Bld.getInt32(1));
591 llvm::Value *NumThreadsSubOne = Bld.CreateNUWSub(NumThreads, Bld.getInt32(1));
592 return Bld.CreateAnd(NumThreadsSubOne, Bld.CreateNot(Mask), "master_tid");
1132 CGBuilderTy &Bld = CGF.Builder;
1142 llvm::Value *IsWorker = Bld.CreateICmpULT(GPUThreadID, ThreadLimit);
1143 Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB);
1152 llvm::Value *IsMaster = Bld.CreateICmpEQ(GPUThreadID, MasterThreadID);
1153 Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB);
1162 Bld.getInt16(/*RequiresOMPRuntime=*/1)};
1258 CGBuilderTy &Bld = CGF.Builder;
1266 Bld.getInt16(RequiresFullRuntime ? 1 : 0)};
1346 CGBuilderTy &Bld = CGF.Builder;
1366 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0));
1375 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
1378 llvm::Value *WorkID = Bld.CreateLoad(WorkFn);
1379 llvm::Value *ShouldTerminate = Bld.CreateIsNull(WorkID, "should_terminate");
1380 Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB);
1385 Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active");
1386 Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB);
1396 llvm::Value *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy);
1399 Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match");
1403 Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB);
1413 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1427 Bld.CreateBitCast(WorkID, ParallelFnTy->getPointerTo());
1433 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1715 CGBuilderTy &Bld = CGF.Builder;
1751 IsTTD = Bld.CreateIsNull(PL);
1753 llvm::Value *IsSPMD = Bld.CreateIsNotNull(
1756 Bld.CreateCondBr(IsSPMD, SPMDBB, NonSPMDBB);
1781 Size = Bld.CreateSelect(
1792 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1795 auto *Phi = Bld.CreatePHI(GlobalRecPtrTy,
1845 llvm::Value *ResAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1868 Bld.CreateConstInBoundsGEP(FrameAddr, Offset).getPointer();
1871 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1887 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1904 Bld.CreatePointerBitCastOrAddrSpaceCast(
1926 {Bld.getInt32(0), getNVPTXLaneID(CGF)});
1944 Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF),
1952 Bld.CreateSelect(I->getSecond().IsInSPMDModeFlag,
1971 CGBuilderTy &Bld = CGF.Builder;
1974 Size = Bld.CreateNUWAdd(
1978 Size = Bld.CreateUDiv(Size, AlignVal);
1979 Size = Bld.CreateNUWMul(Size, AlignVal);
1988 llvm::Value *GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
2022 CGBuilderTy &Bld = CGF.Builder;
2025 Bld.CreateCondBr(I->getSecond().IsInSPMDModeFlag, ExitBB, NonSPMDBB);
2094 CGBuilderTy &Bld = CGF.Builder;
2098 ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy);
2102 llvm::Value *FnPtr = Bld.CreateBitOrPointerCast(OutlinedFn, CGM.Int8PtrTy);
2118 Address Dst = Bld.CreateConstArrayGEP(CapturedVarsAddrs, Idx);
2121 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy);
2123 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy);
2132 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty,
2147 Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(),
2296 CGBuilderTy &Bld = CGF.Builder;
2314 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true);
2326 CGBuilderTy &Bld = CGF.Builder;
2341 Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast(
2342 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy);
2350 Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo());
2352 Bld.CreatePointerBitCastOrAddrSpaceCast(ElemPtr, IntTy->getPointerTo());
2357 llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock();
2360 Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2);
2363 Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2);
2367 llvm::Value *PtrDiff = Bld.CreatePtrDiff(
2368 PtrEnd.getPointer(), Bld.CreatePointerBitCastOrAddrSpaceCast(
2370 Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)),
2382 Address LocalPtr = Bld.CreateConstGEP(Ptr, 1);
2383 Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
2398 Ptr = Bld.CreateConstGEP(Ptr, 1);
2399 ElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
2435 CGBuilderTy &Bld = CGF.Builder;
2462 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
2469 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
2478 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
2485 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
2493 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
2502 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
2504 Bld.CreateNUWAdd(DestBase.getPointer(), CurrentOffset);
2506 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2517 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
2519 Bld.CreateNUWAdd(SrcBase.getPointer(), CurrentOffset);
2521 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2528 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
2538 SrcElementAddr = Bld.CreateElementBitCast(
2540 DestElementAddr = Bld.CreateElementBitCast(DestElementAddr,
2585 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
2598 ScratchpadBasePtr = Bld.CreateNUWAdd(
2600 Bld.CreateNUWMul(ScratchpadWidth, ElementSizeInChars));
2603 ScratchpadBasePtr = Bld.CreateNUWSub(
2605 ScratchpadBasePtr = Bld.CreateUDiv(
2608 ScratchpadBasePtr = Bld.CreateNUWAdd(
2610 ScratchpadBasePtr = Bld.CreateNUWMul(
2667 CGBuilderTy &Bld = CGF.Builder;
2702 Bld.CreatePointerBitCastOrAddrSpaceCast(
2743 Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters));
2744 Bld.CreateCondBr(Cmp, BodyBB, ExitBB);
2756 llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master");
2757 Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
2761 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
2766 ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType);
2768 ElemPtr = Address(Bld.CreateGEP(ElemPtr.getPointer(), Cnt),
2774 llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP(
2780 MediumPtr = Bld.CreateElementBitCast(MediumPtr, CopyType);
2792 Bld.CreateBr(MergeBB);
2795 Bld.CreateBr(MergeBB);
2817 Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread");
2818 Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);
2823 llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP(
2828 SrcMediumPtr = Bld.CreateElementBitCast(SrcMediumPtr, CopyType);
2831 Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
2835 TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType);
2837 TargetElemPtr = Address(Bld.CreateGEP(TargetElemPtr.getPointer(), Cnt),
2846 Bld.CreateBr(W0MergeBB);
2849 Bld.CreateBr(W0MergeBB);
2854 Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1));
2969 CGBuilderTy &Bld = CGF.Builder;
2973 Bld.CreatePointerBitCastOrAddrSpaceCast(
3026 llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal);
3028 llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1));
3029 llvm::Value *CondAlgo1 = Bld.CreateAnd(
3030 Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal));
3032 llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2));
3033 llvm::Value *CondAlgo2 = Bld.CreateAnd(
3034 Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1))));
3035 CondAlgo2 = Bld.CreateAnd(
3036 CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0)));
3038 llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1);
3039 CondReduce = Bld.CreateOr(CondReduce, CondAlgo2);
3044 Bld.CreateCondBr(CondReduce, ThenBB, ElseBB);
3048 llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3050 llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3054 Bld.CreateBr(MergeBB);
3057 Bld.CreateBr(MergeBB);
3063 Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1));
3064 llvm::Value *CondCopy = Bld.CreateAnd(
3065 Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal));
3070 Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
3075 Bld.CreateBr(CpyMergeBB);
3078 Bld.CreateBr(CpyMergeBB);
3124 CGBuilderTy &Bld = CGF.Builder;
3129 Bld.CreatePointerBitCastOrAddrSpaceCast(
3137 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3147 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
3151 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3161 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(
3233 CGBuilderTy &Bld = CGF.Builder;
3239 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3261 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(
3329 CGBuilderTy &Bld = CGF.Builder;
3334 Bld.CreatePointerBitCastOrAddrSpaceCast(
3342 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3353 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
3357 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3367 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(
3439 CGBuilderTy &Bld = CGF.Builder;
3445 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3467 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(
4076 CGBuilderTy &Bld = CGF.Builder;
4102 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
4103 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
4112 Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
4113 TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
4127 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, I + Idx);
4128 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
Indexes created Sat Feb 28 05:31:39 UTC 2026