Lines Matching defs:CGF
47 void Enter(CodeGenFunction &CGF) override {
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
52 ContBlock = CGF.createBasicBlock("omp_if.end");
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
55 CGF.EmitBlock(ThenBlock);
58 void Done(CodeGenFunction &CGF) {
60 CGF.EmitBranch(ContBlock);
61 CGF.EmitBlock(ContBlock, true);
63 void Exit(CodeGenFunction &CGF) override {
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
224 CodeGenFunction &CGF;
243 if (auto *CSI = CGF.CapturedStmtInfo) {
268 if ((!CGF.CapturedStmtInfo ||
269 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) &&
342 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
348 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams,
353 CheckVarsEscapingDeclContext(CodeGenFunction &CGF,
355 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) {
536 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) {
537 CGBuilderTy &Bld = CGF.Builder;
539 CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size_Log2);
540 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
541 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id");
547 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) {
548 CGBuilderTy &Bld = CGF.Builder;
549 unsigned LaneIDMask = CGF.getContext().getTargetInfo().getGridValue(
551 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
552 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask),
561 static llvm::Value *getThreadLimit(CodeGenFunction &CGF,
563 CGBuilderTy &Bld = CGF.Builder;
564 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
567 ThreadLimit = RT.getGPUNumThreads(CGF);
569 llvm::Value *GPUNumThreads = RT.getGPUNumThreads(CGF);
570 llvm::Value *GPUWarpSize = RT.getGPUWarpSize(CGF);
584 static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) {
585 CGBuilderTy &Bld = CGF.Builder;
586 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
587 llvm::Value *NumThreads = RT.getGPUNumThreads(CGF);
589 llvm::Value *Mask = Bld.CreateNUWSub(RT.getGPUWarpSize(CGF), Bld.getInt32(1));
1089 void Enter(CodeGenFunction &CGF) override {
1091 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1092 RT.emitNonSPMDEntryHeader(CGF, EST, WST);
1094 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
1096 void Exit(CodeGenFunction &CGF) override {
1098 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1099 RT.clearLocThreadIdInsertPt(CGF);
1100 RT.emitNonSPMDEntryFooter(CGF, EST);
1129 void CGOpenMPRuntimeGPU::emitNonSPMDEntryHeader(CodeGenFunction &CGF,
1132 CGBuilderTy &Bld = CGF.Builder;
1134 llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker");
1135 llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck");
1136 llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master");
1137 EST.ExitBB = CGF.createBasicBlock(".exit");
1139 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1140 llvm::Value *GPUThreadID = RT.getGPUThreadID(CGF);
1141 llvm::Value *ThreadLimit = getThreadLimit(CGF);
1145 CGF.EmitBlock(WorkerBB);
1146 emitCall(CGF, WST.Loc, WST.WorkerFn);
1147 CGF.EmitBranch(EST.ExitBB);
1149 CGF.EmitBlock(MasterCheckBB);
1150 GPUThreadID = RT.getGPUThreadID(CGF);
1151 llvm::Value *MasterThreadID = getMasterThreadID(CGF);
1155 CGF.EmitBlock(MasterBB);
1161 llvm::Value *Args[] = {getThreadLimit(CGF),
1163 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1168 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1171 emitGenericVarsProlog(CGF, WST.Loc);
1174 void CGOpenMPRuntimeGPU::emitNonSPMDEntryFooter(CodeGenFunction &CGF,
1177 if (!CGF.HaveInsertPoint())
1180 emitGenericVarsEpilog(CGF);
1183 EST.ExitBB = CGF.createBasicBlock(".exit");
1185 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier");
1186 CGF.EmitBranch(TerminateBB);
1188 CGF.EmitBlock(TerminateBB);
1191 llvm::Value *Args[] = {CGF.Builder.getInt16(/*IsOMPRuntimeInitialized=*/1)};
1192 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1196 syncCTAThreads(CGF);
1198 CGF.EmitBranch(EST.ExitBB);
1200 CGF.EmitBlock(EST.ExitBB);
1227 void Enter(CodeGenFunction &CGF) override {
1228 RT.emitSPMDEntryHeader(CGF, EST, D);
1230 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
1232 void Exit(CodeGenFunction &CGF) override {
1233 RT.clearLocThreadIdInsertPt(CGF);
1234 RT.emitSPMDEntryFooter(CGF, EST);
1256 CodeGenFunction &CGF, EntryFunctionState &EST,
1258 CGBuilderTy &Bld = CGF.Builder;
1261 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute");
1262 EST.ExitBB = CGF.createBasicBlock(".exit");
1264 llvm::Value *Args[] = {getThreadLimit(CGF, /*IsInSPMDExecutionMode=*/true),
1267 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1273 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1277 CGF.EmitBranch(ExecuteBB);
1279 CGF.EmitBlock(ExecuteBB);
1284 void CGOpenMPRuntimeGPU::emitSPMDEntryFooter(CodeGenFunction &CGF,
1287 if (!CGF.HaveInsertPoint())
1291 EST.ExitBB = CGF.createBasicBlock(".exit");
1293 llvm::BasicBlock *OMPDeInitBB = CGF.createBasicBlock(".omp.deinit");
1294 CGF.EmitBranch(OMPDeInitBB);
1296 CGF.EmitBlock(OMPDeInitBB);
1299 CGF.Builder.getInt16(RequiresFullRuntime ? 1 : 0)};
1300 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1303 CGF.EmitBranch(EST.ExitBB);
1305 CGF.EmitBlock(EST.ExitBB);
1328 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
1329 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, WST.CGFI, {},
1331 emitWorkerLoop(CGF, WST);
1332 CGF.FinishFunction();
1335 void CGOpenMPRuntimeGPU::emitWorkerLoop(CodeGenFunction &CGF,
1346 CGBuilderTy &Bld = CGF.Builder;
1348 llvm::BasicBlock *AwaitBB = CGF.createBasicBlock(".await.work");
1349 llvm::BasicBlock *SelectWorkersBB = CGF.createBasicBlock(".select.workers");
1350 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute.parallel");
1351 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".terminate.parallel");
1352 llvm::BasicBlock *BarrierBB = CGF.createBasicBlock(".barrier.parallel");
1353 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
1355 CGF.EmitBranch(AwaitBB);
1358 CGF.EmitBlock(AwaitBB);
1360 syncCTAThreads(CGF);
1363 CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn");
1365 CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status");
1366 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0));
1367 CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy));
1372 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1375 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
1383 CGF.EmitBlock(SelectWorkersBB);
1389 CGF.EmitBlock(ExecuteBB);
1391 setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
1401 llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn");
1402 llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next");
1406 CGF.EmitBlock(ExecuteFNBB);
1412 emitCall(CGF, WST.Loc, W,
1413 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1416 CGF.EmitBranch(TerminateBB);
1418 CGF.EmitBlock(CheckNextBB);
1432 emitCall(CGF, WST.Loc, {ParallelFnTy, WorkFnCast},
1433 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1435 CGF.EmitBranch(TerminateBB);
1438 CGF.EmitBlock(TerminateBB);
1439 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1442 CGF.EmitBranch(BarrierBB);
1445 CGF.EmitBlock(BarrierBB);
1447 syncCTAThreads(CGF);
1448 CGF.EmitBranch(AwaitBB);
1451 CGF.EmitBlock(ExitBB);
1453 clearLocThreadIdInsertPt(CGF);
1535 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF,
1542 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc);
1545 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF,
1552 CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc);
1555 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF,
1571 void Enter(CodeGenFunction &CGF) override {
1575 void Exit(CodeGenFunction &CGF) override {
1677 void Enter(CodeGenFunction &CGF) override {
1679 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1681 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first;
1694 Rt.emitGenericVarsProlog(CGF, Loc);
1696 void Exit(CodeGenFunction &CGF) override {
1697 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime())
1698 .emitGenericVarsEpilog(CGF);
1708 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF,
1715 CGBuilderTy &Bld = CGF.Builder;
1717 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
1735 CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo();
1741 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
1742 llvm::BasicBlock *SPMDBB = CGF.createBasicBlock(".spmd");
1743 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd");
1745 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1746 llvm::Value *ThreadID = getThreadID(CGF, Loc);
1747 llvm::Value *PL = CGF.EmitRuntimeCall(
1754 CGF.EmitNounwindRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1758 (void)ApplyDebugLocation::CreateEmpty(CGF);
1759 CGF.EmitBlock(SPMDBB);
1762 CGF.EmitBranch(ExitBB);
1764 (void)ApplyDebugLocation::CreateEmpty(CGF);
1765 CGF.EmitBlock(NonSPMDBB);
1787 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
1788 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
1794 CGF.EmitBlock(ExitBB);
1830 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar(
1842 llvm::Value *Ld = CGF.EmitLoadOfScalar(
1852 CGF.EmitRuntimeCall(
1862 Address FrameAddr = CGF.EmitLoadOfPointer(
1872 GlobalRecValue, CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo());
1880 CGF.Builder.getInt16(UseSharedMemory ? 1 : 0)};
1881 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
1893 CGF.MakeNaturalAlignPointeeAddrLValue(GlobalRecCastAddr, GlobalRecTy);
1902 CGF.ConvertTypeForMem(SecGlobalRecTy)->getPointerTo();
1903 SecBase = CGF.MakeNaturalAlignPointeeAddrLValue(
1914 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
1915 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc);
1917 LValue VarAddr = CGF.EmitLValueForField(Base, Rec.second.FD);
1923 Address Addr = VarAddr.getAddress(CGF);
1924 llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
1926 {Bld.getInt32(0), getNVPTXLaneID(CGF)});
1929 VarAddr = CGF.MakeAddrLValue(
1933 Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
1942 LValue SecVarAddr = CGF.EmitLValueForField(SecBase, SecIt->second.FD);
1944 Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF),
1945 VarAddr.getPointer(CGF)),
1947 Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
1950 Address LocalAddr = CGF.CreateMemTemp(VarTy, Rec.second.FD->getName());
1958 CGF.EmitStoreOfScalar(ParValue, VarAddr);
1959 I->getSecond().MappedParams->setVarAddr(CGF, VD,
1960 VarAddr.getAddress(CGF));
1971 CGBuilderTy &Bld = CGF.Builder;
1972 llvm::Value *Size = CGF.getTypeSize(VD->getType());
1975 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1));
1977 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity());
1983 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
1984 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
1989 GlobalRecValue, CGF.ConvertTypeForMem(VD->getType())->getPointerTo());
1990 LValue Base = CGF.MakeAddrLValue(GlobalRecCastAddr, VD->getType(),
1993 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD),
1994 Base.getAddress(CGF));
1997 I->getSecond().MappedParams->apply(CGF);
2000 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF,
2006 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
2008 I->getSecond().MappedParams->restore(CGF);
2009 if (!CGF.HaveInsertPoint())
2013 CGF.EmitRuntimeCall(
2022 CGBuilderTy &Bld = CGF.Builder;
2023 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
2024 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd");
2027 (void)ApplyDebugLocation::CreateEmpty(CGF);
2028 CGF.EmitBlock(NonSPMDBB);
2029 CGF.EmitRuntimeCall(
2032 CGF.EmitCastToVoidPtr(I->getSecond().GlobalRecordAddr));
2033 CGF.EmitBlock(ExitBB);
2042 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar(
2051 CGF.EmitRuntimeCall(
2057 CGF.EmitRuntimeCall(
2066 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
2071 if (!CGF.HaveInsertPoint())
2074 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2076 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
2078 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer());
2081 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2084 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
2089 if (!CGF.HaveInsertPoint())
2093 IfCond](CodeGenFunction &CGF, PrePostActionTy &Action) {
2094 CGBuilderTy &Bld = CGF.Builder;
2107 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF);
2109 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca(
2115 ASTContext &Ctx = CGF.getContext();
2121 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy);
2123 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy);
2124 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false,
2132 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty,
2135 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1);
2138 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2141 getThreadID(CGF, Loc),
2143 llvm::ConstantInt::get(CGF.Int32Ty, -1),
2144 llvm::ConstantInt::get(CGF.Int32Ty, -1),
2148 CGF.VoidPtrPtrTy),
2150 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2156 RCG(CGF);
2159 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) {
2161 if (!CGF.HaveInsertPoint())
2168 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)};
2169 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2174 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF,
2179 if (!CGF.HaveInsertPoint())
2183 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2184 getThreadID(CGF, Loc)};
2186 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2192 CodeGenFunction &CGF, StringRef CriticalName,
2195 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop");
2196 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test");
2197 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync");
2198 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body");
2199 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit");
2201 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
2204 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2207 llvm::Value *ThreadID = RT.getGPUThreadID(CGF);
2210 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF);
2214 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0);
2215 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter");
2216 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty);
2217 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal,
2221 CGF.EmitBlock(LoopBB);
2222 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc);
2223 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth);
2224 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB);
2228 CGF.EmitBlock(TestBB);
2229 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc);
2231 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal);
2232 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB);
2235 CGF.EmitBlock(BodyBB);
2238 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc,
2245 CGF.EmitBlock(SyncBB);
2247 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2252 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1));
2253 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal);
2254 CGF.EmitBranch(LoopBB);
2257 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2261 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val,
2264 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() &&
2266 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() &&
2268 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy);
2271 if (CGF.getContext().getTypeSizeInChars(ValTy) ==
2272 CGF.getContext().getTypeSizeInChars(CastTy))
2273 return CGF.Builder.CreateBitCast(Val, LLVMCastTy);
2275 return CGF.Builder.CreateIntCast(Val, LLVMCastTy,
2277 Address CastItem = CGF.CreateMemTemp(CastTy);
2278 Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2280 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy,
2283 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc,
2290 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF,
2295 CodeGenModule &CGM = CGF.CGM;
2296 CGBuilderTy &Bld = CGF.Builder;
2301 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType);
2310 QualType CastTy = CGF.getContext().getIntTypeForBitwidth(
2312 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc);
2314 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true);
2316 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall(
2320 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc);
2323 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
2326 CGBuilderTy &Bld = CGF.Builder;
2328 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType);
2342 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy);
2346 QualType IntType = CGF.getContext().getIntTypeForBitwidth(
2347 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)),
2349 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType);
2354 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond");
2355 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then");
2356 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit");
2358 CGF.EmitBlock(PreCondBB);
2369 Ptr.getPointer(), CGF.VoidPtrTy));
2372 CGF.EmitBlock(ThenBB);
2374 CGF,
2375 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc,
2379 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType,
2386 CGF.EmitBranch(PreCondBB);
2387 CGF.EmitBlock(ExitBB);
2390 CGF,
2391 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc,
2395 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType,
2429 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy,
2433 CodeGenModule &CGM = CGF.CGM;
2435 CGBuilderTy &Bld = CGF.Builder;
2463 SrcElementAddr = CGF.EmitLoadOfPointer(
2471 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
2479 SrcElementAddr = CGF.EmitLoadOfPointer(
2486 DestElementAddr = CGF.EmitLoadOfPointer(
2494 SrcElementAddr = CGF.EmitLoadOfPointer(
2500 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2506 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2515 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2521 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
2530 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
2539 SrcElementAddr, CGF.ConvertTypeForMem(Private->getType()));
2546 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
2549 switch (CGF.getEvaluationKind(Private->getType())) {
2551 llvm::Value *Elem = CGF.EmitLoadOfScalar(
2556 CGF.EmitStoreOfScalar(
2562 CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
2563 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
2565 CGF.EmitStoreOfComplex(
2566 Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
2571 CGF.EmitAggregateCopy(
2572 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
2573 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
2585 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
2586 DestElementAddr.getPointer(), CGF.VoidPtrTy),
2597 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2615 DestBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
2617 SrcBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
2664 CodeGenFunction CGF(CGM);
2665 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2667 CGBuilderTy &Bld = CGF.Builder;
2680 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
2692 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
2694 llvm::Value *ThreadID = RT.getGPUThreadID(CGF);
2696 llvm::Value *LaneID = getNVPTXLaneID(CGF);
2698 llvm::Value *WarpID = getNVPTXWarpID(CGF);
2700 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
2703 CGF.EmitLoadOfScalar(
2706 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
2707 CGF.getPointerAlign());
2725 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType);
2732 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr");
2733 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr,
2735 PrecondBB = CGF.createBasicBlock("precond");
2736 ExitBB = CGF.createBasicBlock("exit");
2737 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body");
2739 (void)ApplyDebugLocation::CreateEmpty(CGF);
2740 CGF.EmitBlock(PrecondBB);
2741 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc);
2745 CGF.EmitBlock(BodyBB);
2748 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown,
2751 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then");
2752 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else");
2753 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont");
2758 CGF.EmitBlock(ThenBB);
2762 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
2784 llvm::Value *Elem = CGF.EmitLoadOfScalar(
2788 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType,
2794 CGF.EmitBlock(ElseBB);
2797 CGF.EmitBlock(MergeBB);
2800 CGF, Loc, OMPD_unknown,
2807 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then");
2808 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else");
2809 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont");
2811 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg);
2812 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar(
2820 CGF.EmitBlock(W0ThenBB);
2832 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar(
2843 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc);
2844 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false,
2848 CGF.EmitBlock(W0ElseBB);
2851 CGF.EmitBlock(W0MergeBB);
2855 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy);
2856 CGF.EmitBranch(PrecondBB);
2857 (void)ApplyDebugLocation::CreateEmpty(CGF);
2858 CGF.EmitBlock(ExitBB);
2865 CGF.FinishFunction();
2966 CodeGenFunction CGF(CGM);
2967 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2969 CGBuilderTy &Bld = CGF.Builder;
2971 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
2974 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
2976 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
2977 CGF.getPointerAlign());
2979 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg);
2980 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar(
2983 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg);
2984 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar(
2987 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg);
2988 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar(
2994 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list");
2999 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates,
3041 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then");
3042 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else");
3043 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont");
3046 CGF.EmitBlock(ThenBB);
3049 LocalReduceList.getPointer(), CGF.VoidPtrTy);
3051 RemoteReduceList.getPointer(), CGF.VoidPtrTy);
3053 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr});
3056 CGF.EmitBlock(ElseBB);
3059 CGF.EmitBlock(MergeBB);
3067 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then");
3068 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else");
3069 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont");
3072 CGF.EmitBlock(CpyThenBB);
3073 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates,
3077 CGF.EmitBlock(CpyElseBB);
3080 CGF.EmitBlock(CpyMergeBB);
3082 CGF.FinishFunction();
3121 CodeGenFunction CGF(CGM);
3122 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3124 CGBuilderTy &Bld = CGF.Builder;
3126 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
3127 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
3130 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
3132 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
3133 CGF.getPointerAlign());
3138 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
3140 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty),
3141 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
3148 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
3152 ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo());
3158 CGF.EmitLValueForField(
3159 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD);
3160 Address GlobAddr = GlobLVal.getAddress(CGF);
3164 switch (CGF.getEvaluationKind(Private->getType())) {
3166 llvm::Value *V = CGF.EmitLoadOfScalar(
3169 CGF.EmitStoreOfScalar(V, GlobLVal);
3173 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(
3174 CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc);
3175 CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false);
3179 CGF.EmitAggregateCopy(GlobLVal,
3180 CGF.MakeAddrLValue(ElemPtr, Private->getType()),
3187 CGF.FinishFunction();
3230 CodeGenFunction CGF(CGM);
3231 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3233 CGBuilderTy &Bld = CGF.Builder;
3235 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
3240 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
3246 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
3248 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty),
3249 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
3254 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
3258 LValue GlobLVal = CGF.EmitLValueForField(
3259 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD);
3260 Address GlobAddr = GlobLVal.getAddress(CGF);
3263 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr);
3264 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy);
3268 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
3269 llvm::Value *Size = CGF.Builder.CreateIntCast(
3270 CGF.getVLASize(
3271 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
3273 CGF.SizeTy, /*isSigned=*/false);
3274 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
3281 CGF.EmitCastToVoidPtr(ReductionList.getPointer());
3282 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
3283 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar(
3286 CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr});
3287 CGF.FinishFunction();
3326 CodeGenFunction CGF(CGM);
3327 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3329 CGBuilderTy &Bld = CGF.Builder;
3331 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
3332 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
3335 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
3337 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
3338 CGF.getPointerAlign());
3343 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
3346 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty),
3347 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
3354 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
3358 ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo());
3364 LValue GlobLVal = CGF.EmitLValueForField(
3365 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD);
3366 Address GlobAddr = GlobLVal.getAddress(CGF);
3370 switch (CGF.getEvaluationKind(Private->getType())) {
3372 llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc);
3373 CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(),
3379 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc);
3380 CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()),
3385 CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()),
3393 CGF.FinishFunction();
3436 CodeGenFunction CGF(CGM);
3437 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3439 CGBuilderTy &Bld = CGF.Builder;
3441 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
3446 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
3452 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
3454 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty),
3455 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
3460 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
3464 LValue GlobLVal = CGF.EmitLValueForField(
3465 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD);
3466 Address GlobAddr = GlobLVal.getAddress(CGF);
3469 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr);
3470 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy);
3474 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
3475 llvm::Value *Size = CGF.Builder.CreateIntCast(
3476 CGF.getVLASize(
3477 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
3479 CGF.SizeTy, /*isSigned=*/false);
3480 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
3487 CGF.EmitCastToVoidPtr(ReductionList.getPointer());
3488 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
3489 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar(
3492 CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList});
3493 CGF.FinishFunction();
3740 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
3743 if (!CGF.HaveInsertPoint())
3754 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
3766 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3767 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3784 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
3788 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
3789 CGF.Builder.CreateStore(
3790 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3791 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
3796 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
3797 llvm::Value *Size = CGF.Builder.CreateIntCast(
3798 CGF.getVLASize(
3799 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
3801 CGF.SizeTy, /*isSigned=*/false);
3802 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
3807 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3808 ReductionList.getPointer(), CGF.VoidPtrTy);
3810 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
3812 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
3821 CGF.Builder.getInt32(RHSExprs.size()),
3827 Res = CGF.EmitRuntimeCall(
3850 llvm::Value *GlobalBufferPtr = CGF.EmitLoadOfScalar(
3868 CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum),
3877 Res = CGF.EmitRuntimeCall(
3884 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done");
3885 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then");
3886 llvm::Value *Cond = CGF.Builder.CreateICmpEQ(
3888 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB);
3894 CGF.EmitBlock(ThenBB);
3898 this](CodeGenFunction &CGF, PrePostActionTy &Action) {
3903 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
3918 RCG(CGF);
3920 (void)ApplyDebugLocation::CreateEmpty(CGF);
3921 CGF
3961 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF,
3967 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam);
3973 CGF.getContext().getTargetAddressSpace(NativePointeeTy);
3975 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar(
3978 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3982 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3985 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType);
3986 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false,
3992 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
4008 llvm::Value *TargetArg = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4012 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TargetArg, TargetType));
4014 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs);
4060 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
4061 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs,
4067 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
4069 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
4073 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer());
4076 CGBuilderTy &Bld = CGF.Builder;
4082 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args");
4085 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4094 SharedArgListAddress = CGF.EmitLoadOfPointer(
4095 GlobalArgs, CGF.getContext()
4096 .getPointerType(CGF.getContext().getPointerType(
4097 CGF.getContext().VoidPtrTy))
4104 Src, CGF.SizeTy->getPointerTo());
4105 llvm::Value *LB = CGF.EmitLoadOfScalar(
4108 CGF.getContext().getPointerType(CGF.getContext().getSizeType()),
4114 Src, CGF.SizeTy->getPointerTo());
4115 llvm::Value *UB = CGF.EmitLoadOfScalar(
4118 CGF.getContext().getPointerType(CGF.getContext().getSizeType()),
4124 ASTContext &CGFContext = CGF.getContext();
4129 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)));
4130 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress,
4136 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(),
4143 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args);
4144 CGF.FinishFunction();
4148 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF,
4154 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 &&
4166 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP;
4173 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second);
4183 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first;
4199 CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None);
4213 emitGenericVarsProlog(CGF, D->getBeginLoc(), /*WithSPMDCheck=*/true);
4217 void Emit(CodeGenFunction &CGF, Flags flags) override {
4218 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime())
4219 .emitGenericVarsEpilog(CGF, /*WithSPMDCheck=*/true);
4222 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup);
4226 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF,
4254 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType());
4264 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4274 auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
4295 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) {
4296 FunctionGlobalizedDecls.erase(CGF.CurFn);
4297 CGOpenMPRuntime::functionFinished(CGF);
4301 CodeGenFunction &CGF, const OMPLoopDirective &S,
4304 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
4307 Chunk = CGF.EmitScalarConversion(
4308 RT.getGPUNumThreads(CGF),
4309 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
4314 CGF, S, ScheduleKind, Chunk);
4318 CodeGenFunction &CGF, const OMPLoopDirective &S,
4324 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize,
4325 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
4330 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
4346 Address VDAddr = CGF.GetAddrOfLocalVar(VD);
4349 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType());
4351 VDLVal = CGF.MakeAddrLValue(
4356 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) {
4358 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
4359 llvm::Value *CXXThis = CGF.LoadCXXThis();
4360 CGF.EmitStoreOfScalar(CXXThis, ThisLVal);
4370 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
4371 Address VDAddr = CGF.GetAddrOfLocalVar(VD);
4373 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr,
4375 .getAddress(CGF);
4376 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal);
Indexes created Thu Aug 06 00:25:04 UTC 2026