https://github.com/jsjodin updated https://github.com/llvm/llvm-project/pull/234258
>From 05804cf55dee2d2438dc43806d1b043ffb468a6c Mon Sep 17 00:00:00 2001 From: Jan Leyonberg <[email protected]> Date: Wed, 7 Oct 2026 14:43:58 -0400 Subject: [PATCH] [CIR][OpenMP] Add support for 'distribute parallel for' with 'target teams' Adds support for handling the full '[target] teams distribute parallel for'. Introduces setting the omp.composite attibute for the 'distribute parallel for' composite construct. Assisted-by: Cursor / Claude Sonnet 5 High --- clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 340 ++++++- .../CodeGenOpenMP/distribute-parallel-for.c | 266 ++++++ .../CIR/CodeGenOpenMP/not-yet-implemented.c | 13 + .../CIR/CodeGenOpenMP/region-cleanups-O2.c | 27 + .../target-teams-distribute-parallel-for.c | 876 ++++++++++++++++++ 5 files changed, 1482 insertions(+), 40 deletions(-) create mode 100644 clang/test/CIR/CodeGenOpenMP/distribute-parallel-for.c create mode 100644 clang/test/CIR/CodeGenOpenMP/target-teams-distribute-parallel-for.c diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp index 7d3a9f5e2210bd..1190cb94d98a32 100644 --- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp @@ -100,16 +100,31 @@ findNestedOMPLoopDirective(const OMPExecutableDirective &s) { return findNestedOMPLoopDirective(*nested); } -/// Returns true if \p s is a target SPMD construct: a combined -/// `target parallel for`, or an explicitly nested `target` whose body is a -/// parallel directive. +/// Returns true if \p s is a target SPMD construct. static bool isTargetSPMD(const OMPExecutableDirective &s) { switch (s.getDirectiveKind()) { case llvm::omp::OMPD_target_parallel_for: + case llvm::omp::OMPD_target_teams_distribute_parallel_for: return true; case llvm::omp::OMPD_target: { const OMPExecutableDirective *nested = getSingleNestedOMPDirective(s); - return nested && isOpenMPParallelDirective(nested->getDirectiveKind()); + if (!nested) + return false; + if (isOpenMPParallelDirective(nested->getDirectiveKind())) + return true; + if (nested->getDirectiveKind() != llvm::omp::OMPD_teams) + return false; + const OMPExecutableDirective *distribute = + getSingleNestedOMPDirective(*nested); + return distribute && distribute->getDirectiveKind() == + llvm::omp::OMPD_distribute_parallel_for; + } + case llvm::omp::OMPD_target_teams: { + // `target teams` (combined) with a separately, explicitly nested + // `distribute parallel for` (composite, split off). + const OMPExecutableDirective *distribute = getSingleNestedOMPDirective(s); + return distribute && distribute->getDirectiveKind() == + llvm::omp::OMPD_distribute_parallel_for; } default: return false; @@ -183,13 +198,19 @@ emitOMPRegionBody(CIRGenFunction &cgf, mlir::Block &block, mlir::Location begin, return res; } +/// Emits an omp.parallel op. When \p isComposite is set, this omp.parallel is +/// a leaf of a composite construct, e.g. the `parallel` in `distribute +/// parallel for`, and gets the `composite` attribute set. \p counterMolds is +/// only used by composite callers. Used to create privatized induction +/// variable(s) that can be used inside the loop nest. template <typename DirectiveTy> -static mlir::LogicalResult -emitParallelOp(CIRGenFunction &cgf, const DirectiveTy &s, - const omp::ConstructQueue &queue, - omp::ConstructQueue::const_iterator item, mlir::Location begin, - mlir::Location end, const mlir::omp::ParallelOperands &clauseOps, - llvm::function_ref<mlir::LogicalResult()> emitBody) { +static mlir::LogicalResult emitParallelOp( + CIRGenFunction &cgf, const DirectiveTy &s, const omp::ConstructQueue &queue, + omp::ConstructQueue::const_iterator item, mlir::Location begin, + mlir::Location end, const mlir::omp::ParallelOperands &clauseOps, + llvm::function_ref<mlir::LogicalResult()> emitBody, + bool isComposite = false, + llvm::ArrayRef<std::pair<const VarDecl *, Address>> counterMolds = {}) { CIRGenBuilderTy &builder = cgf.getBuilder(); CIRGenModule &cgm = cgf.getCIRGenModule(); @@ -205,13 +226,38 @@ emitParallelOp(CIRGenFunction &cgf, const DirectiveTy &s, "reduction(task: ...) should already be rejected as NYI"); auto parallelOp = mlir::omp::ParallelOp::create(builder, begin, clauseOps); - if (!omp::isLastItemInQueue(item, queue) || - hasCombinableNestedLeaf(static_cast<const OMPExecutableDirective &>(s))) - parallelOp.setCombined(true); mlir::Block &block = parallelOp.getRegion().emplaceBlock(); - return emitOMPRegionBody<mlir::omp::TerminatorOp>(cgf, block, begin, end, - emitBody); + for (auto &counterMold : counterMolds) + block.addArgument(counterMold.second.getPointer().getType(), begin); + + mlir::LogicalResult res = emitOMPRegionBody<mlir::omp::TerminatorOp>( + cgf, block, begin, end, [&]() -> mlir::LogicalResult { + // Bind each composite loop counter to its private block argument. + for (auto [idx, counterMold] : llvm::enumerate(counterMolds)) { + const VarDecl *vd = counterMold.first; + Address moldAddr = counterMold.second; + Address privAddr(block.getArgument(idx), moldAddr.getElementType(), + moldAddr.getAlignment()); + cgf.replaceAddrOfLocalVar(vd, privAddr); + cgf.symbolTable.insert(vd, privAddr.getPointer()); + } + return emitBody(); + }); + + // omp.composite/omp.combined describe the nested leaves emitted by the body + // above, so they are only attached once that succeeded: a body that bailed + // out leaves an empty region, which omp.composite's verifier rejects with + // an error that would mask the original diagnostic. + if (res.succeeded()) { + if (isComposite) + parallelOp.setComposite(true); + else if (!omp::isLastItemInQueue(item, queue) || + hasCombinableNestedLeaf( + static_cast<const OMPExecutableDirective &>(s))) + parallelOp.setCombined(true); + } + return res; } /// Checks, emits clauses for, and emits an omp.parallel for the `parallel` @@ -465,6 +511,37 @@ emitHostEvalLoopBounds(CIRGenFunction &cgf, const OMPLoopDirective &s) { return hev; } +/// Computes an OMPLoopDirective's normalized `[0, tripCount)` bounds, as +/// builtin-integer values ready for an omp.loop_nest. +/// +/// If \p consumingHostEval is true, the enclosing omp.target already +/// host_eval'd this loop's trip count, so that value is reused here instead +/// of re-emitting the trip-count expression. Otherwise, the caller must have +/// already run preinits (see emitLoopPreInits) before calling this function. +static void emitOMPLoopNestBounds(CIRGenFunction &cgf, + const OMPLoopDirective &s, + bool consumingHostEval, mlir::Value &zero, + mlir::Value &tripCount, mlir::Value &one) { + CIRGenBuilderTy &builder = cgf.getBuilder(); + mlir::Location loc = cgf.getLoc(s.getBeginLoc()); + + if (consumingHostEval) { + // Consume the host_eval bounds forwarded by the enclosing omp.target. + cgf.ompHostEvalBounds->applied = true; + zero = cgf.ompHostEvalBounds->zero; + tripCount = cgf.ompHostEvalBounds->tripCount; + one = cgf.ompHostEvalBounds->one; + } else { + mlir::Value tripCountCir = cgf.emitScalarExpr(s.getNumIterations()); + auto cirIntType = mlir::cast<cir::IntType>(tripCountCir.getType()); + zero = builder.createBuiltinIntCast( + loc, builder.getConstInt(loc, cirIntType, 0)); + one = builder.createBuiltinIntCast(loc, + builder.getConstInt(loc, cirIntType, 1)); + tripCount = builder.createBuiltinIntCast(loc, tripCountCir); + } +} + /// Lowers an OMPLoopDirective to an omp.loop_nest wrapped by an \p OpTy /// (omp.wsloop or omp.distribute). template <typename OpTy, typename OperandsTy> @@ -508,21 +585,7 @@ static mlir::LogicalResult emitOMPLoop(CIRGenFunction &cgf, } mlir::Value zero, tripCount, one; - if (consumingHostEval) { - // Consume the host_eval bounds forwarded by the enclosing omp.target. - cgf.ompHostEvalBounds->applied = true; - zero = cgf.ompHostEvalBounds->zero; - tripCount = cgf.ompHostEvalBounds->tripCount; - one = cgf.ompHostEvalBounds->one; - } else { - mlir::Value tripCountCir = cgf.emitScalarExpr(s.getNumIterations()); - auto cirIntType = mlir::cast<cir::IntType>(tripCountCir.getType()); - zero = builder.createBuiltinIntCast( - loc, builder.getConstInt(loc, cirIntType, 0)); - one = builder.createBuiltinIntCast(loc, - builder.getConstInt(loc, cirIntType, 1)); - tripCount = builder.createBuiltinIntCast(loc, tripCountCir); - } + emitOMPLoopNestBounds(cgf, s, consumingHostEval, zero, tripCount, one); auto loopOp = OpTy::create(builder, loc, clauseOps); mlir::Block *innerBlock = new mlir::Block(); @@ -601,6 +664,160 @@ emitOMPDistributeLoop(CIRGenFunction &cgf, const OMPLoopDirective &s, return emitOMPLoop<mlir::omp::DistributeOp>(cgf, s, clauseOps); } +/// Lowers the `distribute parallel for` composite construct's loop leaves +/// into the stacked omp.distribute + omp.wsloop sharing a single +/// omp.loop_nest, with both wrappers marked composite. This runs inside the +/// composite omp.parallel created by the caller (emitOMPDistributeParallelFor +/// below), which privatizes the shared loop counter(s) on itself (matching +/// Flang -- see distribute-parallel-do.f90): the two loop wrappers here stay +/// bare, consuming the same private block argument(s) already bound by the +/// enclosing omp.parallel. +static mlir::LogicalResult emitOMPDistributeParallelForLoop( + CIRGenFunction &cgf, const OMPLoopDirective &s, + const mlir::omp::DistributeOperands &distributeClauseOps, + const mlir::omp::WsloopOperands &wsloopClauseOps) { + CIRGenBuilderTy &builder = cgf.getBuilder(); + mlir::Location loc = cgf.getLoc(s.getBeginLoc()); + + const CapturedStmt *capturedStmt = s.getInnermostCapturedStmt(); + const auto *forStmt = cast<ForStmt>(capturedStmt->getCapturedStmt()); + + // Allocate storage for Sema's synthetic 0-based loop counter here, inside + // the composite omp.parallel's region (whose insertion point the caller + // has already switched to): like the real per-thread counter(s) bound by + // the enclosing omp.parallel, this storage must live inside the parallel + // region to get a per-thread copy once the region is outlined, rather + // than racing across threads if it were hoisted outside like the + // counter(s)' mold allocas are. + const auto *ivDecl = + cast<VarDecl>(cast<DeclRefExpr>(s.getIterationVariable())->getDecl()); + cgf.emitOMPNoLifetimeVarDecl(*ivDecl); + + // The counter(s) were already privatized on the enclosing omp.parallel by + // the caller, so unlike emitOMPLoop this never runs preinits + // itself: that already happened (or didn't, if host_eval is being + // consumed) before this composite's omp.parallel was created. + bool consumingHostEval = + cgf.ompHostEvalBounds && !cgf.ompHostEvalBounds->applied; + mlir::Value zero, tripCount, one; + emitOMPLoopNestBounds(cgf, s, consumingHostEval, zero, tripCount, one); + + // Stack the composite loop wrappers around a single shared omp.loop_nest: + // omp.distribute {composite} > omp.wsloop {composite} > omp.loop_nest. + mlir::OpBuilder::InsertionGuard guard(builder); + auto distributeOp = + mlir::omp::DistributeOp::create(builder, loc, distributeClauseOps); + distributeOp.getRegion().push_back(new mlir::Block()); + builder.setInsertionPointToStart(&distributeOp.getRegion().front()); + + auto wsloopOp = mlir::omp::WsloopOp::create(builder, loc, wsloopClauseOps); + wsloopOp.getRegion().push_back(new mlir::Block()); + builder.setInsertionPointToStart(&wsloopOp.getRegion().front()); + + // Composite is only attached once the loop body has been emitted: a body + // that bailed out leaves an empty region, which omp.composite's verifier + // rejects with an error that would mask the original diagnostic. + mlir::LogicalResult res = emitOMPLoopNest(cgf, *forStmt, zero, tripCount, one, + ivDecl, s.updates()[0]); + if (res.succeeded()) { + distributeOp.setComposite(/*val=*/true); + wsloopOp.setComposite(/*val=*/true); + } + return res; +} + +/// Emits the `distribute parallel for` composite construct's `distribute`, +/// `parallel` and `for` leaves: \p distributeItem (the first of the three, +/// located in \p queue) and the two leaves following it. Lowers to a +/// composite omp.parallel wrapping the composite omp.distribute + omp.wsloop +/// + omp.loop_nest stack (see emitOMPDistributeParallelForLoop). Shared by +/// emitOMPDistributeParallelForDirective and its `teams`/`target teams` +/// combinations below, which call this once they've peeled off their own +/// (non-composite) outer wrapper(s). +/// +/// Templated on \p DirectiveTy (rather than just taking an OMPLoopDirective, +/// like emitOMPDistributeParallelForLoop below does) so the concrete +/// directive type reaches emitParallelOp, which needs it: hasCancel() and +/// getTaskReductionRefExpr() are declared per-concrete-class, not on the +/// common OMPLoopDirective base. +template <typename DirectiveTy> +static mlir::LogicalResult +emitOMPDistributeParallelFor(CIRGenFunction &cgf, const DirectiveTy &s, + const omp::ConstructQueue &queue, + omp::ConstructQueue::const_iterator distributeItem, + mlir::Location begin, mlir::Location end) { + CIRGenBuilderTy &builder = cgf.getBuilder(); + CIRGenModule &cgm = cgf.getCIRGenModule(); + mlir::Location loc = cgf.getLoc(s.getBeginLoc()); + + assert(distributeItem->id == llvm::omp::OMPD_distribute && + "expected 'distribute' to be the first directive of the composite " + "construct"); + omp::ConstructQueue::const_iterator parallelItem = std::next(distributeItem); + assert(parallelItem != queue.end() && + parallelItem->id == llvm::omp::OMPD_parallel && + "expected a 'parallel' directive nested in 'distribute'"); + omp::ConstructQueue::const_iterator forItem = std::next(parallelItem); + assert(forItem != queue.end() && forItem->id == llvm::omp::OMPD_for && + "expected a 'for' directive nested in 'parallel'"); + + if (mlir::failed(checkSynthesizedClauses(cgf, s, distributeItem))) + return mlir::failure(); + mlir::omp::DistributeOperands distributeOps; + if (emitDistributeClauses(cgf, cgm, builder, loc, distributeItem->clauses, + distributeOps) + .failed()) + return mlir::failure(); + + if (mlir::failed(checkSynthesizedClauses(cgf, s, parallelItem))) + return mlir::failure(); + mlir::omp::ParallelOperands parallelOps; + if (mlir::failed(emitParallelClauses(cgf, cgm, builder, loc, + parallelItem->clauses, parallelOps))) + return mlir::failure(); + + if (mlir::failed(checkSynthesizedClauses(cgf, s, forItem))) + return mlir::failure(); + mlir::omp::WsloopOperands wsloopOps; + if (emitWsloopClauses(cgf, cgm, builder, loc, forItem->clauses, wsloopOps) + .failed()) + return mlir::failure(); + + // If the enclosing omp.target already host_eval'd this loop's trip count, + // the preinits below are skipped, matching emitOMPLoop. + bool consumingHostEval = + cgf.ompHostEvalBounds && !cgf.ompHostEvalBounds->applied; + if (!consumingHostEval && emitLoopPreInits(cgf, s.getPreInits()).failed()) + return mlir::failure(); + + // The composite loop's shared counter(s) are privatized on the enclosing + // omp.parallel (matching Flang: see distribute-parallel-do.f90), not on + // the omp.distribute/omp.wsloop stack below, which stays bare. (The + // synthetic 0-based `.omp.iv` counter is allocated separately, inside the + // omp.parallel region -- see emitOMPDistributeParallelForLoop.) + llvm::SmallVector<CIRGenFunction::DeclMapRevertingRAII, 4> counterShadows; + llvm::SmallVector<std::pair<const VarDecl *, Address>, 4> counterMolds; + counterShadows.reserve(s.counters().size()); + counterMolds.reserve(s.counters().size()); + for (auto [counter, privateCounter] : + llvm::zip_equal(s.counters(), s.private_counters())) { + const auto *vd = cast<VarDecl>(cast<DeclRefExpr>(counter)->getDecl()); + const auto *privateVd = + cast<VarDecl>(cast<DeclRefExpr>(privateCounter)->getDecl()); + counterShadows.emplace_back(cgf, vd); + counterMolds.emplace_back( + vd, emitLoopPrivateCounter(cgf, cgm, loc, privateVd, parallelOps)); + } + + return emitParallelOp( + cgf, s, queue, parallelItem, begin, end, parallelOps, + [&]() -> mlir::LogicalResult { + return emitOMPDistributeParallelForLoop(cgf, s, distributeOps, + wsloopOps); + }, + /*isComposite=*/true, counterMolds); +} + mlir::LogicalResult CIRGenFunction::emitOMPTaskwaitDirective(const OMPTaskwaitDirective &s) { getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPTaskwaitDirective"); @@ -1124,9 +1341,15 @@ CIRGenFunction::emitOMPDistributeDirective(const OMPDistributeDirective &s) { } mlir::LogicalResult CIRGenFunction::emitOMPDistributeParallelForDirective( const OMPDistributeParallelForDirective &s) { - getCIRGenModule().errorNYI(s.getSourceRange(), - "OpenMP OMPDistributeParallelForDirective"); - return mlir::failure(); + mlir::Location begin = getLoc(s.getBeginLoc()); + mlir::Location end = getLoc(s.getEndLoc()); + + // `distribute parallel for` is a composite construct: a composite + // omp.parallel wrapping the composite omp.distribute + omp.wsloop stack. + omp::ConstructQueue queue = + omp::buildConstructQueue(getContext().getLangOpts().OpenMP, s); + return emitOMPDistributeParallelFor(*this, s, queue, queue.begin(), begin, + end); } mlir::LogicalResult CIRGenFunction::emitOMPDistributeParallelForSimdDirective( const OMPDistributeParallelForSimdDirective &s) { @@ -1207,9 +1430,24 @@ CIRGenFunction::emitOMPTeamsDistributeParallelForSimdDirective( } mlir::LogicalResult CIRGenFunction::emitOMPTeamsDistributeParallelForDirective( const OMPTeamsDistributeParallelForDirective &s) { - getCIRGenModule().errorNYI(s.getSourceRange(), - "OpenMP OMPTeamsDistributeParallelForDirective"); - return mlir::failure(); + mlir::Location begin = getLoc(s.getBeginLoc()); + mlir::Location end = getLoc(s.getEndLoc()); + + // `teams distribute parallel for` nests the composite `distribute parallel + // for` stack inside an omp.teams. + omp::ConstructQueue queue = + omp::buildConstructQueue(getContext().getLangOpts().OpenMP, s); + omp::ConstructQueue::const_iterator teamsItem = queue.begin(); + assert(teamsItem->id == llvm::omp::OMPD_teams && + "expected 'teams' to be the outermost directive"); + + return emitTeams(*this, s, queue, teamsItem, begin, end, + [&]() -> mlir::LogicalResult { + omp::ConstructQueue::const_iterator distributeItem = + std::next(teamsItem); + return emitOMPDistributeParallelFor( + *this, s, queue, distributeItem, begin, end); + }); } mlir::LogicalResult CIRGenFunction::emitOMPTeamsGenericLoopDirective( const OMPTeamsGenericLoopDirective &s) { @@ -1277,10 +1515,32 @@ mlir::LogicalResult CIRGenFunction::emitOMPTargetTeamsDistributeDirective( mlir::LogicalResult CIRGenFunction::emitOMPTargetTeamsDistributeParallelForDirective( const OMPTargetTeamsDistributeParallelForDirective &s) { - getCIRGenModule().errorNYI( - s.getSourceRange(), - "OpenMP OMPTargetTeamsDistributeParallelForDirective"); - return mlir::failure(); + mlir::Location begin = getLoc(s.getBeginLoc()); + mlir::Location end = getLoc(s.getEndLoc()); + + // `target teams distribute parallel for` nests the composite `distribute + // parallel for` stack inside omp.teams inside an (SPMD) omp.target. + omp::ConstructQueue queue = + omp::buildConstructQueue(getContext().getLangOpts().OpenMP, s); + omp::ConstructQueue::const_iterator targetItem = queue.begin(); + assert(targetItem->id == llvm::omp::OMPD_target && + "expected 'target' to be the outermost directive"); + + return emitTarget( + *this, s, queue, targetItem, begin, end, [&]() -> mlir::LogicalResult { + omp::ConstructQueue::const_iterator teamsItem = std::next(targetItem); + assert(teamsItem != queue.end() && + teamsItem->id == llvm::omp::OMPD_teams && + "expected a 'teams' directive nested in 'target'"); + + return emitTeams(*this, s, queue, teamsItem, begin, end, + [&]() -> mlir::LogicalResult { + omp::ConstructQueue::const_iterator distributeItem = + std::next(teamsItem); + return emitOMPDistributeParallelFor( + *this, s, queue, distributeItem, begin, end); + }); + }); } mlir::LogicalResult CIRGenFunction::emitOMPTargetTeamsDistributeParallelForSimdDirective( diff --git a/clang/test/CIR/CodeGenOpenMP/distribute-parallel-for.c b/clang/test/CIR/CodeGenOpenMP/distribute-parallel-for.c new file mode 100644 index 00000000000000..8aaa0573e968cf --- /dev/null +++ b/clang/test/CIR/CodeGenOpenMP/distribute-parallel-for.c @@ -0,0 +1,266 @@ +// RUN: %clang_cc1 -fopenmp -emit-cir -fclangir %s -o - | FileCheck %s --check-prefix=CIR + +// Lowering to LLVM IR: mostly identical between CIR and OGCG, modulo how +// each path schedules the distribute and for chunking (see the comments +// below); they also just name the outlined functions differently. +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fclangir -emit-llvm %s -o - \ +// RUN: | FileCheck %s --check-prefixes=LLVM,LLVM-CIR +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -emit-llvm %s -o - \ +// RUN: | FileCheck %s --check-prefixes=LLVM,LLVM-OGCG + +void during(int); + +// `distribute parallel for` is a composite construct. It lowers to a +// composite omp.parallel wrapping the composite omp.distribute + omp.wsloop +// stack around a single shared omp.loop_nest. Each loop wrapper (plus the +// enclosing omp.parallel) carries the omp.composite attribute required by +// the verifier. Unlike the standalone `distribute`/`for` case, the shared loop +// counter is privatized on the omp.parallel itself (matching Flang's +// lowering of the same construct), not on omp.distribute/omp.wsloop, which +// stay bare. Here the composite is closely nested in a standalone `teams` +// region; the `teams` directive is not the innermost directive of its own +// queue (`distribute parallel for`'s decomposition still follows it), so the +// omp.teams carries omp.combined just like the plain `teams`+`distribute` +// nesting. +void distribute_parallel_for() { + // CIR: cir.func{{.*}}@distribute_parallel_for( +#pragma omp teams + { +#pragma omp distribute parallel for + for (int i = 0; i < 10; i++) { + during(i); + } + } + + // CIR: omp.teams { + // The real counter's "mold" alloca must dominate omp.parallel (it becomes + // one of its operands), so it is emitted directly inside omp.teams, before + // omp.parallel. + // CIR-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i> + // CIR-NEXT: omp.parallel private(@{{.*}} %[[I_ALLOCA]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { + // The synthetic normalized counter is written on every iteration, so, + // unlike the mold above, it must live inside omp.parallel's region to + // become per-thread once the region is outlined. + // CIR-NEXT: %[[IV_ALLOCA:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i> + + // tripCount = (10 - (0 - 1 + 1)) / 1 = 10 + // CIR-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i + // CIR-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i + // CIR-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i + // CIR-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i + // CIR-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i + // CIR-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i + // CIR-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i + // CIR-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 + // CIR-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 + // CIR-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 + + // CIR-NEXT: omp.distribute { + // CIR-NEXT: omp.wsloop { + // CIR-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[ZERO]]) to (%[[TRIPCOUNT]]) step (%[[ONE]]) { + // CIR-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i + // CIR-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ALLOCA]] : !s32i, !cir.ptr<!s32i> + // CIR: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> + // CIR: cir.load align(4) %[[I_PRIV]] + // CIR: cir.call @{{.*}}during + // CIR: omp.yield + // CIR: } + // CIR: } {omp.composite} + // CIR: } {omp.composite} + // CIR: omp.terminator + // CIR: } {omp.composite} + // CIR: omp.terminator + // CIR: } {omp.combined} +} + +// The combined `teams distribute parallel for` directive nests the composite +// `distribute parallel for` stack inside an omp.teams. The `teams` directive +// is not the innermost directive, so it carries the omp.combined attribute, +// while the loop wrappers and their enclosing omp.parallel are marked +// omp.composite. +void teams_distribute_parallel_for() { + // CIR: cir.func{{.*}}@teams_distribute_parallel_for( +#pragma omp teams distribute parallel for + for (int i = 0; i < 10; i++) { + during(i); + } + + // CIR: omp.teams { + // CIR-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i> + // CIR-NEXT: omp.parallel private(@{{.*}} %[[I_ALLOCA]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { + // CIR-NEXT: %[[IV_ALLOCA:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i> + + // tripCount = (10 - (0 - 1 + 1)) / 1 = 10 + // CIR-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i + // CIR-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i + // CIR-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i + // CIR-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i + // CIR-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i + // CIR-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i + // CIR-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i + // CIR-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 + // CIR-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i + // CIR-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 + // CIR-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 + + // CIR-NEXT: omp.distribute { + // CIR-NEXT: omp.wsloop { + // CIR-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[ZERO]]) to (%[[TRIPCOUNT]]) step (%[[ONE]]) { + // CIR-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i + // CIR-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ALLOCA]] : !s32i, !cir.ptr<!s32i> + // CIR: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> + // CIR: cir.load align(4) %[[I_PRIV]] + // CIR: cir.call @{{.*}}during + // CIR: omp.yield + // CIR: } + // CIR: } {omp.composite} + // CIR: } {omp.composite} + // CIR: omp.terminator + // CIR: } {omp.composite} + // CIR: omp.terminator + // CIR: } {omp.combined} +} + +// Host wrapper: __kmpc_fork_teams is identical on both paths. +// +// The CIR path's teams-outlined function just forks the parallel region +// directly via __kmpc_fork_call, since CIR's composite lowering privatizes +// the shared counter on the omp.parallel itself rather than chunking inside +// the teams-outlined function the way OGCG does. The forked function then +// schedules both the distribute and the for chunking in a single +// __kmpc_dist_for_static_init_4u call (schedule kind 34, "distribute + +// static for"), ending with a barrier (the composite's implicit `nowait` +// doesn't apply across the whole construct the way plain `distribute`'s +// does). +// +// The OGCG path instead schedules the distribute chunking (schedule kind +// 92) in the teams-outlined function, forking the parallel region once per +// *distribute chunk* via __kmpc_fork_call; the forked function then +// schedules its own static-for chunking (schedule kind 34) over just that +// chunk, with no barrier at the end. +// LLVM-LABEL: define {{.*}} void @distribute_parallel_for( +// LLVM: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr {{.*}}, i32 0, ptr @[[TEAMS_OUTLINED:[^)]+]]) + +// LLVM: define internal void @[[TEAMS_OUTLINED]]( +// LLVM-CIR: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 0, ptr @[[PAR_OUTLINED:[^)]+]]) + +// LLVM-CIR: define internal void @[[PAR_OUTLINED]]( +// LLVM-CIR: store i32 0, ptr %[[LB:.*]], align 4 +// LLVM-CIR-NEXT: store i32 9, ptr %[[UB:.*]], align 4 +// LLVM-CIR-NEXT: store i32 1, ptr %[[STRIDE:.*]], align 4 +// LLVM-CIR-NEXT: %[[TID:.*]] = call i32 @__kmpc_global_thread_num(ptr {{.*}}) +// LLVM-CIR-NEXT: call void @__kmpc_dist_for_static_init_4u( +// LLVM-CIR-SAME: ptr {{.*}}, i32 %[[TID]], i32 34, +// LLVM-CIR-SAME: ptr %{{[^,]+}}, ptr %[[LB]], ptr %[[UB]], ptr %{{[^,]+}}, ptr %[[STRIDE]], +// LLVM-CIR-SAME: i32 1, i32 0) +// LLVM-CIR: %[[IV:.*]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXT:[^,]+]], %{{.*}} ] +// LLVM-CIR: %[[LOOP_CMP:.*]] = icmp ult i32 %[[IV]], %{{.*}} +// LLVM-CIR-NEXT: br i1 %[[LOOP_CMP]], label %[[LOOP_BODY:[^,]+]], label %[[LOOP_EXIT:[^ ,]+]] +// LLVM-CIR: {{^}}[[LOOP_EXIT]]: +// LLVM-CIR-NEXT: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[TID]]) +// LLVM-CIR: call void @__kmpc_barrier( +// LLVM-CIR: {{^}}[[LOOP_BODY]]: +// LLVM-CIR: call void @during(i32 +// LLVM-CIR: %[[NEXT]] = add nuw i32 %[[IV]], 1 + +// LLVM-OGCG: store i32 0, ptr %[[COMB_LB:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 9, ptr %[[COMB_UB:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 1, ptr %[[DIST_STRIDE:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 0, ptr %[[DIST_IS_LAST:.*]], align 4 +// LLVM-OGCG: %[[DIST_TID:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-OGCG-SAME: ptr {{.*}}, i32 %[[DIST_TID]], i32 92, +// LLVM-OGCG-SAME: ptr %[[DIST_IS_LAST]], ptr %[[COMB_LB]], ptr %[[COMB_UB]], ptr %[[DIST_STRIDE]], +// LLVM-OGCG-SAME: i32 1, i32 1) +// LLVM-OGCG: %[[DIST_CMP:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-OGCG-NEXT: br i1 %[[DIST_CMP]], label %[[DIST_BODY:[^,]+]], label %{{.*}} +// LLVM-OGCG: {{^}}[[DIST_BODY]]: +// LLVM-OGCG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @[[PAR_OUTLINED:[^,]+]], +// LLVM-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[DIST_TID]]) +// LLVM-OGCG-NEXT: ret void + +// LLVM-OGCG: define internal void @[[PAR_OUTLINED]]( +// LLVM-OGCG: store i32 0, ptr %[[OMP_LB:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 9, ptr %[[OMP_UB:.*]], align 4 +// LLVM-OGCG: store i32 1, ptr %[[OMP_STRIDE:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 0, ptr %[[OMP_IS_LAST:.*]], align 4 +// LLVM-OGCG: %[[FOR_TID:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-OGCG-SAME: ptr {{.*}}, i32 %[[FOR_TID]], i32 34, +// LLVM-OGCG-SAME: ptr %[[OMP_IS_LAST]], ptr %[[OMP_LB]], ptr %[[OMP_UB]], ptr %[[OMP_STRIDE]], +// LLVM-OGCG-SAME: i32 1, i32 1) +// LLVM-OGCG: %[[FOR_CMP:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-OGCG-NEXT: br i1 %[[FOR_CMP]], label %[[FOR_BODY:[^,]+]], label %{{.*}} +// LLVM-OGCG: {{^}}[[FOR_BODY]]: +// LLVM-OGCG: call void @during(i32 +// LLVM-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[FOR_TID]]) +// LLVM-OGCG-NEXT: ret void + +// `teams_distribute_parallel_for` lowers to the exact same shape as +// `distribute_parallel_for` above; we check the full CFG again here instead +// of just the call sequence, so a future change that alters this shape for +// only one of the two spellings still gets caught. +// LLVM-LABEL: define {{.*}} void @teams_distribute_parallel_for( +// LLVM: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr {{.*}}, i32 0, ptr @[[TEAMS_OUTLINED2:[^)]+]]) + +// LLVM: define internal void @[[TEAMS_OUTLINED2]]( +// LLVM-CIR: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 0, ptr @[[PAR_OUTLINED2:[^)]+]]) + +// LLVM-CIR: define internal void @[[PAR_OUTLINED2]]( +// LLVM-CIR: store i32 0, ptr %[[LB2:.*]], align 4 +// LLVM-CIR-NEXT: store i32 9, ptr %[[UB2:.*]], align 4 +// LLVM-CIR-NEXT: store i32 1, ptr %[[STRIDE2:.*]], align 4 +// LLVM-CIR-NEXT: %[[TID2:.*]] = call i32 @__kmpc_global_thread_num(ptr {{.*}}) +// LLVM-CIR-NEXT: call void @__kmpc_dist_for_static_init_4u( +// LLVM-CIR-SAME: ptr {{.*}}, i32 %[[TID2]], i32 34, +// LLVM-CIR-SAME: ptr %{{[^,]+}}, ptr %[[LB2]], ptr %[[UB2]], ptr %{{[^,]+}}, ptr %[[STRIDE2]], +// LLVM-CIR-SAME: i32 1, i32 0) +// LLVM-CIR: %[[IV2:.*]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXT2:[^,]+]], %{{.*}} ] +// LLVM-CIR: %[[LOOP_CMP2:.*]] = icmp ult i32 %[[IV2]], %{{.*}} +// LLVM-CIR-NEXT: br i1 %[[LOOP_CMP2]], label %[[LOOP_BODY2:[^,]+]], label %[[LOOP_EXIT2:[^ ,]+]] +// LLVM-CIR: {{^}}[[LOOP_EXIT2]]: +// LLVM-CIR-NEXT: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[TID2]]) +// LLVM-CIR: call void @__kmpc_barrier( +// LLVM-CIR: {{^}}[[LOOP_BODY2]]: +// LLVM-CIR: call void @during(i32 +// LLVM-CIR: %[[NEXT2]] = add nuw i32 %[[IV2]], 1 + +// LLVM-OGCG: store i32 0, ptr %[[COMB_LB2:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 9, ptr %[[COMB_UB2:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 1, ptr %[[DIST_STRIDE2:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 0, ptr %[[DIST_IS_LAST2:.*]], align 4 +// LLVM-OGCG: %[[DIST_TID2:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-OGCG-SAME: ptr {{.*}}, i32 %[[DIST_TID2]], i32 92, +// LLVM-OGCG-SAME: ptr %[[DIST_IS_LAST2]], ptr %[[COMB_LB2]], ptr %[[COMB_UB2]], ptr %[[DIST_STRIDE2]], +// LLVM-OGCG-SAME: i32 1, i32 1) +// LLVM-OGCG: %[[DIST_CMP2:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-OGCG-NEXT: br i1 %[[DIST_CMP2]], label %[[DIST_BODY2:[^,]+]], label %{{.*}} +// LLVM-OGCG: {{^}}[[DIST_BODY2]]: +// LLVM-OGCG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @[[PAR_OUTLINED2:[^,]+]], +// LLVM-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[DIST_TID2]]) +// LLVM-OGCG-NEXT: ret void + +// LLVM-OGCG: define internal void @[[PAR_OUTLINED2]]( +// LLVM-OGCG: store i32 0, ptr %[[OMP_LB2:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 9, ptr %[[OMP_UB2:.*]], align 4 +// LLVM-OGCG: store i32 1, ptr %[[OMP_STRIDE2:.*]], align 4 +// LLVM-OGCG-NEXT: store i32 0, ptr %[[OMP_IS_LAST2:.*]], align 4 +// LLVM-OGCG: %[[FOR_TID2:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-OGCG-SAME: ptr {{.*}}, i32 %[[FOR_TID2]], i32 34, +// LLVM-OGCG-SAME: ptr %[[OMP_IS_LAST2]], ptr %[[OMP_LB2]], ptr %[[OMP_UB2]], ptr %[[OMP_STRIDE2]], +// LLVM-OGCG-SAME: i32 1, i32 1) +// LLVM-OGCG: %[[FOR_CMP2:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-OGCG-NEXT: br i1 %[[FOR_CMP2]], label %[[FOR_BODY2:[^,]+]], label %{{.*}} +// LLVM-OGCG: {{^}}[[FOR_BODY2]]: +// LLVM-OGCG: call void @during(i32 +// LLVM-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[FOR_TID2]]) +// LLVM-OGCG-NEXT: ret void diff --git a/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c b/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c index fd58fae5ea4ec0..2f65390cda368d 100644 --- a/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c +++ b/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c @@ -51,4 +51,17 @@ void do_things() { #pragma omp teams distribute collapse(1) for (int j = 0; j < 10; j++) { } + + // The composite `distribute parallel for` construct routes clauses to its + // `distribute` and `for` directives, both of which report every clause as + // not-yet-implemented. + // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenMP DISTRIBUTE 'dist_schedule' clause}} +#pragma omp teams distribute parallel for dist_schedule(static) + for (int j = 0; j < 10; j++) { + } + + // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenMP FOR 'schedule' clause}} +#pragma omp teams distribute parallel for schedule(static) + for (int j = 0; j < 10; j++) { + } } diff --git a/clang/test/CIR/CodeGenOpenMP/region-cleanups-O2.c b/clang/test/CIR/CodeGenOpenMP/region-cleanups-O2.c index 37f118f9799c4d..babf631831a031 100644 --- a/clang/test/CIR/CodeGenOpenMP/region-cleanups-O2.c +++ b/clang/test/CIR/CodeGenOpenMP/region-cleanups-O2.c @@ -162,3 +162,30 @@ void local_in_teams(void) { // LLVM-LABEL: define{{.*}} void @local_in_teams( // LLVM: call void {{.*}}@__kmpc_fork_teams( + +// The composite loop's iteration variable is emitted without lifetime markers, +// so the composite omp.parallel, omp.distribute and omp.wsloop stay directly +// nested and the kernel is still SPMD. +void local_in_teams_distribute_parallel_for(void) { +#pragma omp target teams distribute parallel for + for (int i = 0; i < 10; i++) { + int t = i * 2; + use(t); + } +} + +// CIR-LABEL: cir.func{{.*}}@local_in_teams_distribute_parallel_for +// CIR: omp.target kernel_type(spmd) +// CIR: omp.teams { +// CIR: omp.parallel +// CIR-NOT: cir.lifetime.start +// CIR-NOT: cir.cleanup.scope +// CIR: omp.distribute { +// CIR-NEXT: omp.wsloop { +// CIR-NEXT: omp.loop_nest +// CIR: cir.lifetime.start +// CIR: cir.cleanup.scope { +// CIR: omp.yield + +// LLVM-LABEL: define{{.*}} void @local_in_teams_distribute_parallel_for( +// LLVM: call i32 @__tgt_target_kernel( diff --git a/clang/test/CIR/CodeGenOpenMP/target-teams-distribute-parallel-for.c b/clang/test/CIR/CodeGenOpenMP/target-teams-distribute-parallel-for.c new file mode 100644 index 00000000000000..f1ac9fd177ccda --- /dev/null +++ b/clang/test/CIR/CodeGenOpenMP/target-teams-distribute-parallel-for.c @@ -0,0 +1,876 @@ +// REQUIRES: amdgpu-registered-target + +// Host compilation (x86 host, AMDGPU offload target). +// RUN: %clang_cc1 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -emit-cir -fclangir %s -o - \ +// RUN: | FileCheck %s --check-prefix=CIR-HOST + +// Device compilation (AMDGPU). +// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-target-device \ +// RUN: -emit-cir -fclangir %s -o - \ +// RUN: | FileCheck %s --check-prefix=CIR-DEVICE + +// Host-side lowering to LLVM IR: the Tripcount host_eval and +// __tgt_target_kernel call are identical on both paths, so LLVM-HOST +// checks both; the outlined helper functions differ, checked below under +// LLVM-HOST-CIR/LLVM-HOST-OGCG. +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp \ +// RUN: -fopenmp-targets=amdgcn-amd-amdhsa -fclangir -emit-llvm %s -o - \ +// RUN: | FileCheck %s --check-prefixes=LLVM-HOST,LLVM-HOST-CIR +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp \ +// RUN: -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm %s -o - \ +// RUN: | FileCheck %s --check-prefixes=LLVM-HOST,LLVM-HOST-OGCG + +// Device-side lowering to LLVM IR: compiled in two steps to mirror the +// offloading driver. +// RUN: %clang_cc1 -fopenmp -triple x86_64-linux-gnu -fopenmp-targets=amdgcn-amd-amdhsa \ +// RUN: -fclangir -emit-llvm-bc %s -o %t-cir-host.bc +// RUN: %clang_cc1 -fopenmp -triple x86_64-linux-gnu -fopenmp-targets=amdgcn-amd-amdhsa \ +// RUN: -emit-llvm-bc %s -o %t-ogcg-host.bc +// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-target-device \ +// RUN: -fopenmp-host-ir-file-path %t-cir-host.bc \ +// RUN: -fclangir -emit-llvm %s -o - \ +// RUN: | FileCheck %s --check-prefixes=LLVM-DEVICE,LLVM-DEVICE-CIR +// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-target-device \ +// RUN: -fopenmp-host-ir-file-path %t-ogcg-host.bc \ +// RUN: -emit-llvm %s -o - \ +// RUN: | FileCheck %s --check-prefixes=LLVM-DEVICE,LLVM-DEVICE-OGCG + +void during(int); + +void explicit_target_teams_distribute_parallel_for() { +#pragma omp target +#pragma omp teams +#pragma omp distribute parallel for + for (int i = 0; i < 10; i++) { + during(i); + } +} + +// CIR-HOST: cir.func{{.*}}@explicit_target_teams_distribute_parallel_for( +// tripCount = (10 - (0 - 1 + 1)) / 1 = 10, computed on the host and +// passed in through host_eval. +// CIR-HOST-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-HOST-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-HOST-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-HOST-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-HOST-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-HOST-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: omp.target kernel_type(spmd) host_eval(%[[ZERO]] -> %[[AZERO:.*]], %[[TRIPCOUNT]] -> %[[ATRIPCOUNT:.*]], %[[ONE]] -> %[[AONE:.*]] : i32, i32, i32) { +// CIR-HOST-NEXT: omp.teams { +// CIR-HOST-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.parallel private(@{{.*}} %[[I_ALLOCA]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-HOST-NEXT: %[[IV_ADDR:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.distribute { +// CIR-HOST-NEXT: omp.wsloop { +// CIR-HOST-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[AZERO]]) to (%[[ATRIPCOUNT]]) step (%[[AONE]]) { +// CIR-HOST-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-HOST-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.call @{{.*}}during +// CIR-HOST: omp.yield +// CIR-HOST: } +// CIR-HOST: } {omp.composite} +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} + +// CIR-DEVICE: cir.func{{.*}}@explicit_target_teams_distribute_parallel_for( +// CIR-DEVICE-NEXT: omp.target kernel_type(spmd) { +// CIR-DEVICE-NEXT: omp.teams { +// CIR-DEVICE-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[I_ADDR:.*]] = cir.cast address_space %[[I_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// CIR-DEVICE-NEXT: omp.parallel private(@{{.*}} %[[I_ADDR]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-DEVICE-NEXT: %[[IV_ALLOCA:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[IV_ADDR:.*]] = cir.cast address_space %[[IV_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// The device has no host_eval operands, so it recomputes the same trip count +// inside the region. +// CIR-DEVICE-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-DEVICE-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-DEVICE-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-DEVICE-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-DEVICE-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-DEVICE-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: omp.distribute { +// CIR-DEVICE-NEXT: omp.wsloop { +// CIR-DEVICE-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[ZERO]]) to (%[[TRIPCOUNT]]) step (%[[ONE]]) { +// CIR-DEVICE-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-DEVICE-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.call @{{.*}}during +// CIR-DEVICE: omp.yield +// CIR-DEVICE: } +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} + +void target_teams_distribute_parallel_for() { +#pragma omp target teams distribute parallel for + for (int i = 0; i < 10; i++) { + during(i); + } +} + +// CIR-HOST: cir.func{{.*}}@target_teams_distribute_parallel_for( +// tripCount = (10 - (0 - 1 + 1)) / 1 = 10, computed on the host and +// passed in through host_eval. +// CIR-HOST-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-HOST-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-HOST-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-HOST-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-HOST-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-HOST-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: omp.target kernel_type(spmd) host_eval(%[[ZERO]] -> %[[AZERO:.*]], %[[TRIPCOUNT]] -> %[[ATRIPCOUNT:.*]], %[[ONE]] -> %[[AONE:.*]] : i32, i32, i32) { +// CIR-HOST-NEXT: omp.teams { +// CIR-HOST-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.parallel private(@{{.*}} %[[I_ALLOCA]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-HOST-NEXT: %[[IV_ADDR:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.distribute { +// CIR-HOST-NEXT: omp.wsloop { +// CIR-HOST-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[AZERO]]) to (%[[ATRIPCOUNT]]) step (%[[AONE]]) { +// CIR-HOST-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-HOST-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.call @{{.*}}during +// CIR-HOST: omp.yield +// CIR-HOST: } +// CIR-HOST: } {omp.composite} +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} + +// CIR-DEVICE: cir.func{{.*}}@target_teams_distribute_parallel_for( +// CIR-DEVICE-NEXT: omp.target kernel_type(spmd) { +// CIR-DEVICE-NEXT: omp.teams { +// CIR-DEVICE-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[I_ADDR:.*]] = cir.cast address_space %[[I_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// CIR-DEVICE-NEXT: omp.parallel private(@{{.*}} %[[I_ADDR]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-DEVICE-NEXT: %[[IV_ALLOCA:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[IV_ADDR:.*]] = cir.cast address_space %[[IV_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// The device has no host_eval operands, so it recomputes the same trip count +// inside the region. +// CIR-DEVICE-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-DEVICE-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-DEVICE-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-DEVICE-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-DEVICE-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-DEVICE-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: omp.distribute { +// CIR-DEVICE-NEXT: omp.wsloop { +// CIR-DEVICE-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[ZERO]]) to (%[[TRIPCOUNT]]) step (%[[ONE]]) { +// CIR-DEVICE-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-DEVICE-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.call @{{.*}}during +// CIR-DEVICE: omp.yield +// CIR-DEVICE: } +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} + +void target_teams_split_distribute_parallel_for() { +#pragma omp target teams +#pragma omp distribute parallel for + for (int i = 0; i < 10; i++) { + during(i); + } +} + +// CIR-HOST: cir.func{{.*}}@target_teams_split_distribute_parallel_for( +// tripCount = (10 - (0 - 1 + 1)) / 1 = 10, computed on the host and +// passed in through host_eval. +// CIR-HOST-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-HOST-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-HOST-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-HOST-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-HOST-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-HOST-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: omp.target kernel_type(spmd) host_eval(%[[ZERO]] -> %[[AZERO:.*]], %[[TRIPCOUNT]] -> %[[ATRIPCOUNT:.*]], %[[ONE]] -> %[[AONE:.*]] : i32, i32, i32) { +// CIR-HOST-NEXT: omp.teams { +// CIR-HOST-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.parallel private(@{{.*}} %[[I_ALLOCA]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-HOST-NEXT: %[[IV_ADDR:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.distribute { +// CIR-HOST-NEXT: omp.wsloop { +// CIR-HOST-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[AZERO]]) to (%[[ATRIPCOUNT]]) step (%[[AONE]]) { +// CIR-HOST-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-HOST-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.call @{{.*}}during +// CIR-HOST: omp.yield +// CIR-HOST: } +// CIR-HOST: } {omp.composite} +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} + +// CIR-DEVICE: cir.func{{.*}}@target_teams_split_distribute_parallel_for( +// CIR-DEVICE-NEXT: omp.target kernel_type(spmd) { +// CIR-DEVICE-NEXT: omp.teams { +// CIR-DEVICE-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[I_ADDR:.*]] = cir.cast address_space %[[I_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// CIR-DEVICE-NEXT: omp.parallel private(@{{.*}} %[[I_ADDR]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-DEVICE-NEXT: %[[IV_ALLOCA:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[IV_ADDR:.*]] = cir.cast address_space %[[IV_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// The device has no host_eval operands, so it recomputes the same trip count +// inside the region. +// CIR-DEVICE-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-DEVICE-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-DEVICE-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-DEVICE-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-DEVICE-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-DEVICE-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: omp.distribute { +// CIR-DEVICE-NEXT: omp.wsloop { +// CIR-DEVICE-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[ZERO]]) to (%[[TRIPCOUNT]]) step (%[[ONE]]) { +// CIR-DEVICE-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-DEVICE-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.call @{{.*}}during +// CIR-DEVICE: omp.yield +// CIR-DEVICE: } +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} + +void target_split_teams_distribute_parallel_for() { +#pragma omp target +#pragma omp teams distribute parallel for + for (int i = 0; i < 10; i++) { + during(i); + } +} + +// CIR-HOST: cir.func{{.*}}@target_split_teams_distribute_parallel_for( +// tripCount = (10 - (0 - 1 + 1)) / 1 = 10, computed on the host and +// passed in through host_eval. +// CIR-HOST-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-HOST-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-HOST-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-HOST-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-HOST-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-HOST-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-HOST-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-HOST-NEXT: omp.target kernel_type(spmd) host_eval(%[[ZERO]] -> %[[AZERO:.*]], %[[TRIPCOUNT]] -> %[[ATRIPCOUNT:.*]], %[[ONE]] -> %[[AONE:.*]] : i32, i32, i32) { +// CIR-HOST-NEXT: omp.teams { +// CIR-HOST-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.parallel private(@{{.*}} %[[I_ALLOCA]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-HOST-NEXT: %[[IV_ADDR:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i> +// CIR-HOST-NEXT: omp.distribute { +// CIR-HOST-NEXT: omp.wsloop { +// CIR-HOST-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[AZERO]]) to (%[[ATRIPCOUNT]]) step (%[[AONE]]) { +// CIR-HOST-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-HOST-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-HOST: cir.call @{{.*}}during +// CIR-HOST: omp.yield +// CIR-HOST: } +// CIR-HOST: } {omp.composite} +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.composite} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} +// CIR-HOST: omp.terminator +// CIR-HOST: } {omp.combined} + +// CIR-DEVICE: cir.func{{.*}}@target_split_teams_distribute_parallel_for( +// CIR-DEVICE-NEXT: omp.target kernel_type(spmd) { +// CIR-DEVICE-NEXT: omp.teams { +// CIR-DEVICE-NEXT: %[[I_ALLOCA:.*]] = cir.alloca "i" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[I_ADDR:.*]] = cir.cast address_space %[[I_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// CIR-DEVICE-NEXT: omp.parallel private(@{{.*}} %[[I_ADDR]] -> %[[I_PRIV:.*]] : !cir.ptr<!s32i>) { +// CIR-DEVICE-NEXT: %[[IV_ALLOCA:.*]] = cir.alloca ".omp.iv" align(4) : !cir.ptr<!s32i, target_address_space(5)> +// CIR-DEVICE-NEXT: %[[IV_ADDR:.*]] = cir.cast address_space %[[IV_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i> +// The device has no host_eval operands, so it recomputes the same trip count +// inside the region. +// CIR-DEVICE-NEXT: %[[HI:.*]] = cir.const #cir.int<10> : !s32i +// CIR-DEVICE-NEXT: %[[LO:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[STEP_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[LO_MINUS_STEP:.*]] = cir.sub nsw %[[LO]], %[[STEP_A]] : !s32i +// CIR-DEVICE-NEXT: %[[ONE_A:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ADJ_LO:.*]] = cir.add nsw %[[LO_MINUS_STEP]], %[[ONE_A]] : !s32i +// CIR-DEVICE-NEXT: %[[SPAN:.*]] = cir.sub nsw %[[HI]], %[[ADJ_LO]] : !s32i +// CIR-DEVICE-NEXT: %[[STEP_B:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[TRIPCOUNT_CIR:.*]] = cir.div %[[SPAN]], %[[STEP_B]] : !s32i +// CIR-DEVICE-NEXT: %[[ZERO_CIR:.*]] = cir.const #cir.int<0> : !s32i +// CIR-DEVICE-NEXT: %[[ZERO:.*]] = cir.builtin_int_cast %[[ZERO_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[ONE_CIR:.*]] = cir.const #cir.int<1> : !s32i +// CIR-DEVICE-NEXT: %[[ONE:.*]] = cir.builtin_int_cast %[[ONE_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: %[[TRIPCOUNT:.*]] = cir.builtin_int_cast %[[TRIPCOUNT_CIR]] : !s32i -> i32 +// CIR-DEVICE-NEXT: omp.distribute { +// CIR-DEVICE-NEXT: omp.wsloop { +// CIR-DEVICE-NEXT: omp.loop_nest (%[[IV:.*]]) : i32 = (%[[ZERO]]) to (%[[TRIPCOUNT]]) step (%[[ONE]]) { +// CIR-DEVICE-NEXT: %[[IV_CIR:.*]] = cir.builtin_int_cast %[[IV]] : i32 -> !s32i +// CIR-DEVICE-NEXT: cir.store align(4) %[[IV_CIR]], %[[IV_ADDR]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.store align(4) %{{.*}}, %[[I_PRIV]] : !s32i, !cir.ptr<!s32i> +// CIR-DEVICE: cir.call @{{.*}}during +// CIR-DEVICE: omp.yield +// CIR-DEVICE: } +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.composite} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} +// CIR-DEVICE: omp.terminator +// CIR-DEVICE: } {omp.combined} + +// Host wrappers: both paths store the trip count (10) in the kernel +// arguments. CIR groups all its helper functions together after the host +// wrappers, since the MLIR-to-LLVM translation emits them during the +// lowering pass itself; OGCG's direct AST walk emits each one right after +// its host wrapper. Only the host wrappers' positions differ, so the later +// three are checked separately for each path. +// LLVM-HOST-LABEL: define {{.*}} void @explicit_target_teams_distribute_parallel_for( +// LLVM-HOST: %[[KARGS:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST: %[[TRIPCOUNT_GEP:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS]], i32 0, i32 8 +// LLVM-HOST-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP]], align 8 +// LLVM-HOST: call i32 @__tgt_target_kernel( +// LLVM-HOST: br i1 %{{.*}}, label %[[FAILED:[^,]+]], label %{{.*}} +// LLVM-HOST: {{^}}[[FAILED]]: +// LLVM-HOST-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_explicit_target_teams_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// LLVM-HOST-CIR-LABEL: define {{.*}} void @target_teams_distribute_parallel_for( +// LLVM-HOST-CIR: %[[KARGS2:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST-CIR: %[[TRIPCOUNT_GEP2:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS2]], i32 0, i32 8 +// LLVM-HOST-CIR-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP2]], align 8 +// LLVM-HOST-CIR: call i32 @__tgt_target_kernel( +// LLVM-HOST-CIR: br i1 %{{.*}}, label %[[FAILED2:[^,]+]], label %{{.*}} +// LLVM-HOST-CIR: {{^}}[[FAILED2]]: +// LLVM-HOST-CIR-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// LLVM-HOST-CIR-LABEL: define {{.*}} void @target_teams_split_distribute_parallel_for( +// LLVM-HOST-CIR: %[[KARGS3:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST-CIR: %[[TRIPCOUNT_GEP3:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS3]], i32 0, i32 8 +// LLVM-HOST-CIR-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP3]], align 8 +// LLVM-HOST-CIR: call i32 @__tgt_target_kernel( +// LLVM-HOST-CIR: br i1 %{{.*}}, label %[[FAILED3:[^,]+]], label %{{.*}} +// LLVM-HOST-CIR: {{^}}[[FAILED3]]: +// LLVM-HOST-CIR-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_split_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// LLVM-HOST-CIR-LABEL: define {{.*}} void @target_split_teams_distribute_parallel_for( +// LLVM-HOST-CIR: %[[KARGS4:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST-CIR: %[[TRIPCOUNT_GEP4:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS4]], i32 0, i32 8 +// LLVM-HOST-CIR-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP4]], align 8 +// LLVM-HOST-CIR: call i32 @__tgt_target_kernel( +// LLVM-HOST-CIR: br i1 %{{.*}}, label %[[FAILED4:[^,]+]], label %{{.*}} +// LLVM-HOST-CIR: {{^}}[[FAILED4]]: +// LLVM-HOST-CIR-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_split_teams_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// Outlined target-entry functions: fork the teams region via +// __kmpc_fork_teams on both paths. +// +// The CIR path's teams-outlined function just forks the parallel region +// directly via __kmpc_fork_call; the forked function then schedules both the +// distribute and the for chunking in a single __kmpc_dist_for_static_init_4u +// call (schedule kind 34), ending with a barrier. +// +// The OGCG path instead schedules the distribute chunking (schedule kind 92) +// in the teams-outlined function, forking the parallel region once per +// distribute chunk; the forked function then schedules its own static-for +// chunking (schedule kind 34) over just that chunk. +// LLVM-HOST-LABEL: define internal void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_explicit_target_teams_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-HOST: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr {{.*}}, i32 0, ptr @[[TEAMS_OUTLINED:[^)]+]]) + +// LLVM-HOST: define internal void @[[TEAMS_OUTLINED]]( +// LLVM-HOST-CIR: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 0, ptr @[[PAR_OUTLINED:[^)]+]]) + +// LLVM-HOST-CIR: define internal void @[[PAR_OUTLINED]]( +// LLVM-HOST-CIR: store i32 0, ptr %[[LB:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 9, ptr %[[UB:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 1, ptr %[[STRIDE:.*]], align 4 +// LLVM-HOST-CIR-NEXT: %[[TID:.*]] = call i32 @__kmpc_global_thread_num(ptr {{.*}}) +// LLVM-HOST-CIR-NEXT: call void @__kmpc_dist_for_static_init_4u( +// LLVM-HOST-CIR-SAME: ptr {{.*}}, i32 %[[TID]], i32 34, +// LLVM-HOST-CIR-SAME: ptr %{{[^,]+}}, ptr %[[LB]], ptr %[[UB]], ptr %{{[^,]+}}, ptr %[[STRIDE]], +// LLVM-HOST-CIR-SAME: i32 1, i32 0) +// LLVM-HOST-CIR: %[[IV:.*]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXT:[^,]+]], %{{.*}} ] +// LLVM-HOST-CIR: %[[LOOP_CMP:.*]] = icmp ult i32 %[[IV]], %{{.*}} +// LLVM-HOST-CIR-NEXT: br i1 %[[LOOP_CMP]], label %[[LOOP_BODY:[^,]+]], label %[[LOOP_EXIT:[^ ,]+]] +// LLVM-HOST-CIR: {{^}}[[LOOP_EXIT]]: +// LLVM-HOST-CIR-NEXT: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[TID]]) +// LLVM-HOST-CIR: call void @__kmpc_barrier( +// LLVM-HOST-CIR: {{^}}[[LOOP_BODY]]: +// LLVM-HOST-CIR: call void @during(i32 +// LLVM-HOST-CIR: %[[NEXT]] = add nuw i32 %[[IV]], 1 + +// LLVM-HOST-OGCG: store i32 0, ptr %[[COMB_LB:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[COMB_UB:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 1, ptr %[[DIST_STRIDE:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[DIST_IS_LAST:.*]], align 4 +// LLVM-HOST-OGCG: %[[DIST_TID:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[DIST_TID]], i32 92, +// LLVM-HOST-OGCG-SAME: ptr %[[DIST_IS_LAST]], ptr %[[COMB_LB]], ptr %[[COMB_UB]], ptr %[[DIST_STRIDE]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[DIST_CMP:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[DIST_CMP]], label %[[DIST_BODY:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[DIST_BODY]]: +// LLVM-HOST-OGCG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @[[PAR_OUTLINED:[^,]+]], +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[DIST_TID]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG: define internal void @[[PAR_OUTLINED]]( +// LLVM-HOST-OGCG: store i32 0, ptr %[[OMP_LB:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[OMP_UB:.*]], align 4 +// LLVM-HOST-OGCG: store i32 1, ptr %[[OMP_STRIDE:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[OMP_IS_LAST:.*]], align 4 +// LLVM-HOST-OGCG: %[[FOR_TID:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[FOR_TID]], i32 34, +// LLVM-HOST-OGCG-SAME: ptr %[[OMP_IS_LAST]], ptr %[[OMP_LB]], ptr %[[OMP_UB]], ptr %[[OMP_STRIDE]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[FOR_CMP:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[FOR_CMP]], label %[[FOR_BODY:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FOR_BODY]]: +// LLVM-HOST-OGCG: call void @during(i32 +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[FOR_TID]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG-LABEL: define {{.*}} void @target_teams_distribute_parallel_for( +// LLVM-HOST-OGCG: %[[KARGS2:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST-OGCG: %[[TRIPCOUNT_GEP2:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS2]], i32 0, i32 8 +// LLVM-HOST-OGCG-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP2]], align 8 +// LLVM-HOST-OGCG: call i32 @__tgt_target_kernel( +// LLVM-HOST-OGCG: br i1 %{{.*}}, label %[[FAILED2:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FAILED2]]: +// LLVM-HOST-OGCG-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// LLVM-HOST-LABEL: define internal void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-HOST: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr {{.*}}, i32 0, ptr @[[TEAMS_OUTLINED2:[^)]+]]) + +// LLVM-HOST: define internal void @[[TEAMS_OUTLINED2]]( +// LLVM-HOST-CIR: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 0, ptr @[[PAR_OUTLINED2:[^)]+]]) + +// LLVM-HOST-CIR: define internal void @[[PAR_OUTLINED2]]( +// LLVM-HOST-CIR: store i32 0, ptr %[[LB2:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 9, ptr %[[UB2:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 1, ptr %[[STRIDE2:.*]], align 4 +// LLVM-HOST-CIR-NEXT: %[[TID2:.*]] = call i32 @__kmpc_global_thread_num(ptr {{.*}}) +// LLVM-HOST-CIR-NEXT: call void @__kmpc_dist_for_static_init_4u( +// LLVM-HOST-CIR-SAME: ptr {{.*}}, i32 %[[TID2]], i32 34, +// LLVM-HOST-CIR-SAME: ptr %{{[^,]+}}, ptr %[[LB2]], ptr %[[UB2]], ptr %{{[^,]+}}, ptr %[[STRIDE2]], +// LLVM-HOST-CIR-SAME: i32 1, i32 0) +// LLVM-HOST-CIR: %[[IV2:.*]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXT2:[^,]+]], %{{.*}} ] +// LLVM-HOST-CIR: %[[LOOP_CMP2:.*]] = icmp ult i32 %[[IV2]], %{{.*}} +// LLVM-HOST-CIR-NEXT: br i1 %[[LOOP_CMP2]], label %[[LOOP_BODY2:[^,]+]], label %[[LOOP_EXIT2:[^ ,]+]] +// LLVM-HOST-CIR: {{^}}[[LOOP_EXIT2]]: +// LLVM-HOST-CIR-NEXT: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[TID2]]) +// LLVM-HOST-CIR: call void @__kmpc_barrier( +// LLVM-HOST-CIR: {{^}}[[LOOP_BODY2]]: +// LLVM-HOST-CIR: call void @during(i32 +// LLVM-HOST-CIR: %[[NEXT2]] = add nuw i32 %[[IV2]], 1 + +// LLVM-HOST-OGCG: store i32 0, ptr %[[COMB_LB2:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[COMB_UB2:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 1, ptr %[[DIST_STRIDE2:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[DIST_IS_LAST2:.*]], align 4 +// LLVM-HOST-OGCG: %[[DIST_TID2:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[DIST_TID2]], i32 92, +// LLVM-HOST-OGCG-SAME: ptr %[[DIST_IS_LAST2]], ptr %[[COMB_LB2]], ptr %[[COMB_UB2]], ptr %[[DIST_STRIDE2]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[DIST_CMP2:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[DIST_CMP2]], label %[[DIST_BODY2:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[DIST_BODY2]]: +// LLVM-HOST-OGCG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @[[PAR_OUTLINED2:[^,]+]], +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[DIST_TID2]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG: define internal void @[[PAR_OUTLINED2]]( +// LLVM-HOST-OGCG: store i32 0, ptr %[[OMP_LB2:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[OMP_UB2:.*]], align 4 +// LLVM-HOST-OGCG: store i32 1, ptr %[[OMP_STRIDE2:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[OMP_IS_LAST2:.*]], align 4 +// LLVM-HOST-OGCG: %[[FOR_TID2:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[FOR_TID2]], i32 34, +// LLVM-HOST-OGCG-SAME: ptr %[[OMP_IS_LAST2]], ptr %[[OMP_LB2]], ptr %[[OMP_UB2]], ptr %[[OMP_STRIDE2]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[FOR_CMP2:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[FOR_CMP2]], label %[[FOR_BODY2:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FOR_BODY2]]: +// LLVM-HOST-OGCG: call void @during(i32 +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[FOR_TID2]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG-LABEL: define {{.*}} void @target_teams_split_distribute_parallel_for( +// LLVM-HOST-OGCG: %[[KARGS3:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST-OGCG: %[[TRIPCOUNT_GEP3:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS3]], i32 0, i32 8 +// LLVM-HOST-OGCG-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP3]], align 8 +// LLVM-HOST-OGCG: call i32 @__tgt_target_kernel( +// LLVM-HOST-OGCG: br i1 %{{.*}}, label %[[FAILED3:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FAILED3]]: +// LLVM-HOST-OGCG-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_split_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// LLVM-HOST-LABEL: define internal void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_split_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-HOST: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr {{.*}}, i32 0, ptr @[[TEAMS_OUTLINED3:[^)]+]]) + +// LLVM-HOST: define internal void @[[TEAMS_OUTLINED3]]( +// LLVM-HOST-CIR: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 0, ptr @[[PAR_OUTLINED3:[^)]+]]) + +// LLVM-HOST-CIR: define internal void @[[PAR_OUTLINED3]]( +// LLVM-HOST-CIR: store i32 0, ptr %[[LB3:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 9, ptr %[[UB3:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 1, ptr %[[STRIDE3:.*]], align 4 +// LLVM-HOST-CIR-NEXT: %[[TID3:.*]] = call i32 @__kmpc_global_thread_num(ptr {{.*}}) +// LLVM-HOST-CIR-NEXT: call void @__kmpc_dist_for_static_init_4u( +// LLVM-HOST-CIR-SAME: ptr {{.*}}, i32 %[[TID3]], i32 34, +// LLVM-HOST-CIR-SAME: ptr %{{[^,]+}}, ptr %[[LB3]], ptr %[[UB3]], ptr %{{[^,]+}}, ptr %[[STRIDE3]], +// LLVM-HOST-CIR-SAME: i32 1, i32 0) +// LLVM-HOST-CIR: %[[IV3:.*]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXT3:[^,]+]], %{{.*}} ] +// LLVM-HOST-CIR: %[[LOOP_CMP3:.*]] = icmp ult i32 %[[IV3]], %{{.*}} +// LLVM-HOST-CIR-NEXT: br i1 %[[LOOP_CMP3]], label %[[LOOP_BODY3:[^,]+]], label %[[LOOP_EXIT3:[^ ,]+]] +// LLVM-HOST-CIR: {{^}}[[LOOP_EXIT3]]: +// LLVM-HOST-CIR-NEXT: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[TID3]]) +// LLVM-HOST-CIR: call void @__kmpc_barrier( +// LLVM-HOST-CIR: {{^}}[[LOOP_BODY3]]: +// LLVM-HOST-CIR: call void @during(i32 +// LLVM-HOST-CIR: %[[NEXT3]] = add nuw i32 %[[IV3]], 1 + +// LLVM-HOST-OGCG: store i32 0, ptr %[[COMB_LB3:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[COMB_UB3:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 1, ptr %[[DIST_STRIDE3:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[DIST_IS_LAST3:.*]], align 4 +// LLVM-HOST-OGCG: %[[DIST_TID3:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[DIST_TID3]], i32 92, +// LLVM-HOST-OGCG-SAME: ptr %[[DIST_IS_LAST3]], ptr %[[COMB_LB3]], ptr %[[COMB_UB3]], ptr %[[DIST_STRIDE3]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[DIST_CMP3:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[DIST_CMP3]], label %[[DIST_BODY3:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[DIST_BODY3]]: +// LLVM-HOST-OGCG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @[[PAR_OUTLINED3:[^,]+]], +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[DIST_TID3]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG: define internal void @[[PAR_OUTLINED3]]( +// LLVM-HOST-OGCG: store i32 0, ptr %[[OMP_LB3:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[OMP_UB3:.*]], align 4 +// LLVM-HOST-OGCG: store i32 1, ptr %[[OMP_STRIDE3:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[OMP_IS_LAST3:.*]], align 4 +// LLVM-HOST-OGCG: %[[FOR_TID3:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[FOR_TID3]], i32 34, +// LLVM-HOST-OGCG-SAME: ptr %[[OMP_IS_LAST3]], ptr %[[OMP_LB3]], ptr %[[OMP_UB3]], ptr %[[OMP_STRIDE3]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[FOR_CMP3:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[FOR_CMP3]], label %[[FOR_BODY3:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FOR_BODY3]]: +// LLVM-HOST-OGCG: call void @during(i32 +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[FOR_TID3]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG-LABEL: define {{.*}} void @target_split_teams_distribute_parallel_for( +// LLVM-HOST-OGCG: %[[KARGS4:.*]] = alloca %struct.__tgt_kernel_arguments, align 8 +// LLVM-HOST-OGCG: %[[TRIPCOUNT_GEP4:.*]] = getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr %[[KARGS4]], i32 0, i32 8 +// LLVM-HOST-OGCG-NEXT: store i64 10, ptr %[[TRIPCOUNT_GEP4]], align 8 +// LLVM-HOST-OGCG: call i32 @__tgt_target_kernel( +// LLVM-HOST-OGCG: br i1 %{{.*}}, label %[[FAILED4:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FAILED4]]: +// LLVM-HOST-OGCG-NEXT: call void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_split_teams_distribute_parallel_for_l{{[0-9]+}}(ptr null) + +// LLVM-HOST-LABEL: define internal void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_split_teams_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-HOST: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr {{.*}}, i32 0, ptr @[[TEAMS_OUTLINED4:[^)]+]]) + +// LLVM-HOST: define internal void @[[TEAMS_OUTLINED4]]( +// LLVM-HOST-CIR: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 0, ptr @[[PAR_OUTLINED4:[^)]+]]) + +// LLVM-HOST-CIR: define internal void @[[PAR_OUTLINED4]]( +// LLVM-HOST-CIR: store i32 0, ptr %[[LB4:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 9, ptr %[[UB4:.*]], align 4 +// LLVM-HOST-CIR-NEXT: store i32 1, ptr %[[STRIDE4:.*]], align 4 +// LLVM-HOST-CIR-NEXT: %[[TID4:.*]] = call i32 @__kmpc_global_thread_num(ptr {{.*}}) +// LLVM-HOST-CIR-NEXT: call void @__kmpc_dist_for_static_init_4u( +// LLVM-HOST-CIR-SAME: ptr {{.*}}, i32 %[[TID4]], i32 34, +// LLVM-HOST-CIR-SAME: ptr %{{[^,]+}}, ptr %[[LB4]], ptr %[[UB4]], ptr %{{[^,]+}}, ptr %[[STRIDE4]], +// LLVM-HOST-CIR-SAME: i32 1, i32 0) +// LLVM-HOST-CIR: %[[IV4:.*]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXT4:[^,]+]], %{{.*}} ] +// LLVM-HOST-CIR: %[[LOOP_CMP4:.*]] = icmp ult i32 %[[IV4]], %{{.*}} +// LLVM-HOST-CIR-NEXT: br i1 %[[LOOP_CMP4]], label %[[LOOP_BODY4:[^,]+]], label %[[LOOP_EXIT4:[^ ,]+]] +// LLVM-HOST-CIR: {{^}}[[LOOP_EXIT4]]: +// LLVM-HOST-CIR-NEXT: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[TID4]]) +// LLVM-HOST-CIR: call void @__kmpc_barrier( +// LLVM-HOST-CIR: {{^}}[[LOOP_BODY4]]: +// LLVM-HOST-CIR: call void @during(i32 +// LLVM-HOST-CIR: %[[NEXT4]] = add nuw i32 %[[IV4]], 1 + +// LLVM-HOST-OGCG: store i32 0, ptr %[[COMB_LB4:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[COMB_UB4:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 1, ptr %[[DIST_STRIDE4:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[DIST_IS_LAST4:.*]], align 4 +// LLVM-HOST-OGCG: %[[DIST_TID4:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[DIST_TID4]], i32 92, +// LLVM-HOST-OGCG-SAME: ptr %[[DIST_IS_LAST4]], ptr %[[COMB_LB4]], ptr %[[COMB_UB4]], ptr %[[DIST_STRIDE4]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[DIST_CMP4:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[DIST_CMP4]], label %[[DIST_BODY4:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[DIST_BODY4]]: +// LLVM-HOST-OGCG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @[[PAR_OUTLINED4:[^,]+]], +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[DIST_TID4]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// LLVM-HOST-OGCG: define internal void @[[PAR_OUTLINED4]]( +// LLVM-HOST-OGCG: store i32 0, ptr %[[OMP_LB4:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 9, ptr %[[OMP_UB4:.*]], align 4 +// LLVM-HOST-OGCG: store i32 1, ptr %[[OMP_STRIDE4:.*]], align 4 +// LLVM-HOST-OGCG-NEXT: store i32 0, ptr %[[OMP_IS_LAST4:.*]], align 4 +// LLVM-HOST-OGCG: %[[FOR_TID4:.*]] = load i32, ptr %{{.*}}, align 4 +// LLVM-HOST-OGCG-NEXT: call void @__kmpc_for_static_init_4( +// LLVM-HOST-OGCG-SAME: ptr {{.*}}, i32 %[[FOR_TID4]], i32 34, +// LLVM-HOST-OGCG-SAME: ptr %[[OMP_IS_LAST4]], ptr %[[OMP_LB4]], ptr %[[OMP_UB4]], ptr %[[OMP_STRIDE4]], +// LLVM-HOST-OGCG-SAME: i32 1, i32 1) +// LLVM-HOST-OGCG: %[[FOR_CMP4:.*]] = icmp sle i32 %{{.*}}, %{{.*}} +// LLVM-HOST-OGCG-NEXT: br i1 %[[FOR_CMP4]], label %[[FOR_BODY4:[^,]+]], label %{{.*}} +// LLVM-HOST-OGCG: {{^}}[[FOR_BODY4]]: +// LLVM-HOST-OGCG: call void @during(i32 +// LLVM-HOST-OGCG: call void @__kmpc_for_static_fini(ptr {{.*}}, i32 %[[FOR_TID4]]) +// LLVM-HOST-OGCG-NEXT: ret void + +// Device kernel entry: __kmpc_target_init/__kmpc_target_deinit and the +// branch into the user code are identical on both paths. Both paths fork +// the parallel region with __kmpc_parallel_60 and no wrapper (SPMD). +// +// On the CIR path, the parallel region calls a distribute-outlined function +// that runs the loop through the callback-based +// __kmpc_distribute_for_static_loop_4u, passing the trip count (10). OGCG +// does the distribute chunking (schedule kind 91) in the teams-outlined +// function and the for chunking (schedule kind 33) in the parallel one. +// LLVM-DEVICE-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_explicit_target_teams_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-DEVICE: call i32 @__kmpc_target_init( +// LLVM-DEVICE: br i1 %{{.*}}, label %[[USER_CODE:[^,]+]], label %{{.*}} +// LLVM-DEVICE: {{^}}[[USER_CODE]]: +// LLVM-DEVICE: call void @[[DEV_TEAMS:__omp_offloading_[^(]+]]( +// LLVM-DEVICE: call void @__kmpc_target_deinit() +// LLVM-DEVICE-NEXT: ret void + +// LLVM-DEVICE-CIR: define internal void @[[DEV_TEAMS]]( +// LLVM-DEVICE-CIR: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR:[^,]+]], ptr null, +// LLVM-DEVICE-CIR: define internal void @[[DEV_PAR]]( +// LLVM-DEVICE-CIR: call void @[[DEV_DIST:__omp_offloading_[^(]+]]( +// LLVM-DEVICE-CIR: define internal void @[[DEV_DIST]]( +// LLVM-DEVICE-CIR: call void @__kmpc_distribute_for_static_loop_4u(ptr {{.*}}, ptr @[[DEV_BODY:[^,]+]], ptr {{.*}}, i32 10, i32 %{{.*}}, i32 0, i32 0, i8 0) +// LLVM-DEVICE-CIR: call void @__kmpc_barrier( +// LLVM-DEVICE-CIR: define internal void @[[DEV_BODY]]( +// LLVM-DEVICE-CIR: call void @during(i32 + +// LLVM-DEVICE-OGCG: define internal void @[[DEV_TEAMS]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_COMB_LB:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_COMB_UB:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 1, ptr %[[DEV_DIST_STRIDE:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_DIST_IS_LAST:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 91, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_DIST_IS_LAST]], ptr %[[DEV_COMB_LB]], ptr %[[DEV_COMB_UB]], ptr %[[DEV_DIST_STRIDE]], +// LLVM-DEVICE-OGCG: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR:[^,]+]], ptr null, +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_fini( +// LLVM-DEVICE-OGCG: define internal void @[[DEV_PAR]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_LB:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_UB:.*]], align 4 +// LLVM-DEVICE-OGCG: store i32 1, ptr %[[DEV_STRIDE:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_IS_LAST:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 33, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_IS_LAST]], ptr %[[DEV_LB]], ptr %[[DEV_UB]], ptr %[[DEV_STRIDE]], +// LLVM-DEVICE-OGCG: call void @during(i32 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_fini( + +// LLVM-DEVICE-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-DEVICE: call i32 @__kmpc_target_init( +// LLVM-DEVICE: br i1 %{{.*}}, label %[[USER_CODE2:[^,]+]], label %{{.*}} +// LLVM-DEVICE: {{^}}[[USER_CODE2]]: +// LLVM-DEVICE: call void @[[DEV_TEAMS2:__omp_offloading_[^(]+]]( +// LLVM-DEVICE: call void @__kmpc_target_deinit() +// LLVM-DEVICE-NEXT: ret void + +// LLVM-DEVICE-CIR: define internal void @[[DEV_TEAMS2]]( +// LLVM-DEVICE-CIR: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR2:[^,]+]], ptr null, +// LLVM-DEVICE-CIR: define internal void @[[DEV_PAR2]]( +// LLVM-DEVICE-CIR: call void @[[DEV_DIST2:__omp_offloading_[^(]+]]( +// LLVM-DEVICE-CIR: define internal void @[[DEV_DIST2]]( +// LLVM-DEVICE-CIR: call void @__kmpc_distribute_for_static_loop_4u(ptr {{.*}}, ptr @[[DEV_BODY2:[^,]+]], ptr {{.*}}, i32 10, i32 %{{.*}}, i32 0, i32 0, i8 0) +// LLVM-DEVICE-CIR: call void @__kmpc_barrier( +// LLVM-DEVICE-CIR: define internal void @[[DEV_BODY2]]( +// LLVM-DEVICE-CIR: call void @during(i32 + +// LLVM-DEVICE-OGCG: define internal void @[[DEV_TEAMS2]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_COMB_LB2:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_COMB_UB2:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 1, ptr %[[DEV_DIST_STRIDE2:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_DIST_IS_LAST2:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 91, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_DIST_IS_LAST2]], ptr %[[DEV_COMB_LB2]], ptr %[[DEV_COMB_UB2]], ptr %[[DEV_DIST_STRIDE2]], +// LLVM-DEVICE-OGCG: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR2:[^,]+]], ptr null, +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_fini( +// LLVM-DEVICE-OGCG: define internal void @[[DEV_PAR2]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_LB2:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_UB2:.*]], align 4 +// LLVM-DEVICE-OGCG: store i32 1, ptr %[[DEV_STRIDE2:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_IS_LAST2:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 33, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_IS_LAST2]], ptr %[[DEV_LB2]], ptr %[[DEV_UB2]], ptr %[[DEV_STRIDE2]], +// LLVM-DEVICE-OGCG: call void @during(i32 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_fini( + +// LLVM-DEVICE-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_teams_split_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-DEVICE: call i32 @__kmpc_target_init( +// LLVM-DEVICE: br i1 %{{.*}}, label %[[USER_CODE3:[^,]+]], label %{{.*}} +// LLVM-DEVICE: {{^}}[[USER_CODE3]]: +// LLVM-DEVICE: call void @[[DEV_TEAMS3:__omp_offloading_[^(]+]]( +// LLVM-DEVICE: call void @__kmpc_target_deinit() +// LLVM-DEVICE-NEXT: ret void + +// LLVM-DEVICE-CIR: define internal void @[[DEV_TEAMS3]]( +// LLVM-DEVICE-CIR: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR3:[^,]+]], ptr null, +// LLVM-DEVICE-CIR: define internal void @[[DEV_PAR3]]( +// LLVM-DEVICE-CIR: call void @[[DEV_DIST3:__omp_offloading_[^(]+]]( +// LLVM-DEVICE-CIR: define internal void @[[DEV_DIST3]]( +// LLVM-DEVICE-CIR: call void @__kmpc_distribute_for_static_loop_4u(ptr {{.*}}, ptr @[[DEV_BODY3:[^,]+]], ptr {{.*}}, i32 10, i32 %{{.*}}, i32 0, i32 0, i8 0) +// LLVM-DEVICE-CIR: call void @__kmpc_barrier( +// LLVM-DEVICE-CIR: define internal void @[[DEV_BODY3]]( +// LLVM-DEVICE-CIR: call void @during(i32 + +// LLVM-DEVICE-OGCG: define internal void @[[DEV_TEAMS3]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_COMB_LB3:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_COMB_UB3:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 1, ptr %[[DEV_DIST_STRIDE3:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_DIST_IS_LAST3:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 91, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_DIST_IS_LAST3]], ptr %[[DEV_COMB_LB3]], ptr %[[DEV_COMB_UB3]], ptr %[[DEV_DIST_STRIDE3]], +// LLVM-DEVICE-OGCG: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR3:[^,]+]], ptr null, +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_fini( +// LLVM-DEVICE-OGCG: define internal void @[[DEV_PAR3]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_LB3:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_UB3:.*]], align 4 +// LLVM-DEVICE-OGCG: store i32 1, ptr %[[DEV_STRIDE3:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_IS_LAST3:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 33, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_IS_LAST3]], ptr %[[DEV_LB3]], ptr %[[DEV_UB3]], ptr %[[DEV_STRIDE3]], +// LLVM-DEVICE-OGCG: call void @during(i32 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_fini( + +// LLVM-DEVICE-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{[0-9a-f]+_[0-9a-f]+}}_target_split_teams_distribute_parallel_for_l{{[0-9]+}}( +// LLVM-DEVICE: call i32 @__kmpc_target_init( +// LLVM-DEVICE: br i1 %{{.*}}, label %[[USER_CODE4:[^,]+]], label %{{.*}} +// LLVM-DEVICE: {{^}}[[USER_CODE4]]: +// LLVM-DEVICE: call void @[[DEV_TEAMS4:__omp_offloading_[^(]+]]( +// LLVM-DEVICE: call void @__kmpc_target_deinit() +// LLVM-DEVICE-NEXT: ret void + +// LLVM-DEVICE-CIR: define internal void @[[DEV_TEAMS4]]( +// LLVM-DEVICE-CIR: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR4:[^,]+]], ptr null, +// LLVM-DEVICE-CIR: define internal void @[[DEV_PAR4]]( +// LLVM-DEVICE-CIR: call void @[[DEV_DIST4:__omp_offloading_[^(]+]]( +// LLVM-DEVICE-CIR: define internal void @[[DEV_DIST4]]( +// LLVM-DEVICE-CIR: call void @__kmpc_distribute_for_static_loop_4u(ptr {{.*}}, ptr @[[DEV_BODY4:[^,]+]], ptr {{.*}}, i32 10, i32 %{{.*}}, i32 0, i32 0, i8 0) +// LLVM-DEVICE-CIR: call void @__kmpc_barrier( +// LLVM-DEVICE-CIR: define internal void @[[DEV_BODY4]]( +// LLVM-DEVICE-CIR: call void @during(i32 + +// LLVM-DEVICE-OGCG: define internal void @[[DEV_TEAMS4]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_COMB_LB4:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_COMB_UB4:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 1, ptr %[[DEV_DIST_STRIDE4:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_DIST_IS_LAST4:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 91, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_DIST_IS_LAST4]], ptr %[[DEV_COMB_LB4]], ptr %[[DEV_COMB_UB4]], ptr %[[DEV_DIST_STRIDE4]], +// LLVM-DEVICE-OGCG: call void @__kmpc_parallel_60(ptr {{.*}}, i32 {{.*}}, i32 1, i32 -1, i32 -1, ptr @[[DEV_PAR4:[^,]+]], ptr null, +// LLVM-DEVICE-OGCG: call void @__kmpc_distribute_static_fini( +// LLVM-DEVICE-OGCG: define internal void @[[DEV_PAR4]]( +// LLVM-DEVICE-OGCG: store i32 0, ptr %[[DEV_LB4:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 9, ptr %[[DEV_UB4:.*]], align 4 +// LLVM-DEVICE-OGCG: store i32 1, ptr %[[DEV_STRIDE4:.*]], align 4 +// LLVM-DEVICE-OGCG-NEXT: store i32 0, ptr %[[DEV_IS_LAST4:.*]], align 4 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_init_4( +// LLVM-DEVICE-OGCG-SAME: ptr {{.*}}, i32 %{{.*}}, i32 33, +// LLVM-DEVICE-OGCG-SAME: ptr %[[DEV_IS_LAST4]], ptr %[[DEV_LB4]], ptr %[[DEV_UB4]], ptr %[[DEV_STRIDE4]], +// LLVM-DEVICE-OGCG: call void @during(i32 +// LLVM-DEVICE-OGCG: call void @__kmpc_for_static_fini( _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
