https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/239501
>From 0da07d3ff41edfba328fbbe50804747f9788b442 Mon Sep 17 00:00:00 2001 From: skc7 <[email protected]> Date: Fri, 6 Nov 2026 13:00:47 +0530 Subject: [PATCH] [CIR][AMDGPU] Lower Indirect arguments and sret returns in addrspace(5) --- .../Dialect/Transforms/CIRABIClassifier.cpp | 3 - .../TargetLowering/CIRABIRewriteContext.cpp | 96 ++++++++++++------- .../TargetLowering/CIRABIRewriteContext.h | 11 ++- .../abi-lowering/amdgpu-indirect-nyi.cir | 22 ----- .../abi-lowering/amdgpu-indirect.cir | 93 ++++++++++++++++++ 5 files changed, 161 insertions(+), 64 deletions(-) delete mode 100644 clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir create mode 100644 clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir diff --git a/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp b/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp index a95672a5081219..614cf1e93ac881 100644 --- a/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp +++ b/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp @@ -649,9 +649,6 @@ convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx, mlir::Type extendedTy = abiTypeToCIR(info.getCoerceToType(), ctx); return ArgClassification::getExtend(extendedTy, info.isSignExt()); } - // The rewriter has no address space for the Indirect pointer yet. - if (info.isIndirect() && info.getIndirectAddrSpace() != 0) - return std::nullopt; if (info.isIndirect()) return ArgClassification::getIndirect(info.getIndirectAlign(), info.getIndirectByVal(), diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp index a36ae03bc7dbe3..eb125d91033777 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp @@ -82,6 +82,14 @@ cir::PointerType getByRefPtrTy(mlir::Type pointee, return getPtrTyInAddrSpace(pointee, ac.indirectAddrSpace); } +/// The pointer passed in place of an Indirect argument or sret return of type +/// \p pointee. +cir::PointerType getIndirectPtrTy(mlir::Type pointee, + const ArgClassification &ac) { + assert(ac.kind == ArgKind::Indirect && "expected an indirect classification"); + return getPtrTyInAddrSpace(pointee, ac.indirectAddrSpace); +} + /// Return the coerced RecordType for a Direct classification that should be /// flattened into individual scalar arguments, or a null type if the /// classification does not call for flattening. @@ -168,7 +176,7 @@ buildNewArgTypes(ArrayRef<mlir::Type> oldArgTypes, : origTy); break; case ArgKind::Indirect: - newArgTypes.push_back(cir::PointerType::get(origTy)); + newArgTypes.push_back(getIndirectPtrTy(origTy, ac)); break; case ArgKind::IndirectAliased: newArgTypes.push_back(getByRefPtrTy(origTy, ac)); @@ -1099,6 +1107,15 @@ static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) { return store; } +/// The alloca a parameter spill writes to through \p addr, either directly or +/// through an address_space cast of it. Null otherwise. +static cir::AllocaOp getSpillSlot(mlir::Value addr) { + if (auto view = addr.getDefiningOp<cir::CastOp>(); + view && view.getKind() == cir::CastKind::address_space) + addr = view.getSrc(); + return addr.getDefiningOp<cir::AllocaOp>(); +} + /// The store that spills non-byval indirect parameter \p blockArg, and the /// slot it spills into. prepareNonByvalParameters has already established /// that the spill is the block argument's only use and that it stores into @@ -1111,7 +1128,9 @@ findParamSpill(mlir::BlockArgument blockArg) { return {}; cir::StoreOp store = maybeFindParamSpill(blockArg); assert(store && "non-byval arg's only use must be its CIRGen param spill"); - return {store, cast<cir::AllocaOp>(store.getAddr().getDefiningOp())}; + cir::AllocaOp slot = getSpillSlot(store.getAddr()); + assert(slot && "non-byval arg's spill must store to an alloca"); + return {store, slot}; } /// For each Direct arg with a coerced type, change the block argument's type to @@ -1312,18 +1331,16 @@ insertArgCoercion(mlir::FunctionOpInterface funcOp, return mlir::failure(); } else if (ac.kind == ArgKind::Indirect || ac.kind == ArgKind::IndirectAliased) { - // byval and non-byval both lower to !cir.ptr<T>, and which it is shows - // up only in the attrs updateArgAttrs applies. Body lowering differs: - // the body addresses a byval parameter through the CIRGen slot it was - // spilled into, so that slot has to be filled from the incoming - // pointer, while a non-byval parameter's slot is replaced by the - // pointer so the body works on the caller's storage in place. A byref - // parameter takes the byval path, through a pointer in its own address - // space. + // byval and non-byval both lower to a pointer to T in the classified + // address space, and which it is shows up only in the attrs + // updateArgAttrs applies. Body lowering differs: the body addresses a + // byval parameter through the CIRGen slot it was spilled into, so that + // slot has to be filled from the incoming pointer, while a non-byval + // parameter's slot is replaced by the pointer so the body works on the + // caller's storage in place. A byref parameter takes the byval path. bool byRef = ac.kind == ArgKind::IndirectAliased; - cir::PointerType ptrTy = byRef - ? getByRefPtrTy(blockArg.getType(), ac) - : cir::PointerType::get(blockArg.getType()); + cir::PointerType ptrTy = byRef ? getByRefPtrTy(blockArg.getType(), ac) + : getIndirectPtrTy(blockArg.getType(), ac); if (!byRef && !ac.byVal) { // Without byval, drop the spill store and let the slot's uses read the @@ -1630,7 +1647,7 @@ rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc, mlir::OpBuilder &builder, const mlir::DataLayout &dl, bool enableNoundefAttrs) { mlir::MLIRContext *ctx = call->getContext(); - auto ptrTy = cir::PointerType::get(origRetTy); + cir::PointerType ptrTy = getIndirectPtrTy(origRetTy, fc.returnInfo); builder.setInsertionPoint(call); uint64_t sretAlign = fc.returnInfo.indirectAlign.value(); @@ -1644,18 +1661,25 @@ rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc, // %dest as the sret slot directly so construction flows into it, // matching classic CodeGen's "pass %s as sret" pattern. %dest must // dominate the call so the rewritten call (which takes it as operand 0) - // does not use a value before its definition. + // does not use a value before its definition. An address_space cast of a + // slot that already has the sret pointer's type stands for that slot. mlir::Value sretSlot = nullptr; cir::StoreOp reuseStore = nullptr; if (call.getResult().hasOneUse()) { mlir::Operation *user = *call.getResult().getUsers().begin(); - if (auto store = mlir::dyn_cast<cir::StoreOp>(user)) - if (store.getValue() == call.getResult() && - store.getAddr().getType() == ptrTy && - mlir::DominanceInfo().properlyDominates(store.getAddr(), call)) { - sretSlot = store.getAddr(); + auto store = mlir::dyn_cast<cir::StoreOp>(user); + if (store && store.getValue() == call.getResult()) { + mlir::Value dest = store.getAddr(); + if (auto view = dest.getDefiningOp<cir::CastOp>(); + view && view.getKind() == cir::CastKind::address_space && + view.getSrc().getType() == ptrTy) + dest = view.getSrc(); + if (dest.getType() == ptrTy && + mlir::DominanceInfo().properlyDominates(dest, call)) { + sretSlot = dest; reuseStore = store; } + } } if (!sretSlot) { auto alloca = cir::AllocaOp::create( @@ -1738,6 +1762,8 @@ static mlir::LogicalResult prepareOneNonByvalParameter( cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac, mlir::BlockArgument blockArg, mlir::DominanceInfo &dom, SmallPtrSetImpl<mlir::Operation *> &claimedSlots) { + cir::PointerType paramPtrTy = getIndirectPtrTy(blockArg.getType(), ac); + // The spill is the store that writes the parameter itself. Any other use // consumes the record value. At -O1 and above such a use comes from // cir-simplify: CIRGen marks a const-qualified parameter's slot const, so @@ -1795,21 +1821,21 @@ static mlir::LogicalResult prepareOneNonByvalParameter( mlir::OpBuilder builder(funcOp.getContext()); builder.setInsertionPointToStart(blockArg.getOwner()); auto synthesized = cir::AllocaOp::create( - builder, funcOp.getLoc(), cir::PointerType::get(blockArg.getType()), + builder, funcOp.getLoc(), paramPtrTy, builder.getStringAttr("nonbyval.param"), builder.getI64IntegerAttr(ac.indirectAlign.value())); spill = cir::StoreOp::create(builder, funcOp.getLoc(), blockArg, synthesized); } - // The incoming pointer replaces the slot itself, in the default address - // space. Retargeting a cast of it would leave the allocation's other views - // reading storage nothing writes. Storage in another address space, or - // storage that is not a local alloca at all, is not something the incoming - // pointer can be substituted for. - cir::AllocaOp slot = spill.getAddr().getDefiningOp<cir::AllocaOp>(); - if (!slot || - spill.getAddr().getType() != cir::PointerType::get(blockArg.getType())) + // The incoming pointer replaces the slot itself, which also covers a spill + // through an address_space cast of it. Retargeting only the cast would + // leave the allocation's other views reading storage nothing writes. + // Storage in another address space than the incoming pointer, or storage + // that is not a local alloca at all, is not something the incoming pointer + // can be substituted for. + cir::AllocaOp slot = getSpillSlot(spill.getAddr()); + if (!slot || slot.getType() != paramPtrTy) return spill->emitOpError() << "non-byval parameter " << argNo << " spilled to storage that cannot take the incoming pointer is " @@ -1940,8 +1966,11 @@ mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition( // below. bool hasSRet = fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty(); - if (hasSRet) - newArgTypes.insert(newArgTypes.begin(), cir::PointerType::get(origRetTy)); + cir::PointerType sretPtrTy; + if (hasSRet) { + sretPtrTy = getIndirectPtrTy(origRetTy, fc.returnInfo); + newArgTypes.insert(newArgTypes.begin(), sretPtrTy); + } if (funcOp.isDefinition()) { mlir::Region &body = funcOp->getRegion(0); @@ -1950,8 +1979,7 @@ mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition( // through it before any index-based argument handling below (which // then accounts for the +1 offset). if (hasSRet) { - body.front().insertArgument(0u, cir::PointerType::get(origRetTy), - funcOp.getLoc()); + body.front().insertArgument(0u, sretPtrTy, funcOp.getLoc()); insertSRetStores(funcOp, origRetTy, builder); } @@ -2227,7 +2255,7 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp, bool byRef = ac.kind == ArgKind::IndirectAliased; bool byCopy = byRef || ac.byVal; cir::PointerType ptrTy = byRef ? getByRefPtrTy(arg.getType(), ac) - : cir::PointerType::get(arg.getType()); + : getIndirectPtrTy(arg.getType(), ac); // A record operand that names storage is passed from that storage // rather than from the loaded value: a record value carries only the diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h index f22c80dec2d648..671be6c185a83c 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h @@ -81,11 +81,12 @@ class CIRABIRewriteContext : public mlir::abi::ABIRewriteContext { /// Bring each non-byval indirect parameter of \p funcOp into the shape the /// rest of the rewrite assumes: the parameter's only use, if it has one, is - /// a single store into an alloca of the matching pointer type that no other - /// non-byval indirect parameter spills to, and that alloca states the - /// alignment the ABI promises rather than the one CIRGen picked for a local - /// copy. A use of the parameter as a call argument is routed through a load - /// of that slot, so that it names the storage an argument has to name. + /// a single store into an alloca of the incoming pointer's type, directly or + /// through an address_space cast of it. No other non-byval indirect + /// parameter spills to that alloca, and it states the alignment the ABI + /// promises rather than the one CIRGen picked for a local copy. A use of + /// the parameter as a call argument is routed through a load of that slot, + /// so that it names the storage an argument has to name. /// /// Call for every function before any definition or call site is rewritten. /// findParamSpill asserts this shape while the enclosing definition is diff --git a/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir b/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir deleted file mode 100644 index c0521d18a08bda..00000000000000 --- a/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir +++ /dev/null @@ -1,22 +0,0 @@ -// RUN: not cir-opt %s -cir-call-conv-lowering=target=amdgpu 2>&1 | FileCheck %s - -// AMDGPU Indirect classifications whose pointer is not in the default address -// space are not lowered yet, so they are reported rather than misclassified. - -!s32i = !cir.int<s, 32> -!rec_Big = !cir.struct<"Big" {data !cir.array<!s32i x 17>}> - -module attributes { - dlti.dl_spec = #dlti.dl_spec< - #dlti.dl_entry<i32, dense<32>: vector<2xi64>>, - #dlti.dl_entry<i64, dense<64>: vector<2xi64>>> -} { - - // A return past the register budget goes through an sret in addrspace(5). - cir.func @ret_big(%arg0: !cir.ptr<!rec_Big>) -> !rec_Big requires_call_conv_lowering { - %0 = cir.load %arg0 : !cir.ptr<!rec_Big>, !rec_Big - cir.return %0 : !rec_Big - } - - // CHECK: not yet implemented for the ABI coercion of type '!cir.struct<"Big" -} diff --git a/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir b/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir new file mode 100644 index 00000000000000..1699f504ec0831 --- /dev/null +++ b/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir @@ -0,0 +1,93 @@ +// RUN: cir-opt %s -cir-call-conv-lowering=target=amdgpu | FileCheck %s + +// Big returns and non-trivial arguments are passed through a pointer in +// addrspace(5) on AMDGPU, as in classic CodeGen, without extra copies. + +!s32i = !cir.int<s, 32> +!rec_Big = !cir.struct<"Big" {data !cir.array<!s32i x 17>}> +!rec_NT = !cir.struct<"NT" {data !s32i}> + +module attributes { + cir.record_layouts = {NT = #cir.record_layout< + arg_passing_kind = cannot_pass_in_regs, has_trivial_dtor = false, + record_align = 4>}, + dlti.dl_spec = #dlti.dl_spec< + #dlti.dl_entry<i32, dense<32>: vector<2xi64>>, + #dlti.dl_entry<i64, dense<64>: vector<2xi64>>, + #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>> +} { + + cir.func private @fill(!cir.ptr<!rec_Big>) + cir.func private @make_big() -> !rec_Big requires_call_conv_lowering + cir.func private @copy_nt(!cir.ptr<!rec_NT>, !cir.ptr<!rec_NT>) + cir.func private @use_nt(!cir.ptr<!rec_NT>) + + // CHECK: cir.func private @make_big(!cir.ptr<!rec_Big, target_address_space(5)> {llvm.align = 4 : i64, llvm.dead_on_unwind, llvm.sret = !rec_Big, llvm.writable}) + + // The callee builds its result directly in the caller's sret memory. + cir.func @ret_big() -> !rec_Big requires_call_conv_lowering { + %0 = cir.alloca "__retval" align(4) : !cir.ptr<!rec_Big, target_address_space(5)> + %1 = cir.cast address_space %0 : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big> + cir.call @fill(%1) : (!cir.ptr<!rec_Big>) -> () + %2 = cir.load %0 : !cir.ptr<!rec_Big, target_address_space(5)>, !rec_Big + cir.return %2 : !rec_Big + } + + // CHECK: cir.func @ret_big(%[[SRET:.*]]: !cir.ptr<!rec_Big, target_address_space(5)> {llvm.align = 4 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_Big, llvm.writable}) { + // CHECK-NEXT: %[[SRET_VIEW:.*]] = cir.cast address_space %[[SRET]] : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big> + // CHECK-NEXT: cir.call @fill(%[[SRET_VIEW]]) : (!cir.ptr<!rec_Big>) -> () + // CHECK-NEXT: cir.return + // CHECK-NEXT: } + + // The callee writes straight into `b`, with no temporary copy. + cir.func @use_big() requires_call_conv_lowering { + %0 = cir.alloca "b" align(4) init : !cir.ptr<!rec_Big, target_address_space(5)> + %1 = cir.cast address_space %0 : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big> + %2 = cir.call @make_big() : () -> !rec_Big + cir.store %2, %1 : !rec_Big, !cir.ptr<!rec_Big> + cir.call @fill(%1) : (!cir.ptr<!rec_Big>) -> () + cir.return + } + + // CHECK: cir.func @use_big() { + // CHECK-NEXT: %[[B_SLOT:.*]] = cir.alloca "b" align(4) init : !cir.ptr<!rec_Big, target_address_space(5)> + // CHECK-NEXT: %[[B_VIEW:.*]] = cir.cast address_space %[[B_SLOT]] : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big> + // CHECK-NEXT: cir.call @make_big(%[[B_SLOT]]) : (!cir.ptr<!rec_Big, target_address_space(5)> {llvm.align = 4 : i64, llvm.dead_on_unwind, llvm.sret = !rec_Big, llvm.writable}) -> () + // CHECK-NEXT: cir.call @fill(%[[B_VIEW]]) : (!cir.ptr<!rec_Big>) -> () + // CHECK-NEXT: cir.return + // CHECK-NEXT: } + + // The callee works on the caller's object in place, not on a copy. + cir.func @take_nt(%arg0: !rec_NT) requires_call_conv_lowering { + %0 = cir.alloca "n" align(4) init : !cir.ptr<!rec_NT, target_address_space(5)> + %1 = cir.cast address_space %0 : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT> + cir.store %arg0, %1 : !rec_NT, !cir.ptr<!rec_NT> + cir.call @use_nt(%1) : (!cir.ptr<!rec_NT>) -> () + cir.return + } + + // CHECK: cir.func @take_nt(%[[N_PTR:.*]]: !cir.ptr<!rec_NT, target_address_space(5)> {llvm.align = 4 : i64, llvm.dereferenceable = 4 : i64, llvm.nofreeobj, llvm.noundef}) { + // CHECK-NEXT: %[[N_VIEW:.*]] = cir.cast address_space %[[N_PTR]] : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT> + // CHECK-NEXT: cir.call @use_nt(%[[N_VIEW]]) : (!cir.ptr<!rec_NT>) -> () + // CHECK-NEXT: cir.return + // CHECK-NEXT: } + + // The callee gets the copy-constructed temporary itself, not a byte copy. + cir.func @call_nt(%arg0: !cir.ptr<!rec_NT>) requires_call_conv_lowering { + %0 = cir.alloca "agg.tmp0" align(4) : !cir.ptr<!rec_NT, target_address_space(5)> + %1 = cir.cast address_space %0 : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT> + cir.call @copy_nt(%1, %arg0) : (!cir.ptr<!rec_NT>, !cir.ptr<!rec_NT>) -> () + %2 = cir.load %1 : !cir.ptr<!rec_NT>, !rec_NT + cir.call @take_nt(%2) : (!rec_NT) -> () + cir.return + } + + // CHECK: cir.func @call_nt(%[[SRC:.*]]: !cir.ptr<!rec_NT>) { + // CHECK-NEXT: %[[TMP:.*]] = cir.alloca "agg.tmp0" align(4) : !cir.ptr<!rec_NT, target_address_space(5)> + // CHECK-NEXT: %[[TMP_VIEW:.*]] = cir.cast address_space %[[TMP]] : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT> + // CHECK-NEXT: cir.call @copy_nt(%[[TMP_VIEW]], %[[SRC]]) : (!cir.ptr<!rec_NT>, !cir.ptr<!rec_NT>) -> () + // CHECK-NEXT: cir.call @take_nt(%[[TMP]]) : (!cir.ptr<!rec_NT, target_address_space(5)> {llvm.align = 4 : i64, llvm.dereferenceable = 4 : i64, llvm.nofreeobj, llvm.noundef}) -> () + // CHECK-NEXT: cir.return + // CHECK-NEXT: } + +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
