https://github.com/skc7 created https://github.com/llvm/llvm-project/pull/204283
Model LLVM's `!invariant.load` metadata on `cir.load` via an optional `invariant` UnitAttr, lower it to the `llvm.load isInvariant` flag. Update existing LoadOp::create call sites with new parameter. >From efbfb2c1de8785abb56ac391db210273c2a85a8a Mon Sep 17 00:00:00 2001 From: skc7 <[email protected]> Date: Wed, 17 Jun 2026 10:18:51 +0530 Subject: [PATCH] [CIR] Add invariant attribute to cir.load --- .../CIR/Dialect/Builder/CIRBaseBuilder.h | 8 ++++--- clang/include/clang/CIR/Dialect/IR/CIROps.td | 10 ++++++++- clang/lib/CIR/CodeGen/CIRGenBuilder.h | 6 ++++-- .../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 6 ++++-- .../TargetLowering/CIRABIRewriteContext.cpp | 3 ++- .../TargetLowering/LowerItaniumCXXABI.cpp | 15 ++++++++----- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 2 +- clang/test/CIR/IR/load-invariant.cir | 20 ++++++++++++++++++ clang/test/CIR/Lowering/load-invariant.cir | 21 +++++++++++++++++++ 9 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 clang/test/CIR/IR/load-invariant.cir create mode 100644 clang/test/CIR/Lowering/load-invariant.cir diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h index 121eed5f8ba9a..e6b041ebc1601 100644 --- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h +++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h @@ -229,11 +229,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { } cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr, - bool isVolatile = false, uint64_t alignment = 0) { + bool isVolatile = false, uint64_t alignment = 0, + bool isInvariant = false) { mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment); return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, isVolatile, alignmentAttr, cir::SyncScopeKindAttr{}, - cir::MemOrderAttr{}); + cir::MemOrderAttr{}, /*invariant=*/isInvariant); } mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr, @@ -427,7 +428,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { auto addr = createAlloca(loc, getPointerTo(type), {}, alignmentAttr); return cir::LoadOp::create(*this, loc, addr, /*isDeref=*/false, /*isVolatile=*/false, alignmentAttr, - /*sync_scope=*/{}, /*mem_order=*/{}); + /*sync_scope=*/{}, /*mem_order=*/{}, + /*invariant=*/false); } cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base, diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index 9dae3534991e5..08b6aa3f0525e 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -671,6 +671,9 @@ def CIR_LoadOp : CIR_Op<"load", [ `alignment` can be used to specify an alignment that's different from the default, which is computed from `result`'s type ABI data layout. + A unit attribute `invariant` can be used to indicate that the loaded memory + never changes, mapping to LLVM IR's `!invariant.load` metadata. + Example: ``` @@ -685,6 +688,9 @@ def CIR_LoadOp : CIR_Op<"load", [ // Perform a volatile load from address in %0. %4 = cir.load volatile %0 : !cir.ptr<i32>, i32 + // Perform an invariant load from address in %0. + %5 = cir.load invariant %0 : !cir.ptr<i32>, i32 + // Others %x = cir.load align(16) atomic(seq_cst) %0 : !cir.ptr<i32>, i32 ``` @@ -696,12 +702,14 @@ def CIR_LoadOp : CIR_Op<"load", [ UnitAttr:$is_volatile, OptionalAttr<I64Attr>:$alignment, OptionalAttr<CIR_SyncScopeKind>:$sync_scope, - OptionalAttr<CIR_MemOrder>:$mem_order); + OptionalAttr<CIR_MemOrder>:$mem_order, + UnitAttr:$invariant); let results = (outs CIR_AnyType:$result); let assemblyFormat = [{ (`deref` $isDeref^)? (`volatile` $is_volatile^)? + (`invariant` $invariant^)? (`align` `(` $alignment^ `)`)? (`syncscope` `(` $sync_scope^ `)`)? (`atomic` `(` $mem_order^ `)`)? diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h index 3188b096579be..27ab2718183a9 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h +++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h @@ -582,7 +582,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy { return cir::LoadOp::create(*this, loc, addr.getPointer(), /*isDeref=*/false, isVolatile, /*alignment=*/align, /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr{}); + /*mem_order=*/cir::MemOrderAttr{}, + /*invariant=*/false); } cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty, @@ -594,7 +595,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy { return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, /*isVolatile=*/false, alignAttr, /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr{}); + /*mem_order=*/cir::MemOrderAttr{}, + /*invariant=*/false); } cir::LoadOp diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp index ddeeb98fee820..9f9e7a14d418f 100644 --- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp +++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp @@ -980,7 +980,8 @@ class CIRCleanupScopeOpFlattening auto loaded = cir::LoadOp::create( rewriter, loc, alloca, /*isDeref=*/false, /*isVolatile=*/false, /*alignment=*/mlir::IntegerAttr(), - cir::SyncScopeKindAttr(), cir::MemOrderAttr()); + cir::SyncScopeKindAttr(), cir::MemOrderAttr(), + /*invariant=*/false); returnValues.push_back(loaded); } } @@ -1293,7 +1294,8 @@ class CIRCleanupScopeOpFlattening auto slotValue = cir::LoadOp::create( rewriter, loc, destSlot, /*isDeref=*/false, /*isVolatile=*/false, /*alignment=*/mlir::IntegerAttr(), - cir::SyncScopeKindAttr(), cir::MemOrderAttr()); + cir::SyncScopeKindAttr(), cir::MemOrderAttr(), + /*invariant=*/false); // Create destination blocks for each exit and collect switch case info. llvm::SmallVector<mlir::APInt, 8> caseValues; diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp index b252ed188c408..00a41456b303b 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp @@ -534,7 +534,8 @@ void rewriteIndirectReturnCall(cir::CallOp call, /*isVolatile=*/mlir::UnitAttr(), /*alignment=*/mlir::IntegerAttr(), /*sync_scope=*/cir::SyncScopeKindAttr(), - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/mlir::UnitAttr()); call.getResult().replaceAllUsesWith(load); } call->erase(); diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp index 0e246c8612f25..8dbe9918e53d4 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp @@ -379,7 +379,8 @@ void LowerItaniumCXXABI::lowerGetMethod( /*isVolatile=*/false, /*alignment=*/mlir::IntegerAttr(), /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); // Apply the offset. // On ARM64, to reserve extra space in virtual member function pointers, @@ -406,7 +407,8 @@ void LowerItaniumCXXABI::lowerGetMethod( /*isDeref=*/false, /*isVolatile=*/false, /*alignment=*/mlir::IntegerAttr(), /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); cir::YieldOp::create(b, loc, fnPtr.getResult()); assert(!cir::MissingFeatures::emitCFICheck()); @@ -779,7 +781,8 @@ static mlir::Value buildDynamicCastToVoidAfterNullCheck( /*is_volatile=*/false, /*alignment=*/builder.getI64IntegerAttr(vtableElemAlign), /*sync_scope=*/cir::SyncScopeKindAttr(), - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); mlir::Value elementPtr = cir::CastOp::create(builder, loc, vtableElemPtrTy, cir::CastKind::bitcast, vptr); mlir::Value minusTwo = @@ -792,7 +795,8 @@ static mlir::Value buildDynamicCastToVoidAfterNullCheck( /*is_volatile=*/false, /*alignment=*/builder.getI64IntegerAttr(vtableElemAlign), /*sync_scope=*/cir::SyncScopeKindAttr(), - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); auto voidPtrTy = cir::PointerType::get(cir::VoidType::get(builder.getContext())); @@ -900,7 +904,8 @@ mlir::Value LowerItaniumCXXABI::readArrayCookieImpl( return cir::LoadOp::create( builder, loc, countPtr, /*isDeref=*/false, /*isVolatile=*/false, builder.getI64IntegerAttr(countAlignment.getQuantity()), - cir::SyncScopeKindAttr(), cir::MemOrderAttr()); + cir::SyncScopeKindAttr(), cir::MemOrderAttr(), + /*invariant=*/false); } } // namespace cir diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index c844375a000e0..1a5c3271b3a4d 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -1863,7 +1863,7 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite( mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create( rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment, op.getIsVolatile(), /*isNonTemporal=*/false, - /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering, + /*isInvariant=*/op.getInvariant(), /*isInvariantGroup=*/false, ordering, llvmSyncScope.value_or(std::string())); // Convert adapted result to its original type if needed. diff --git a/clang/test/CIR/IR/load-invariant.cir b/clang/test/CIR/IR/load-invariant.cir new file mode 100644 index 0000000000000..4b014666db3a9 --- /dev/null +++ b/clang/test/CIR/IR/load-invariant.cir @@ -0,0 +1,20 @@ +// RUN: cir-opt %s --verify-roundtrip | FileCheck %s + +!s32i = !cir.int<s, 32> + +module { + // CHECK-LABEL: cir.func @invariant_load + // CHECK: %{{.*}} = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i + cir.func @invariant_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } + + // A plain load must not print the invariant keyword. + // CHECK-LABEL: cir.func @plain_load + // CHECK: %{{.*}} = cir.load %arg0 : !cir.ptr<!s32i>, !s32i + cir.func @plain_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } +} diff --git a/clang/test/CIR/Lowering/load-invariant.cir b/clang/test/CIR/Lowering/load-invariant.cir new file mode 100644 index 0000000000000..40c929ba227df --- /dev/null +++ b/clang/test/CIR/Lowering/load-invariant.cir @@ -0,0 +1,21 @@ +// RUN: cir-opt %s -cir-to-llvm -o - | FileCheck %s + +// The invariant flag on cir.load lowers to the LLVM dialect invariant load. + +!s32i = !cir.int<s, 32> + +module { + // CHECK-LABEL: llvm.func @invariant_load + // CHECK: llvm.load %{{.*}} invariant {{.*}}: !llvm.ptr -> i32 + cir.func @invariant_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } + + // CHECK-LABEL: llvm.func @plain_load + // CHECK-NOT: invariant + cir.func @plain_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
