https://github.com/erichkeane created https://github.com/llvm/llvm-project/pull/222730
This only works for types where a compare-exchange instruction (or just a lock-free implementation matters) is possible, and a loop isn't necessary. This is basically just a conversion of classic-codegen, except it clarifies some of the logic (particularly around bools). >From 467fe91c7ca075f61f48c708a1c7cd000729e95a Mon Sep 17 00:00:00 2001 From: erichkeane <[email protected]> Date: Thu, 10 Sep 2026 07:41:53 -0700 Subject: [PATCH] [CIR] Implement 'simple' atomic inc/dec. This only works for types where a compare-exchange instruction (or just a lock-free implementation matters) is possible, and a loop isn't necessary. This is basically just a conversion of classic-codegen, except it clarifies some of the logic (particularly around bools). --- clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 84 +++++++- clang/test/CIR/CodeGen/atomic.c | 215 +++++++++++++++++++++ 2 files changed, 296 insertions(+), 3 deletions(-) diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp index fb833241c1f81..3a5e928c51c45 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp @@ -628,10 +628,88 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> { mlir::Value value; mlir::Value input; - if (type->getAs<AtomicType>()) { + if (const AtomicType *atomicTy = type->getAs<AtomicType>()) { + QualType valType = atomicTy->getValueType(); + mlir::Location loc = cgf.getLoc(e->getSourceRange()); + bool isInc = e->isIncrementOp(); + bool isPre = e->isPrefix(); + + // Bools are always set-to-true, as decrement isn't legal on bools. + if (valType->isBooleanType()) { + assert(isInc); + // Atomic operations require an integer type; reinterpret the bool + // pointer as a pointer to its underlying storage integer type. + cir::IntType intTy = builder.getUInt8Ty(); + mlir::Value one = builder.getConstInt(loc, intTy, 1); + Address intAddr = lv.getAddress().withElementType(builder, intTy); + + if (isPre) { + // Pre-increment: atomically store true, return true. + cir::StoreOp store = builder.createStore(loc, one, intAddr); + store.setMemOrder(cir::MemOrder::SequentiallyConsistent); + if (lv.isVolatileQualified()) + store.setIsVolatile(true); + return builder.getTrue(loc); + } + // Post-increment: atomically exchange with true, return old value. + auto xchg = cir::AtomicXchgOp::create( + builder, loc, intAddr.getPointer(), one, + cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System, + lv.isVolatileQualified()); + return builder.createCast(cir::CastKind::int_to_bool, xchg.getResult(), + builder.getBoolTy()); + } + + // Special case for atomic increment / decrement on integers, emit + // atomicrmw instructions. We skip this if we want to be doing overflow + // checking, and fall into the slow path with the atomic cmpxchg loop. + if (!valType->isBooleanType() && valType->isIntegerType() && + !(valType->isUnsignedIntegerType() && + cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) && + cgf.getLangOpts().getSignedOverflowBehavior() != + LangOptions::SOB_Trapping) { + mlir::Type intTy = cgf.convertType(valType); + mlir::Value one = builder.getConstInt(loc, intTy, 1); + cir::AtomicFetchKind kind = + isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub; + auto rmw = cir::AtomicFetchOp::create( + builder, loc, lv.getPointer(), one, kind, + cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System, + lv.isVolatileQualified(), /*fetch_first=*/true); + mlir::Value oldVal = rmw->getResult(0); + // Prefix returns new value; postfix returns old value. + return isPre ? emitIncOrDec(e, oldVal) : oldVal; + } + + // Special case for atomic increment/decrement on floats. + // Bail out non-power-of-2-sized floating point types (e.g., x86_fp80). + if (valType->isFloatingType()) { + CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e); + mlir::Type fpTy = cgf.convertType(valType); + auto fpType = mlir::cast<cir::FPTypeInterface>(fpTy); + // Bail on non-power-of-2 types (e.g., x86_fp80 is 80 bits). + if (llvm::has_single_bit(fpType.getWidth())) { + mlir::Value amt = builder.getConstFP( + loc, fpTy, llvm::APFloat(fpType.getFloatSemantics(), 1)); + cir::AtomicFetchKind kind = + isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub; + auto rmw = cir::AtomicFetchOp::create( + builder, loc, lv.getPointer(), amt, kind, + cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System, + lv.isVolatileQualified(), /*fetch_first=*/true); + mlir::Value oldVal = rmw->getResult(0); + if (isPre) + return isInc ? builder.createFAdd(loc, oldVal, amt) + : builder.createFSub(loc, oldVal, amt); + return oldVal; + } else { + cgf.cgm.errorNYI(e->getSourceRange(), + "Atomic inc/dec of non-power-of-2 float"); + } + } + + // Otherwise we need a loop on compare-exchange. cgf.cgm.errorNYI(e->getSourceRange(), "Atomic inc/dec"); - // TODO(cir): This is not correct, but it will produce reasonable code - // until atomic operations are implemented. value = cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue(); input = value; } else { diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c index 21b99331fb417..f3913ea28072e 100644 --- a/clang/test/CIR/CodeGen/atomic.c +++ b/clang/test/CIR/CodeGen/atomic.c @@ -4022,3 +4022,218 @@ void store_atomic_different_size(S a) { // OGCG: %[[ATOMIC_TMP:.*]] = load i32, ptr %[[ATOMIC_TMP_ADDR]], align 4 // OGCG: store atomic i32 %[[ATOMIC_TMP]], ptr %[[B_ADDR]] seq_cst, align 4 } + +void atomic_pre_inc(_Atomic(int) *p) { ++(*p); } + +// CIR-LABEL: @atomic_pre_inc +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i +// CIR: cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i + +// LLVM-LABEL: @atomic_pre_inc +// LLVM: atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4 + +// OGCG-LABEL: @atomic_pre_inc +// OGCG: atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4 + +int atomic_post_inc(_Atomic(int) *p) { return (*p)++; } + +// CIR-LABEL: @atomic_post_inc +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i +// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i +// CIR: cir.store %[[OLD]], %{{.+}} : !s32i, !cir.ptr<!s32i> + +// LLVM-LABEL: @atomic_post_inc +// LLVM: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4 +// LLVM: store i32 %[[OLD]], ptr %{{.+}}, align 4 +// LLVM: %[[RET:.+]] = load i32, ptr %{{.+}}, align 4 +// LLVM: ret i32 %[[RET]] + +// OGCG-LABEL: @atomic_post_inc +// OGCG: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4 +// OGCG: ret i32 %[[OLD]] + +void atomic_pre_dec(_Atomic(int) *p) { --(*p); } + +// CIR-LABEL: @atomic_pre_dec +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i +// CIR: cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i + +// LLVM-LABEL: @atomic_pre_dec +// LLVM: atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4 + +// OGCG-LABEL: @atomic_pre_dec +// OGCG: atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4 + +int atomic_post_dec(_Atomic(int) *p) { return (*p)--; } + +// CIR-LABEL: @atomic_post_dec +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i +// CIR: %[[OLD:.+]] = cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i +// CIR: cir.store %[[OLD]], %{{.+}} : !s32i, !cir.ptr<!s32i> + +// LLVM-LABEL: @atomic_post_dec +// LLVM: %[[OLD:.+]] = atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4 +// LLVM: store i32 %[[OLD]], ptr %{{.+}}, align 4 +// LLVM: %[[RET:.+]] = load i32, ptr %{{.+}}, align 4 +// LLVM: ret i32 %[[RET]] + +// OGCG-LABEL: @atomic_post_dec +// OGCG: %[[OLD:.+]] = atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4 +// OGCG: ret i32 %[[OLD]] + +int atomic_pre_inc_used(_Atomic(int) *p) { return ++(*p); } + +// CIR-LABEL: @atomic_pre_inc_used +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i +// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i +// CIR: %[[NEW:.+]] = cir.inc %[[OLD]] : !s32i +// CIR: cir.store %[[NEW]], %{{.+}} : !s32i, !cir.ptr<!s32i> + +// LLVM-LABEL: @atomic_pre_inc_used +// LLVM: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4 +// LLVM: %[[NEW:.+]] = add i32 %[[OLD]], 1 +// LLVM: store i32 %[[NEW]], ptr %{{.+}}, align 4 +// LLVM: %[[RET:.+]] = load i32, ptr %{{.+}}, align 4 +// LLVM: ret i32 %[[RET]] + +// OGCG-LABEL: @atomic_pre_inc_used +// OGCG: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4 +// OGCG: %[[NEW:.+]] = add i32 %[[OLD]], 1 +// OGCG: ret i32 %[[NEW]] + +// Tests for atomic increment on _Bool. + +void atomic_bool_pre_inc_void(_Atomic _Bool *p) { ++(*p); } + +// CIR-LABEL: @atomic_bool_pre_inc_void +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.bool>>, !cir.ptr<!cir.bool> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !u8i +// CIR: %[[IPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!cir.bool> -> !cir.ptr<!u8i> +// CIR: cir.store {{.*}} atomic(seq_cst) %[[ONE]], %[[IPTR]] : !u8i, !cir.ptr<!u8i> + +// LLVM-LABEL: @atomic_bool_pre_inc_void +// LLVM: store atomic i8 1, ptr %{{.+}} seq_cst, align 1 + +// OGCG-LABEL: @atomic_bool_pre_inc_void +// OGCG: store atomic i8 1, ptr %{{.+}} seq_cst, align 1 + +_Bool atomic_bool_pre_inc(_Atomic _Bool *p) { return ++(*p); } + +// CIR-LABEL: @atomic_bool_pre_inc +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.bool>>, !cir.ptr<!cir.bool> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !u8i +// CIR: %[[IPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!cir.bool> -> !cir.ptr<!u8i> +// CIR: cir.store {{.*}} atomic(seq_cst) %[[ONE]], %[[IPTR]] : !u8i, !cir.ptr<!u8i> +// CIR: cir.const #true + +// LLVM-LABEL: @atomic_bool_pre_inc +// LLVM: store atomic i8 1, ptr %{{.+}} seq_cst, align 1 + +// OGCG-LABEL: @atomic_bool_pre_inc +// OGCG: store atomic i8 1, ptr %{{.+}} seq_cst, align 1 +// OGCG: ret i1 true + +_Bool atomic_bool_post_inc(_Atomic _Bool *p) { return (*p)++; } + +// CIR-LABEL: @atomic_bool_post_inc +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.bool>>, !cir.ptr<!cir.bool> +// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !u8i +// CIR: %[[IPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!cir.bool> -> !cir.ptr<!u8i> +// CIR: %[[OLD:.+]] = cir.atomic.xchg seq_cst syncscope(system) %[[IPTR]], %[[ONE]] : (!cir.ptr<!u8i>, !u8i) -> !u8i +// CIR: cir.cast int_to_bool %[[OLD]] : !u8i -> !cir.bool + +// LLVM-LABEL: @atomic_bool_post_inc +// LLVM: %[[OLD:.+]] = atomicrmw xchg ptr %{{.+}}, i8 1 seq_cst, align 1 + +// OGCG-LABEL: @atomic_bool_post_inc +// OGCG: %[[OLD:.+]] = atomicrmw xchg ptr %{{.+}}, i8 1 seq_cst, align 1 + +// Tests for atomic increment/decrement on float. + +void atomic_float_pre_inc(_Atomic(float) *p) { ++(*p); } + +// CIR-LABEL: @atomic_float_pre_inc +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float> +// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float +// CIR: cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float + +// LLVM-LABEL: @atomic_float_pre_inc +// LLVM: atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 + +// OGCG-LABEL: @atomic_float_pre_inc +// OGCG: atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 + +float atomic_float_post_inc(_Atomic(float) *p) { return (*p)++; } + +// CIR-LABEL: @atomic_float_post_inc +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float> +// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float +// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float +// CIR: cir.store %[[OLD]], %{{.+}} : !cir.float, !cir.ptr<!cir.float> + +// LLVM-LABEL: @atomic_float_post_inc +// LLVM: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 +// LLVM: store float %[[OLD]], ptr %{{.+}}, align 4 +// LLVM: %[[RET:.+]] = load float, ptr %{{.+}}, align 4 +// LLVM: ret float %[[RET]] + +// OGCG-LABEL: @atomic_float_post_inc +// OGCG: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 +// OGCG: ret float %[[OLD]] + +float atomic_float_pre_inc_used(_Atomic(float) *p) { return ++(*p); } + +// CIR-LABEL: @atomic_float_pre_inc_used +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float> +// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float +// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float +// CIR: %[[NEW:.+]] = cir.fadd %[[OLD]], %[[AMT]] : !cir.float +// CIR: cir.store %[[NEW]], %{{.+}} : !cir.float, !cir.ptr<!cir.float> + +// LLVM-LABEL: @atomic_float_pre_inc_used +// LLVM: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 +// LLVM: %[[NEW:.+]] = fadd float %[[OLD]], 1.000000e+00 +// LLVM: store float %[[NEW]], ptr %{{.+}}, align 4 +// LLVM: %[[RET:.+]] = load float, ptr %{{.+}}, align 4 +// LLVM: ret float %[[RET]] + +// OGCG-LABEL: @atomic_float_pre_inc_used +// OGCG: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 +// OGCG: %[[NEW:.+]] = fadd float %[[OLD]], 1.000000e+00 +// OGCG: ret float %[[NEW]] + +void atomic_float_pre_dec(_Atomic(float) *p) { --(*p); } + +// CIR-LABEL: @atomic_float_pre_dec +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float> +// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float +// CIR: cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float + +// LLVM-LABEL: @atomic_float_pre_dec +// LLVM: atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 + +// OGCG-LABEL: @atomic_float_pre_dec +// OGCG: atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 + +float atomic_float_post_dec(_Atomic(float) *p) { return (*p)--; } + +// CIR-LABEL: @atomic_float_post_dec +// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float> +// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float +// CIR: %[[OLD:.+]] = cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float +// CIR: cir.store %[[OLD]], %{{.+}} : !cir.float, !cir.ptr<!cir.float> + +// LLVM-LABEL: @atomic_float_post_dec +// LLVM: %[[OLD:.+]] = atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 +// LLVM: store float %[[OLD]], ptr %{{.+}}, align 4 +// LLVM: %[[RET:.+]] = load float, ptr %{{.+}}, align 4 +// LLVM: ret float %[[RET]] + +// OGCG-LABEL: @atomic_float_post_dec +// OGCG: %[[OLD:.+]] = atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4 +// OGCG: ret float %[[OLD]] _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
