Author: Kerry McLaughlin Date: 2026-09-24T14:26:38+01:00 New Revision: 34f5f2c594111e6a3edf65673c3ea2849e96e2cb
URL: https://github.com/llvm/llvm-project/commit/34f5f2c594111e6a3edf65673c3ea2849e96e2cb DIFF: https://github.com/llvm/llvm-project/commit/34f5f2c594111e6a3edf65673c3ea2849e96e2cb.diff LOG: [AArch64] Implement the atomic store with hint intrinsic (#198316) Adds the following ACLE intrinsic as described in [1]: void __arm_atomic_store_with_hint(type *ptr, type data, int memory_order, int hint); An atomic store instruction is emitted in Clang for this builtin with additional metadata attached with !mem.cache_hint, which ensures the instruction is recognised as atomic by later passes in LLVM. When an atomic store is found with this metadata, it is lowered to the ATOMIC_STORE_HINT pseudo which is later expanded into an STSHH instruction plus an atomic store. The hint is a suggestion to the compiler and maps directly to a specific hint instruction variant in the ISA. The compiler may use this hint when selecting code sequences, but it is not required to emit a specific hint instruction or a specific instruction sequence. Supported atomic store hints are currently: HINT_STSHH_KEEP: Requests retention of the updated location in the local cache of the updating PE. HINT_STSHH_STRM: Requests that the updated location not be retained in the local cache of the updating PE. [1] https://github.com/ARM-software/acle/pull/432 Added: clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp llvm/include/llvm/Support/AArch64MemoryHints.h llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir Modified: clang/include/clang/Basic/BuiltinsAArch64.td clang/include/clang/Basic/DiagnosticSemaKinds.td clang/include/clang/Sema/Sema.h clang/include/clang/Sema/SemaARM.h clang/lib/CodeGen/TargetBuiltins/ARM.cpp clang/lib/Headers/arm_acle.h clang/lib/Sema/SemaARM.cpp clang/lib/Sema/SemaChecking.cpp clang/test/CodeGen/arm_acle.c clang/test/CodeGen/builtins-arm64.c clang/test/Sema/builtins-arm64.c llvm/lib/CodeGen/AtomicExpandPass.cpp llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp llvm/lib/Target/AArch64/AArch64ISelLowering.cpp llvm/lib/Target/AArch64/AArch64InstrAtomics.td llvm/lib/Target/AArch64/AArch64InstrInfo.cpp llvm/lib/Target/AArch64/AArch64InstrInfo.h Removed: ################################################################################ diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td index 4c49c874133fb6..30aa3d526cbbb7 100644 --- a/clang/include/clang/Basic/BuiltinsAArch64.td +++ b/clang/include/clang/Basic/BuiltinsAArch64.td @@ -171,6 +171,10 @@ let Attributes = [NoThrow], Features = "ls64" in { def st64bv0 : AArch64TargetBuiltin<"uint64_t (void *, uint64_t const *)">; } +let Attributes = [NoThrow, CustomTypeChecking] in { + def atomic_store_with_hint : AArch64Builtin<"void(...)">; +} + // Armv9.3-A Guarded Control Stack let Attributes = [NoThrow], Features = "gcs" in { def gcspopm : AArch64TargetBuiltin<"uint64_t (uint64_t)">; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 36a18473f4d4cc..e854652a9efb85 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9615,6 +9615,13 @@ def err_atomic_builtin_must_be_pointer : Error< def err_atomic_builtin_must_be_pointer_intptr : Error< "address argument to atomic builtin must be a pointer to integer or pointer" " (%0 invalid)">; + +def err_atomic_hint_builtin_must_be_pointer : Error< + "address argument to atomic hint builtin must be a pointer to a scalar " + "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">; +def err_atomic_hint_has_invalid_hint_type : Error< + "invalid hint type argument to atomic hint operation (%0)">; + def err_atomic_builtin_cannot_be_const : Error< "address argument to atomic builtin cannot be const-qualified (%0 invalid)">; def err_atomic_builtin_must_be_pointer_intfltptr : Error< @@ -9683,6 +9690,13 @@ def err_atomic_op_needs_atomic_int : Error< def err_atomic_op_needs_atomic_fp : Error<"address argument to atomic operation must be a pointer to " "%select{|atomic }0floating point type (%1 invalid)">; +def err_atomic_op_hint_data_size : Error< + "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">; +def err_atomic_hint_has_invalid_memory_order : Error< + "invalid memory order argument to atomic hint operation (%0 invalid)">; +def warn_atomic_hint_has_invalid_hint_type : Warning< + "unrecognised hint type argument to atomic hint operation (%0)">, + InGroup<DiagGroup<"atomic-hint-type">>; def warn_atomic_op_has_invalid_memory_order : Warning< "%select{|success |failure }0memory order argument to atomic operation is invalid">, InGroup<DiagGroup<"atomic-memory-ordering">>; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 5f24adbf9e9cf3..eb3b908351448d 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -2915,6 +2915,8 @@ class Sema final : public SemaBase { /// This is useful when doing custom type-checking. Returns true on error. bool checkArgCount(CallExpr *Call, unsigned DesiredArgCount); + bool convertArgumentToType(Expr *&Value, QualType Ty); + /// Returns true if the argument consists of one contiguous run of 1s with any /// number of 0s on either side. The 1s are allowed to wrap from LSB to MSB, /// so 0x000FFF0, 0x0000FFFF, 0xFF0000FF, 0x0 are all runs. 0x0F0F0000 is not, diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h index af8e0e9047171d..b0a01c40ffecef 100644 --- a/clang/include/clang/Sema/SemaARM.h +++ b/clang/include/clang/Sema/SemaARM.h @@ -70,6 +70,7 @@ class SemaARM : public SemaBase { bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum, unsigned ExpectedFieldNum, bool AllowName); bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall); + bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall); bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName); bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName); diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index 621e5d18ebfeeb..52a7564789fb77 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -20,6 +20,7 @@ #include "llvm/IR/IntrinsicsAArch64.h" #include "llvm/IR/IntrinsicsARM.h" #include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/Support/AArch64MemoryHints.h" #include "llvm/TargetParser/AArch64TargetParser.h" #include <numeric> @@ -2065,6 +2066,67 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, Ops); } +static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, + unsigned BuiltinID, + const CallExpr *E) { + CodeGen::CGBuilderTy &Builder = CGF.Builder; + CodeGen::CodeGenModule &CGM = CGF.CGM; + Expr::EvalResult Result; + if (!E->getArg(2)->EvaluateAsInt(Result, CGM.getContext())) + llvm_unreachable( + "Expected integer policy argument to atomic store with hint."); + + const Expr *Ptr = E->getArg(0); + Address Addr = CGF.EmitPointerWithAlignment(Ptr); + Addr = Addr.withElementType( + CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType())); + + const Expr *Data = E->getArg(1); + Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType()); + + StoreInst *Store = Builder.CreateStore(DataVal, Addr); + Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified()); + + AtomicOrdering Ordering; + unsigned OrderingArg = Result.Val.getInt().getExtValue(); + assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering"); + + switch (static_cast<AtomicOrderingCABI>(OrderingArg)) { + default: + llvm_unreachable("Unsupported atomic ordering found."); + case AtomicOrderingCABI::relaxed: + Ordering = AtomicOrdering::Monotonic; + break; + case AtomicOrderingCABI::release: + Ordering = AtomicOrdering::Release; + break; + case AtomicOrderingCABI::seq_cst: + Ordering = AtomicOrdering::SequentiallyConsistent; + break; + } + Store->setAtomic(Ordering); + + if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext())) + llvm_unreachable( + "Expected integer hint argument to atomic store with hint."); + unsigned HintArg = Result.Val.getInt().getExtValue(); + + // Attach the hint if valid + if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::HINT_NONE) { + LLVMContext &Ctx = CGM.getLLVMContext(); + MDNode *MemHint = MDNode::get( + Ctx, {MDString::get(Ctx, "aarch64.mem_hint"), + llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))}); + MDNode *HintNode = MDNode::get( + CGM.getLLVMContext(), + {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint}); + + Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode); + } + + return Store; +} + /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra /// argument that specifies the vector type. The additional argument is meant /// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function @@ -4840,6 +4902,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x) return EmitRangePrefetchBuiltin(*this, BuiltinID, E); + if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint) + return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E); + // Memory Tagging Extensions (MTE) Intrinsics Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic; switch (BuiltinID) { diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h index 73b1e841b6719d..715fe851b0c322 100644 --- a/clang/lib/Headers/arm_acle.h +++ b/clang/lib/Headers/arm_acle.h @@ -742,6 +742,14 @@ __arm_st64bv0(void *__addr, data512_t __value) { } #endif +/* Atomic store with hints */ +#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE +#define HINT_STSHH_KEEP 0 +#define HINT_STSHH_STRM 1 +#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint) +#endif + /* 11.1 Special register intrinsics */ #define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg) #define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg) diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp index f38a35ec08fb0a..2bf6901bcc07f2 100644 --- a/clang/lib/Sema/SemaARM.cpp +++ b/clang/lib/Sema/SemaARM.cpp @@ -17,6 +17,7 @@ #include "clang/Sema/Initialization.h" #include "clang/Sema/ParsedAttr.h" #include "clang/Sema/Sema.h" +#include "llvm/Support/AArch64MemoryHints.h" namespace clang { @@ -322,6 +323,126 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, return false; } +bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, + CallExpr *TheCall) { + if (SemaRef.checkArgCount(TheCall, 4)) + return true; + + // Arg 0 should be the pointer type. The pointee type must be a + // scalar integral or floating-point type of 8, 16, 32 or 64 bits. + ASTContext &Context = getASTContext(); + auto PtrArgRes = + SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0)); + if (PtrArgRes.isInvalid()) + return true; + auto *PtrArg = PtrArgRes.get(); + auto *PtrTy = PtrArg->getType()->getAs<PointerType>(); + if (!PtrTy) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_hint_builtin_must_be_pointer) + << PtrArg->getType() << 0 << PtrArg->getSourceRange(); + TheCall->setArg(0, PtrArg); + + QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType()); + if (PtrQT.isConstQualified()) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_op_needs_non_const_pointer) + << PtrQT << PtrArg->getSourceRange(); + + PtrQT = PtrQT.getUnqualifiedType(); + if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() && + !PtrQT->isMFloat8Type()) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_op_needs_atomic_int_or_fp) + << 0 << PtrQT << PtrArg->getSourceRange(); + + if (PtrQT->isBitIntType()) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_builtin_bit_int_prohibit) + << PtrQT << PtrArg->getSourceRange(); + + unsigned TySize = Context.getTypeSize(PtrQT); + if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64) + return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size) + << PtrArg->getSourceRange(); + + // Arg 1 is the data to be stored. The type must match the pointee + // type found above. + auto DataArgRes = + SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1)); + if (DataArgRes.isInvalid()) + return true; + auto *DataArg = DataArgRes.get(); + QualType DataQT = + Context.getCanonicalType(DataArg->getType()).getUnqualifiedType(); + TheCall->setArg(1, DataArg); + + if (PtrQT != DataQT) + return Diag(TheCall->getBeginLoc(), + diag::err_typecheck_call_ diff erent_arg_types) + << PtrQT << DataQT; + + // Arg 2 is the memory order, which must be relaxed, release or seq_cst + auto MemOrdArg = + SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)); + if (MemOrdArg.isInvalid()) + return true; + auto *MemOrd = MemOrdArg.get(); + if (SemaRef.convertArgumentToType(MemOrd, Context.IntTy)) + return true; + TheCall->setArg(2, MemOrd); + + if (!MemOrd->isValueDependent()) { + std::optional<llvm::APSInt> MemOrdAP = + MemOrd->getIntegerConstantExpr(Context); + if (!MemOrdAP) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_hint_has_invalid_memory_order) + << MemOrd->getType() << MemOrd->getSourceRange(); + + unsigned Ordering = MemOrdAP->getZExtValue(); + if (!llvm::isValidAtomicOrderingCABI(Ordering)) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_hint_has_invalid_memory_order) + << *MemOrdAP << MemOrd->getSourceRange(); + + auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering); + if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed && + AtomicOrdering != llvm::AtomicOrderingCABI::release && + AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_hint_has_invalid_memory_order) + << *MemOrdAP << MemOrd->getSourceRange(); + } + + // Arg 3 is the hint type. Only values represented by AArch64MemoryHint + // are valid. + auto HintArg = + SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)); + if (HintArg.isInvalid()) + return true; + auto Hint = HintArg.get(); + if (SemaRef.convertArgumentToType(Hint, Context.IntTy)) + return true; + TheCall->setArg(3, Hint); + + if (!Hint->isValueDependent()) { + std::optional<llvm::APSInt> HintAP = Hint->getIntegerConstantExpr(Context); + if (!HintAP) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_hint_has_invalid_hint_type) + << Hint->getType() << Hint->getSourceRange(); + + if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) == + llvm::AArch64MemoryHint::HINT_NONE) { + Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type) + << *HintAP << Hint->getSourceRange(); + return false; + } + } + return false; +} + /// getNeonEltType - Return the QualType corresponding to the elements of /// the vector type specified by the NeonTypeFlags. This is used to check /// the pointer arguments for Neon load/store intrinsics. @@ -1170,6 +1291,9 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, BuiltinID == AArch64::BI__builtin_arm_wsrp) return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true); + if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint) + return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall); + // Only check the valid encoding range. Any constant in this range would be // converted to a register of the form S2_2_C3_C4_5. Let the hardware throw // an exception for incorrect registers. This matches MSVC behavior. diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index e4cb8ba0885d09..687af75c1496be 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -201,14 +201,14 @@ static bool checkBuiltinVerboseTrap(CallExpr *Call, Sema &S) { return !HasError; } -static bool convertArgumentToType(Sema &S, Expr *&Value, QualType Ty) { +bool Sema::convertArgumentToType(Expr *&Value, QualType Ty) { if (Value->isTypeDependent()) return false; InitializedEntity Entity = - InitializedEntity::InitializeParameter(S.Context, Ty, false); + InitializedEntity::InitializeParameter(Context, Ty, false); ExprResult Result = - S.PerformCopyInitialization(Entity, SourceLocation(), Value); + PerformCopyInitialization(Entity, SourceLocation(), Value); if (Result.isInvalid()) return true; Value = Result.get(); @@ -1702,7 +1702,7 @@ static bool checkPointerAuthEnabled(Sema &S, Expr *E) { static bool checkPointerAuthKey(Sema &S, Expr *&Arg) { // Convert it to type 'int'. - if (convertArgumentToType(S, Arg, S.Context.IntTy)) + if (S.convertArgumentToType(Arg, S.Context.IntTy)) return true; // Value-dependent expressions are okay; wait for template instantiation. @@ -1838,7 +1838,7 @@ static bool checkPointerAuthValue(Sema &S, Expr *&Arg, PointerAuthOpKind OpKind, // Convert to that type. This should just be an lvalue-to-rvalue // conversion. - if (convertArgumentToType(S, Arg, ExpectedTy)) + if (S.convertArgumentToType(Arg, ExpectedTy)) return true; if (!RequireConstant) { @@ -6710,7 +6710,7 @@ bool Sema::BuiltinPrefetch(CallExpr *TheCall) { // Argument 0 is checked for us and the remaining arguments must be // constant integers. for (unsigned i = 1; i != NumArgs; ++i) { - if (convertArgumentToType(*this, TheCall->getArgs()[i], Context.IntTy)) + if (convertArgumentToType(TheCall->getArgs()[i], Context.IntTy)) return true; if (BuiltinConstantArgRange(TheCall, i, 0, i == 1 ? 1 : 3)) return true; @@ -6825,7 +6825,7 @@ bool Sema::BuiltinAssumeAligned(CallExpr *TheCall) { if (NumArgs > 2) { Expr *ThirdArg = TheCall->getArg(2); - if (convertArgumentToType(*this, ThirdArg, Context.getSizeType())) + if (convertArgumentToType(ThirdArg, Context.getSizeType())) return true; TheCall->setArg(2, ThirdArg); } diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp new file mode 100644 index 00000000000000..ecf1e10a09b5da --- /dev/null +++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp @@ -0,0 +1,63 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 6 +// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-elf -target-feature +v9.4a -O0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s + +#include <arm_acle.h> + +// REQUIRES: aarch64-registered-target + +template <unsigned Order> +void test_atomic_store_hint_template_order(int *ptr, int val) { +// CHECK-LABEL: define weak_odr void @_Z37test_atomic_store_hint_template_orderILj3EEvPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] comdat { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] release, align 4, !mem.cache_hint [[META4:![0-9]+]] +// CHECK-NEXT: ret void + __builtin_arm_atomic_store_with_hint(ptr, val, Order, 0); +} + +template void test_atomic_store_hint_template_order<__ATOMIC_RELEASE>(int *, int); + +template <unsigned Hint> +void test_atomic_store_hint_template_hint(int *ptr, int val) { +// CHECK-LABEL: define weak_odr void @_Z36test_atomic_store_hint_template_hintILj1EEvPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] seq_cst, align 4, !mem.cache_hint [[META6:![0-9]+]] +// CHECK-NEXT: ret void + __builtin_arm_atomic_store_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint); +} + +template void test_atomic_store_hint_template_hint<HINT_STSHH_STRM>(int *, int); + +struct MemVal { + constexpr operator int() const { return __ATOMIC_RELAXED; } +}; + +void test_atomic_store_hint_array_consexpr_order(int *ptr, int value) { +// CHECK-LABEL: define dso_local void @_Z43test_atomic_store_hint_array_consexpr_orderPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VALUE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: store atomic i32 [[VALUE]], ptr [[PTR]] monotonic, align 4, !mem.cache_hint [[META4]] +// CHECK-NEXT: ret void + __builtin_arm_atomic_store_with_hint(ptr, value, MemVal{}, 0); +} + +struct HintVal { + constexpr operator int() const { return 1; } +}; + +void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) { +// CHECK-LABEL: define dso_local void @_Z42test_atomic_store_hint_array_consexpr_hintPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VALUE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: store atomic i32 [[VALUE]], ptr [[PTR]] monotonic, align 4, !mem.cache_hint [[META6]] +// CHECK-NEXT: ret void + __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{}); +} + +//. +// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]} +// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0} +// CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]} +// CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1} +//. diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c index e1bbc3d745bcd0..60ec3e48e94980 100644 --- a/clang/test/CodeGen/arm_acle.c +++ b/clang/test/CodeGen/arm_acle.c @@ -8,6 +8,7 @@ #include <arm_acle.h> +#include <stdbool.h> // REQUIRES: arm-registered-target,aarch64-registered-target @@ -1761,3 +1762,236 @@ int test_rndrrs(uint64_t *__addr) { return __rndrrs(__addr); } #endif + +#if defined(__ARM_64BIT_STATE) + +// AArch64-LABEL: @test_atomic_store_hint_char( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1:![0-9]+]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_char(char *ptr, char data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0); +} + +// AArch64-LABEL: @test_atomic_store_hint_uchar( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_schar( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_schar(signed char *ptr, signed char data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_bool( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8 +// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0 +// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8 +// AArch64-NEXT: store atomic i8 [[STOREDV1]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_bool(bool *ptr, bool data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_mfloat( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic <1 x i8> [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_mfloat(__mfp8 *ptr, __mfp8 data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_bfloat( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3:![0-9]+]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1); +} + +// AArch64-LABEL: @test_atomic_store_hint_half( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_STRM); +} + +// AArch64-LABEL: @test_atomic_store_hint_short( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_short(short *ptr, short data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_ushort( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_int( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_int(int *ptr, int data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_unsigned( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_u32( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_s32( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_float( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_float(float *ptr, float data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_s64( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_STRM); +} + +// AArch64-LABEL: @test_atomic_store_hint_long( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_long(long *ptr, long data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_ulong( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_long_long_int( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_long_long_uint( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_double( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_double(double *ptr, double data) { + __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_STRM); +} + +typedef int aliased_int; + +// AArch64-LABEL: @test_atomic_store_hint_typedef_ptr( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i32 [[VALUE:%.*]], ptr [[PTR:%.*]] monotonic, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_typedef_ptr(aliased_int *ptr, int value) { + __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_typedef_val( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic i32 [[VALUE:%.*]], ptr [[PTR:%.*]] release, align 4, !mem.cache_hint [[HINT3]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_typedef_val(int *ptr, aliased_int value) { + __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELEASE, HINT_STSHH_STRM); +} + +// AArch64-LABEL: @test_atomic_store_hint_volatile( +// AArch64-NEXT: entry: +// AArch64-NEXT: store atomic volatile i32 [[VALUE:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_volatile(volatile int *ptr, int value) { + __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); +} + +// AArch64-LABEL: @test_atomic_store_hint_array_arg( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4 +// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0 +// AArch64-NEXT: store atomic i32 0, ptr [[ARRAYDECAY]] monotonic, align 4, !mem.cache_hint [[HINT3]] +// AArch64-NEXT: ret void +// +void test_atomic_store_hint_array_arg() { + int storage[1]; + __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM); +} + +// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]} +// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0} + +// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]} +// AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1} +#endif diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index 1800ff77a57a52..fb297cd3eec2c7 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -2,6 +2,7 @@ // RUN: %clang_cc1 -triple aarch64-windows -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-WIN // RUN: %clang_cc1 -triple arm64_32-apple-ios13 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s #include <stdint.h> +#include <arm_acle.h> void f0(void *a, void *b) { __clear_cache(a,b); @@ -216,4 +217,26 @@ void trap() { __builtin_arm_trap(42); } +void atomic_store_with_hint(int64_t *a, int64_t b) { + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M1:[0-9]]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M1]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_STSHH_STRM); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]] + + // Invalid hint should be dropped + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 +} + // CHECK: ![[M0]] = !{!"1:2:3:4:5"} +// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]} +// CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0} +// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]} +// CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1} diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c index 41cffd7ebb1a0d..9a6e0715c4c788 100644 --- a/clang/test/Sema/builtins-arm64.c +++ b/clang/test/Sema/builtins-arm64.c @@ -1,6 +1,8 @@ // RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify %s // RUN: %clang_cc1 -triple arm64-apple-ios -DTEST1 -fsyntax-only -verify %s +#include <arm_acle.h> + #ifdef TEST1 void __clear_cache(void *start, void *end); #endif @@ -51,3 +53,29 @@ void test_trap(short s, unsigned short us) { __builtin_arm_trap(s); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}} __builtin_arm_trap(us); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}} } + +void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr, + char c_data, __int128 inv_data, float f_data, + int inv_int, const char *const_c_ptr, + unsigned _BitInt(7) *bit_ptr, unsigned _BitInt(7) *bit_data) { + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}} + + __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral or floating-point type of 8, 16, 32, or 64 bits ('int' invalid)}} + __builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of diff erent types ('char' vs 'float')}} + __builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}} + + __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}} + + __builtin_arm_atomic_store_with_hint(bit_ptr, bit_data, __ATOMIC_RELAXED, HINT_STSHH_STRM); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}} + + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, __ATOMIC_RELAXED, HINT_STSHH_INVALID); // expected-error {{use of undeclared identifier 'HINT_STSHH_INVALID'}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}} + + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}} + + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} + // expected-error@-1 {{invalid hint type argument to atomic hint operation ('int')}} +} diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h new file mode 100644 index 00000000000000..44ac64e481e80b --- /dev/null +++ b/llvm/include/llvm/Support/AArch64MemoryHints.h @@ -0,0 +1,35 @@ +//===-- AArch64MemoryHints.h - AArch64 Memory Hint Attributes ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_AARCH64MEMORYHINTS_H +#define LLVM_SUPPORT_AARCH64MEMORYHINTS_H + +namespace llvm { +enum class AArch64MemoryHint { + HINT_NONE = 0, + HINT_STSHH_KEEP = 1, + HINT_STSHH_STRM = 2, +}; + +template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) { + return (Int)AArch64MemoryHint::HINT_STSHH_KEEP <= I && + I <= (Int)AArch64MemoryHint::HINT_STSHH_STRM; +} + +template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { + switch (I) { + case 0: + return AArch64MemoryHint::HINT_STSHH_KEEP; + case 1: + return AArch64MemoryHint::HINT_STSHH_STRM; + default: + return AArch64MemoryHint::HINT_NONE; + } +} +} // namespace llvm +#endif // LLVM_SUPPORT_AARCH64MEMORYHINTS_H diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 50e09e40f6c757..b3d186fdf5f011 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -246,6 +246,7 @@ static void copyMetadataForAtomic(Instruction &Dest, case LLVMContext::MD_tbaa: case LLVMContext::MD_tbaa_struct: case LLVMContext::MD_alias_scope: + case LLVMContext::MD_mem_cache_hint: case LLVMContext::MD_noalias: case LLVMContext::MD_noalias_addrspace: case LLVMContext::MD_access_group: @@ -759,6 +760,7 @@ StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) { Value *Addr = SI->getPointerOperand(); StoreInst *NewSI = Builder.CreateStore(NewVal, Addr, SI->getProperties()); + copyMetadataForAtomic(*NewSI, *SI); LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n"); SI->eraseFromParent(); return NewSI; diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index bc8c5510ea5154..5a502f785e0f40 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -290,6 +290,11 @@ class AArch64AsmPrinter : public AsmPrinter { // Emit expansion of Compare-and-branch pseudo instructions void emitCBPseudoExpansion(const MachineInstr *MI); + // Emit expansion of atomic store with hint pseudo instructions + void emitAtomicHintPseudoExpansion(const MachineInstr *MI); + void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI); + void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI); + void EmitToStreamer(MCStreamer &S, const MCInst &Inst); void EmitToStreamer(const MCInst &Inst) { EmitToStreamer(*OutStreamer, Inst); @@ -3249,6 +3254,161 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, Inst); } +void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) { + + unsigned StOpc; + unsigned Relaxed = MI->getOperand(2).getImm(); + assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds"); + switch (MI->getOpcode()) { + case AArch64::ATOMIC_STORE_HINT_B: + StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB; + break; + case AArch64::ATOMIC_STORE_HINT_H: + StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH; + break; + case AArch64::ATOMIC_STORE_HINT_W: + StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW; + break; + case AArch64::ATOMIC_STORE_HINT_X: + StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX; + break; + default: + llvm_unreachable("Unexpected atomic hint size."); + } + + EmitToStreamer( + MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(3).getImm())); + + MCInst Store; + Store.setOpcode(StOpc); + Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); + Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); + Store.setFlags(MI->getFlags()); + if (Relaxed) + Store.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, Store); +} + +void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO( + const MachineInstr *MI) { + unsigned StOpc; + assert(MI->getOperand(5).getImm() == 1 && + "Atomic store addressing mode only supports relaxed stores"); + + switch (MI->getOpcode()) { + case AArch64::ATOMIC_STORE_HINT_BroW: + StOpc = AArch64::STRBBroW; + break; + case AArch64::ATOMIC_STORE_HINT_HroW: + StOpc = AArch64::STRHHroW; + break; + case AArch64::ATOMIC_STORE_HINT_WroW: + StOpc = AArch64::STRWroW; + break; + case AArch64::ATOMIC_STORE_HINT_XroW: + StOpc = AArch64::STRXroW; + break; + case AArch64::ATOMIC_STORE_HINT_SroW: + StOpc = AArch64::STRSroW; + break; + case AArch64::ATOMIC_STORE_HINT_DroW: + StOpc = AArch64::STRDroW; + break; + case AArch64::ATOMIC_STORE_HINT_BroX: + StOpc = AArch64::STRBBroX; + break; + case AArch64::ATOMIC_STORE_HINT_HroX: + StOpc = AArch64::STRHHroX; + break; + case AArch64::ATOMIC_STORE_HINT_WroX: + StOpc = AArch64::STRWroX; + break; + case AArch64::ATOMIC_STORE_HINT_XroX: + StOpc = AArch64::STRXroX; + break; + case AArch64::ATOMIC_STORE_HINT_SroX: + StOpc = AArch64::STRSroX; + break; + case AArch64::ATOMIC_STORE_HINT_DroX: + StOpc = AArch64::STRDroX; + break; + default: + llvm_unreachable("Unexpected atomic hint opcode."); + } + + EmitToStreamer( + MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(6).getImm())); + + MCInst Store; + Store.setOpcode(StOpc); + Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn + Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Rm + Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data + Store.addOperand(MCOperand::createImm(MI->getOperand(3).getImm())); // Signed + Store.addOperand(MCOperand::createImm(MI->getOperand(4).getImm())); // Shift + Store.setFlags(MI->getFlags()); + EmitToStreamer(*OutStreamer, Store); +} + +void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm( + const MachineInstr *MI) { + unsigned StOpc; + assert(MI->getOperand(3).getImm() == 1 && + "Atomic store addressing mode only supports relaxed stores"); + + switch (MI->getOpcode()) { + case AArch64::ATOMIC_STORE_HINT_Bui: + StOpc = AArch64::STRBBui; + break; + case AArch64::ATOMIC_STORE_HINT_Hui: + StOpc = AArch64::STRHHui; + break; + case AArch64::ATOMIC_STORE_HINT_Wui: + StOpc = AArch64::STRWui; + break; + case AArch64::ATOMIC_STORE_HINT_Xui: + StOpc = AArch64::STRXui; + break; + case AArch64::ATOMIC_STORE_HINT_Sui: + StOpc = AArch64::STRSui; + break; + case AArch64::ATOMIC_STORE_HINT_Dui: + StOpc = AArch64::STRDui; + break; + case AArch64::ATOMIC_STORE_HINT_Bi: + StOpc = AArch64::STURBBi; + break; + case AArch64::ATOMIC_STORE_HINT_Hi: + StOpc = AArch64::STURHHi; + break; + case AArch64::ATOMIC_STORE_HINT_Wi: + StOpc = AArch64::STURWi; + break; + case AArch64::ATOMIC_STORE_HINT_Xi: + StOpc = AArch64::STURXi; + break; + case AArch64::ATOMIC_STORE_HINT_Si: + StOpc = AArch64::STURSi; + break; + case AArch64::ATOMIC_STORE_HINT_Di: + StOpc = AArch64::STURDi; + break; + default: + llvm_unreachable("Unexpected atomic hint opcode."); + } + + EmitToStreamer( + MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm())); + + MCInst Store; + Store.setOpcode(StOpc); + Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn + Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data + Store.addOperand(MCOperand::createImm(MI->getOperand(2).getImm())); // Imm + Store.setFlags(MI->getFlags()); + EmitToStreamer(*OutStreamer, Store); +} + // Simple pseudo-instructions have their lowering (with expansion to real // instructions) auto-generated. #include "AArch64GenMCPseudoLowering.inc" @@ -3966,6 +4126,40 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { case AArch64::CBXPrr: emitCBPseudoExpansion(MI); return; + case AArch64::ATOMIC_STORE_HINT_B: + case AArch64::ATOMIC_STORE_HINT_H: + case AArch64::ATOMIC_STORE_HINT_W: + case AArch64::ATOMIC_STORE_HINT_X: + emitAtomicHintPseudoExpansion(MI); + return; + case AArch64::ATOMIC_STORE_HINT_BroW: + case AArch64::ATOMIC_STORE_HINT_HroW: + case AArch64::ATOMIC_STORE_HINT_WroW: + case AArch64::ATOMIC_STORE_HINT_XroW: + case AArch64::ATOMIC_STORE_HINT_SroW: + case AArch64::ATOMIC_STORE_HINT_DroW: + case AArch64::ATOMIC_STORE_HINT_BroX: + case AArch64::ATOMIC_STORE_HINT_HroX: + case AArch64::ATOMIC_STORE_HINT_WroX: + case AArch64::ATOMIC_STORE_HINT_XroX: + case AArch64::ATOMIC_STORE_HINT_SroX: + case AArch64::ATOMIC_STORE_HINT_DroX: + emitAtomicHintPseudoExpansionRO(MI); + return; + case AArch64::ATOMIC_STORE_HINT_Bui: + case AArch64::ATOMIC_STORE_HINT_Hui: + case AArch64::ATOMIC_STORE_HINT_Wui: + case AArch64::ATOMIC_STORE_HINT_Xui: + case AArch64::ATOMIC_STORE_HINT_Sui: + case AArch64::ATOMIC_STORE_HINT_Dui: + case AArch64::ATOMIC_STORE_HINT_Bi: + case AArch64::ATOMIC_STORE_HINT_Hi: + case AArch64::ATOMIC_STORE_HINT_Wi: + case AArch64::ATOMIC_STORE_HINT_Xi: + case AArch64::ATOMIC_STORE_HINT_Si: + case AArch64::ATOMIC_STORE_HINT_Di: + emitAtomicHintPseudoExpansionImm(MI); + return; } if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol())) diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index feb8f465837940..ee87070c127272 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAArch64.h" +#include "llvm/Support/AArch64MemoryHints.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" @@ -516,6 +517,10 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { bool SelectCMP_SWAP(SDNode *N); + AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const; + bool isAtomicSTSHH_KEEP(SDNode *N) const; + bool isAtomicSTSHH_STRM(SDNode *N) const; + bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate); bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm, @@ -4614,6 +4619,34 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) { return true; } +AArch64MemoryHint +AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const { + int MemoryHint = -1; + const MDNode *MemCacheHint = MMO->getMemCacheHint(); + if (!MemCacheHint) + return AArch64MemoryHint::HINT_NONE; + + for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) { + if (MemCacheHint->getOperand(I).equalsStr("aarch64.mem_hint")) { + const Metadata *Val = MemCacheHint->getOperand(I + 1).get(); + MemoryHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue()) + ->getZExtValue(); + } + } + + return toAArch64MemoryHint(MemoryHint); +} + +bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::HINT_STSHH_KEEP; +} + +bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::HINT_STSHH_STRM; +} + bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate) { if (!isa<ConstantSDNode>(N)) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d90030ee5519e7..d5135b47f4ef19 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -41,6 +41,7 @@ #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/CallingConvLower.h" +#include "llvm/CodeGen/CodeGenCommonISel.h" #include "llvm/CodeGen/ComplexDeinterleavingPass.h" #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" #include "llvm/CodeGen/GlobalISel/Utils.h" @@ -34068,6 +34069,14 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const { return true; } } + + // The !mem.cache_hint metadata is not supported by GISel and will be dropped. + // TODO: Remove this and handle atomic store hints in GISel once supported. + if (auto *Store = dyn_cast<StoreInst>(&Inst)) { + if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1)) + return true; + } + return false; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 2187f21abb70fd..7e0e75c7a3c6b8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -282,6 +282,156 @@ def : Pat<(relaxed_store<atomic_store_64> (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>; +//===---------------------------------- +// Atomic store with hint pseudos +//===---------------------------------- + +let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in { + class BaseStoreHintPseudo<RegisterClass regtype> + : Pseudo<(outs), (ins regtype:$data, GPR64sp:$addr, + i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>; + + class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt> + : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, GPR32:$Rm, WExt:$extend, + i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>; + + class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt> + : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, GPR64:$Rm, XExt:$extend, + i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>; + + class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp> + : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, UImmOp:$offset, + i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>; + + class BaseStoreHintPseudoImm<RegisterClass regtype> + : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, simm9:$offset, + i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>; +} + +multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand XExt, Operand UImmOp> { + def NAME : BaseStoreHintPseudo<regtype>; + def NAME # roW : BaseStoreHintPseudoROW<regtype, WExt>; + def NAME # roX : BaseStoreHintPseudoROX<regtype, XExt>; + def NAME # ui : BaseStoreHintPseudoUImm<regtype, UImmOp>; + def NAME # i : BaseStoreHintPseudoImm<regtype>; +} + +defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xextend8, uimm12s1>; +defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>; +defm ATOMIC_STORE_HINT_W : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>; +defm ATOMIC_STORE_HINT_X : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>; +defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<FPR32, ro_Wextend32, ro_Xextend32, uimm12s4>; +defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<FPR64, ro_Wextend64, ro_Xextend64, uimm12s8>; + +class atomic_hint_store<PatFrag Base, int Relaxed, code Pred> + : PatFrag<(ops node:$val, node:$ptr), + (Base node:$val, node:$ptr), Pred> { + let IsAtomic = 1; + let IsAtomicOrderingReleaseOrStronger = !eq(Relaxed, 0); + + // TODO: Change once mem.cache_hint supported in GISel + let GISelPredicateCode = [{ return false; }]; +} + +multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> { + def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred> + GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)), + (ATOMIC_STORE_HINT_BroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred> + GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)), + (ATOMIC_STORE_HINT_HroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)), + (ATOMIC_STORE_HINT_WroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)), + (ATOMIC_STORE_HINT_XroW GPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)), + (ATOMIC_STORE_HINT_SroW FPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)), + (ATOMIC_STORE_HINT_DroW FPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>; + + def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred> + GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)), + (ATOMIC_STORE_HINT_BroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred> + GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)), + (ATOMIC_STORE_HINT_HroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)), + (ATOMIC_STORE_HINT_WroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)), + (ATOMIC_STORE_HINT_XroX GPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)), + (ATOMIC_STORE_HINT_SroX FPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)), + (ATOMIC_STORE_HINT_DroX FPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>; + + def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred> + GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)), + (ATOMIC_STORE_HINT_Bui GPR32:$data, GPR64sp:$addr, uimm12s1:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred> + GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)), + (ATOMIC_STORE_HINT_Hui GPR32:$data, GPR64sp:$addr, uimm12s2:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)), + (ATOMIC_STORE_HINT_Wui GPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)), + (ATOMIC_STORE_HINT_Xui GPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + (i32 (bitconvert (f32 FPR32Op:$data))), (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)), + (ATOMIC_STORE_HINT_Sui FPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + (i64 (bitconvert (f64 FPR64Op:$data))), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)), + (ATOMIC_STORE_HINT_Dui FPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>; + + def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred> + GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)), + (ATOMIC_STORE_HINT_Bi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred> + GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)), + (ATOMIC_STORE_HINT_Hi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)), + (ATOMIC_STORE_HINT_Wi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)), + (ATOMIC_STORE_HINT_Xi GPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred> + (i32 (bitconvert (f32 FPR32Op:$data))), (am_unscaled32 GPR64sp:$addr, simm9:$offset)), + (ATOMIC_STORE_HINT_Si FPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred> + (i64 (bitconvert (f64 FPR64Op:$data))), (am_unscaled64 GPR64sp:$addr, simm9:$offset)), + (ATOMIC_STORE_HINT_Di FPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>; +} + +multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> { + def : Pat<(atomic_hint_store<atomic_store_8, Relaxed, Pred> GPR32:$data, GPR64sp:$addr), + (ATOMIC_STORE_HINT_B GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_16, Relaxed, Pred> GPR32:$data, GPR64sp:$addr), + (ATOMIC_STORE_HINT_H GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_32, Relaxed, Pred> GPR32:$data, GPR64sp:$addr), + (ATOMIC_STORE_HINT_W GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>; + def : Pat<(atomic_hint_store<atomic_store_64, Relaxed, Pred> GPR64:$data, GPR64sp:$addr), + (ATOMIC_STORE_HINT_X GPR64:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>; +} + +let AddedComplexity = 15 in { + defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>; + defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>; + + defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; + defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; + defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>; + defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>; +} + //===---------------------------------- // Low-level exclusive operations //===---------------------------------- diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index b8e74f8ad496d9..6962d66dedaf01 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -3151,6 +3151,18 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) { case AArch64::STZ2Gi: case AArch64::STZGi: case AArch64::TAGPstack: + case AArch64::ATOMIC_STORE_HINT_Bi: + case AArch64::ATOMIC_STORE_HINT_Hi: + case AArch64::ATOMIC_STORE_HINT_Wi: + case AArch64::ATOMIC_STORE_HINT_Si: + case AArch64::ATOMIC_STORE_HINT_Xi: + case AArch64::ATOMIC_STORE_HINT_Di: + case AArch64::ATOMIC_STORE_HINT_Bui: + case AArch64::ATOMIC_STORE_HINT_Hui: + case AArch64::ATOMIC_STORE_HINT_Wui: + case AArch64::ATOMIC_STORE_HINT_Sui: + case AArch64::ATOMIC_STORE_HINT_Xui: + case AArch64::ATOMIC_STORE_HINT_Dui: return 2; case AArch64::LD1B_D_IMM: case AArch64::LD1B_H_IMM: @@ -4761,6 +4773,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::STRXui: case AArch64::STRDui: case AArch64::PRFMui: + case AArch64::ATOMIC_STORE_HINT_Xui: + case AArch64::ATOMIC_STORE_HINT_Dui: Scale = Width = TypeSize::getFixed(8); MinOffset = 0; MaxOffset = 4095; @@ -4770,6 +4784,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::LDRSWui: case AArch64::STRWui: case AArch64::STRSui: + case AArch64::ATOMIC_STORE_HINT_Wui: + case AArch64::ATOMIC_STORE_HINT_Sui: Scale = Width = TypeSize::getFixed(4); MinOffset = 0; MaxOffset = 4095; @@ -4780,6 +4796,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::LDRSHXui: case AArch64::STRHui: case AArch64::STRHHui: + case AArch64::ATOMIC_STORE_HINT_Hui: Scale = Width = TypeSize::getFixed(2); MinOffset = 0; MaxOffset = 4095; @@ -4790,6 +4807,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::LDRSBXui: case AArch64::STRBui: case AArch64::STRBBui: + case AArch64::ATOMIC_STORE_HINT_Bui: Scale = Width = TypeSize::getFixed(1); MinOffset = 0; MaxOffset = 4095; @@ -4868,6 +4886,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::STURDi: case AArch64::STLURXi: case AArch64::PRFUMi: + case AArch64::ATOMIC_STORE_HINT_Xi: + case AArch64::ATOMIC_STORE_HINT_Di: Scale = TypeSize::getFixed(1); Width = TypeSize::getFixed(8); MinOffset = -256; @@ -4881,6 +4901,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::STURWi: case AArch64::STURSi: case AArch64::STLURWi: + case AArch64::ATOMIC_STORE_HINT_Wi: + case AArch64::ATOMIC_STORE_HINT_Si: Scale = TypeSize::getFixed(1); Width = TypeSize::getFixed(4); MinOffset = -256; @@ -4896,6 +4918,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::STURHi: case AArch64::STURHHi: case AArch64::STLURHi: + case AArch64::ATOMIC_STORE_HINT_Hi: Scale = TypeSize::getFixed(1); Width = TypeSize::getFixed(2); MinOffset = -256; @@ -4911,6 +4934,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale, case AArch64::STURBi: case AArch64::STURBBi: case AArch64::STLURBi: + case AArch64::ATOMIC_STORE_HINT_Bi: Scale = Width = TypeSize::getFixed(1); MinOffset = -256; MaxOffset = 255; diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 99e0142824d438..6a8346395eea1a 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -16,6 +16,7 @@ #include "AArch64.h" #include "AArch64RegisterInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/Support/AArch64MemoryHints.h" #include "llvm/Support/TypeSize.h" #include <optional> diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll new file mode 100644 index 00000000000000..bfa9fc49cf7522 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -0,0 +1,339 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s + +; +; STSHH: Keep, Relaxed +; + +define void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_relaxed_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_relaxed_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_relaxed_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +; +; STSHH: Keep, Release +; + +define void @test_atomic_store_keep_release_mfloat(ptr %ptr, <1 x i8> %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_release_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlrb w8, [x0] +; CHECK-NEXT: ret + store atomic <1 x i8> %val, ptr %ptr release, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_release_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlrh w8, [x0] +; CHECK-NEXT: ret + store atomic bfloat %val, ptr %ptr release, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_release_half: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlrh w8, [x0] +; CHECK-NEXT: ret + store atomic half %val, ptr %ptr release, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_release_float: +; CHECK: // %bb.0: +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlr w8, [x0] +; CHECK-NEXT: ret + store atomic float %val, ptr %ptr release, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_release_double: +; CHECK: // %bb.0: +; CHECK-NEXT: fmov x8, d0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlr x8, [x0] +; CHECK-NEXT: ret + store atomic double %val, ptr %ptr release, align 8, !mem.cache_hint !0 + ret void +} + +; +; STSHH: Keep, SequentiallyConsistent +; + +define void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_seqcst_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_seqcst_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_seqcst_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_keep_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0 + ret void +} + +; +; STSHH: Stream, Relaxed +; + +define void @test_atomic_store_strm_relaxed_mfloat(ptr %ptr, <1 x i8> %val) nounwind { +; CHECK-LABEL: test_atomic_store_strm_relaxed_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strb w8, [x0] +; CHECK-NEXT: ret + store atomic <1 x i8> %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind { +; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strh w8, [x0] +; CHECK-NEXT: ret + store atomic bfloat %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind { +; CHECK-LABEL: test_atomic_store_strm_relaxed_half: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strh w8, [x0] +; CHECK-NEXT: ret + store atomic half %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind { +; CHECK-LABEL: test_atomic_store_strm_relaxed_float: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str s0, [x0] +; CHECK-NEXT: ret + store atomic float %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind { +; CHECK-LABEL: test_atomic_store_strm_relaxed_double: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str d0, [x0] +; CHECK-NEXT: ret + store atomic double %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +; +; STSHH: Stream, Release +; + +define void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_release_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_release_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_release_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !2 + ret void +} + +; +; STSHH: Stream, SequentiallyConsistent +; + +define void @test_atomic_store_stream_seqcst_mfloat(ptr %ptr, <1 x i8> %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_seqcst_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlrb w8, [x0] +; CHECK-NEXT: ret + store atomic <1 x i8> %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlrh w8, [x0] +; CHECK-NEXT: ret + store atomic bfloat %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_seqcst_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlrh w8, [x0] +; CHECK-NEXT: ret + store atomic half %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_seqcst_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlr w8, [x0] +; CHECK-NEXT: ret + store atomic float %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2 + ret void +} + +define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind { +; CHECK-LABEL: test_atomic_store_stream_seqcst_double: +; CHECK: // %bb.0: +; CHECK-NEXT: fmov x8, d0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: stlr x8, [x0] +; CHECK-NEXT: ret + store atomic double %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2 + ret void +} + +; +; Invalid Hint +; + +define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_invalid_hint: +; CHECK: // %bb.0: +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + ret void +} + +!0 = !{i32 1, !1} +!1 = !{!"aarch64.mem_hint", i32 0} +!2 = !{i32 1, !3} +!3 = !{!"aarch64.mem_hint", i32 1} +!4 = !{i32 1, !5} +!5 = !{!"aarch64.mem_hint", i32 2} diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll new file mode 100644 index 00000000000000..5f5bc430397782 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll @@ -0,0 +1,462 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s + +; +; W register offset +; + +; Int + +define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strb w2, [x0, w1, sxtw] +; CHECK-NEXT: ret + %addr = getelementptr i8, ptr %ptr, i32 %offset + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roW_i16(ptr %ptr, i32 %offset, i16 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strh w2, [x0, w1, sxtw #1] +; CHECK-NEXT: ret + %addr = getelementptr i16, ptr %ptr, i32 %offset + store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roW_i32(ptr %ptr, i32 %offset, i32 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str w2, [x0, w1, sxtw #2] +; CHECK-NEXT: ret + %addr = getelementptr i32, ptr %ptr, i32 %offset + store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roW_i64(ptr %ptr, i32 %offset, i64 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str x2, [x0, w1, sxtw #3] +; CHECK-NEXT: ret + %addr = getelementptr i64, ptr %ptr, i32 %offset + store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +; FP + +define void @relaxed_store_hint_roW_mfloat(ptr %ptr, i32 %offset, <1 x i8> %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strb w8, [x0, w1, sxtw] +; CHECK-NEXT: ret + %addr = getelementptr <1 x i8>, ptr %ptr, i32 %offset + store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strh w8, [x0, w1, sxtw #1] +; CHECK-NEXT: ret + %addr = getelementptr bfloat, ptr %ptr, i32 %offset + store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_half: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strh w8, [x0, w1, sxtw #1] +; CHECK-NEXT: ret + %addr = getelementptr half, ptr %ptr, i32 %offset + store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_float: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str s0, [x0, w1, sxtw #2] +; CHECK-NEXT: ret + %addr = getelementptr float, ptr %ptr, i32 %offset + store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roW_double: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str d0, [x0, w1, sxtw #3] +; CHECK-NEXT: ret + %addr = getelementptr double, ptr %ptr, i32 %offset + store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +; +; X register offset +; + +; Int + +define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strb w2, [x0, x1] +; CHECK-NEXT: ret + %addr = getelementptr i8, ptr %ptr, i64 %offset + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roX_i16(ptr %ptr, i64 %offset, i16 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strh w2, [x0, x1, lsl #1] +; CHECK-NEXT: ret + %addr = getelementptr i16, ptr %ptr, i64 %offset + store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roX_i32(ptr %ptr, i64 %offset, i32 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str w2, [x0, x1, lsl #2] +; CHECK-NEXT: ret + %addr = getelementptr i32, ptr %ptr, i64 %offset + store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_roX_i64(ptr %ptr, i64 %offset, i64 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str x2, [x0, x1, lsl #3] +; CHECK-NEXT: ret + %addr = getelementptr i64, ptr %ptr, i64 %offset + store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +; FP + +define void @relaxed_store_hint_roX_mfloat(ptr %ptr, i64 %offset, <1 x i8> %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strb w8, [x0, x1] +; CHECK-NEXT: ret + %addr = getelementptr <1 x i8>, ptr %ptr, i64 %offset + store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strh w8, [x0, x1, lsl #1] +; CHECK-NEXT: ret + %addr = getelementptr bfloat, ptr %ptr, i64 %offset + store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_half: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strh w8, [x0, x1, lsl #1] +; CHECK-NEXT: ret + %addr = getelementptr half, ptr %ptr, i64 %offset + store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_float: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str s0, [x0, x1, lsl #2] +; CHECK-NEXT: ret + %addr = getelementptr float, ptr %ptr, i64 %offset + store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_roX_double: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str d0, [x0, x1, lsl #3] +; CHECK-NEXT: ret + %addr = getelementptr double, ptr %ptr, i64 %offset + store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +; +; Unsigned immediate offset +; + +; Int + +define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strb w1, [x0, #4095] +; CHECK-NEXT: ret + %addr = getelementptr i8, ptr %ptr, i32 4095 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_uimm_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: strh w1, [x0, #4096] +; CHECK-NEXT: ret + %addr = getelementptr i16, ptr %ptr, i32 2048 + store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_uimm_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str w1, [x0, #4096] +; CHECK-NEXT: ret + %addr = getelementptr i32, ptr %ptr, i32 1024 + store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_uimm_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str x1, [x0, #4096] +; CHECK-NEXT: ret + %addr = getelementptr i64, ptr %ptr, i32 512 + store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +; FP + +define void @relaxed_store_hint_uimm_mfloat(ptr %ptr, <1 x i8> %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strb w8, [x0, #2048] +; CHECK-NEXT: ret + %addr = getelementptr <1 x i8>, ptr %ptr, i32 2048 + store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strh w8, [x0, #4096] +; CHECK-NEXT: ret + %addr = getelementptr bfloat, ptr %ptr, i32 2048 + store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_half: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh keep +; CHECK-NEXT: strh w8, [x0, #4096] +; CHECK-NEXT: ret + %addr = getelementptr half, ptr %ptr, i32 2048 + store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_float: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str s0, [x0, #8192] +; CHECK-NEXT: ret + %addr = getelementptr float, ptr %ptr, i32 2048 + store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_uimm_double: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: str d0, [x0, #16384] +; CHECK-NEXT: ret + %addr = getelementptr double, ptr %ptr, i32 2048 + store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +; +; Signed immediate offset +; + +; Int + +define void @relaxed_store_hint_imm_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: sturb w1, [x0, #-256] +; CHECK-NEXT: ret + %addr = getelementptr i8, ptr %ptr, i32 -256 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_imm_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: sturh w1, [x0, #-256] +; CHECK-NEXT: ret + %addr = getelementptr i16, ptr %ptr, i32 -128 + store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_imm_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stur w1, [x0, #-256] +; CHECK-NEXT: ret + %addr = getelementptr i32, ptr %ptr, i32 -64 + store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +define void @relaxed_store_hint_imm_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stshh keep +; CHECK-NEXT: stur x1, [x0, #-256] +; CHECK-NEXT: ret + %addr = getelementptr i64, ptr %ptr, i32 -32 + store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + ret void +} + +; FP + +define void @relaxed_store_hint_imm_mfloat(ptr %ptr, <1 x i8> %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_mfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w8, v0.b[0] +; CHECK-NEXT: stshh strm +; CHECK-NEXT: sturb w8, [x0, #-128] +; CHECK-NEXT: ret + %addr = getelementptr <1 x i8>, ptr %ptr, i32 -128 + store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: sturh w8, [x0, #-256] +; CHECK-NEXT: ret + %addr = getelementptr bfloat, ptr %ptr, i32 -128 + store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_half: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: sturh w8, [x0, #-256] +; CHECK-NEXT: ret + %addr = getelementptr half, ptr %ptr, i32 -128 + store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_float: +; CHECK: // %bb.0: +; CHECK-NEXT: sub x8, x0, #512 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str s0, [x8] +; CHECK-NEXT: ret + %addr = getelementptr float, ptr %ptr, i32 -128 + store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind { +; CHECK-LABEL: relaxed_store_hint_imm_double: +; CHECK: // %bb.0: +; CHECK-NEXT: sub x8, x0, #1024 +; CHECK-NEXT: stshh strm +; CHECK-NEXT: str d0, [x8] +; CHECK-NEXT: ret + %addr = getelementptr double, ptr %ptr, i32 -128 + store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + ret void +} + +!0 = !{i32 1, !1} +!1 = !{!"aarch64.mem_hint", i32 0} +!2 = !{i32 1, !3} +!3 = !{!"aarch64.mem_hint", i32 1} diff --git a/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir b/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir new file mode 100644 index 00000000000000..31f82a33de8eb3 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir @@ -0,0 +1,54 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=prolog-epilog -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s + +--- | + define void @test_atomic_store_pseudo() nounwind { entry: unreachable } +... +--- +name: test_atomic_store_pseudo +tracksRegLiveness: true +liveins: + - { reg: '$w0' } + - { reg: '$s0' } + - { reg: '$x0' } + - { reg: '$d0' } +stack: + - { id: 0, size: 8, alignment: 8 } +body: | + bb.0: + liveins: $w0, $s0, $x0, $d0 + + ; CHECK-LABEL: name: test_atomic_store_pseudo + ; CHECK: liveins: $w0, $s0, $x0, $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Bi $w0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Hi $w0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Wi $w0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Si $s0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Xi $x0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Di $d0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Bui $w0, $sp, 8, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Hui $w0, $sp, 4, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Wui $w0, $sp, 2, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Sui $s0, $sp, 2, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Xui $x0, $sp, 1, 1, 48 + ; CHECK-NEXT: ATOMIC_STORE_HINT_Dui $d0, $sp, 1, 1, 48 + ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0 + ; CHECK-NEXT: RET_ReallyLR + ATOMIC_STORE_HINT_Bi $w0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Hi $w0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Wi $w0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Si $s0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Xi $x0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Di $d0, %stack.0, 0, 1, 48 + + ATOMIC_STORE_HINT_Bui $w0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Hui $w0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Wui $w0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Sui $s0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Xui $x0, %stack.0, 0, 1, 48 + ATOMIC_STORE_HINT_Dui $d0, %stack.0, 0, 1, 48 + + RET_ReallyLR +... _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
