llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang Author: Hendrik Hübner (HendrikHuebner) <details> <summary>Changes</summary> This PR upstreams support for several `__sync_<OP>_and_fetch` builtins. Additionally, some needed helper methods are added. --- Patch is 56.18 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/168347.diff 6 Files Affected: - (modified) clang/lib/CIR/CodeGen/Address.h (+26-2) - (added) clang/lib/CIR/CodeGen/CIREHScopeStack.h (+279) - (modified) clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp (+268) - (modified) clang/lib/CIR/CodeGen/CIRGenExpr.cpp (+23) - (modified) clang/lib/CIR/CodeGen/CIRGenFunction.h (+5) - (modified) clang/test/CIR/CodeGen/atomic.c (+491) ``````````diff diff --git a/clang/lib/CIR/CodeGen/Address.h b/clang/lib/CIR/CodeGen/Address.h index c8ce530a7b0d3..02a24a86b3c84 100644 --- a/clang/lib/CIR/CodeGen/Address.h +++ b/clang/lib/CIR/CodeGen/Address.h @@ -45,8 +45,12 @@ class Address { public: Address(mlir::Value pointer, mlir::Type elementType, clang::CharUnits alignment) - : pointerAndKnownNonNull(pointer, false), elementType(elementType), - alignment(alignment) { + : Address(pointer, elementType, alignment, false) {} + + Address(mlir::Value pointer, mlir::Type elementType, + clang::CharUnits alignment, bool pointerAndKnownNonNull) + : pointerAndKnownNonNull(pointer, pointerAndKnownNonNull), + elementType(elementType), alignment(alignment) { assert(pointer && "Pointer cannot be null"); assert(elementType && "Element type cannot be null"); assert(!alignment.isZero() && "Alignment cannot be zero"); @@ -77,6 +81,13 @@ class Address { return Address(newPtr, getElementType(), getAlignment()); } + /// Return address with different alignment, but same pointer and element + /// type. + Address withAlignment(clang::CharUnits newAlignment) const { + return Address(getPointer(), getElementType(), newAlignment, + isKnownNonNull()); + } + /// Return address with different element type, a bitcast pointer, and /// the same alignment. Address withElementType(CIRGenBuilderTy &builder, mlir::Type ElemTy) const; @@ -133,6 +144,19 @@ class Address { template <typename OpTy> OpTy getDefiningOp() const { return mlir::dyn_cast_or_null<OpTy>(getDefiningOp()); } + + /// Whether the pointer is known not to be null. + bool isKnownNonNull() const { + assert(isValid() && "Invalid address"); + return static_cast<bool>(pointerAndKnownNonNull.getInt()); + } + + /// Set the non-null bit. + Address setKnownNonNull() { + assert(isValid() && "Invalid address"); + pointerAndKnownNonNull.setInt(true); + return *this; + } }; } // namespace clang::CIRGen diff --git a/clang/lib/CIR/CodeGen/CIREHScopeStack.h b/clang/lib/CIR/CodeGen/CIREHScopeStack.h new file mode 100644 index 0000000000000..c7b86a06339a8 --- /dev/null +++ b/clang/lib/CIR/CodeGen/CIREHScopeStack.h @@ -0,0 +1,279 @@ +//===-- EHScopeStack.h - Stack for cleanup CIR generation -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// These classes should be the minimum interface required for other parts of +// CIR CodeGen to emit cleanups. The implementation is in CIRGenCleanup.cpp and +// other implemenentation details that are not widely needed are in +// CIRGenCleanup.h. +// +// TODO(cir): this header should be shared between LLVM and CIR codegen. +// +//===----------------------------------------------------------------------===// + +#ifndef CLANG_LIB_CIR_CODEGEN_EHSCOPESTACK_H +#define CLANG_LIB_CIR_CODEGEN_EHSCOPESTACK_H + +#include "clang/CIR/Dialect/IR/CIRDialect.h" +#include "clang/lib/CodeGen/EHScopeStack.h" + +namespace clang::CIRGen { + +class CIRGenFunction; + +/// A branch fixup. These are required when emitting a goto to a +/// label which hasn't been emitted yet. The goto is optimistically +/// emitted as a branch to the basic block for the label, and (if it +/// occurs in a scope with non-trivial cleanups) a fixup is added to +/// the innermost cleanup. When a (normal) cleanup is popped, any +/// unresolved fixups in that scope are threaded through the cleanup. +struct BranchFixup { + /// The block containing the terminator which needs to be modified + /// into a switch if this fixup is resolved into the current scope. + /// If null, LatestBranch points directly to the destination. + mlir::Block *optimisticBranchBlock = nullptr; + + /// The ultimate destination of the branch. + /// + /// This can be set to null to indicate that this fixup was + /// successfully resolved. + mlir::Block *destination = nullptr; + + /// The destination index value. + unsigned destinationIndex = 0; + + /// The initial branch of the fixup. + cir::BrOp initialBranch = {}; +}; + +enum CleanupKind : unsigned { + /// Denotes a cleanup that should run when a scope is exited using exceptional + /// control flow (a throw statement leading to stack unwinding, ). + EHCleanup = 0x1, + + /// Denotes a cleanup that should run when a scope is exited using normal + /// control flow (falling off the end of the scope, return, goto, ...). + NormalCleanup = 0x2, + + NormalAndEHCleanup = EHCleanup | NormalCleanup, + + LifetimeMarker = 0x8, + NormalEHLifetimeMarker = LifetimeMarker | NormalAndEHCleanup, +}; + +/// A stack of scopes which respond to exceptions, including cleanups +/// and catch blocks. +class EHScopeStack { + friend class CIRGenFunction; + +public: + // TODO(ogcg): Switch to alignof(uint64_t) instead of 8 + enum { ScopeStackAlignment = 8 }; + + /// A saved depth on the scope stack. This is necessary because + /// pushing scopes onto the stack invalidates iterators. + class stable_iterator { + friend class EHScopeStack; + + /// Offset from startOfData to endOfBuffer. + ptrdiff_t size = -1; + + explicit stable_iterator(ptrdiff_t size) : size(size) {} + + public: + static stable_iterator invalid() { return stable_iterator(-1); } + stable_iterator() = default; + + bool isValid() const { return size >= 0; } + + /// Returns true if this scope encloses I. + /// Returns false if I is invalid. + /// This scope must be valid. + bool encloses(stable_iterator other) const { return size <= other.size; } + + /// Returns true if this scope strictly encloses I: that is, + /// if it encloses I and is not I. + /// Returns false is I is invalid. + /// This scope must be valid. + bool strictlyEncloses(stable_iterator I) const { return size < I.size; } + + friend bool operator==(stable_iterator A, stable_iterator B) { + return A.size == B.size; + } + friend bool operator!=(stable_iterator A, stable_iterator B) { + return A.size != B.size; + } + }; + + /// Information for lazily generating a cleanup. Subclasses must be + /// POD-like: cleanups will not be destructed, and they will be + /// allocated on the cleanup stack and freely copied and moved + /// around. + /// + /// Cleanup implementations should generally be declared in an + /// anonymous namespace. + class LLVM_MOVABLE_POLYMORPHIC_TYPE Cleanup { + // Anchor the construction vtable. + virtual void anchor(); + + public: + Cleanup(const Cleanup &) = default; + Cleanup(Cleanup &&) {} + Cleanup() = default; + + virtual ~Cleanup() = default; + + /// Emit the cleanup. For normal cleanups, this is run in the + /// same EH context as when the cleanup was pushed, i.e. the + /// immediately-enclosing context of the cleanup scope. For + /// EH cleanups, this is run in a terminate context. + /// + // \param flags cleanup kind. + virtual void emit(CIRGenFunction &cgf) = 0; + }; + +private: + // The implementation for this class is in CIRGenCleanup.h and + // CIRGenCleanup.cpp; the definition is here because it's used as a + // member of CIRGenFunction. + + /// The start of the scope-stack buffer, i.e. the allocated pointer + /// for the buffer. All of these pointers are either simultaneously + /// null or simultaneously valid. + std::unique_ptr<char[]> startOfBuffer; + + /// The end of the buffer. + char *endOfBuffer = nullptr; + + /// The first valid entry in the buffer. + char *startOfData = nullptr; + + /// The innermost normal cleanup on the stack. + stable_iterator innermostNormalCleanup = stable_end(); + + /// The innermost EH scope on the stack. + stable_iterator innermostEHScope = stable_end(); + + /// The CGF this Stack belong to + CIRGenFunction *cgf = nullptr; + + /// The current set of branch fixups. A branch fixup is a jump to + /// an as-yet unemitted label, i.e. a label for which we don't yet + /// know the EH stack depth. Whenever we pop a cleanup, we have + /// to thread all the current branch fixups through it. + /// + /// Fixups are recorded as the Use of the respective branch or + /// switch statement. The use points to the final destination. + /// When popping out of a cleanup, these uses are threaded through + /// the cleanup and adjusted to point to the new cleanup. + /// + /// Note that branches are allowed to jump into protected scopes + /// in certain situations; e.g. the following code is legal: + /// struct A { ~A(); }; // trivial ctor, non-trivial dtor + /// goto foo; + /// A a; + /// foo: + /// bar(); + llvm::SmallVector<BranchFixup> branchFixups; + + // This class uses a custom allocator for maximum efficiency because cleanups + // are allocated and freed very frequently. It's basically a bump pointer + // allocator, but we can't use LLVM's BumpPtrAllocator because we use offsets + // into the buffer as stable iterators. + char *allocate(size_t size); + void deallocate(size_t size); + + void *pushCleanup(CleanupKind kind, size_t dataSize); + +public: + EHScopeStack() = default; + ~EHScopeStack() = default; + + /// Push a lazily-created cleanup on the stack. + template <class T, class... As> void pushCleanup(CleanupKind kind, As... a) { + static_assert(alignof(T) <= ScopeStackAlignment, + "Cleanup's alignment is too large."); + void *buffer = pushCleanup(kind, sizeof(T)); + [[maybe_unused]] Cleanup *obj = new (buffer) T(a...); + } + + void setCGF(CIRGenFunction *inCGF) { cgf = inCGF; } + + /// Pops a cleanup scope off the stack. This is private to CIRGenCleanup.cpp. + void popCleanup(); + + /// Push a set of catch handlers on the stack. The catch is + /// uninitialized and will need to have the given number of handlers + /// set on it. + class EHCatchScope *pushCatch(unsigned numHandlers); + + /// Pops a catch scope off the stack. This is private to CIRGenException.cpp. + void popCatch(); + + /// Determines whether the exception-scopes stack is empty. + bool empty() const { return startOfData == endOfBuffer; } + + /// Determines whether there are any normal cleanups on the stack. + bool hasNormalCleanups() const { + return innermostNormalCleanup != stable_end(); + } + + /// Returns the innermost normal cleanup on the stack, or + /// stable_end() if there are no normal cleanups. + stable_iterator getInnermostNormalCleanup() const { + return innermostNormalCleanup; + } + stable_iterator getInnermostActiveNormalCleanup() const; + + stable_iterator getInnermostEHScope() const { return innermostEHScope; } + + /// An unstable reference to a scope-stack depth. Invalidated by + /// pushes but not pops. + class iterator; + + /// Returns an iterator pointing to the innermost EH scope. + iterator begin() const; + + /// Returns an iterator pointing to the outermost EH scope. + iterator end() const; + + /// Create a stable reference to the top of the EH stack. The + /// returned reference is valid until that scope is popped off the + /// stack. + stable_iterator stable_begin() const { + return stable_iterator(endOfBuffer - startOfData); + } + + /// Create a stable reference to the bottom of the EH stack. + static stable_iterator stable_end() { return stable_iterator(0); } + + /// Turn a stable reference to a scope depth into a unstable pointer + /// to the EH stack. + iterator find(stable_iterator savePoint) const; + + /// Add a branch fixup to the current cleanup scope. + BranchFixup &addBranchFixup() { + assert(hasNormalCleanups() && "adding fixup in scope without cleanups"); + branchFixups.push_back(BranchFixup()); + return branchFixups.back(); + } + + unsigned getNumBranchFixups() const { return branchFixups.size(); } + BranchFixup &getBranchFixup(unsigned i) { + assert(i < getNumBranchFixups()); + return branchFixups[i]; + } + + /// Pops lazily-removed fixups from the end of the list. This + /// should only be called by procedures which have just popped a + /// cleanup or resolved one or more fixups. + void popNullFixups(); +}; + +} // namespace clang::CIRGen + +#endif // CLANG_LIB_CIR_CODEGEN_EHSCOPESTACK_H diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index 77f19343653db..a0a350ebe031c 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -21,6 +21,7 @@ #include "clang/AST/Expr.h" #include "clang/AST/GlobalDecl.h" #include "clang/Basic/Builtins.h" +#include "clang/Basic/DiagnosticFrontend.h" #include "clang/CIR/Dialect/IR/CIRTypes.h" #include "clang/CIR/MissingFeatures.h" #include "llvm/Support/ErrorHandling.h" @@ -58,6 +59,107 @@ static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e, return RValue::get(result); } +/// Emit the conversions required to turn the given value into an +/// integer of the given size. +static mlir::Value emitToInt(CIRGenFunction &cgf, mlir::Value v, QualType t, + cir::IntType intType) { + v = cgf.emitToMemory(v, t); + + if (isa<cir::PointerType>(v.getType())) + return cgf.getBuilder().createPtrToInt(v, intType); + + assert(v.getType() == intType); + return v; +} + +static mlir::Value emitFromInt(CIRGenFunction &cgf, mlir::Value v, QualType t, + mlir::Type resultType) { + v = cgf.emitFromMemory(v, t); + + if (isa<cir::PointerType>(resultType)) + return cgf.getBuilder().createIntToPtr(v, resultType); + + assert(v.getType() == resultType); + return v; +} + +static Address checkAtomicAlignment(CIRGenFunction &cgf, const CallExpr *e) { + ASTContext &astContext = cgf.getContext(); + Address ptr = cgf.emitPointerWithAlignment(e->getArg(0)); + unsigned bytes = + isa<cir::PointerType>(ptr.getElementType()) + ? astContext.getTypeSizeInChars(astContext.VoidPtrTy).getQuantity() + : cgf.cgm.getDataLayout().getTypeSizeInBits(ptr.getElementType()) / 8; + unsigned align = ptr.getAlignment().getQuantity(); + if (align % bytes != 0) { + DiagnosticsEngine &diags = cgf.cgm.getDiags(); + diags.Report(e->getBeginLoc(), diag::warn_sync_op_misaligned); + // Force address to be at least naturally-aligned. + return ptr.withAlignment(CharUnits::fromQuantity(bytes)); + } + return ptr; +} + +/// Utility to insert an atomic instruction based on Intrinsic::ID +/// and the expression node. +static mlir::Value makeBinaryAtomicValue( + CIRGenFunction &cgf, cir::AtomicFetchKind kind, const CallExpr *expr, + mlir::Value *neededValP = nullptr, + cir::MemOrder ordering = cir::MemOrder::SequentiallyConsistent) { + + QualType type = expr->getType(); + QualType ptrType = expr->getArg(0)->getType(); + + assert(ptrType->isPointerType()); + assert( + cgf.getContext().hasSameUnqualifiedType(type, ptrType->getPointeeType())); + assert(cgf.getContext().hasSameUnqualifiedType(type, + expr->getArg(1)->getType())); + + Address destAddr = checkAtomicAlignment(cgf, expr); + CIRGenBuilderTy &builder = cgf.getBuilder(); + cir::IntType intType = + ptrType->getPointeeType()->isUnsignedIntegerType() + ? builder.getUIntNTy(cgf.getContext().getTypeSize(type)) + : builder.getSIntNTy(cgf.getContext().getTypeSize(type)); + mlir::Value val = cgf.emitScalarExpr(expr->getArg(1)); + mlir::Type valueType = val.getType(); + val = emitToInt(cgf, val, type, intType); + + // This output argument is needed for post atomic fetch operations + // that calculate the result of the operation as return value of + // <binop>_and_fetch builtins. The `AtomicFetch` operation only updates the + // memory location and returns the old value. + if (neededValP) { + *neededValP = val; + } + + auto rmwi = cir::AtomicFetchOp::create( + builder, cgf.getLoc(expr->getSourceRange()), destAddr.emitRawPointer(), + val, kind, ordering, false, /* is volatile */ + true); /* fetch first */ + return emitFromInt(cgf, rmwi->getResult(0), type, valueType); +} + +static RValue emitBinaryAtomicPost(CIRGenFunction &cgf, + cir::AtomicFetchKind atomicOpkind, + const CallExpr *e, cir::BinOpKind binopKind, + bool invert = false) { + mlir::Value val; + clang::QualType typ = e->getType(); + mlir::Value result = makeBinaryAtomicValue(cgf, atomicOpkind, e, &val); + clang::CIRGen::CIRGenBuilderTy &builder = cgf.getBuilder(); + result = cir::BinOp::create(builder, result.getLoc(), binopKind, result, val); + + if (invert) { + result = cir::UnaryOp::create(builder, result.getLoc(), + cir::UnaryOpKind::Not, result); + } + + result = emitFromInt(cgf, result, typ, val.getType()); + return RValue::get(result); +} + RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) { mlir::Value input = emitScalarExpr(e->getArg(0)); mlir::Value amount = emitScalarExpr(e->getArg(1)); @@ -520,6 +622,172 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID, cir::PrefetchOp::create(builder, loc, address, locality, isWrite); return RValue::get(nullptr); } + case Builtin::BI__sync_fetch_and_add: + case Builtin::BI__sync_fetch_and_sub: + case Builtin::BI__sync_fetch_and_or: + case Builtin::BI__sync_fetch_and_and: + case Builtin::BI__sync_fetch_and_xor: + case Builtin::BI__sync_fetch_and_nand: + case Builtin::BI__sync_add_and_fetch: + case Builtin::BI__sync_sub_and_fetch: + case Builtin::BI__sync_and_and_fetch: + case Builtin::BI__sync_or_and_fetch: + case Builtin::BI__sync_xor_and_fetch: + case Builtin::BI__sync_nand_and_fetch: + case Builtin::BI__sync_val_compare_and_swap: + case Builtin::BI__sync_bool_compare_and_swap: + case Builtin::BI__sync_lock_test_and_set: + case Builtin::BI__sync_lock_release: + case Builtin::BI__sync_swap: + llvm_unreachable("Shouldn't make it through sema"); + + case Builtin::BI__sync_fetch_and_add_1: + case Builtin::BI__sync_fetch_and_add_2: + case Builtin::BI__sync_fetch_and_add_4: + case Builtin::BI__sync_fetch_and_add_8: + case Builtin::BI__sync_fetch_and_add_16: + llvm_unreachable("BI__sync_fetch_and_add NYI"); + case Builtin::BI__sync_fetch_and_sub_1: + case Builtin::BI__sync_fetch_and_sub_2: + case Builtin::BI__sync_fetch_and_sub_4: + case Builtin::BI__sync_fetch_and_sub_8: + case Builtin::BI__sync_fetch_and_sub_16: + llvm_unreachable("BI__sync_fetch_and_sub NYI"); + + case Builtin::BI__sync_fetch_and_or_1: + case Builtin::BI__sync_fetch_and_or_2: + case Builtin::BI__sync_fetch_and_or_4: + case Builtin::BI__sync_fetch_and_or_8: + case Builtin::BI__sync_fetch_and_or_16: + llvm_unreachable("BI__sync_fetch_and_or NYI"); + case Builtin::BI__sync_fetch_and_and_1: + case Builtin::BI__sync_fetch_and_and_2: + case Builtin::BI__sync_fetch_and_and_4: + case Builtin::BI__sync_fetch_and_and_8: + case Builtin::BI__sync_fetch_and_and_16: + llvm_unreachable("BI__sync_fetch_and_and NYI"); + case Builtin::BI__sync_fetch_and_xor_1: + case Builtin::BI__sync_fetch_and_xor_2: + case Builtin::BI__sync_fetch_and_xor_4: + case Builtin::BI__sync_fetch_and_xor_8: + case Builtin::BI__sync_fetch_and_xor_16: + llvm_unreachable("BI__sync_fetch_and_xor NYI"); + case Builtin::BI__sync_fetch_and_nand_1: + case Builtin::BI__sync_fetch_and_nand_2: + case Builtin::BI__sync_fetch_and_nand_4: + case Builtin::BI__sync_fetch_and_nand_8: + case Builtin::BI__sync_fetch_and_nand_16: + llvm_unreachable("BI__sync_fetch_and_nand NYI"); + + // Clang extensions: not overloaded yet. + case Builtin::BI__sync_fetch_and_min: + llvm_unreachable("BI__sync_fetch_and_min NYI"); + case Builtin::BI__sync_fetch_and_max: + llvm_unreachable("BI__sync_fetch_and_max NYI"); + case Builtin::BI__sync_fetch_and_umin: + llvm_unreachable("BI__sync_fetch_and_umin NYI"); + case Builtin::BI__sync_fetch_and_umax: + llvm_unreachable("BI__sync_fetch_and_umax NYI"); + + case Builtin::BI__sync_add_and_fetch_1: + case Builtin::BI__sy... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/168347 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
