================ @@ -0,0 +1,635 @@ +//===- ConcurrencySanitizer.cpp - watchpoint race detector ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements instrumentation for ConcurrencySanitizer, a sampling +// data-race detector. Memory accesses are preceded by runtime probes while the +// original operations remain unchanged. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h" +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/bit.h" +#include "llvm/Analysis/CaptureTracking.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/AMDGPUAddrSpace.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/NVPTXAddrSpace.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/EscapeEnumerator.h" +#include "llvm/Transforms/Utils/Instrumentation.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "csan" + +static cl::opt<bool> ClInstrumentMemoryAccesses( + "csan-instrument-memory-accesses", cl::init(true), + cl::desc("Instrument memory accesses"), cl::Hidden); +static cl::opt<bool> + ClInstrumentFuncEntryExit("csan-instrument-func-entry-exit", cl::init(true), + cl::desc("Instrument function entry and exit"), + cl::Hidden); +static cl::opt<bool> ClHandleCxxExceptions( + "csan-handle-cxx-exceptions", cl::init(true), + cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), + cl::Hidden); +static cl::opt<bool> ClInstrumentAtomics("csan-instrument-atomics", + cl::init(true), + cl::desc("Instrument atomics"), + cl::Hidden); +static cl::opt<bool> ClInstrumentMemIntrinsics( + "csan-instrument-memintrinsics", cl::init(true), + cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); +static cl::opt<bool> ClDistinguishVolatile( + "csan-distinguish-volatile", cl::init(false), + cl::desc("Treat aligned volatile accesses up to 8 bytes as atomic"), + cl::Hidden); +STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); +STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); +STATISTIC(NumOmittedReadsBeforeWrite, + "Number of reads ignored due to following writes"); + +static constexpr char kCsanModuleCtorName[] = "csan.module_ctor"; +static constexpr char kCsanInitName[] = "__csan_init"; + +// Accesses sizes are powers of two: 1, 2, 4, 8, 16. +static constexpr size_t kNumAccessSizes = 5; +static constexpr StringLiteral kCsanReadNames[kNumAccessSizes] = { + "__csan_read1", "__csan_read2", "__csan_read4", "__csan_read8", + "__csan_read16"}; +static constexpr StringLiteral kCsanWriteNames[kNumAccessSizes] = { + "__csan_write1", "__csan_write2", "__csan_write4", "__csan_write8", + "__csan_write16"}; +static constexpr StringLiteral kCsanUnalignedReadNames[kNumAccessSizes] = { + "__csan_unaligned_read1", "__csan_unaligned_read2", + "__csan_unaligned_read4", "__csan_unaligned_read8", + "__csan_unaligned_read16"}; +static constexpr StringLiteral kCsanUnalignedWriteNames[kNumAccessSizes] = { + "__csan_unaligned_write1", "__csan_unaligned_write2", + "__csan_unaligned_write4", "__csan_unaligned_write8", + "__csan_unaligned_write16"}; +static constexpr StringLiteral kCsanCompoundRWNames[kNumAccessSizes] = { + "__csan_read_write1", "__csan_read_write2", "__csan_read_write4", + "__csan_read_write8", "__csan_read_write16"}; +static constexpr StringLiteral kCsanUnalignedCompoundRWNames[kNumAccessSizes] = + {"__csan_unaligned_read_write1", "__csan_unaligned_read_write2", + "__csan_unaligned_read_write4", "__csan_unaligned_read_write8", + "__csan_unaligned_read_write16"}; + +namespace { + +/// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h. +enum class AccessFlags : unsigned { + None = 0, + Atomic = 1u << 0, + Compound = 1u << 1, + LLVM_MARK_AS_BITMASK_ENUM(Compound), +}; + +static bool isAtomicMemoryAccess(const Instruction *I) { + auto SSID = getAtomicSyncScopeID(I); + if (!SSID) + return false; + if (isa<LoadInst>(I) || isa<StoreInst>(I)) + return *SSID != SyncScope::SingleThread; + return true; +} + +static Value *getCallbackAddress(IRBuilderBase &IRB, Value *Addr) { + return IRB.CreateAddrSpaceCast(Addr, IRB.getPtrTy()); +} + +static ConstantInt *createOrdering(IRBuilderBase &IRB, AtomicOrdering Ord) { + uint32_t Value = 0; + switch (Ord) { + case AtomicOrdering::NotAtomic: + llvm_unreachable("unexpected atomic ordering"); + case AtomicOrdering::Unordered: + case AtomicOrdering::Monotonic: + Value = 0; + break; + case AtomicOrdering::Acquire: + Value = 2; + break; + case AtomicOrdering::Release: + Value = 3; + break; + case AtomicOrdering::AcquireRelease: + Value = 4; + break; + case AtomicOrdering::SequentiallyConsistent: + Value = 5; + break; + } + return IRB.getInt32(Value); +} + +static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) { + assert(Ty->isSized()); + if (Ty->isScalableTy()) + return -1; + uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty); + if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 && + TypeSize != 128) + return -1; + unsigned Idx = llvm::countr_zero(TypeSize / 8); + return static_cast<int>(Idx); +} + +static bool addressSpaceMayRace(const Triple &T, unsigned AS) { + if (T.isAMDGPU()) { + // GDS and buffer fat pointers cannot form a generic watchpoint key. + return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS || + AS == AMDGPUAS::LOCAL_ADDRESS; + } + if (T.isNVPTX()) { + return AS == NVPTXAS::ADDRESS_SPACE_GENERIC || + AS == NVPTXAS::ADDRESS_SPACE_GLOBAL || + AS == NVPTXAS::ADDRESS_SPACE_SHARED || + AS == NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER; + } + if (T.isSPIRV()) { + // FIXME: No exposed address spaces for SPIR-V. + return false; + } + return AS == 0; +} + +struct ConcurrencySanitizer { + bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); + +private: + struct MemoryAccessLists { + SmallVector<Instruction *, 8> LoadsAndStores; + SmallVector<Instruction *, 8> AtomicAccesses; + SmallVector<IntrinsicInst *, 8> MemIntrinCalls; + bool HasCalls = false; + }; + + void initialize(Module &M, const TargetLibraryInfo &TLI); + void collectMemoryAccesses(Function &F, MemoryAccessLists &Out); + bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); + bool instrumentAtomic(Instruction *I, const DataLayout &DL); + bool instrumentMemIntrinsic(IntrinsicInst *II, const DataLayout &DL); + bool insertAccessProbe(Instruction *I, Value *Addr, Type *AccessTy, + const DataLayout &DL, bool IsWrite, bool IsCompound, + bool IsAtomic); + bool shouldInstrumentAddress(Value *Addr) const; + bool shouldInstrumentAccess(Instruction *I) const; + void insertFuncEntryExit(Function &F); + void insertRuntimeIgnores(Function &F); + Constant *getFlags(AccessFlags Flags) const { + return ConstantInt::get(FlagsTy, to_underlying(Flags)); + } + + Module *Mod = nullptr; + Type *IntptrTy = nullptr; + IntegerType *FlagsTy = nullptr; + + // void __csan_readN(ptr, i32); + FunctionCallee CsanRead[kNumAccessSizes]; + // void __csan_writeN(ptr, i32); + FunctionCallee CsanWrite[kNumAccessSizes]; + // void __csan_unaligned_readN(ptr, i32); + FunctionCallee CsanUnalignedRead[kNumAccessSizes]; + // void __csan_unaligned_writeN(ptr, i32); + FunctionCallee CsanUnalignedWrite[kNumAccessSizes]; + // void __csan_read_writeN(ptr, i32); + FunctionCallee CsanCompoundRW[kNumAccessSizes]; + // void __csan_unaligned_read_writeN(ptr, i32); + FunctionCallee CsanUnalignedCompoundRW[kNumAccessSizes]; + // void __csan_func_entry(ptr); + FunctionCallee CsanFuncEntry; + // void __csan_func_exit(); + FunctionCallee CsanFuncExit; + // void __csan_ignore_thread_begin(); + FunctionCallee CsanIgnoreBegin; + // void __csan_ignore_thread_end(); + FunctionCallee CsanIgnoreEnd; + // void __csan_read_range(ptr, intptr_t, i32); + FunctionCallee CsanReadRange; + // void __csan_write_range(ptr, intptr_t, i32); + FunctionCallee CsanWriteRange; + // void __csan_atomic_thread_fence(i32); + FunctionCallee CsanAtomicThreadFence; + // void __csan_atomic_signal_fence(i32); + FunctionCallee CsanAtomicSignalFence; +}; + +void insertModuleCtor(Module &M) { + getOrCreateSanitizerCtorAndInitFunctions( + M, kCsanModuleCtorName, kCsanInitName, /*InitArgTypes=*/{}, + /*InitArgs=*/{}, + [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); +} + +} // namespace + +PreservedAnalyses ConcurrencySanitizerPass::run(Function &F, + FunctionAnalysisManager &FAM) { + ConcurrencySanitizer CSan; + if (CSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +PreservedAnalyses ModuleConcurrencySanitizerPass::run(Module &M, + ModuleAnalysisManager &) { + if (checkIfAlreadyInstrumented(M, "nosanitize_concurrency")) + return PreservedAnalyses::all(); + insertModuleCtor(M); + return PreservedAnalyses::none(); +} + +void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) { + LLVMContext &Ctx = M.getContext(); + Mod = &M; + IntptrTy = M.getDataLayout().getIntPtrType(Ctx); + FlagsTy = Type::getInt32Ty(Ctx); + + AttributeList Attr = AttributeList().addFnAttribute(Ctx, Attribute::NoUnwind); + IRBuilder<> IRB(Ctx); + Type *VoidTy = IRB.getVoidTy(); + Type *PtrTy = IRB.getPtrTy(); + CsanFuncEntry = + M.getOrInsertFunction("__csan_func_entry", Attr, VoidTy, PtrTy); + CsanFuncExit = M.getOrInsertFunction("__csan_func_exit", Attr, VoidTy); + CsanIgnoreBegin = + M.getOrInsertFunction("__csan_ignore_thread_begin", Attr, VoidTy); + CsanIgnoreEnd = + M.getOrInsertFunction("__csan_ignore_thread_end", Attr, VoidTy); + auto AccessFn = [&](StringRef Name) { + return M.getOrInsertFunction(Name, Attr, VoidTy, PtrTy, FlagsTy); + }; + for (unsigned I = 0; I < kNumAccessSizes; ++I) { + CsanRead[I] = AccessFn(kCsanReadNames[I]); + CsanWrite[I] = AccessFn(kCsanWriteNames[I]); + CsanUnalignedRead[I] = AccessFn(kCsanUnalignedReadNames[I]); + CsanUnalignedWrite[I] = AccessFn(kCsanUnalignedWriteNames[I]); + CsanCompoundRW[I] = AccessFn(kCsanCompoundRWNames[I]); + CsanUnalignedCompoundRW[I] = AccessFn(kCsanUnalignedCompoundRWNames[I]); + } + IntegerType *OrdTy = IRB.getInt32Ty(); + CsanReadRange = M.getOrInsertFunction("__csan_read_range", Attr, VoidTy, + PtrTy, IntptrTy, FlagsTy); + CsanWriteRange = M.getOrInsertFunction("__csan_write_range", Attr, VoidTy, + PtrTy, IntptrTy, FlagsTy); + CsanAtomicThreadFence = M.getOrInsertFunction( + "__csan_atomic_thread_fence", + TLI.getAttrList(&Ctx, {0}, /*Signed=*/true, /*Ret=*/false, Attr), VoidTy, + OrdTy); + CsanAtomicSignalFence = M.getOrInsertFunction( + "__csan_atomic_signal_fence", + TLI.getAttrList(&Ctx, {0}, /*Signed=*/true, /*Ret=*/false, Attr), VoidTy, + OrdTy); +} + +bool ConcurrencySanitizer::sanitizeFunction(Function &F, + const TargetLibraryInfo &TLI) { + // This is required to prevent instrumenting call to __csan_init from within + // the module constructor. + if (F.getName() == kCsanModuleCtorName) + return false; + // Naked functions can not have prologue/epilogue + // (__csan_func_entry/__csan_func_exit) generated, so don't instrument them at + // all. + if (F.hasFnAttribute(Attribute::Naked)) + return false; + + // __attribute__(disable_sanitizer_instrumentation) prevents all kinds of + // instrumentation. + if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation)) + return false; + + Mod = F.getParent(); + MemoryAccessLists Acc; + collectMemoryAccesses(F, Acc); + + const bool SuppressChecking = + F.hasFnAttribute("sanitize_concurrency_no_checking_at_run_time"); + const bool SanitizeFunction = + F.hasFnAttribute(Attribute::SanitizeConcurrency) && !SuppressChecking; + const bool MayInstrument = + SanitizeFunction && + ((ClInstrumentMemoryAccesses && !Acc.LoadsAndStores.empty()) || + (ClInstrumentAtomics && !Acc.AtomicAccesses.empty()) || + (ClInstrumentMemIntrinsics && !Acc.MemIntrinCalls.empty())); + const bool NeedsRuntimeIgnores = SuppressChecking && Acc.HasCalls; + const bool NeedsFuncEntryExit = ClInstrumentFuncEntryExit && Acc.HasCalls; + if (!MayInstrument && !NeedsRuntimeIgnores && !NeedsFuncEntryExit) + return false; + + initialize(*Mod, TLI); + bool Res = false; + const DataLayout &DL = F.getDataLayout(); + + if (ClInstrumentMemoryAccesses && SanitizeFunction) + for (Instruction *I : Acc.LoadsAndStores) + Res |= instrumentLoadOrStore(I, DL); + + if (ClInstrumentAtomics && SanitizeFunction) + for (Instruction *I : Acc.AtomicAccesses) + Res |= instrumentAtomic(I, DL); + + if (ClInstrumentMemIntrinsics && SanitizeFunction) + for (IntrinsicInst *II : Acc.MemIntrinCalls) + Res |= instrumentMemIntrinsic(II, DL); + + if (NeedsRuntimeIgnores) { + insertRuntimeIgnores(F); + Res = true; + } + + if ((Res || Acc.HasCalls) && ClInstrumentFuncEntryExit) { + insertFuncEntryExit(F); + Res = true; + } + return Res; +} + +bool ConcurrencySanitizer::shouldInstrumentAddress(Value *Addr) const { + Value *BaseAddr = Addr->stripInBoundsOffsets(); + if (auto *GV = dyn_cast<GlobalVariable>(BaseAddr)) { + if (GV->hasSection()) { + StringRef SectionName = GV->getSection(); + auto OF = Mod->getTargetTriple().getObjectFormat(); + if (SectionName.ends_with( + getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) + return false; + } + } + + Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); + unsigned AS = PtrTy->getPointerAddressSpace(); + if (Mod->getDataLayout().getPointerSizeInBits(AS) > 64) + return false; + return addressSpaceMayRace(Mod->getTargetTriple(), AS); +} + +bool ConcurrencySanitizer::shouldInstrumentAccess(Instruction *I) const { + const bool IsWrite = isa<StoreInst>(I); + Value *Addr = getLoadStorePointerOperand(I); + if (!shouldInstrumentAddress(Addr)) + return false; + + if (!IsWrite) + if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr))) + if (GV->isConstant()) + return false; + + const AllocaInst *AI = findAllocaForValue(Addr); + if (!AI || PointerMayBeCaptured(AI, /*ReturnCaptures=*/true)) + return true; + + const Triple &T = AI->getModule()->getTargetTriple(); + return T.isGPU() && addressSpaceMayRace(T, AI->getAddressSpace()); +} + +void ConcurrencySanitizer::collectMemoryAccesses(Function &F, + MemoryAccessLists &Out) { + SmallVector<Instruction *, 8> LocalLoadsAndStores; + auto FlushLocalAccesses = [&] { + DenseSet<Value *> WriteTargets; + for (Instruction *Inst : reverse(LocalLoadsAndStores)) { + const bool IsWrite = isa<StoreInst>(Inst); + Value *Addr = getLoadStorePointerOperand(Inst); + if (!IsWrite && WriteTargets.contains(Addr)) { + ++NumOmittedReadsBeforeWrite; + continue; + } + + Out.LoadsAndStores.push_back(Inst); + if (IsWrite) + WriteTargets.insert(Addr); + } + LocalLoadsAndStores.clear(); + }; + + for (BasicBlock &BB : F) { + for (Instruction &Inst : BB) { + // Skip instructions inserted by another instrumentation. + if (Inst.hasMetadata(LLVMContext::MD_nosanitize)) + continue; + if (isAtomicMemoryAccess(&Inst)) ---------------- yxsamliu wrote:
Nit: makes sense to match TSan. Could you add a short comment that we may drop the read even when the write is smaller or an acquire is in between? https://github.com/llvm/llvm-project/pull/225780 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
