================ @@ -0,0 +1,617 @@ +//===- ConcurrencySanitizer.cpp - watchpoint race detector ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements instrumentation for ConcurrencySanitizer, a sampling +// data-race detector. Memory accesses are preceded by runtime probes while the +// original operations remain unchanged. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h" +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/bit.h" +#include "llvm/Analysis/CaptureTracking.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/AMDGPUAddrSpace.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/NVPTXAddrSpace.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/EscapeEnumerator.h" +#include "llvm/Transforms/Utils/Instrumentation.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include <optional> + +using namespace llvm; + +#define DEBUG_TYPE "csan" + +static cl::opt<bool> ClInstrumentMemoryAccesses( + "csan-instrument-memory-accesses", cl::init(true), + cl::desc("Instrument memory accesses"), cl::Hidden); +static cl::opt<bool> + ClInstrumentFuncEntryExit("csan-instrument-func-entry-exit", cl::init(true), + cl::desc("Instrument function entry and exit"), + cl::Hidden); +static cl::opt<bool> ClHandleCxxExceptions( + "csan-handle-cxx-exceptions", cl::init(true), + cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), + cl::Hidden); +static cl::opt<bool> ClInstrumentAtomics("csan-instrument-atomics", + cl::init(true), + cl::desc("Instrument atomics"), + cl::Hidden); +static cl::opt<bool> ClInstrumentMemIntrinsics( + "csan-instrument-memintrinsics", cl::init(true), + cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); +static cl::opt<bool> ClDistinguishVolatile( + "csan-distinguish-volatile", cl::init(false), + cl::desc("Treat aligned volatile accesses up to 8 bytes as atomic"), + cl::Hidden); +STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); +STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); +STATISTIC(NumOmittedReadsBeforeWrite, + "Number of reads ignored due to following writes"); + +static constexpr char kCsanModuleCtorName[] = "csan.module_ctor"; +static constexpr char kCsanInitName[] = "__csan_init"; + +// Accesses sizes are powers of two: 1, 2, 4, 8, 16. +static constexpr size_t kNumAccessSizes = 5; +static constexpr StringLiteral kCsanReadNames[kNumAccessSizes] = { + "__csan_read1", "__csan_read2", "__csan_read4", "__csan_read8", + "__csan_read16"}; +static constexpr StringLiteral kCsanWriteNames[kNumAccessSizes] = { + "__csan_write1", "__csan_write2", "__csan_write4", "__csan_write8", + "__csan_write16"}; +static constexpr StringLiteral kCsanUnalignedReadNames[kNumAccessSizes] = { + "__csan_unaligned_read1", "__csan_unaligned_read2", + "__csan_unaligned_read4", "__csan_unaligned_read8", + "__csan_unaligned_read16"}; +static constexpr StringLiteral kCsanUnalignedWriteNames[kNumAccessSizes] = { + "__csan_unaligned_write1", "__csan_unaligned_write2", + "__csan_unaligned_write4", "__csan_unaligned_write8", + "__csan_unaligned_write16"}; +static constexpr StringLiteral kCsanCompoundRWNames[kNumAccessSizes] = { + "__csan_read_write1", "__csan_read_write2", "__csan_read_write4", + "__csan_read_write8", "__csan_read_write16"}; +static constexpr StringLiteral kCsanUnalignedCompoundRWNames[kNumAccessSizes] = + {"__csan_unaligned_read_write1", "__csan_unaligned_read_write2", + "__csan_unaligned_read_write4", "__csan_unaligned_read_write8", + "__csan_unaligned_read_write16"}; + +namespace { + +/// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h. +enum class AccessFlags : unsigned { + None = 0, + Atomic = 1u << 0, + Compound = 1u << 1, + LLVM_MARK_AS_BITMASK_ENUM(Compound), +}; + +static bool isAtomicMemoryAccess(const Instruction *I) { + std::optional<SyncScope::ID> SSID = getAtomicSyncScopeID(I); + if (!SSID) + return false; + if (isa<LoadInst>(I) || isa<StoreInst>(I)) + return *SSID != SyncScope::SingleThread; + return true; +} + +static Value *getCallbackAddress(IRBuilderBase &IRB, Value *Addr) { + return IRB.CreateAddrSpaceCast(Addr, IRB.getPtrTy()); +} + +static ConstantInt *createOrdering(IRBuilderBase &IRB, AtomicOrdering Ord) { + return IRB.getInt32(static_cast<uint32_t>(toCABI(Ord))); +} + +static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) { + assert(Ty->isSized()); + if (Ty->isScalableTy()) + return -1; + uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty); + if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 && + TypeSize != 128) + return -1; + unsigned Idx = llvm::countr_zero(TypeSize / 8); + return static_cast<int>(Idx); +} + +// A later write of the same pointer drops an earlier read in this window. +static void selectMemoryAccesses(SmallVectorImpl<Instruction *> &Local, + SmallVectorImpl<Instruction *> &Out) { + DenseSet<Value *> WriteTargets; + for (Instruction *Inst : reverse(Local)) { + const bool IsWrite = isa<StoreInst>(Inst); + Value *Addr = getLoadStorePointerOperand(Inst); + if (!IsWrite && WriteTargets.contains(Addr)) { ---------------- yxsamliu wrote:
Nit: TSan keeps the read check here if `-tsan-distinguish-volatile` is on and either access is volatile. Was it left out on purpose? Without it, a plain read before a volatile write only gets an atomic write check, so the race can't be caught. https://github.com/llvm/llvm-project/pull/225780 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
