================
@@ -431,10 +441,135 @@ static Value *getSlotPtr(const AArch64Options &CLOpts,
IRBuilder<> &IRB,
return nullptr;
}
+void AArch64StackTagging::createDynamicAllocasInitStorage() {
+ IRBuilder<> IRB(&*F->getEntryBlock().getFirstInsertionPt());
+ Type *IntptrTy = IRB.getIntPtrTy(*DL);
+ DynamicAllocaLayout =
+ IRB.CreateAlloca(IntptrTy, nullptr, "dynamic.alloca.layout");
+ // 0 means that no alloca has been made.
+ IRB.CreateStore(Constant::getNullValue(IntptrTy), DynamicAllocaLayout);
+ // Its address is the end of the memory untagged on return, which must be
+ // granule aligned.
+ DynamicAllocaLayout->setAlignment(kTagGranuleSize);
+}
+
+// Replace the dynamic alloca with one whose size is a multiple of the tag
+// granule, and tag the memory it returns.
+void AArch64StackTagging::handleDynamicAllocaCall(memtag::AllocaInfo &Info,
+ Value *Base, unsigned Tag) {
+ AllocaInst *AI = Info.AI;
+ IRBuilder<> IRB(AI);
+ Type *IntptrTy = IRB.getIntPtrTy(*DL);
+ const uint64_t Granule = kTagGranuleSize.value();
+ Value *Size = IRB.CreateAllocationSize(IntptrTy, AI);
+ // Rounded up to a multiple of the granule.
+ Value *PaddedSize = IRB.CreateAnd(
+ IRB.CreateAdd(Size, ConstantInt::get(IntptrTy, Granule - 1)),
+ ConstantInt::get(IntptrTy, ~(Granule - 1)));
+
+ AllocaInst *NewAI =
+ IRB.CreateAlloca(IRB.getInt8Ty(), AI->getAddressSpace(), PaddedSize);
+ NewAI->setAlignment(std::max(AI->getAlign(), kTagGranuleSize));
+ NewAI->takeName(AI);
+ NewAI->copyMetadata(*AI);
+ AI->replaceAllUsesWith(NewAI);
+ AI->eraseFromParent();
+ Info.AI = NewAI;
+
+ // Replace alloca with tagp(alloca).
+ IRB.SetInsertPoint(NewAI->getNextNode());
+ Instruction *TagPCall = IRB.CreateIntrinsicWithoutFolding(
+ Intrinsic::aarch64_tagp, {NewAI->getType()},
+ {Constant::getNullValue(NewAI->getType()), Base, IRB.getInt64(Tag)});
+ if (NewAI->hasName())
+ TagPCall->setName(NewAI->getName() + ".tag");
+ // Does not replace metadata, so we don't have to handle DbgVariableRecords.
+ NewAI->replaceUsesWithIf(TagPCall, [&](const Use &U) {
+ return !isa<LifetimeIntrinsic>(U.getUser());
+ });
+ TagPCall->setOperand(0, NewAI);
+
+ // Store the last alloca's address to DynamicAllocaLayout. This points to the
+ // most recently executed dynamic alloca, which is used when untagging. It is
+ // granule aligned.
+ IRB.CreateStore(IRB.CreatePtrToInt(NewAI, IntptrTy), DynamicAllocaLayout);
+
+ // Tag the memory at llvm.lifetime.start if there is one, and untag it at
+ // llvm.lifetime.end. NewAI, and so TagPCall, is granule aligned, and TagSize
+ // is a multiple of the granule.
+ Value *TagSize = IRB.CreateZExtOrTrunc(PaddedSize, IRB.getInt64Ty());
+ SmallVector<LifetimeIntrinsic *, 4> Lifetimes;
+ for (User *U : NewAI->users())
+ if (auto *II = dyn_cast<LifetimeIntrinsic>(U))
+ Lifetimes.push_back(II);
+ if (none_of(Lifetimes, [](LifetimeIntrinsic *II) {
+ return II->getIntrinsicID() == Intrinsic::lifetime_start;
+ }))
+ IRB.CreateCall(SetTagFunc, {TagPCall, TagSize});
+ for (LifetimeIntrinsic *II : Lifetimes) {
+ if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
+ IRB.SetInsertPoint(II->getNextNode());
+ IRB.CreateCall(SetTagFunc, {TagPCall, TagSize});
+ } else {
+ // NewAI has tag 0.
+ IRB.SetInsertPoint(II);
+ IRB.CreateCall(SetTagFunc, {NewAI, TagSize});
+ }
+ }
+}
+
+// Untag the memory from the most recent dynamic alloca up to End, a granule
+// aligned address, unless the range is empty or no alloca has been made.
+void AArch64StackTagging::untagDynamicAllocasBeforeInst(Instruction
*InstBefore,
+ Value *End) {
+ IRBuilder<> IRB(InstBefore);
+ Type *IntptrTy = IRB.getIntPtrTy(*DL);
+ // Only 0 or the address of an alloca is stored to DynamicAllocaLayout, so
+ // this is granule aligned, and the size a multiple of the granule.
+ Value *LastAlloca =
+ IRB.CreateLoad(IntptrTy, DynamicAllocaLayout, "last.alloca");
+ Value *Zero = Constant::getNullValue(IntptrTy);
+ Value *NonZero = IRB.CreateICmpNE(LastAlloca, Zero);
+ Value *InRange = IRB.CreateICmpULE(LastAlloca, End);
+ Value *Cond = IRB.CreateAnd(NonZero, InRange);
+ Value *Size = IRB.CreateSelect(Cond, IRB.CreateSub(End, LastAlloca), Zero,
+ "untag.size");
+ // Tag 0.
+ IRB.CreateCall(SetTagFunc, {IRB.CreateIntToPtr(LastAlloca, IRB.getPtrTy()),
+ IRB.CreateZExtOrTrunc(Size, IRB.getInt64Ty())});
+}
+
+// Untag dynamic allocas.
+void AArch64StackTagging::untagDynamicAllocas(const memtag::StackInfo &SInfo) {
----------------
vitalybuka wrote:
Should this be:
```cpp
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -302,7 +302,8 @@ private:
void createDynamicAllocasInitStorage();
void handleDynamicAllocaCall(memtag::AllocaInfo &Info, Value *Base,
unsigned Tag);
- void untagDynamicAllocasBeforeInst(Instruction *InstBefore, Value *End);
+ void untagDynamicAllocasBeforeInst(Instruction *InstBefore, Value *End,
+ bool UpdateLayout);
void untagDynamicAllocas(const memtag::StackInfo &SInfo);
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -521,11 +522,12 @@ void
AArch64StackTagging::handleDynamicAllocaCall(memtag::AllocaInfo &Info,
// Untag the memory from the most recent dynamic alloca up to End, a granule
// aligned address, unless the range is empty or no alloca has been made.
void AArch64StackTagging::untagDynamicAllocasBeforeInst(Instruction
*InstBefore,
- Value *End) {
+ Value *End,
+ bool UpdateLayout) {
IRBuilder<> IRB(InstBefore);
Type *IntptrTy = IRB.getIntPtrTy(*DL);
- // Only 0 or the address of an alloca is stored to DynamicAllocaLayout, so
- // this is granule aligned, and the size a multiple of the granule.
+ // Only 0, the address of an alloca, or a restored SP is stored to
+ // DynamicAllocaLayout, so this is granule aligned, and the size a multiple
of
+ // the granule.
Value *LastAlloca =
IRB.CreateLoad(IntptrTy, DynamicAllocaLayout, "last.alloca");
Value *Zero = Constant::getNullValue(IntptrTy);
@@ -537,16 +539,20 @@ void
AArch64StackTagging::untagDynamicAllocasBeforeInst(Instruction *InstBefore,
// Tag 0.
IRB.CreateCall(SetTagFunc, {IRB.CreateIntToPtr(LastAlloca, IRB.getPtrTy()),
IRB.CreateZExtOrTrunc(Size, IRB.getInt64Ty())});
+ if (UpdateLayout)
+ IRB.CreateStore(IRB.CreateSelect(Cond, End, LastAlloca),
+ DynamicAllocaLayout);
}
// Untag dynamic allocas.
void AArch64StackTagging::untagDynamicAllocas(const memtag::StackInfo &SInfo) {
Type *IntptrTy = DL->getIntPtrType(F->getContext());
// On return, untag up to DynamicAllocaLayout, which is above every dynamic
// alloca and granule aligned.
for (Instruction *Ret : SInfo.RetVec) {
IRBuilder<> IRB(Ret);
untagDynamicAllocasBeforeInst(
- Ret, IRB.CreatePtrToInt(DynamicAllocaLayout, IntptrTy, "untag.end"));
+ Ret, IRB.CreatePtrToInt(DynamicAllocaLayout, IntptrTy, "untag.end"),
+ /*UpdateLayout=*/false);
}
// When we untag before @llvm.stackrestore, we need to adjust extracted SP to
@@ -562,7 +568,8 @@ void AArch64StackTagging::untagDynamicAllocas(const
memtag::StackInfo &SInfo) {
Value *DynamicAreaPtr = IRB.CreateAdd(
IRB.CreatePtrToInt(StackRestoreInst->getOperand(0), IntptrTy),
DynamicAreaOffset, "untag.end");
- untagDynamicAllocasBeforeInst(StackRestoreInst, DynamicAreaPtr);
+ untagDynamicAllocasBeforeInst(StackRestoreInst, DynamicAreaPtr,
+ /*UpdateLayout=*/true);
}
}
```
https://github.com/llvm/llvm-project/pull/236278
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits