[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -139,64 +262,59 @@ bool
SLSHardeningInserter::hardenReturnsAndBRs(MachineModuleInfo ,
return Modified;
}
-static const unsigned NumPermittedRegs = 29;
-static const struct ThunkNameAndReg {
- const char* Name;
- Register Reg;
-} SLSBLRThunks[NumPermittedRegs] = {
-{"__llvm_slsblr_thunk_x0", AArch64::X0},
-{"__llvm_slsblr_thunk_x1", AArch64::X1},
-{"__llvm_slsblr_thunk_x2", AArch64::X2},
-{"__llvm_slsblr_thunk_x3", AArch64::X3},
-{"__llvm_slsblr_thunk_x4", AArch64::X4},
-{"__llvm_slsblr_thunk_x5", AArch64::X5},
-{"__llvm_slsblr_thunk_x6", AArch64::X6},
-{"__llvm_slsblr_thunk_x7", AArch64::X7},
-{"__llvm_slsblr_thunk_x8", AArch64::X8},
-{"__llvm_slsblr_thunk_x9", AArch64::X9},
-{"__llvm_slsblr_thunk_x10", AArch64::X10},
-{"__llvm_slsblr_thunk_x11", AArch64::X11},
-{"__llvm_slsblr_thunk_x12", AArch64::X12},
-{"__llvm_slsblr_thunk_x13", AArch64::X13},
-{"__llvm_slsblr_thunk_x14", AArch64::X14},
-{"__llvm_slsblr_thunk_x15", AArch64::X15},
-// X16 and X17 are deliberately missing, as the mitigation requires those
-// register to not be used in BLR. See comment in ConvertBLRToBL for more
-// details.
-{"__llvm_slsblr_thunk_x18", AArch64::X18},
-{"__llvm_slsblr_thunk_x19", AArch64::X19},
-{"__llvm_slsblr_thunk_x20", AArch64::X20},
-{"__llvm_slsblr_thunk_x21", AArch64::X21},
-{"__llvm_slsblr_thunk_x22", AArch64::X22},
-{"__llvm_slsblr_thunk_x23", AArch64::X23},
-{"__llvm_slsblr_thunk_x24", AArch64::X24},
-{"__llvm_slsblr_thunk_x25", AArch64::X25},
-{"__llvm_slsblr_thunk_x26", AArch64::X26},
-{"__llvm_slsblr_thunk_x27", AArch64::X27},
-{"__llvm_slsblr_thunk_x28", AArch64::X28},
-{"__llvm_slsblr_thunk_x29", AArch64::FP},
-// X30 is deliberately missing, for similar reasons as X16 and X17 are
-// missing.
-{"__llvm_slsblr_thunk_x31", AArch64::XZR},
-};
+// Currently, the longest possible thunk name is
+// __llvm_slsblr_thunk_aa_xNN_xMM
+// which is 31 characters (without the '\0' character).
+static SmallString<32> createThunkName(const ThunkKind , Register Xn,
+ Register Xm) {
+ unsigned N = ThunksSet::indexOfXReg(Xn);
+ if (!Kind.HasXmOperand)
+return formatv("{0}{1}x{2}", CommonNamePrefix, Kind.NameInfix, N);
+
+ unsigned M = ThunksSet::indexOfXReg(Xm);
+ return formatv("{0}{1}x{2}_x{3}", CommonNamePrefix, Kind.NameInfix, N, M);
+}
-unsigned getThunkIndex(Register Reg) {
- for (unsigned I = 0; I < NumPermittedRegs; ++I)
-if (SLSBLRThunks[I].Reg == Reg)
- return I;
- llvm_unreachable("Unexpected register");
+static const ThunkKind (StringRef ThunkName, Register ,
kbeyls wrote:
Again a very minor nitpick: I was a bit confused initially that the 3 results
of this function are returned split between function return value (ThunkKind)
and 2 pass-by-reference function arguments (Xn, Xm).
Would it be better to either return all 3 of them through a function return
value, or return all 3 of them through pass-by-reference function arguments?
I'm just sharing it in case you think it's worthwhile to make such a change.
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -221,13 +339,19 @@ void SLSHardeningInserter::populateThunk(MachineFunction
) {
// __llvm_slsblr_thunk_xN:
// BR xN
// barrierInsts
kbeyls wrote:
As part of reviewing this, I was wondering why the actual thunk content is
```
// __llvm_slsblr_thunk_{aa|ab|aaz|abz|}_xN_{xM}:
// MOV X16, Xn
// BR X16 | BRA{A|B} X16, Xm | BRA{A|B}Z X16
// barrierInsts
```
I had to use git blame to remind myself of why I changed this about 4 years ago,
pointing to this commit:
https://github.com/llvm/llvm-project/commit/d938ec4509c47d461377527fc2877ae14b91275c
I think it would be useful to add an explanation similar to the one on that
commit message
to the comment here to explain why the `mov X16, Xn` is needed, as
it is non-trivial. The explanation on the original commit message is:
```
A "BTI c" instruction only allows jumping/calling to using a BLR* instruction.
However, the SLSBLR mitigation changes a BLR to a BR to implement the
function call. Therefore, a "BTI c" check that passed before could
trigger after the BLR->BR change done by the SLSBLR mitigation.
However, if the register used in BR is X16 or X17, this trigger will not
fire (see ArmARM for further details).
```
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -221,13 +339,19 @@ void SLSHardeningInserter::populateThunk(MachineFunction
) {
// __llvm_slsblr_thunk_xN:
// BR xN
// barrierInsts
kbeyls wrote:
Maybe something like
```
// __llvm_slsblr_thunk_{aa|ab|aaz|abz|}_xN_{xM}:
// BR Xn | BRA{A|B} Xn, Xm | BRA{A|B}Z xN
// barrierInsts
```
?
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -68,6 +156,57 @@ struct SLSHardeningInserter :
ThunkInserter {
} // end anonymous namespace
+const ThunkKind ThunkKind::BR = {ThunkBR, "", false, false, AArch64::BR};
+const ThunkKind ThunkKind::BRAA = {ThunkBRAA, "aa_", true, true,
AArch64::BRAA};
+const ThunkKind ThunkKind::BRAB = {ThunkBRAB, "ab_", true, true,
AArch64::BRAB};
+const ThunkKind ThunkKind::BRAAZ = {ThunkBRAAZ, "aaz_", false, true,
+AArch64::BRAAZ};
+const ThunkKind ThunkKind::BRABZ = {ThunkBRABZ, "abz_", false, true,
+AArch64::BRABZ};
+
+static const ThunkKind *getThunkKind(unsigned OriginalOpcode) {
kbeyls wrote:
very minor comment: I thought I'd just share that if I would've written this
function, I'd probably use std::optional rather than a "pointer to" return
value. But this is very nit-picky, so if you think "pointer to" is better,
let's leave it as is.
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -68,6 +156,57 @@ struct SLSHardeningInserter :
ThunkInserter {
} // end anonymous namespace
+const ThunkKind ThunkKind::BR = {ThunkBR, "", false, false, AArch64::BR};
+const ThunkKind ThunkKind::BRAA = {ThunkBRAA, "aa_", true, true,
AArch64::BRAA};
+const ThunkKind ThunkKind::BRAB = {ThunkBRAB, "ab_", true, true,
AArch64::BRAB};
+const ThunkKind ThunkKind::BRAAZ = {ThunkBRAAZ, "aaz_", false, true,
+AArch64::BRAAZ};
+const ThunkKind ThunkKind::BRABZ = {ThunkBRABZ, "abz_", false, true,
+AArch64::BRABZ};
+
+static const ThunkKind *getThunkKind(unsigned OriginalOpcode) {
+ switch (OriginalOpcode) {
+ case AArch64::BLR:
+ case AArch64::BLRNoIP:
+return ::BR;
+ case AArch64::BLRAA:
+return ::BRAA;
+ case AArch64::BLRAB:
+return ::BRAB;
+ case AArch64::BLRAAZ:
+return ::BRAAZ;
+ case AArch64::BLRABZ:
+return ::BRABZ;
+ }
+ return nullptr;
+}
+
+static bool isBLR(const MachineInstr ) {
+ return getThunkKind(MI.getOpcode()) != nullptr;
+}
+
+unsigned ThunksSet::indexOfXReg(Register Reg) {
+ assert(AArch64::GPR64RegClass.contains(Reg));
+ assert(Reg != AArch64::X16 && Reg != AArch64::X17 && Reg != AArch64::LR);
+
+ // Most Xn registers have consequent ids, except for FP and XZR.
kbeyls wrote:
s/consequent/consecutive/?
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -221,13 +339,19 @@ void SLSHardeningInserter::populateThunk(MachineFunction
) {
// __llvm_slsblr_thunk_xN:
// BR xN
// barrierInsts
kbeyls wrote:
I think it would be useful to update this comment to make it clear what the
different kinds of thunk bodies are that are intended to be created here.
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -0,0 +1,210 @@ +# RUN: llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu \ +# RUN: -start-before aarch64-sls-hardening -o - %s \ +# RUN: -asm-verbose=0 \ +# RUN: | FileCheck %s \ +# RUN: --implicit-check-not=__llvm_slsblr_thunk_aa_x5_x8 \ +# RUN: --implicit-check-not=__llvm_slsblr_thunk_ab_x5_x8 \ +# RUN: --implicit-check-not=__llvm_slsblr_thunk_aaz_x5 \ +# RUN: --implicit-check-not=__llvm_slsblr_thunk_abz_x5 + +# Pointer Authentication extension introduces more branch-with-link instructions kbeyls wrote: Maybe say "branch-with-link-to-register" as that's the name the ArmARM (Arm Architecture reference manual) uses for BLR* instructions. "branch-with-link" to me refers to the BL instruction. https://github.com/llvm/llvm-project/pull/97605 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -274,40 +398,31 @@ void SLSHardeningInserter::convertBLRToBL(
MachineInstr = *MBBI;
assert(isBLR(BLR));
- unsigned BLOpcode;
- Register Reg;
- bool RegIsKilled;
- switch (BLR.getOpcode()) {
- case AArch64::BLR:
- case AArch64::BLRNoIP:
-BLOpcode = AArch64::BL;
-Reg = BLR.getOperand(0).getReg();
-assert(Reg != AArch64::X16 && Reg != AArch64::X17 && Reg != AArch64::LR);
-RegIsKilled = BLR.getOperand(0).isKill();
-break;
- case AArch64::BLRAA:
- case AArch64::BLRAB:
- case AArch64::BLRAAZ:
- case AArch64::BLRABZ:
-llvm_unreachable("BLRA instructions cannot yet be produced by LLVM, "
- "therefore there is no need to support them for now.");
- default:
-llvm_unreachable("unhandled BLR");
- }
+ const ThunkKind = *getThunkKind(BLR.getOpcode());
kbeyls wrote:
In the 20-line-ish long comment above (which I cannot comment on directly in
github),
I think it would also be useful to indicate which branch instructions might be
the authenticating ones. I think that might help the reader a bit better with
more quickly understanding what the intended transform is. E.g. something like:
```
// Before:
// |-|
// | ...|
// | instI |
// | BLR{A|B|}{Z} xN{, xM} |
// | instJ |
// | ...|
// |-|
//
// After:
// |-|
// | ...|
// | instI |
// | BL __llvm_slsblr_thunk_xN |
// | instJ |
// | ...|
// |-|
//
// __llvm_slsblr_thunk_xN:
// |-|
// | BR{A|B|}{Z} xN{, xM} |
// | barrierInsts |
// |-|
//
```
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/kbeyls commented: Thank you, this mostly looks good to me. I've only added very minor comments; feel free to disagree with them. https://github.com/llvm/llvm-project/pull/97605 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/kbeyls edited https://github.com/llvm/llvm-project/pull/97605 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
@@ -68,6 +156,57 @@ struct SLSHardeningInserter :
ThunkInserter {
} // end anonymous namespace
+const ThunkKind ThunkKind::BR = {ThunkBR, "", false, false, AArch64::BR};
+const ThunkKind ThunkKind::BRAA = {ThunkBRAA, "aa_", true, true,
AArch64::BRAA};
+const ThunkKind ThunkKind::BRAB = {ThunkBRAB, "ab_", true, true,
AArch64::BRAB};
+const ThunkKind ThunkKind::BRAAZ = {ThunkBRAAZ, "aaz_", false, true,
+AArch64::BRAAZ};
+const ThunkKind ThunkKind::BRABZ = {ThunkBRABZ, "abz_", false, true,
+AArch64::BRABZ};
kbeyls wrote:
very minor nitpick: These probably can go directly after the definition of
`struct ThunkKind`?
If so, it's easier to understand what all the "false" and "true"s mean here
without having to scroll a lot to where `struct ThunkKind` is defined.
https://github.com/llvm/llvm-project/pull/97605
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97605
>From 84e7eb3c36b99ac7f673d9a7ad0c88c469f7f45d Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH 1/2] [AArch64][PAC] Support BLRA* instructions in SLS
Hardening pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 00ba31b3e500d..4ae2e48af337f 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet |=(const ThunksSet ) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t (ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseThunk(const
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97605
>From 84e7eb3c36b99ac7f673d9a7ad0c88c469f7f45d Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH] [AArch64][PAC] Support BLRA* instructions in SLS Hardening
pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 00ba31b3e500d..4ae2e48af337f 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet |=(const ThunksSet ) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t (ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseThunk(const MachineFunction )
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97605
>From f49c32c8465e9e68d7345fa82ae1294cc2faf0e7 Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH] [AArch64][PAC] Support BLRA* instructions in SLS Hardening
pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 24f023a3d70e7..4ae2e48af337f 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet |=(const ThunksSet ) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t (ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseThunk(const MachineFunction )
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
llvmbot wrote:
@llvm/pr-subscribers-backend-aarch64
Author: Anatoly Trosinenko (atrosinenko)
Changes
Make SLS Hardening pass handle BLRA* instructions the same way it handles BLR.
The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear thunk
function's name lookup and parse the name instead.
---
Patch is 23.27 KiB, truncated to 20.00 KiB below, full version:
https://github.com/llvm/llvm-project/pull/97605.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64SLSHardening.cpp (+222-104)
- (added) llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir (+210)
``diff
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index feb166f30127a..d93fe2a875845 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet |=(const ThunksSet ) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t (ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseThunk(const MachineFunction ) {
// FIXME: ComdatThunks is only accumulated until the first thunk is
created.
ComdatThunks &= !MF.getSubtarget().hardenSlsNoComdat();
@@ -69,6
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko created
https://github.com/llvm/llvm-project/pull/97605
Make SLS Hardening pass handle BLRA* instructions the same way it handles BLR.
The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear thunk
function's name lookup and parse the name instead.
>From b389284b8e92f5bf09cea38f3f9a53974a84dc29 Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH] [AArch64][PAC] Support BLRA* instructions in SLS Hardening
pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index feb166f30127a..d93fe2a875845 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet |=(const ThunksSet ) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t (ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
