[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
https://github.com/atrosinenko created
https://github.com/llvm/llvm-project/pull/97472
In preparation for implementing hardening of BLRA* instructions, restrict thunk
function generation to only the thunks being actually called from any function.
As described in the existing comments, emitting all possible thunks for BLRAA
and BLRAB instructions would mean adding about 1800 functions in total, most of
which are likely not to be called.
This commit merges AArch64SLSHardening class into SLSBLRThunkInserter, so
thunks can be created as needed while rewriting a machine function. The usages
of TII, TRI and ST fields of AArch64SLSHardening class are replaced with
requesting them in-place, as ThunkInserter assumes multiple "entry points" in
contrast to the only runOnMachineFunction method of AArch64SLSHardening.
The runOnMachineFunction method essentially replaces pre-existing insertThunks
implementation as there is no more need to insert all possible thunks
unconditionally. Instead, thunks are created on first use from inside of
insertThunks method.
>From a246cfe705b326c520d6b36882a17bd90b622e5d Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Fri, 28 Jun 2024 21:50:24 +0300
Subject: [PATCH] [AArch64] Only create called thunks when hardening against
SLS
In preparation for implementing hardening of BLRA* instructions,
restrict thunk function generation to only the thunks being actually
called from any function. As described in the existing comments,
emitting all possible thunks for BLRAA and BLRAB instructions would
mean adding about 1800 functions in total, most of which are likely
not to be called.
This commit merges AArch64SLSHardening class into SLSBLRThunkInserter,
so thunks can be created as needed while rewriting a machine function.
The usages of TII, TRI and ST fields of AArch64SLSHardening class are
replaced with requesting them in-place, as ThunkInserter assumes
multiple "entry points" in contrast to the only runOnMachineFunction
method of AArch64SLSHardening.
The runOnMachineFunction method essentially replaces pre-existing
insertThunks implementation as there is no more need to insert all
possible thunks unconditionally. Instead, thunks are created on first
use from inside of insertThunks method.
---
llvm/lib/Target/AArch64/AArch64.h | 1 -
.../Target/AArch64/AArch64SLSHardening.cpp| 188 +++---
.../Target/AArch64/AArch64TargetMachine.cpp | 1 -
llvm/test/CodeGen/AArch64/O0-pipeline.ll | 1 -
llvm/test/CodeGen/AArch64/O3-pipeline.ll | 1 -
.../AArch64/arm64-opt-remarks-lazy-bfi.ll | 8 -
.../speculation-hardening-sls-blr-bti.mir | 20 --
.../AArch64/speculation-hardening-sls-blr.mir | 20 +-
8 files changed, 75 insertions(+), 165 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64.h
b/llvm/lib/Target/AArch64/AArch64.h
index 6f2aeb83a451a..66ad701d83958 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -40,7 +40,6 @@ FunctionPass *createAArch64ISelDag(AArch64TargetMachine &TM,
FunctionPass *createAArch64StorePairSuppressPass();
FunctionPass *createAArch64ExpandPseudoPass();
FunctionPass *createAArch64SLSHardeningPass();
-FunctionPass *createAArch64IndirectThunks();
FunctionPass *createAArch64SpeculationHardeningPass();
FunctionPass *createAArch64LoadStoreOptimizationPass();
ModulePass *createAArch64LowerHomogeneousPrologEpilogPass();
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index d35386eaeab12..b4ebd7d5377c2 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,20 +13,16 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
-#include "Utils/AArch64BaseInfo.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
-#include "llvm/Support/CodeGen.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -36,38 +32,43 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
+static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+
namespace {
-class AArch64SLSHardening : public MachineFunctionPass {
-public:
- const TargetInstrInfo *TII;
- const TargetRegisterInfo *TRI;
- const AArch64Subtarget *ST;
+// Set of inserted thunks: bitmask with bits corresponding to
+// indexes in SLSBLRThunks array.
+typedef uint32_t ThunksSet;
- static char ID;
-
- AArch64SLSHardening() : MachineFunctionPass(ID) {
-
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97472
>From a246cfe705b326c520d6b36882a17bd90b622e5d Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Fri, 28 Jun 2024 21:50:24 +0300
Subject: [PATCH 1/2] [AArch64] Only create called thunks when hardening
against SLS
In preparation for implementing hardening of BLRA* instructions,
restrict thunk function generation to only the thunks being actually
called from any function. As described in the existing comments,
emitting all possible thunks for BLRAA and BLRAB instructions would
mean adding about 1800 functions in total, most of which are likely
not to be called.
This commit merges AArch64SLSHardening class into SLSBLRThunkInserter,
so thunks can be created as needed while rewriting a machine function.
The usages of TII, TRI and ST fields of AArch64SLSHardening class are
replaced with requesting them in-place, as ThunkInserter assumes
multiple "entry points" in contrast to the only runOnMachineFunction
method of AArch64SLSHardening.
The runOnMachineFunction method essentially replaces pre-existing
insertThunks implementation as there is no more need to insert all
possible thunks unconditionally. Instead, thunks are created on first
use from inside of insertThunks method.
---
llvm/lib/Target/AArch64/AArch64.h | 1 -
.../Target/AArch64/AArch64SLSHardening.cpp| 188 +++---
.../Target/AArch64/AArch64TargetMachine.cpp | 1 -
llvm/test/CodeGen/AArch64/O0-pipeline.ll | 1 -
llvm/test/CodeGen/AArch64/O3-pipeline.ll | 1 -
.../AArch64/arm64-opt-remarks-lazy-bfi.ll | 8 -
.../speculation-hardening-sls-blr-bti.mir | 20 --
.../AArch64/speculation-hardening-sls-blr.mir | 20 +-
8 files changed, 75 insertions(+), 165 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64.h
b/llvm/lib/Target/AArch64/AArch64.h
index 6f2aeb83a451a..66ad701d83958 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -40,7 +40,6 @@ FunctionPass *createAArch64ISelDag(AArch64TargetMachine &TM,
FunctionPass *createAArch64StorePairSuppressPass();
FunctionPass *createAArch64ExpandPseudoPass();
FunctionPass *createAArch64SLSHardeningPass();
-FunctionPass *createAArch64IndirectThunks();
FunctionPass *createAArch64SpeculationHardeningPass();
FunctionPass *createAArch64LoadStoreOptimizationPass();
ModulePass *createAArch64LowerHomogeneousPrologEpilogPass();
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index d35386eaeab12..b4ebd7d5377c2 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,20 +13,16 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
-#include "Utils/AArch64BaseInfo.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
-#include "llvm/Support/CodeGen.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -36,38 +32,43 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
+static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+
namespace {
-class AArch64SLSHardening : public MachineFunctionPass {
-public:
- const TargetInstrInfo *TII;
- const TargetRegisterInfo *TRI;
- const AArch64Subtarget *ST;
+// Set of inserted thunks: bitmask with bits corresponding to
+// indexes in SLSBLRThunks array.
+typedef uint32_t ThunksSet;
- static char ID;
-
- AArch64SLSHardening() : MachineFunctionPass(ID) {
-initializeAArch64SLSHardeningPass(*PassRegistry::getPassRegistry());
+struct SLSBLRThunkInserter : ThunkInserter {
+public:
+ const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ bool mayUseThunk(const MachineFunction &MF) {
+// FIXME: ComdatThunks is only accumulated until the first thunk is
created.
+ComdatThunks &= !MF.getSubtarget().hardenSlsNoComdat();
+// We are inserting barriers aside from thunk calls, so
+// check hardenSlsRetBr() as well.
+return MF.getSubtarget().hardenSlsBlr() ||
+ MF.getSubtarget().hardenSlsRetBr();
}
+ ThunksSet insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,
+ ThunksSet ExistingThunks);
+ void populateThunk(MachineFunction &MF);
- bool runOnMachineFunction(MachineFunction &Fn) override;
+private:
+ bool ComdatThunks = true;
- StringRef getPassName() const override { return AARCH64_SLS_HARDENING_NAME; }
+ bool hardenReturnsAndBRs(MachineModuleInfo &
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
@@ -36,38 +32,43 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
+static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+
namespace {
-class AArch64SLSHardening : public MachineFunctionPass {
-public:
- const TargetInstrInfo *TII;
- const TargetRegisterInfo *TRI;
- const AArch64Subtarget *ST;
+// Set of inserted thunks: bitmask with bits corresponding to
+// indexes in SLSBLRThunks array.
+typedef uint32_t ThunksSet;
atrosinenko wrote:
Yes, `ThunkSet` will be class.
https://github.com/llvm/llvm-project/pull/97472
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
@@ -46,13 +40,5 @@ body: | ... -name:__llvm_slsblr_thunk_x8 -tracksRegLiveness: true -body: | - bb.0.entry: -liveins: $x8 -BR $x8 atrosinenko wrote: The above lines are **input** to `llc`, so I removed them to not conflict with the thunks that are inserted by the pass. Unfortunately, MIR syntax is quite verbose, so I added a few basic CHECK lines testing MIR output and duplicated the RUN lines to generate and check assembly output as well. Now there is no separation between function-rewriting and thunk-inserting passes, and I assumed it is safe to expect that initially there are no thunks in the module. I guess the original version of thunk insertion logic would assert on pre-existing thunks as well (due to unknown thunk names with `.1` suffix). https://github.com/llvm/llvm-project/pull/97472 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
https://github.com/atrosinenko edited https://github.com/llvm/llvm-project/pull/97472 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
https://github.com/atrosinenko commented: @kbeyls thank you for the comments! https://github.com/llvm/llvm-project/pull/97472 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
@@ -36,38 +32,43 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
+static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+
namespace {
-class AArch64SLSHardening : public MachineFunctionPass {
-public:
- const TargetInstrInfo *TII;
- const TargetRegisterInfo *TRI;
- const AArch64Subtarget *ST;
+// Set of inserted thunks: bitmask with bits corresponding to
+// indexes in SLSBLRThunks array.
+typedef uint32_t ThunksSet;
- static char ID;
-
- AArch64SLSHardening() : MachineFunctionPass(ID) {
-initializeAArch64SLSHardeningPass(*PassRegistry::getPassRegistry());
+struct SLSBLRThunkInserter : ThunkInserter {
atrosinenko wrote:
Renamed it to `SLSHardeningInserter`, so the `AArch64SLSHardening` name is kept
for the pass itself.
https://github.com/llvm/llvm-project/pull/97472
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
https://github.com/atrosinenko edited https://github.com/llvm/llvm-project/pull/97472 ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko created
https://github.com/llvm/llvm-project/pull/97605
Make SLS Hardening pass handle BLRA* instructions the same way it handles BLR.
The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear thunk
function's name lookup and parse the name instead.
>From b389284b8e92f5bf09cea38f3f9a53974a84dc29 Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH] [AArch64][PAC] Support BLRA* instructions in SLS Hardening
pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index feb166f30127a..d93fe2a875845 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet &operator|=(const ThunksSet &Other) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t &getBitmask(ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+ca
[llvm-branch-commits] [llvm] [AArch64] Only create called thunks when hardening against SLS (PR #97472)
@@ -36,38 +32,43 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
+static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+
namespace {
-class AArch64SLSHardening : public MachineFunctionPass {
-public:
- const TargetInstrInfo *TII;
- const TargetRegisterInfo *TRI;
- const AArch64Subtarget *ST;
+// Set of inserted thunks: bitmask with bits corresponding to
+// indexes in SLSBLRThunks array.
+typedef uint32_t ThunksSet;
atrosinenko wrote:
Here is the PR: #97605
https://github.com/llvm/llvm-project/pull/97472
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97605
>From f49c32c8465e9e68d7345fa82ae1294cc2faf0e7 Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH] [AArch64][PAC] Support BLRA* instructions in SLS Hardening
pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 24f023a3d70e7..4ae2e48af337f 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet &operator|=(const ThunksSet &Other) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t &getBitmask(ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseThunk
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97605
>From 84e7eb3c36b99ac7f673d9a7ad0c88c469f7f45d Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH] [AArch64][PAC] Support BLRA* instructions in SLS Hardening
pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 00ba31b3e500d..4ae2e48af337f 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet &operator|=(const ThunksSet &Other) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t &getBitmask(ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseThunk
[llvm-branch-commits] [llvm] [AArch64][PAC] Support BLRA* instructions in SLS Hardening pass (PR #97605)
https://github.com/atrosinenko updated
https://github.com/llvm/llvm-project/pull/97605
>From 84e7eb3c36b99ac7f673d9a7ad0c88c469f7f45d Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko
Date: Mon, 1 Jul 2024 20:13:54 +0300
Subject: [PATCH 1/2] [AArch64][PAC] Support BLRA* instructions in SLS
Hardening pass
Make SLS Hardening pass handle BLRA* instructions the same way it
handles BLR. The thunk names have the form
__llvm_slsblr_thunk_xNfor BLR thunks
__llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
__llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
Now there are about 1800 possible thunk names, so do not rely on linear
thunk function's name lookup and parse the name instead.
---
.../Target/AArch64/AArch64SLSHardening.cpp| 326 --
.../speculation-hardening-sls-blra.mir| 210 +++
2 files changed, 432 insertions(+), 104 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/speculation-hardening-sls-blra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 00ba31b3e500d..4ae2e48af337f 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IndirectThunks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetMachine.h"
#include
@@ -32,17 +34,103 @@ using namespace llvm;
#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
-static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
+// Common name prefix of all thunks generated by this pass.
+//
+// The generic form is
+// __llvm_slsblr_thunk_xNfor BLR thunks
+// __llvm_slsblr_thunk_(aaz|abz)_xN for BLRAAZ and BLRABZ thunks
+// __llvm_slsblr_thunk_(aa|ab)_xN_xM for BLRAA and BLRAB thunks
+static constexpr StringRef CommonNamePrefix = "__llvm_slsblr_thunk_";
namespace {
-// Set of inserted thunks: bitmask with bits corresponding to
-// indexes in SLSBLRThunks array.
-typedef uint32_t ThunksSet;
+struct ThunkKind {
+ enum ThunkKindId {
+ThunkBR,
+ThunkBRAA,
+ThunkBRAB,
+ThunkBRAAZ,
+ThunkBRABZ,
+ };
+
+ ThunkKindId Id;
+ StringRef NameInfix;
+ bool HasXmOperand;
+ bool NeedsPAuth;
+
+ // Opcode to perform indirect jump from inside the thunk.
+ unsigned BROpcode;
+
+ static const ThunkKind BR;
+ static const ThunkKind BRAA;
+ static const ThunkKind BRAB;
+ static const ThunkKind BRAAZ;
+ static const ThunkKind BRABZ;
+};
+
+// Set of inserted thunks.
+class ThunksSet {
+public:
+ static constexpr unsigned NumXRegisters = 32;
+
+ // Given Xn register, returns n.
+ static unsigned indexOfXReg(Register Xn);
+ // Given n, returns Xn register.
+ static Register xRegByIndex(unsigned N);
+
+ ThunksSet &operator|=(const ThunksSet &Other) {
+BLRThunks |= Other.BLRThunks;
+BLRAAZThunks |= Other.BLRAAZThunks;
+BLRABZThunks |= Other.BLRABZThunks;
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRAAThunks[I] |= Other.BLRAAThunks[I];
+for (unsigned I = 0; I < NumXRegisters; ++I)
+ BLRABThunks[I] |= Other.BLRABThunks[I];
+
+return *this;
+ }
+
+ bool get(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+return getBitmask(Kind, Xm) & XnBit;
+ }
+
+ void set(ThunkKind::ThunkKindId Kind, Register Xn, Register Xm) {
+uint32_t XnBit = 1u << indexOfXReg(Xn);
+getBitmask(Kind, Xm) |= XnBit;
+ }
+
+private:
+ // Bitmasks representing operands used, with n-th bit corresponding to Xn
+ // register operand. If the instruction has a second operand (Xm), an array
+ // of bitmasks is used, indexed by m.
+ // Indexes corresponding to the forbidden x16, x17 and x30 registers are
+ // always unset, for simplicity there are no holes.
+ uint32_t BLRThunks = 0;
+ uint32_t BLRAAZThunks = 0;
+ uint32_t BLRABZThunks = 0;
+ uint32_t BLRAAThunks[NumXRegisters] = {};
+ uint32_t BLRABThunks[NumXRegisters] = {};
+
+ uint32_t &getBitmask(ThunkKind::ThunkKindId Kind, Register Xm) {
+switch (Kind) {
+case ThunkKind::ThunkBR:
+ return BLRThunks;
+case ThunkKind::ThunkBRAAZ:
+ return BLRAAZThunks;
+case ThunkKind::ThunkBRABZ:
+ return BLRABZThunks;
+case ThunkKind::ThunkBRAA:
+ return BLRAAThunks[indexOfXReg(Xm)];
+case ThunkKind::ThunkBRAB:
+ return BLRABThunks[indexOfXReg(Xm)];
+}
+ }
+};
struct SLSHardeningInserter : ThunkInserter {
public:
- const char *getThunkPrefix() { return SLSBLRNamePrefix; }
+ const char *getThunkPrefix() { return CommonNamePrefix.data(); }
bool mayUseT
