samitolvanen updated this revision to Diff 514708.
samitolvanen added a comment.
Addressed most of the feedback.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D148385/new/
https://reviews.llvm.org/D148385
Files:
clang/lib/CodeGen/BackendUtil.cpp
llvm/lib/Target/RISCV/CMakeLists.txt
llvm/lib/Target/RISCV/RISCV.h
llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.h
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
llvm/lib/Target/RISCV/RISCVInstrInfo.h
llvm/lib/Target/RISCV/RISCVInstrInfo.td
llvm/lib/Target/RISCV/RISCVKCFI.cpp
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
llvm/test/CodeGen/RISCV/O0-pipeline.ll
llvm/test/CodeGen/RISCV/O3-pipeline.ll
llvm/test/CodeGen/RISCV/kcfi-isel.mir
llvm/test/CodeGen/RISCV/kcfi-patchable-function-prefix.ll
llvm/test/CodeGen/RISCV/kcfi.ll
llvm/test/CodeGen/RISCV/kcfi.mir
Index: llvm/test/CodeGen/RISCV/kcfi.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/RISCV/kcfi.mir
@@ -0,0 +1,185 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=riscv64 -stop-after=riscv-kcfi -o - %s | FileCheck %s
+--- |
+ ; ModuleID = '<stdin>'
+ source_filename = "<stdin>"
+ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+ target triple = "riscv64"
+
+ define void @f1(ptr noundef %x) !kcfi_type !1 {
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+ }
+
+ define void @f2(ptr noundef %x) #0 {
+ tail call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+ }
+
+ attributes #0 = { "patchable-function-entry"="2" }
+
+ !llvm.module.flags = !{!0}
+
+ !0 = !{i32 4, !"kcfi", i32 1}
+ !1 = !{i32 12345678}
+
+...
+---
+name: f1
+alignment: 4
+exposesReturnsTwice: false
+legalized: false
+regBankSelected: false
+selected: false
+failedISel: false
+tracksRegLiveness: true
+hasWinCFI: false
+callsEHReturn: false
+callsUnwindInit: false
+hasEHCatchret: false
+hasEHScopes: false
+hasEHFunclets: false
+isOutlined: false
+debugInstrRef: false
+failsVerification: false
+tracksDebugUserValues: true
+registers: []
+liveins:
+ - { reg: '$x10', virtual-reg: '' }
+frameInfo:
+ isFrameAddressTaken: false
+ isReturnAddressTaken: false
+ hasStackMap: false
+ hasPatchPoint: false
+ stackSize: 16
+ offsetAdjustment: 0
+ maxAlignment: 8
+ adjustsStack: true
+ hasCalls: true
+ stackProtector: ''
+ functionContext: ''
+ maxCallFrameSize: 0
+ cvBytesOfCalleeSavedRegisters: 0
+ hasOpaqueSPAdjustment: false
+ hasVAStart: false
+ hasMustTailInVarArgFunc: false
+ hasTailCall: false
+ localFrameSize: 0
+ savePoint: ''
+ restorePoint: ''
+fixedStack: []
+stack:
+ - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
+ stack-id: default, callee-saved-register: '$x1', callee-saved-restored: true,
+ debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites: []
+debugValueSubstitutions: []
+constants: []
+machineFunctionInfo:
+ varArgsFrameIndex: 0
+ varArgsSaveSize: 0
+body: |
+ bb.0 (%ir-block.0):
+ liveins: $x10, $x1
+
+ ; CHECK-LABEL: name: f1
+ ; CHECK: liveins: $x1, $x10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -16
+ ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+ ; CHECK-NEXT: SD $x1, $x2, 8 :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
+ ; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -16
+ ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+ ; CHECK-NEXT: SD $x1, $x2, 8 :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
+ ; CHECK-NEXT: BUNDLE implicit-def dead $x6, implicit-def dead $x7, implicit-def dead $x28, implicit-def dead $x29, implicit-def dead $x30, implicit-def dead $x31, implicit-def dead $x1, implicit-def $x2, implicit $x10 {
+ ; CHECK-NEXT: KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
+ ; CHECK-NEXT: PseudoCALLIndirect $x10, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2
+ ; CHECK-NEXT: }
+ ; CHECK-NEXT: dead $x1 = LD $x2, 8 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 16
+ ; CHECK-NEXT: $x1 = LD $x2, 8 :: (load (s64) from %stack.1)
+ ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 16
+ ; CHECK-NEXT: PseudoRET
+ $x2 = frame-setup ADDI $x2, -16
+ frame-setup CFI_INSTRUCTION def_cfa_offset 16
+ SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
+ frame-setup CFI_INSTRUCTION offset $x1, -8
+ BUNDLE implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31, implicit-def dead $x1, implicit-def $x2, implicit killed $x10 {
+ KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
+ PseudoCALLIndirect killed $x10, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2
+ }
+ $x1 = LD $x2, 8 :: (load (s64) from %stack.0)
+ $x2 = frame-destroy ADDI $x2, 16
+ PseudoRET
+
+...
+---
+name: f2
+alignment: 4
+exposesReturnsTwice: false
+legalized: false
+regBankSelected: false
+selected: false
+failedISel: false
+tracksRegLiveness: true
+hasWinCFI: false
+callsEHReturn: false
+callsUnwindInit: false
+hasEHCatchret: false
+hasEHScopes: false
+hasEHFunclets: false
+isOutlined: false
+debugInstrRef: false
+failsVerification: false
+tracksDebugUserValues: true
+registers: []
+liveins:
+ - { reg: '$x10', virtual-reg: '' }
+frameInfo:
+ isFrameAddressTaken: false
+ isReturnAddressTaken: false
+ hasStackMap: false
+ hasPatchPoint: false
+ stackSize: 0
+ offsetAdjustment: 0
+ maxAlignment: 1
+ adjustsStack: false
+ hasCalls: false
+ stackProtector: ''
+ functionContext: ''
+ maxCallFrameSize: 0
+ cvBytesOfCalleeSavedRegisters: 0
+ hasOpaqueSPAdjustment: false
+ hasVAStart: false
+ hasMustTailInVarArgFunc: false
+ hasTailCall: true
+ localFrameSize: 0
+ savePoint: ''
+ restorePoint: ''
+fixedStack: []
+stack: []
+callSites: []
+debugValueSubstitutions: []
+constants: []
+machineFunctionInfo:
+ varArgsFrameIndex: 0
+ varArgsSaveSize: 0
+body: |
+ bb.0 (%ir-block.0):
+ liveins: $x10
+
+ ; CHECK-LABEL: name: f2
+ ; CHECK: liveins: $x10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: BUNDLE implicit-def dead $x6, implicit-def dead $x7, implicit-def dead $x28, implicit-def dead $x29, implicit-def dead $x30, implicit-def dead $x31, implicit $x10, implicit $x2 {
+ ; CHECK-NEXT: KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
+ ; CHECK-NEXT: PseudoTAILIndirect $x10, implicit $x2
+ ; CHECK-NEXT: }
+ BUNDLE implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31, implicit killed $x10, implicit $x2 {
+ KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
+ PseudoTAILIndirect killed $x10, implicit $x2
+ }
+
+...
Index: llvm/test/CodeGen/RISCV/kcfi.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/RISCV/kcfi.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -riscv-no-aliases < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -riscv-no-aliases < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,RV64
+
+; CHECK: .word 12345678
+define void @f1(ptr noundef %x) !kcfi_type !1 {
+; CHECK-LABEL: f1:
+; CHECK: # %bb.0:
+; CHECK: lw t1, -4(a0)
+; CHECK-NEXT: lui t2, 3014
+; RV32-NEXT: addi t2, t2, 334
+; RV64-NEXT: addiw t2, t2, 334
+; CHECK-NEXT: beq t1, t2, .Ltmp0
+; CHECK-NEXT: .Ltmp1:
+; CHECK-NEXT: ebreak
+; CHECK-NEXT: .section .kcfi_traps,"ao",@progbits,.text
+; CHECK-NEXT: .Ltmp2:
+; CHECK-NEXT: .word .Ltmp1-.Ltmp2
+; CHECK-NEXT: .text
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT: jalr ra, 0(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; CHECK-NOT: .word:
+define void @f2(ptr noundef %x) #0 {
+; CHECK-LABEL: f2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi zero, zero, 0
+; CHECK-NEXT: addi zero, zero, 0
+; CHECK-NEXT: lw t1, -4(a0)
+; CHECK-NEXT: lui t2, 3014
+; RV32-NEXT: addi t2, t2, 334
+; RV64-NEXT: addiw t2, t2, 334
+; CHECK-NEXT: beq t1, t2, .Ltmp3
+; CHECK-NEXT: .Ltmp4:
+; CHECK-NEXT: ebreak
+; CHECK-NEXT: .section .kcfi_traps,"ao",@progbits,.text
+; CHECK-NEXT: .Ltmp5:
+; CHECK-NEXT: .word .Ltmp4-.Ltmp5
+; CHECK-NEXT: .text
+; CHECK-NEXT: .Ltmp3:
+; CHECK-NEXT: jalr zero, 0(a0)
+ tail call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+attributes #0 = { "patchable-function-entry"="2" }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 4, !"kcfi", i32 1}
+!1 = !{i32 12345678}
Index: llvm/test/CodeGen/RISCV/kcfi-patchable-function-prefix.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/RISCV/kcfi-patchable-function-prefix.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOC
+; RUN: llc -mtriple=riscv64 -mattr=+c -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,C
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK-NOT: nop
+; CHECK: .word 12345678
+; CHECK-LABEL: f1:
+define void @f1(ptr noundef %x) !kcfi_type !1 {
+; CHECK: lw t1, -4(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK-NOT: .word
+; CHECK-NOT: nop
+; CHECK-LABEL: f2:
+define void @f2(ptr noundef %x) {
+; CHECK: lw t1, -4(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK: .word 12345678
+; CHECK-COUNT-11: nop
+; CHECK-LABEL: f3:
+define void @f3(ptr noundef %x) #0 !kcfi_type !1 {
+; NOC: lw t1, -48(a0)
+; C: lw t1, -26(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+; NOC: .p2align 2
+; C: .p2align 1
+; CHECK-NOT: .word
+; CHECK-COUNT-11: nop
+; CHECK-LABEL: f4:
+define void @f4(ptr noundef %x) #0 {
+; NOC: lw t1, -48(a0)
+; C: lw t1, -26(a0)
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+attributes #0 = { "patchable-function-prefix"="11" }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 4, !"kcfi", i32 1}
+!1 = !{i32 12345678}
Index: llvm/test/CodeGen/RISCV/kcfi-isel.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/RISCV/kcfi-isel.mir
@@ -0,0 +1,175 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=riscv64 -stop-after=finalize-isel -o - %s | FileCheck %s
+--- |
+ ; ModuleID = '<stdin>'
+ source_filename = "<stdin>"
+ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+ target triple = "riscv64"
+
+ define void @f1(ptr noundef %x) !kcfi_type !1 {
+ call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+ }
+
+ define void @f2(ptr noundef %x) #0 {
+ tail call void %x() [ "kcfi"(i32 12345678) ]
+ ret void
+ }
+
+ attributes #0 = { "patchable-function-entry"="2" }
+
+ !llvm.module.flags = !{!0}
+
+ !0 = !{i32 4, !"kcfi", i32 1}
+ !1 = !{i32 12345678}
+
+...
+---
+name: f1
+alignment: 4
+exposesReturnsTwice: false
+legalized: false
+regBankSelected: false
+selected: false
+failedISel: false
+tracksRegLiveness: true
+hasWinCFI: false
+callsEHReturn: false
+callsUnwindInit: false
+hasEHCatchret: false
+hasEHScopes: false
+hasEHFunclets: false
+isOutlined: false
+debugInstrRef: false
+failsVerification: false
+tracksDebugUserValues: false
+registers:
+ - { id: 0, class: gprjalr, preferred-register: '' }
+liveins:
+ - { reg: '$x10', virtual-reg: '%0' }
+frameInfo:
+ isFrameAddressTaken: false
+ isReturnAddressTaken: false
+ hasStackMap: false
+ hasPatchPoint: false
+ stackSize: 0
+ offsetAdjustment: 0
+ maxAlignment: 1
+ adjustsStack: false
+ hasCalls: true
+ stackProtector: ''
+ functionContext: ''
+ maxCallFrameSize: 4294967295
+ cvBytesOfCalleeSavedRegisters: 0
+ hasOpaqueSPAdjustment: false
+ hasVAStart: false
+ hasMustTailInVarArgFunc: false
+ hasTailCall: false
+ localFrameSize: 0
+ savePoint: ''
+ restorePoint: ''
+fixedStack: []
+stack: []
+callSites: []
+debugValueSubstitutions: []
+constants: []
+machineFunctionInfo:
+ varArgsFrameIndex: 0
+ varArgsSaveSize: 0
+body: |
+ bb.0 (%ir-block.0):
+ liveins: $x10
+
+ ; CHECK-LABEL: name: f1
+ ; CHECK: liveins: $x10, $x10, $x10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprjalr = COPY $x10
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gprjalr = COPY $x10
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gprjalr = COPY $x10
+ ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $x2, implicit $x2
+ ; CHECK-NEXT: PseudoCALLIndirect [[COPY2]], csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2, cfi-type 12345678
+ ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $x2, implicit $x2
+ ; CHECK-NEXT: PseudoRET
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: .1 (%ir-block.0):
+ ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $x2, implicit $x2
+ ; CHECK-NEXT: PseudoCALLIndirect [[COPY]], csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2, cfi-type 12345678
+ ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $x2, implicit $x2
+ ; CHECK-NEXT: PseudoRET
+ %0:gprjalr = COPY $x10
+ ADJCALLSTACKDOWN 0, 0, implicit-def dead $x2, implicit $x2
+ PseudoCALLIndirect %0, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2, cfi-type 12345678
+ ADJCALLSTACKUP 0, 0, implicit-def dead $x2, implicit $x2
+ PseudoRET
+
+...
+---
+name: f2
+alignment: 4
+exposesReturnsTwice: false
+legalized: false
+regBankSelected: false
+selected: false
+failedISel: false
+tracksRegLiveness: true
+hasWinCFI: false
+callsEHReturn: false
+callsUnwindInit: false
+hasEHCatchret: false
+hasEHScopes: false
+hasEHFunclets: false
+isOutlined: false
+debugInstrRef: false
+failsVerification: false
+tracksDebugUserValues: false
+registers:
+ - { id: 0, class: gprtc, preferred-register: '' }
+liveins:
+ - { reg: '$x10', virtual-reg: '%0' }
+frameInfo:
+ isFrameAddressTaken: false
+ isReturnAddressTaken: false
+ hasStackMap: false
+ hasPatchPoint: false
+ stackSize: 0
+ offsetAdjustment: 0
+ maxAlignment: 1
+ adjustsStack: false
+ hasCalls: false
+ stackProtector: ''
+ functionContext: ''
+ maxCallFrameSize: 4294967295
+ cvBytesOfCalleeSavedRegisters: 0
+ hasOpaqueSPAdjustment: false
+ hasVAStart: false
+ hasMustTailInVarArgFunc: false
+ hasTailCall: true
+ localFrameSize: 0
+ savePoint: ''
+ restorePoint: ''
+fixedStack: []
+stack: []
+callSites: []
+debugValueSubstitutions: []
+constants: []
+machineFunctionInfo:
+ varArgsFrameIndex: 0
+ varArgsSaveSize: 0
+body: |
+ bb.0 (%ir-block.0):
+ liveins: $x10
+
+ ; CHECK-LABEL: name: f2
+ ; CHECK: liveins: $x10, $x10, $x10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprtc = COPY $x10
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gprtc = COPY $x10
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gprtc = COPY $x10
+ ; CHECK-NEXT: PseudoTAILIndirect [[COPY2]], implicit $x2, cfi-type 12345678
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: .1 (%ir-block.0):
+ ; CHECK-NEXT: PseudoTAILIndirect [[COPY]], implicit $x2, cfi-type 12345678
+ %0:gprtc = COPY $x10
+ PseudoTAILIndirect %0, implicit $x2, cfi-type 12345678
+
+...
Index: llvm/test/CodeGen/RISCV/O3-pipeline.ll
===================================================================
--- llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -154,6 +154,7 @@
; CHECK-NEXT: Tail Duplication
; CHECK-NEXT: Machine Copy Propagation Pass
; CHECK-NEXT: Post-RA pseudo instruction expansion pass
+; CHECK-NEXT: Insert KCFI indirect call checks
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: Post RA top-down list latency scheduler
@@ -179,6 +180,7 @@
; CHECK-NEXT: RISC-V pseudo instruction expansion pass
; CHECK-NEXT: RISC-V insert NTLH instruction pass
; CHECK-NEXT: RISC-V atomic pseudo instruction expansion pass
+; CHECK-NEXT: Unpack machine instruction bundles
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: RISC-V Assembly Printer
Index: llvm/test/CodeGen/RISCV/O0-pipeline.ll
===================================================================
--- llvm/test/CodeGen/RISCV/O0-pipeline.ll
+++ llvm/test/CodeGen/RISCV/O0-pipeline.ll
@@ -50,6 +50,7 @@
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: Prologue/Epilogue Insertion & Frame Finalization
; CHECK-NEXT: Post-RA pseudo instruction expansion pass
+; CHECK-NEXT: Insert KCFI indirect call checks
; CHECK-NEXT: Analyze Machine Code For Garbage Collection
; CHECK-NEXT: Insert fentry calls
; CHECK-NEXT: Insert XRay ops
@@ -66,6 +67,7 @@
; CHECK-NEXT: RISC-V pseudo instruction expansion pass
; CHECK-NEXT: RISC-V insert NTLH instruction pass
; CHECK-NEXT: RISC-V atomic pseudo instruction expansion pass
+; CHECK-NEXT: Unpack machine instruction bundles
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: RISC-V Assembly Printer
Index: llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -87,6 +87,7 @@
initializeRISCVInsertVSETVLIPass(*PR);
initializeRISCVDAGToDAGISelPass(*PR);
initializeRISCVInitUndefPass(*PR);
+ initializeRISCVKCFIPass(*PR);
}
static StringRef computeDataLayout(const Triple &TT) {
@@ -332,7 +333,10 @@
return false;
}
-void RISCVPassConfig::addPreSched2() {}
+void RISCVPassConfig::addPreSched2() {
+ // Emit KCFI checks for indirect calls.
+ addPass(createRISCVKCFIPass());
+}
void RISCVPassConfig::addPreEmitPass() {
addPass(&BranchRelaxationPassID);
@@ -355,6 +359,11 @@
// possibility for other passes to break the requirements for forward
// progress in the LR/SC block.
addPass(createRISCVExpandAtomicPseudoPass());
+
+ // KCFI indirect call checks are lowered to a bundle.
+ addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
+ return MF.getFunction().getParent()->getModuleFlag("kcfi");
+ }));
}
void RISCVPassConfig::addMachineSSAOptimization() {
Index: llvm/lib/Target/RISCV/RISCVKCFI.cpp
===================================================================
--- /dev/null
+++ llvm/lib/Target/RISCV/RISCVKCFI.cpp
@@ -0,0 +1,113 @@
+//===---- RISCVKCFI.cpp - Implements KCFI ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements KCFI indirect call checking.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVInstrInfo.h"
+#include "RISCVSubtarget.h"
+#include "RISCVTargetMachine.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-kcfi"
+#define RISCV_KCFI_PASS_NAME "Insert KCFI indirect call checks"
+
+STATISTIC(NumKCFIChecksAdded, "Number of indirect call checks added");
+
+namespace {
+class RISCVKCFI : public MachineFunctionPass {
+public:
+ static char ID;
+
+ RISCVKCFI() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override { return RISCV_KCFI_PASS_NAME; }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+private:
+ /// Machine instruction info used throughout the class.
+ const RISCVInstrInfo *TII = nullptr;
+
+ /// Emits a KCFI check before an indirect call.
+ /// \returns true if the check was added and false otherwise.
+ bool emitCheck(MachineBasicBlock &MBB,
+ MachineBasicBlock::instr_iterator I) const;
+};
+
+char RISCVKCFI::ID = 0;
+} // end anonymous namespace
+
+INITIALIZE_PASS(RISCVKCFI, DEBUG_TYPE, RISCV_KCFI_PASS_NAME, false, false)
+
+FunctionPass *llvm::createRISCVKCFIPass() { return new RISCVKCFI(); }
+
+bool RISCVKCFI::emitCheck(MachineBasicBlock &MBB,
+ MachineBasicBlock::instr_iterator MBBI) const {
+ assert(TII && "Target instruction info was not initialized");
+
+ // If the call instruction is bundled, we can only emit a check safely if
+ // it's the first instruction in the bundle.
+ if (MBBI->isBundled() && !std::prev(MBBI)->isBundle())
+ report_fatal_error("Cannot emit a KCFI check for a bundled call");
+
+ switch (MBBI->getOpcode()) {
+ case RISCV::PseudoCALLIndirect:
+ case RISCV::PseudoTAILIndirect:
+ break;
+ default:
+ llvm_unreachable("Unexpected CFI call opcode");
+ }
+
+ MachineOperand &Target = MBBI->getOperand(0);
+ assert(Target.isReg() && "Invalid target operand for an indirect call");
+ Target.setIsRenamable(false);
+
+ MachineInstr *Check =
+ BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(RISCV::KCFI_CHECK))
+ .addReg(Target.getReg())
+ .addImm(MBBI->getCFIType())
+ .getInstr();
+ MBBI->setCFIType(*MBB.getParent(), 0);
+
+ // If not already bundled, bundle the check and the call to prevent
+ // further changes.
+ if (!MBBI->isBundled())
+ finalizeBundle(MBB, Check->getIterator(), std::next(MBBI->getIterator()));
+
+ ++NumKCFIChecksAdded;
+ return true;
+}
+
+bool RISCVKCFI::runOnMachineFunction(MachineFunction &MF) {
+ const Module *M = MF.getMMI().getModule();
+ if (!M->getModuleFlag("kcfi"))
+ return false;
+
+ const auto &SubTarget = MF.getSubtarget<RISCVSubtarget>();
+ TII = SubTarget.getInstrInfo();
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineBasicBlock::instr_iterator MII = MBB.instr_begin(),
+ MIE = MBB.instr_end();
+ MII != MIE; ++MII) {
+ if (MII->isCall() && MII->getCFIType())
+ Changed |= emitCheck(MBB, MII);
+ }
+ }
+
+ return Changed;
+}
Index: llvm/lib/Target/RISCV/RISCVInstrInfo.td
===================================================================
--- llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1891,6 +1891,13 @@
[(int_hwasan_check_memaccess_shortgranules X5, GPRJALR:$ptr,
(i32 timm:$accessinfo))]>;
+// This gets lowered into a 20-byte instruction sequence (at most)
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0,
+ Defs = [ X6, X7, X28, X29, X30, X31 ], Size = 20 in {
+def KCFI_CHECK
+ : Pseudo<(outs), (ins GPRJALR:$ptr, i32imm:$type), []>, Sched<[]>;
+}
+
/// Simple optimization
def : Pat<(add GPR:$rs1, (AddiPair:$rs2)),
(ADDI (ADDI GPR:$rs1, (AddiPairImmLarge AddiPair:$rs2)),
Index: llvm/lib/Target/RISCV/RISCVInstrInfo.h
===================================================================
--- llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -229,6 +229,9 @@
protected:
const RISCVSubtarget &STI;
+
+private:
+ unsigned getInstBundleLength(const MachineInstr &MI) const;
};
namespace RISCV {
Index: llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -1257,6 +1257,9 @@
}
}
+ if (Opcode == TargetOpcode::BUNDLE)
+ return getInstBundleLength(MI);
+
if (MI.getParent() && MI.getParent()->getParent()) {
if (isCompressibleInst(MI, STI))
return 2;
@@ -1264,6 +1267,17 @@
return get(Opcode).getSize();
}
+unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
+ unsigned Size = 0;
+ MachineBasicBlock::const_instr_iterator I = MI.getIterator();
+ MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
+ while (++I != E && I->isInsideBundle()) {
+ assert(!I->isBundle() && "No nested bundle!");
+ Size += getInstSizeInBytes(*I);
+ }
+ return Size;
+}
+
bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
const unsigned Opcode = MI.getOpcode();
switch (Opcode) {
Index: llvm/lib/Target/RISCV/RISCVISelLowering.h
===================================================================
--- llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -702,6 +702,8 @@
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
unsigned Factor) const override;
+ bool supportKCFIBundles() const override { return true; }
+
private:
/// RISCVCCAssignFn - This target-specific function extends the default
/// CCValAssign with additional information used to lower RISC-V calling
Index: llvm/lib/Target/RISCV/RISCVISelLowering.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13921,6 +13921,7 @@
SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
+ bool IsCFICall = CLI.CB && CLI.CB->isIndirectCall() && CLI.CFIType;
bool &IsTailCall = CLI.IsTailCall;
CallingConv::ID CallConv = CLI.CallConv;
bool IsVarArg = CLI.IsVarArg;
@@ -14166,10 +14167,15 @@
if (IsTailCall) {
MF.getFrameInfo().setHasTailCall();
- return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
+ SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
+ if (IsCFICall)
+ Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
+ return Ret;
}
Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
+ if (IsCFICall)
+ Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
Glue = Chain.getValue(1);
Index: llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
===================================================================
--- llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -18,6 +18,7 @@
#include "RISCVMachineFunctionInfo.h"
#include "RISCVTargetMachine.h"
#include "TargetInfo/RISCVTargetInfo.h"
+#include "llvm/ADT/APInt.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/AsmPrinter.h"
@@ -71,6 +72,7 @@
typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
+ void LowerKCFI_CHECK(const MachineInstr &MI);
void EmitHwasanMemaccessSymbols(Module &M);
// Wrapper needed for tblgenned pseudo lowering.
@@ -113,6 +115,9 @@
case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
LowerHWASAN_CHECK_MEMACCESS(*MI);
return;
+ case RISCV::KCFI_CHECK:
+ LowerKCFI_CHECK(*MI);
+ return;
case RISCV::PseudoRVVInitUndefM1:
case RISCV::PseudoRVVInitUndefM2:
case RISCV::PseudoRVVInitUndefM4:
@@ -268,6 +273,92 @@
EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
}
+void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
+ Register AddrReg = MI.getOperand(0).getReg();
+ assert(std::next(MI.getIterator())->isCall() &&
+ "KCFI_CHECK not followed by a call instruction");
+ assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
+ "KCFI_CHECK call target doesn't match call operand");
+
+ // Temporary registers for comparing comparing the hashes. If a register is
+ // used for the call target, or reserved by the user, we can clobber another
+ // temporary register as the check is immediately followed by the call. The
+ // check defaults to X6/X7, but can fall back to X28-X31 if needed.
+ unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
+ unsigned NextReg = RISCV::X28;
+ auto isRegAvailable = [&](unsigned Reg) {
+ return (Reg != AddrReg && !STI->isRegisterReservedByUser(Reg));
+ };
+ for (auto &Reg : ScratchRegs) {
+ if (isRegAvailable(Reg))
+ continue;
+ while (!isRegAvailable(NextReg))
+ ++NextReg;
+ Reg = NextReg++;
+ if (Reg > RISCV::X31)
+ report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
+ }
+ assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
+ "Invalid scratch registers for KCFI_CHECK");
+
+ if (AddrReg == RISCV::X0) {
+ // Checking X0 makes no sense. Instead of emitting a load, zero
+ // ScratchRegs[0].
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
+ .addReg(ScratchRegs[0])
+ .addReg(RISCV::X0)
+ .addImm(0));
+ } else {
+ // Adjust the offset for patchable-function-prefix. This assumes that
+ // patchable-function-prefix is the same for all functions.
+ int NopSize =
+ STI->getInstrInfo()->getNop().getOpcode() == RISCV::C_NOP ? 2 : 4;
+ int64_t PrefixNops = 0;
+ (void)MI.getMF()
+ ->getFunction()
+ .getFnAttribute("patchable-function-prefix")
+ .getValueAsString()
+ .getAsInteger(10, PrefixNops);
+
+ // Load the target function type hash.
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
+ .addReg(ScratchRegs[0])
+ .addReg(AddrReg)
+ .addImm(-(PrefixNops * NopSize + 4)));
+ }
+
+ // Load the expected type hash.
+ const int64_t Type = MI.getOperand(1).getImm();
+ const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
+ const int64_t Lo12 = SignExtend64<12>(Type);
+ if (Hi20)
+ EmitToStreamer(
+ *OutStreamer,
+ MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
+ if (Lo12 || Hi20 == 0)
+ EmitToStreamer(*OutStreamer,
+ MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20)
+ ? RISCV::ADDIW
+ : RISCV::ADDI)
+ .addReg(ScratchRegs[1])
+ .addReg(ScratchRegs[1])
+ .addImm(Lo12));
+
+ // Compare the hashes and trap if there's a mismatch.
+ MCSymbol *Pass = OutContext.createTempSymbol();
+ EmitToStreamer(*OutStreamer,
+ MCInstBuilder(RISCV::BEQ)
+ .addReg(ScratchRegs[0])
+ .addReg(ScratchRegs[1])
+ .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
+
+ MCSymbol *Trap = OutContext.createTempSymbol();
+ OutStreamer->emitLabel(Trap);
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
+ emitKCFITrapEntry(*MI.getMF(), Trap);
+ OutStreamer->emitLabel(Pass);
+}
+
void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
if (HwasanMemaccessSymbols.empty())
return;
Index: llvm/lib/Target/RISCV/RISCV.h
===================================================================
--- llvm/lib/Target/RISCV/RISCV.h
+++ llvm/lib/Target/RISCV/RISCV.h
@@ -75,6 +75,9 @@
void initializeRISCVInitUndefPass(PassRegistry &);
extern char &RISCVInitUndefID;
+FunctionPass *createRISCVKCFIPass();
+void initializeRISCVKCFIPass(PassRegistry &);
+
InstructionSelector *createRISCVInstructionSelector(const RISCVTargetMachine &,
RISCVSubtarget &,
RISCVRegisterBankInfo &);
Index: llvm/lib/Target/RISCV/CMakeLists.txt
===================================================================
--- llvm/lib/Target/RISCV/CMakeLists.txt
+++ llvm/lib/Target/RISCV/CMakeLists.txt
@@ -31,6 +31,7 @@
RISCVInstrInfo.cpp
RISCVISelDAGToDAG.cpp
RISCVISelLowering.cpp
+ RISCVKCFI.cpp
RISCVMachineFunctionInfo.cpp
RISCVMacroFusion.cpp
RISCVMCInstLower.cpp
Index: clang/lib/CodeGen/BackendUtil.cpp
===================================================================
--- clang/lib/CodeGen/BackendUtil.cpp
+++ clang/lib/CodeGen/BackendUtil.cpp
@@ -628,7 +628,7 @@
PassBuilder &PB) {
// If the back-end supports KCFI operand bundle lowering, skip KCFIPass.
if (TargetTriple.getArch() == llvm::Triple::x86_64 ||
- TargetTriple.isAArch64(64))
+ TargetTriple.isAArch64(64) || TargetTriple.isRISCV())
return;
// Ensure we lower KCFI operand bundles with -O0.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
