craig.topper updated this revision to Diff 257465. craig.topper added a comment.
Address some of the review comments. Primarily the ones in ImmutableGraph. I did de-templatize the method in X86LoadValueInjectionLoadHardening.cpp CHANGES SINCE LAST ACTION https://reviews.llvm.org/D75936/new/ https://reviews.llvm.org/D75936 Files: clang/include/clang/Driver/Options.td clang/lib/Driver/ToolChains/Arch/X86.cpp clang/test/Driver/x86-target-features.c llvm/lib/Target/X86/CMakeLists.txt llvm/lib/Target/X86/ImmutableGraph.h llvm/lib/Target/X86/X86.h llvm/lib/Target/X86/X86.td llvm/lib/Target/X86/X86ISelLowering.cpp llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp llvm/lib/Target/X86/X86Subtarget.h llvm/lib/Target/X86/X86TargetMachine.cpp llvm/test/CodeGen/X86/O0-pipeline.ll llvm/test/CodeGen/X86/O3-pipeline.ll llvm/test/CodeGen/X86/lvi-hardening-gadget-graph.ll
Index: llvm/test/CodeGen/X86/lvi-hardening-gadget-graph.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/lvi-hardening-gadget-graph.ll @@ -0,0 +1,129 @@ +; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown -x86-lvi-load-dot-verify -o %t < %s | FileCheck %s + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @test(i32* %untrusted_user_ptr, i32* %secret, i32 %secret_size) #0 { +entry: + %untrusted_user_ptr.addr = alloca i32*, align 8 + %secret.addr = alloca i32*, align 8 + %secret_size.addr = alloca i32, align 4 + %ret_val = alloca i32, align 4 + %i = alloca i32, align 4 + store i32* %untrusted_user_ptr, i32** %untrusted_user_ptr.addr, align 8 + store i32* %secret, i32** %secret.addr, align 8 + store i32 %secret_size, i32* %secret_size.addr, align 4 + store i32 0, i32* %ret_val, align 4 + call void @llvm.x86.sse2.lfence() + store i32 0, i32* %i, align 4 + br label %for.cond + +for.cond: ; preds = %for.inc, %entry + %0 = load i32, i32* %i, align 4 + %1 = load i32, i32* %secret_size.addr, align 4 + %cmp = icmp slt i32 %0, %1 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %2 = load i32, i32* %i, align 4 + %rem = srem i32 %2, 2 + %cmp1 = icmp eq i32 %rem, 0 + br i1 %cmp1, label %if.then, label %if.else + +if.then: ; preds = %for.body + %3 = load i32*, i32** %secret.addr, align 8 + %4 = load i32, i32* %ret_val, align 4 + %idxprom = sext i32 %4 to i64 + %arrayidx = getelementptr inbounds i32, i32* %3, i64 %idxprom + %5 = load i32, i32* %arrayidx, align 4 + %6 = load i32*, i32** %untrusted_user_ptr.addr, align 8 + store i32 %5, i32* %6, align 4 + br label %if.end + +if.else: ; preds = %for.body + %7 = load i32*, i32** %secret.addr, align 8 + %8 = load i32, i32* %ret_val, align 4 + %idxprom2 = sext i32 %8 to i64 + %arrayidx3 = getelementptr inbounds i32, i32* %7, i64 %idxprom2 + store i32 42, i32* %arrayidx3, align 4 + br label %if.end + +if.end: ; preds = %if.else, %if.then + %9 = load i32*, i32** %untrusted_user_ptr.addr, align 8 + %10 = load i32, i32* %9, align 4 + store i32 %10, i32* %ret_val, align 4 + br label %for.inc + +for.inc: ; preds = %if.end + %11 = load i32, i32* %i, align 4 + %inc = add nsw i32 %11, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + %12 = load i32, i32* %ret_val, align 4 + ret i32 %12 +} + +; CHECK: digraph "Speculative gadgets for \"test\" function" { +; CHECK-NEXT: label="Speculative gadgets for \"test\" function"; +; CHECK: Node0x{{[0-9a-f]+}} [shape=record,color = green,label="{LFENCE\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm %stack.4.i, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.i)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JCC_1 %bb.6, 13, implicit killed $eflags\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{CMP32rm killed renamable $eax, %stack.2.secret_size.addr, 1, $noreg, 0, $noreg, implicit-def $eflags :: (dereferenceable load 4 from %ir.secret_size.addr)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm %stack.4.i, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.i)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JCC_1 %bb.4, 5, implicit killed $eflags\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rax = MOV64rm %stack.1.secret.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.secret.addr)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm killed renamable $rax, 4, killed renamable $rcx, 0, $noreg :: (load 4 from %ir.arrayidx)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rcx = MOVSX64rm32 %stack.3.ret_val, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.ret_val)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rcx = MOV64rm %stack.0.untrusted_user_ptr.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.untrusted_user_ptr.addr)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{MOV32mr killed renamable $rcx, 1, $noreg, 0, $noreg, killed renamable $eax :: (store 4 into %ir.6)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rax = MOV64rm %stack.1.secret.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.secret.addr)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{MOV32mi killed renamable $rax, 4, killed renamable $rcx, 0, $noreg, 42 :: (store 4 into %ir.arrayidx3)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rcx = MOVSX64rm32 %stack.3.ret_val, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.ret_val)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rax = MOV64rm %stack.0.untrusted_user_ptr.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.untrusted_user_ptr.addr)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm killed renamable $rax, 1, $noreg, 0, $noreg :: (load 4 from %ir.9)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,color = blue,label="{ARGS}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{MOV64mr %stack.0.untrusted_user_ptr.addr, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %ir.untrusted_user_ptr.addr)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JMP_1 %bb.5\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JMP_1 %bb.1\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm %stack.3.ret_val, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.ret_val)\n}"]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0]; +; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{RET 0, $eax\n}"]; +; CHECK-NEXT: } + +; Function Attrs: nounwind +declare void @llvm.x86.sse2.lfence() #1 + +attributes #0 = { "target-features"="+lvi-cfi" + "target-features"="+lvi-load-hardening" } +attributes #1 = { nounwind } Index: llvm/test/CodeGen/X86/O3-pipeline.ll =================================================================== --- llvm/test/CodeGen/X86/O3-pipeline.ll +++ /dev/null @@ -1,197 +0,0 @@ -; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each -; pass. Ignore it with 'grep -v'. -; RUN: llc -mtriple=x86_64-- -O3 -debug-pass=Structure < %s -o /dev/null 2>&1 \ -; RUN: | grep -v 'Verify generated machine code' | FileCheck %s - -; REQUIRES: asserts - -; CHECK-LABEL: Pass Arguments: -; CHECK-NEXT: Target Library Information -; CHECK-NEXT: Target Pass Configuration -; CHECK-NEXT: Machine Module Information -; CHECK-NEXT: Target Transform Information -; CHECK-NEXT: Type-Based Alias Analysis -; CHECK-NEXT: Scoped NoAlias Alias Analysis -; CHECK-NEXT: Assumption Cache Tracker -; CHECK-NEXT: Profile summary info -; CHECK-NEXT: Create Garbage Collector Module Metadata -; CHECK-NEXT: Machine Branch Probability Analysis -; CHECK-NEXT: ModulePass Manager -; CHECK-NEXT: Pre-ISel Intrinsic Lowering -; CHECK-NEXT: FunctionPass Manager -; CHECK-NEXT: Expand Atomic instructions -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) -; CHECK-NEXT: Module Verifier -; CHECK-NEXT: Natural Loop Information -; CHECK-NEXT: Canonicalize natural loops -; CHECK-NEXT: Scalar Evolution Analysis -; CHECK-NEXT: Loop Pass Manager -; CHECK-NEXT: Induction Variable Users -; CHECK-NEXT: Loop Strength Reduction -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) -; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Merge contiguous icmps into a memcmp -; CHECK-NEXT: Natural Loop Information -; CHECK-NEXT: Lazy Branch Probability Analysis -; CHECK-NEXT: Lazy Block Frequency Analysis -; CHECK-NEXT: Expand memcmp() to load/stores -; CHECK-NEXT: Lower Garbage Collection Instructions -; CHECK-NEXT: Shadow Stack GC Lowering -; CHECK-NEXT: Lower constant intrinsics -; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Natural Loop Information -; CHECK-NEXT: Branch Probability Analysis -; CHECK-NEXT: Block Frequency Analysis -; CHECK-NEXT: Constant Hoisting -; CHECK-NEXT: Partially inline calls to library functions -; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) -; CHECK-NEXT: Scalarize Masked Memory Intrinsics -; CHECK-NEXT: Expand reduction intrinsics -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Interleaved Access Pass -; CHECK-NEXT: X86 Partial Reduction -; CHECK-NEXT: Expand indirectbr instructions -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Natural Loop Information -; CHECK-NEXT: CodeGen Prepare -; CHECK-NEXT: Rewrite Symbols -; CHECK-NEXT: FunctionPass Manager -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Exception handling preparation -; CHECK-NEXT: Safe Stack instrumentation pass -; CHECK-NEXT: Insert stack protectors -; CHECK-NEXT: Module Verifier -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) -; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Natural Loop Information -; CHECK-NEXT: Branch Probability Analysis -; CHECK-NEXT: Lazy Branch Probability Analysis -; CHECK-NEXT: Lazy Block Frequency Analysis -; CHECK-NEXT: X86 DAG->DAG Instruction Selection -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Local Dynamic TLS Access Clean-up -; CHECK-NEXT: X86 PIC Global Base Reg Initialization -; CHECK-NEXT: Finalize ISel and expand pseudo-instructions -; CHECK-NEXT: X86 Domain Reassignment Pass -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Early Tail Duplication -; CHECK-NEXT: Optimize machine instruction PHIs -; CHECK-NEXT: Slot index numbering -; CHECK-NEXT: Merge disjoint stack slots -; CHECK-NEXT: Local Stack Slot Allocation -; CHECK-NEXT: Remove dead machine instructions -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Machine Trace Metrics -; CHECK-NEXT: Early If-Conversion -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine InstCombiner -; CHECK-NEXT: X86 cmov Conversion -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Early Machine Loop Invariant Code Motion -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Block Frequency Analysis -; CHECK-NEXT: Machine Common Subexpression Elimination -; CHECK-NEXT: MachinePostDominator Tree Construction -; CHECK-NEXT: Machine code sinking -; CHECK-NEXT: Peephole Optimizations -; CHECK-NEXT: Remove dead machine instructions -; CHECK-NEXT: Live Range Shrink -; CHECK-NEXT: X86 Fixup SetCC -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: X86 LEA Optimize -; CHECK-NEXT: X86 Optimize Call Frame -; CHECK-NEXT: X86 Avoid Store Forwarding Block -; CHECK-NEXT: X86 speculative load hardening -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: X86 EFLAGS copy lowering -; CHECK-NEXT: X86 WinAlloca Expander -; CHECK-NEXT: Detect Dead Lanes -; CHECK-NEXT: Process Implicit Definitions -; CHECK-NEXT: Remove unreachable machine basic blocks -; CHECK-NEXT: Live Variable Analysis -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Eliminate PHI nodes for register allocation -; CHECK-NEXT: Two-Address instruction pass -; CHECK-NEXT: Slot index numbering -; CHECK-NEXT: Live Interval Analysis -; CHECK-NEXT: Simple Register Coalescing -; CHECK-NEXT: Rename Disconnected Subregister Components -; CHECK-NEXT: Machine Instruction Scheduler -; CHECK-NEXT: Machine Block Frequency Analysis -; CHECK-NEXT: Debug Variable Analysis -; CHECK-NEXT: Live Stack Slot Analysis -; CHECK-NEXT: Virtual Register Map -; CHECK-NEXT: Live Register Matrix -; CHECK-NEXT: Bundle Machine CFG Edges -; CHECK-NEXT: Spill Code Placement Analysis -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine Optimization Remark Emitter -; CHECK-NEXT: Greedy Register Allocator -; CHECK-NEXT: Virtual Register Rewriter -; CHECK-NEXT: Stack Slot Coloring -; CHECK-NEXT: Machine Copy Propagation Pass -; CHECK-NEXT: Machine Loop Invariant Code Motion -; CHECK-NEXT: Bundle Machine CFG Edges -; CHECK-NEXT: X86 FP Stackifier -; CHECK-NEXT: Fixup Statepoint Caller Saved -; CHECK-NEXT: PostRA Machine Sink -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Machine Block Frequency Analysis -; CHECK-NEXT: MachinePostDominator Tree Construction -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine Optimization Remark Emitter -; CHECK-NEXT: Shrink Wrapping analysis -; CHECK-NEXT: Prologue/Epilogue Insertion & Frame Finalization -; CHECK-NEXT: Control Flow Optimizer -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Tail Duplication -; CHECK-NEXT: Machine Copy Propagation Pass -; CHECK-NEXT: Post-RA pseudo instruction expansion pass -; CHECK-NEXT: X86 pseudo instruction expansion pass -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Post RA top-down list latency scheduler -; CHECK-NEXT: Analyze Machine Code For Garbage Collection -; CHECK-NEXT: Machine Block Frequency Analysis -; CHECK-NEXT: MachinePostDominator Tree Construction -; CHECK-NEXT: Branch Probability Basic Block Placement -; CHECK-NEXT: Insert fentry calls -; CHECK-NEXT: Insert XRay ops -; CHECK-NEXT: Implement the 'patchable-function' attribute -; CHECK-NEXT: ReachingDefAnalysis -; CHECK-NEXT: X86 Execution Dependency Fix -; CHECK-NEXT: BreakFalseDeps -; CHECK-NEXT: X86 Indirect Branch Tracking -; CHECK-NEXT: X86 vzeroupper inserter -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: X86 Byte/Word Instruction Fixup -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: X86 Atom pad short functions -; CHECK-NEXT: X86 LEA Fixup -; CHECK-NEXT: Compressing EVEX instrs to VEX encoding when possible -; CHECK-NEXT: X86 Discriminate Memory Operands -; CHECK-NEXT: X86 Insert Cache Prefetches -; CHECK-NEXT: X86 insert wait instruction -; CHECK-NEXT: Contiguously Lay Out Funclets -; CHECK-NEXT: StackMap Liveness Analysis -; CHECK-NEXT: Live DEBUG_VALUE analysis -; CHECK-NEXT: X86 Indirect Thunks -; CHECK-NEXT: Check CFA info and insert CFI instructions if needed -; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine Optimization Remark Emitter -; CHECK-NEXT: X86 Assembly Printer -; CHECK-NEXT: Free MachineFunction - -define void @f() { - ret void -} Index: llvm/test/CodeGen/X86/O0-pipeline.ll =================================================================== --- llvm/test/CodeGen/X86/O0-pipeline.ll +++ /dev/null @@ -1,86 +0,0 @@ -; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each -; pass. Ignore it with 'grep -v'. -; RUN: llc -mtriple=x86_64-- -O0 -debug-pass=Structure < %s -o /dev/null 2>&1 \ -; RUN: | grep -v 'Verify generated machine code' | FileCheck %s - -; REQUIRES: asserts - -; CHECK-LABEL: Pass Arguments: -; CHECK-NEXT: Target Library Information -; CHECK-NEXT: Target Pass Configuration -; CHECK-NEXT: Machine Module Information -; CHECK-NEXT: Target Transform Information -; CHECK-NEXT: Type-Based Alias Analysis -; CHECK-NEXT: Scoped NoAlias Alias Analysis -; CHECK-NEXT: Assumption Cache Tracker -; CHECK-NEXT: Create Garbage Collector Module Metadata -; CHECK-NEXT: Profile summary info -; CHECK-NEXT: Machine Branch Probability Analysis -; CHECK-NEXT: ModulePass Manager -; CHECK-NEXT: Pre-ISel Intrinsic Lowering -; CHECK-NEXT: FunctionPass Manager -; CHECK-NEXT: Expand Atomic instructions -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) -; CHECK-NEXT: Module Verifier -; CHECK-NEXT: Lower Garbage Collection Instructions -; CHECK-NEXT: Shadow Stack GC Lowering -; CHECK-NEXT: Lower constant intrinsics -; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) -; CHECK-NEXT: Scalarize Masked Memory Intrinsics -; CHECK-NEXT: Expand reduction intrinsics -; CHECK-NEXT: Expand indirectbr instructions -; CHECK-NEXT: Rewrite Symbols -; CHECK-NEXT: FunctionPass Manager -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Exception handling preparation -; CHECK-NEXT: Safe Stack instrumentation pass -; CHECK-NEXT: Insert stack protectors -; CHECK-NEXT: Module Verifier -; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Natural Loop Information -; CHECK-NEXT: Lazy Branch Probability Analysis -; CHECK-NEXT: Lazy Block Frequency Analysis -; CHECK-NEXT: X86 DAG->DAG Instruction Selection -; CHECK-NEXT: X86 PIC Global Base Reg Initialization -; CHECK-NEXT: Finalize ISel and expand pseudo-instructions -; CHECK-NEXT: Local Stack Slot Allocation -; CHECK-NEXT: X86 speculative load hardening -; CHECK-NEXT: MachineDominator Tree Construction -; CHECK-NEXT: X86 EFLAGS copy lowering -; CHECK-NEXT: X86 WinAlloca Expander -; CHECK-NEXT: Eliminate PHI nodes for register allocation -; CHECK-NEXT: Two-Address instruction pass -; CHECK-NEXT: Fast Register Allocator -; CHECK-NEXT: Bundle Machine CFG Edges -; CHECK-NEXT: X86 FP Stackifier -; CHECK-NEXT: Fixup Statepoint Caller Saved -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine Optimization Remark Emitter -; CHECK-NEXT: Prologue/Epilogue Insertion & Frame Finalization -; CHECK-NEXT: Post-RA pseudo instruction expansion pass -; CHECK-NEXT: X86 pseudo instruction expansion pass -; CHECK-NEXT: Analyze Machine Code For Garbage Collection -; CHECK-NEXT: Insert fentry calls -; CHECK-NEXT: Insert XRay ops -; CHECK-NEXT: Implement the 'patchable-function' attribute -; CHECK-NEXT: X86 Indirect Branch Tracking -; CHECK-NEXT: X86 vzeroupper inserter -; CHECK-NEXT: X86 Discriminate Memory Operands -; CHECK-NEXT: X86 Insert Cache Prefetches -; CHECK-NEXT: X86 insert wait instruction -; CHECK-NEXT: Contiguously Lay Out Funclets -; CHECK-NEXT: StackMap Liveness Analysis -; CHECK-NEXT: Live DEBUG_VALUE analysis -; CHECK-NEXT: X86 Indirect Thunks -; CHECK-NEXT: Check CFA info and insert CFI instructions if needed -; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine Optimization Remark Emitter -; CHECK-NEXT: X86 Assembly Printer -; CHECK-NEXT: Free MachineFunction - -define void @f() { - ret void -} Index: llvm/lib/Target/X86/X86TargetMachine.cpp =================================================================== --- llvm/lib/Target/X86/X86TargetMachine.cpp +++ llvm/lib/Target/X86/X86TargetMachine.cpp @@ -83,6 +83,7 @@ initializeX86SpeculativeLoadHardeningPassPass(PR); initializeX86FlagsCopyLoweringPassPass(PR); initializeX86CondBrFoldingPassPass(PR); + initializeX86LoadValueInjectionLoadHardeningPassPass(PR); initializeX86LoadValueInjectionRetHardeningPassPass(PR); initializeX86OptimizeLEAPassPass(PR); initializeX86PartialReductionPass(PR); @@ -494,6 +495,7 @@ void X86PassConfig::addPostRegAlloc() { addPass(createX86FloatingPointStackifierPass()); + addPass(createX86LoadValueInjectionLoadHardeningPass()); } void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); } Index: llvm/lib/Target/X86/X86Subtarget.h =================================================================== --- llvm/lib/Target/X86/X86Subtarget.h +++ llvm/lib/Target/X86/X86Subtarget.h @@ -437,6 +437,10 @@ /// POP+LFENCE+JMP sequence. bool UseLVIControlFlowIntegrity = false; + /// Insert LFENCE instructions to prevent data speculatively injected into + /// loads from being used maliciously. + bool UseLVILoadHardening = false; + /// Use software floating point for code generation. bool UseSoftFloat = false; @@ -739,6 +743,7 @@ bool preferMaskRegisters() const { return PreferMaskRegisters; } bool useGLMDivSqrtCosts() const { return UseGLMDivSqrtCosts; } bool useLVIControlFlowIntegrity() const { return UseLVIControlFlowIntegrity; } + bool useLVILoadHardening() const { return UseLVILoadHardening; } unsigned getPreferVectorWidth() const { return PreferVectorWidth; } unsigned getRequiredVectorWidth() const { return RequiredVectorWidth; } Index: llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp =================================================================== --- /dev/null +++ llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp @@ -0,0 +1,583 @@ +//==-- X86LoadValueInjectionLoadHardening.cpp - LVI load hardening for x86 --=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Description: This pass finds Load Value Injection (LVI) gadgets consisting +/// of a load from memory (i.e., SOURCE), and any operation that may transmit +/// the value loaded from memory over a covert channel, or use the value loaded +/// from memory to determine a branch/call target (i.e., SINK). +/// +//===----------------------------------------------------------------------===// + +#include "ImmutableGraph.h" +#include "X86.h" +#include "X86Subtarget.h" +#include "X86TargetMachine.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineDominanceFrontier.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RDFGraph.h" +#include "llvm/CodeGen/RDFLiveness.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/DOTGraphTraits.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/GraphWriter.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +#define PASS_KEY "x86-lvi-load" +#define DEBUG_TYPE PASS_KEY + +STATISTIC(NumFunctionsConsidered, "Number of functions analyzed"); +STATISTIC(NumFunctionsMitigated, "Number of functions for which mitigations " + "were deployed"); +STATISTIC(NumGadgets, "Number of LVI gadgets detected during analysis"); + +static cl::opt<bool> NoConditionalBranches( + PASS_KEY "-no-cbranch", + cl::desc("Don't treat conditional branches as disclosure gadgets. This " + "may improve performance, at the cost of security."), + cl::init(false), cl::Hidden); + +static cl::opt<bool> EmitDot( + PASS_KEY "-dot", + cl::desc( + "For each function, emit a dot graph depicting potential LVI gadgets"), + cl::init(false), cl::Hidden); + +static cl::opt<bool> EmitDotOnly( + PASS_KEY "-dot-only", + cl::desc("For each function, emit a dot graph depicting potential LVI " + "gadgets, and do not insert any fences"), + cl::init(false), cl::Hidden); + +static cl::opt<bool> EmitDotVerify( + PASS_KEY "-dot-verify", + cl::desc("For each function, emit a dot graph to stdout depicting " + "potential LVI gadgets, used for testing purposes only"), + cl::init(false), cl::Hidden); + +static cl::opt<bool> NoFixedLoads( + PASS_KEY "-no-fixed", + cl::desc("Don't mitigate RIP-relative or RSP-relative loads. This " + "may improve performance, at the cost of security."), + cl::init(false), cl::Hidden); + +namespace { + +struct MachineGadgetGraph : ImmutableGraph<MachineInstr *, int> { + static constexpr int GadgetEdgeSentinel = -1; + static constexpr MachineInstr *const ArgNodeSentinel = nullptr; + + using GraphT = ImmutableGraph<MachineInstr *, int>; + using Node = typename GraphT::Node; + using Edge = typename GraphT::Edge; + using size_type = typename GraphT::size_type; + MachineGadgetGraph(std::unique_ptr<Node[]> Nodes, + std::unique_ptr<Edge[]> Edges, size_type NodesSize, + size_type EdgesSize, int NumFences = 0, int NumGadgets = 0) + : GraphT(std::move(Nodes), std::move(Edges), NodesSize, EdgesSize), + NumFences(NumFences), NumGadgets(NumGadgets) {} + static inline bool isCFGEdge(const Edge &E) { + return E.getValue() != GadgetEdgeSentinel; + } + static inline bool isGadgetEdge(const Edge &E) { + return E.getValue() == GadgetEdgeSentinel; + } + int NumFences; + int NumGadgets; +}; + +class X86LoadValueInjectionLoadHardeningPass : public MachineFunctionPass { +public: + X86LoadValueInjectionLoadHardeningPass() : MachineFunctionPass(ID) {} + + StringRef getPassName() const override { + return "X86 Load Value Injection (LVI) Load Hardening"; + } + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnMachineFunction(MachineFunction &MF) override; + + static char ID; + +private: + using GraphBuilder = ImmutableGraphBuilder<MachineGadgetGraph>; + using EdgeSet = MachineGadgetGraph::EdgeSet; + using Gadget = std::pair<MachineInstr *, MachineInstr *>; + + const X86Subtarget *STI; + const TargetInstrInfo *TII; + const TargetRegisterInfo *TRI; + + int hardenLoads(MachineFunction &MF, bool Fixed) const; + std::unique_ptr<MachineGadgetGraph> + getGadgetGraph(MachineFunction &MF, const MachineLoopInfo &MLI, + const MachineDominatorTree &MDT, + const MachineDominanceFrontier &MDF, bool FixedLoads) const; + + bool instrUsesRegToAccessMemory(const MachineInstr &I, unsigned Reg) const; + bool instrUsesRegToBranch(const MachineInstr &I, unsigned Reg) const; + bool instrAccessesStackSlot(const MachineInstr &MI) const; + bool instrAccessesConstantPool(const MachineInstr &MI) const; + bool instrAccessesGOT(const MachineInstr &MI) const; + inline bool instrIsFixedAccess(const MachineInstr &MI) const { + return instrAccessesConstantPool(MI) || instrAccessesStackSlot(MI) || + instrAccessesGOT(MI); + } + inline bool isFence(const MachineInstr *MI) const { + return MI && (MI->getOpcode() == X86::LFENCE || + (STI->useLVIControlFlowIntegrity() && MI->isCall())); + } +}; + +} // end anonymous namespace + +namespace llvm { + +template <> +struct GraphTraits<MachineGadgetGraph *> + : GraphTraits<ImmutableGraph<MachineInstr *, int> *> {}; + +template <> +struct DOTGraphTraits<MachineGadgetGraph *> : DefaultDOTGraphTraits { + using GraphType = MachineGadgetGraph; + using Traits = llvm::GraphTraits<GraphType *>; + using NodeRef = typename Traits::NodeRef; + using EdgeRef = typename Traits::EdgeRef; + using ChildIteratorType = typename Traits::ChildIteratorType; + using ChildEdgeIteratorType = typename Traits::ChildEdgeIteratorType; + + DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} + + std::string getNodeLabel(NodeRef Node, GraphType *) { + if (Node->getValue() == MachineGadgetGraph::ArgNodeSentinel) + return "ARGS"; + + std::string Str; + raw_string_ostream OS(Str); + OS << *Node->getValue(); + return OS.str(); + } + + static std::string getNodeAttributes(NodeRef Node, GraphType *) { + MachineInstr *MI = Node->getValue(); + if (MI == MachineGadgetGraph::ArgNodeSentinel) + return "color = blue"; + if (MI->getOpcode() == X86::LFENCE) + return "color = green"; + return ""; + } + + static std::string getEdgeAttributes(NodeRef, ChildIteratorType E, + GraphType *) { + int EdgeVal = (*E.getCurrent()).getValue(); + return EdgeVal >= 0 ? "label = " + std::to_string(EdgeVal) + : "color = red, style = \"dashed\""; + } +}; + +} // end namespace llvm + +constexpr MachineInstr *MachineGadgetGraph::ArgNodeSentinel; +constexpr int MachineGadgetGraph::GadgetEdgeSentinel; + +char X86LoadValueInjectionLoadHardeningPass::ID = 0; + +void X86LoadValueInjectionLoadHardeningPass::getAnalysisUsage( + AnalysisUsage &AU) const { + MachineFunctionPass::getAnalysisUsage(AU); + AU.addRequired<MachineLoopInfo>(); + AU.addRequired<MachineDominatorTree>(); + AU.addRequired<MachineDominanceFrontier>(); + AU.setPreservesCFG(); +} + +bool X86LoadValueInjectionLoadHardeningPass::runOnMachineFunction( + MachineFunction &MF) { + LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName() + << " *****\n"); + STI = &MF.getSubtarget<X86Subtarget>(); + if (!STI->useLVILoadHardening()) + return false; + + // FIXME: support 32-bit + if (!STI->is64Bit()) { + report_fatal_error("LVI load hardening is only supported on 64-bit " + "targets."); + } + + // Don't skip functions with the "optnone" attr but participate in opt-bisect. + const Function &F = MF.getFunction(); + if (!F.hasOptNone() && skipFunction(F)) + return false; + + ++NumFunctionsConsidered; + TII = STI->getInstrInfo(); + TRI = STI->getRegisterInfo(); + LLVM_DEBUG(dbgs() << "Hardening data-dependent loads...\n"); + hardenLoads(MF, false); + LLVM_DEBUG(dbgs() << "Hardening data-dependent loads... Done\n"); + if (!NoFixedLoads) { + LLVM_DEBUG(dbgs() << "Hardening fixed loads...\n"); + hardenLoads(MF, true); + LLVM_DEBUG(dbgs() << "Hardening fixed loads... Done\n"); + } + return false; +} + +static void WriteGadgetGraph(raw_ostream &OS, MachineFunction &MF, + MachineGadgetGraph *G) { + WriteGraph(OS, G, /*ShortNames*/ false, + "Speculative gadgets for \"" + MF.getName() + "\" function"); +} + +// Apply the mitigation to `MF`, return the number of fences inserted. +// If `FixedLoads` is `true`, then the mitigation will be applied to fixed +// loads; otherwise, mitigation will be applied to non-fixed loads. +int X86LoadValueInjectionLoadHardeningPass::hardenLoads(MachineFunction &MF, + bool FixedLoads) const { + LLVM_DEBUG(dbgs() << "Building gadget graph...\n"); + const auto &MLI = getAnalysis<MachineLoopInfo>(); + const auto &MDT = getAnalysis<MachineDominatorTree>(); + const auto &MDF = getAnalysis<MachineDominanceFrontier>(); + std::unique_ptr<MachineGadgetGraph> Graph = + getGadgetGraph(MF, MLI, MDT, MDF, FixedLoads); + LLVM_DEBUG(dbgs() << "Building gadget graph... Done\n"); + if (Graph == nullptr) + return 0; // didn't find any gadgets + + if (EmitDotVerify) { + WriteGadgetGraph(outs(), MF, Graph.get()); + return 0; + } + + if (EmitDot || EmitDotOnly) { + LLVM_DEBUG(dbgs() << "Emitting gadget graph...\n"); + std::error_code FileError; + std::string FileName = "lvi."; + if (FixedLoads) + FileName += "fixed."; + FileName += MF.getName(); + FileName += ".dot"; + raw_fd_ostream FileOut(FileName, FileError); + if (FileError) + errs() << FileError.message(); + WriteGadgetGraph(FileOut, MF, Graph.get()); + FileOut.close(); + LLVM_DEBUG(dbgs() << "Emitting gadget graph... Done\n"); + if (EmitDotOnly) + return 0; + } + + return 0; +} + +std::unique_ptr<MachineGadgetGraph> +X86LoadValueInjectionLoadHardeningPass::getGadgetGraph( + MachineFunction &MF, const MachineLoopInfo &MLI, + const MachineDominatorTree &MDT, const MachineDominanceFrontier &MDF, + bool FixedLoads) const { + using namespace rdf; + + // Build the Register Dataflow Graph using the RDF framework + TargetOperandInfo TOI{*TII}; + DataFlowGraph DFG{MF, *TII, *TRI, MDT, MDF, TOI}; + DFG.build(); + Liveness L{MF.getRegInfo(), DFG}; + L.computePhiInfo(); + + GraphBuilder Builder; + using GraphIter = typename GraphBuilder::BuilderNodeRef; + DenseMap<MachineInstr *, GraphIter> NodeMap; + int FenceCount = 0; + auto MaybeAddNode = [&NodeMap, &Builder](MachineInstr *MI) { + auto Ref = NodeMap.find(MI); + if (Ref == NodeMap.end()) { + auto I = Builder.addVertex(MI); + NodeMap[MI] = I; + return std::pair<GraphIter, bool>{I, true}; + } + return std::pair<GraphIter, bool>{Ref->getSecond(), false}; + }; + + DenseSet<std::pair<GraphIter, GraphIter>> GadgetEdgeSet; + auto AnalyzeUse = [&](NodeAddr<UseNode *> Use, MachineInstr *MI) { + assert(!(Use.Addr->getFlags() & NodeAttrs::PhiRef)); + MachineOperand &UseMO = Use.Addr->getOp(); + MachineInstr &UseMI = *UseMO.getParent(); + assert(UseMO.isReg()); + // We naively assume that an instruction propagates any loaded Uses + // to all Defs, unless the instruction is a call + if (UseMI.isCall()) + return false; + if (instrUsesRegToAccessMemory(UseMI, UseMO.getReg()) || + (!NoConditionalBranches && + instrUsesRegToBranch(UseMI, UseMO.getReg()))) { // found a gadget! + // add the root of this chain + auto GadgetBegin = MaybeAddNode(MI); + // and the instruction that (transitively) discloses the root + auto GadgetEnd = MaybeAddNode(&UseMI); + if (GadgetEdgeSet.insert({GadgetBegin.first, GadgetEnd.first}).second) + Builder.addEdge(MachineGadgetGraph::GadgetEdgeSentinel, + GadgetBegin.first, GadgetEnd.first); + if (UseMI.mayLoad()) // FIXME: This should be more precise + return false; // stop traversing further uses of `Reg` + } + return true; + }; + + // Analyze all machine instructions to find gadgets and LFENCEs, adding + // each interesting value to `Nodes` + auto AnalyzeDef = [&](NodeAddr<DefNode *> Def) { + MachineInstr *MI = Def.Addr->getFlags() & NodeAttrs::PhiRef + ? MachineGadgetGraph::ArgNodeSentinel + : Def.Addr->getOp().getParent(); + SmallSet<NodeId, 8> NodesVisited; + std::function<void(NodeAddr<DefNode *>)> AnalyzeDefUseChain = + [&](NodeAddr<DefNode *> Def) { + if (Def.Addr->getAttrs() & NodeAttrs::Dead) + return; + RegisterRef DefReg = DFG.getPRI().normalize(Def.Addr->getRegRef(DFG)); + NodeList Uses; + for (auto UseID : L.getAllReachedUses(DefReg, Def)) { + auto Use = DFG.addr<UseNode *>(UseID); + if (Use.Addr->getFlags() & NodeAttrs::PhiRef) { // phi node + NodeAddr<PhiNode *> Phi = Use.Addr->getOwner(DFG); + for (auto I : L.getRealUses(Phi.Id)) { + if (DFG.getPRI().alias(RegisterRef(I.first), DefReg)) { + for (auto UA : I.second) { + auto PhiUse = DFG.addr<UseNode *>(UA.first); + Uses.push_back(PhiUse); + } + } + } + } else { // not a phi node + Uses.push_back(Use); + } + } + for (auto N : Uses) { + NodeAddr<UseNode *> Use{N}; + if (NodesVisited.insert(Use.Id).second && AnalyzeUse(Use, MI)) { + NodeAddr<InstrNode *> Owner{Use.Addr->getOwner(DFG)}; + NodeList Defs = Owner.Addr->members_if(DataFlowGraph::IsDef, DFG); + llvm::for_each(Defs, AnalyzeDefUseChain); + } + } + }; + AnalyzeDefUseChain(Def); + }; + + LLVM_DEBUG(dbgs() << "Analyzing def-use chains to find gadgets\n"); + // Analyze function arguments + if (!FixedLoads) { // only need to analyze function args once + NodeAddr<BlockNode *> EntryBlock = DFG.getFunc().Addr->getEntryBlock(DFG); + for (NodeAddr<PhiNode *> ArgPhi : + EntryBlock.Addr->members_if(DataFlowGraph::IsPhi, DFG)) { + NodeList Defs = ArgPhi.Addr->members_if(DataFlowGraph::IsDef, DFG); + llvm::for_each(Defs, AnalyzeDef); + } + } + // Analyze every instruction in MF + for (NodeAddr<BlockNode *> BA : DFG.getFunc().Addr->members(DFG)) { + for (NodeAddr<StmtNode *> SA : + BA.Addr->members_if(DataFlowGraph::IsCode<NodeAttrs::Stmt>, DFG)) { + MachineInstr *MI = SA.Addr->getCode(); + if (isFence(MI)) { + MaybeAddNode(MI); + ++FenceCount; + } else if (MI->mayLoad() && ((FixedLoads && instrIsFixedAccess(*MI)) || + (!FixedLoads && !instrIsFixedAccess(*MI)))) { + NodeList Defs = SA.Addr->members_if(DataFlowGraph::IsDef, DFG); + llvm::for_each(Defs, AnalyzeDef); + } + } + } + int GadgetCount = static_cast<int>(GadgetEdgeSet.size()); + LLVM_DEBUG(dbgs() << "Found " << FenceCount << " fences\n"); + LLVM_DEBUG(dbgs() << "Found " << GadgetCount << " gadgets\n"); + if (GadgetCount == 0) + return nullptr; + NumGadgets += GadgetCount; + + // Traverse CFG to build the rest of the graph + SmallSet<MachineBasicBlock *, 8> BlocksVisited; + std::function<void(MachineBasicBlock *, GraphIter, unsigned)> TraverseCFG = + [&](MachineBasicBlock *MBB, GraphIter GI, unsigned ParentDepth) { + unsigned LoopDepth = MLI.getLoopDepth(MBB); + if (!MBB->empty()) { + // Always add the first instruction in each block + auto NI = MBB->begin(); + auto BeginBB = MaybeAddNode(&*NI); + Builder.addEdge(ParentDepth, GI, BeginBB.first); + if (!BlocksVisited.insert(MBB).second) + return; + + // Add any instructions within the block that are gadget components + GI = BeginBB.first; + while (++NI != MBB->end()) { + auto Ref = NodeMap.find(&*NI); + if (Ref != NodeMap.end()) { + Builder.addEdge(LoopDepth, GI, Ref->getSecond()); + GI = Ref->getSecond(); + } + } + + // Always add the terminator instruction, if one exists + auto T = MBB->getFirstTerminator(); + if (T != MBB->end()) { + auto EndBB = MaybeAddNode(&*T); + if (EndBB.second) + Builder.addEdge(LoopDepth, GI, EndBB.first); + GI = EndBB.first; + } + } + for (MachineBasicBlock *Succ : MBB->successors()) + TraverseCFG(Succ, GI, LoopDepth); + }; + // ArgNodeSentinel is a pseudo-instruction that represents MF args in the + // GadgetGraph + GraphIter ArgNode = MaybeAddNode(MachineGadgetGraph::ArgNodeSentinel).first; + TraverseCFG(&MF.front(), ArgNode, 0); + std::unique_ptr<MachineGadgetGraph> G{Builder.get(FenceCount, GadgetCount)}; + LLVM_DEBUG(dbgs() << "Found " << G->nodes_size() << " nodes\n"); + return G; +} + +bool X86LoadValueInjectionLoadHardeningPass::instrUsesRegToAccessMemory( + const MachineInstr &MI, unsigned Reg) const { + if (!MI.mayLoadOrStore() || MI.getOpcode() == X86::MFENCE || + MI.getOpcode() == X86::SFENCE || MI.getOpcode() == X86::LFENCE) + return false; + + // FIXME: This does not handle pseudo loading instruction like TCRETURN* + const MCInstrDesc &Desc = MI.getDesc(); + int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); + if (MemRefBeginIdx < 0) { + LLVM_DEBUG(dbgs() << "Warning: unable to obtain memory operand for loading " + "instruction:\n"; + MI.print(dbgs()); dbgs() << '\n';); + return false; + } + MemRefBeginIdx += X86II::getOperandBias(Desc); + + const MachineOperand &BaseMO = + MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg); + const MachineOperand &IndexMO = + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg); + return (BaseMO.isReg() && BaseMO.getReg() != X86::NoRegister && + TRI->regsOverlap(BaseMO.getReg(), Reg)) || + (IndexMO.isReg() && IndexMO.getReg() != X86::NoRegister && + TRI->regsOverlap(IndexMO.getReg(), Reg)); +} + +bool X86LoadValueInjectionLoadHardeningPass::instrUsesRegToBranch( + const MachineInstr &MI, unsigned Reg) const { + if (!MI.isConditionalBranch()) + return false; + for (const MachineOperand &Use : MI.uses()) + if (Use.isReg() && Use.getReg() == Reg) + return true; + return false; +} + +static bool hasLoadFrom( const MachineInstr &MI, PseudoSourceValue::PSVKind K) { + for (auto &MMO : MI.memoperands()) { + const PseudoSourceValue *PSV = MMO->getPseudoValue(); + if (PSV && PSV->kind() == K && MMO->isLoad()) + return true; + } + return false; +} + +bool X86LoadValueInjectionLoadHardeningPass::instrAccessesStackSlot( + const MachineInstr &MI) const { + // Check the PSV first + if (hasLoadFrom(MI, PseudoSourceValue::PSVKind::FixedStack)) + return true; + // Some loads are not marked with a PSV, so we always need to double check + const MCInstrDesc &Desc = MI.getDesc(); + int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); + if (MemRefBeginIdx < 0) + return false; + MemRefBeginIdx += X86II::getOperandBias(Desc); + return MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).isFI() && + MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).isImm() && + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).isReg() && + MI.getOperand(MemRefBeginIdx + X86::AddrDisp).isImm() && + MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).getImm() == 1 && + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).getReg() == + X86::NoRegister && + MI.getOperand(MemRefBeginIdx + X86::AddrDisp).getImm() == 0; +} + +bool X86LoadValueInjectionLoadHardeningPass::instrAccessesConstantPool( + const MachineInstr &MI) const { + if (hasLoadFrom(MI, PseudoSourceValue::PSVKind::ConstantPool)) + return true; + const MCInstrDesc &Desc = MI.getDesc(); + int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); + if (MemRefBeginIdx < 0) + return false; + MemRefBeginIdx += X86II::getOperandBias(Desc); + return MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).isReg() && + MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).isImm() && + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).isReg() && + MI.getOperand(MemRefBeginIdx + X86::AddrDisp).isCPI() && + (MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).getReg() == + X86::RIP || + MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).getReg() == + X86::NoRegister) && + MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).getImm() == 1 && + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).getReg() == + X86::NoRegister; +} + +bool X86LoadValueInjectionLoadHardeningPass::instrAccessesGOT( + const MachineInstr &MI) const { + if (hasLoadFrom(MI, PseudoSourceValue::PSVKind::GOT)) + return true; + const MCInstrDesc &Desc = MI.getDesc(); + int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); + if (MemRefBeginIdx < 0) + return false; + MemRefBeginIdx += X86II::getOperandBias(Desc); + return MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).isReg() && + MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).isImm() && + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).isReg() && + MI.getOperand(MemRefBeginIdx + X86::AddrDisp).getTargetFlags() == + X86II::MO_GOTPCREL && + MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).getReg() == + X86::RIP && + MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).getImm() == 1 && + MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).getReg() == + X86::NoRegister; +} + +INITIALIZE_PASS_BEGIN(X86LoadValueInjectionLoadHardeningPass, PASS_KEY, + "X86 LVI load hardening", false, false) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) +INITIALIZE_PASS_DEPENDENCY(MachineDominanceFrontier) +INITIALIZE_PASS_END(X86LoadValueInjectionLoadHardeningPass, PASS_KEY, + "X86 LVI load hardening", false, false) + +FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() { + return new X86LoadValueInjectionLoadHardeningPass(); +} Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -8623,6 +8623,7 @@ unsigned NumElts = VT.getVectorNumElements(); unsigned NumUndefElts = UndefElements.count(); if (!Ld || (NumElts - NumUndefElts) <= 1) { + dbgs() << "ctopper1\n"; APInt SplatValue, Undef; unsigned SplatBitSize; bool HasUndef; @@ -8630,15 +8631,18 @@ if (BVOp->isConstantSplat(SplatValue, Undef, SplatBitSize, HasUndef) && SplatBitSize > VT.getScalarSizeInBits() && SplatBitSize < VT.getSizeInBits()) { + dbgs() << "ctopper2\n"; // Avoid replacing with broadcast when it's a use of a shuffle // instruction to preserve the present custom lowering of shuffles. if (isFoldableUseOfShuffle(BVOp)) return SDValue(); + dbgs() << "ctopper3\n"; // replace BUILD_VECTOR with broadcast of the repeated constants. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); LLVMContext *Ctx = DAG.getContext(); MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); if (Subtarget.hasAVX()) { + dbgs() << "ctopper4\n"; if (SplatBitSize == 32 || SplatBitSize == 64 || (SplatBitSize < 32 && Subtarget.hasAVX2())) { // Splatted value can fit in one INTEGER constant in constant pool. Index: llvm/lib/Target/X86/X86.td =================================================================== --- llvm/lib/Target/X86/X86.td +++ llvm/lib/Target/X86/X86.td @@ -444,6 +444,13 @@ "LFENCE instruction to serialize control flow. Also decompose RET " "instructions into a POP+LFENCE+JMP sequence.">; +// Mitigate LVI attacks against data loads +def FeatureLVILoadHardening + : SubtargetFeature< + "lvi-load-hardening", "UseLVILoadHardening", "true", + "Insert LFENCE instructions to prevent data speculatively injected " + "into loads from being used maliciously.">; + // Direct Move instructions. def FeatureMOVDIRI : SubtargetFeature<"movdiri", "HasMOVDIRI", "true", "Support movdiri instruction">; Index: llvm/lib/Target/X86/X86.h =================================================================== --- llvm/lib/Target/X86/X86.h +++ llvm/lib/Target/X86/X86.h @@ -142,6 +142,7 @@ X86Subtarget &, X86RegisterBankInfo &); +FunctionPass *createX86LoadValueInjectionLoadHardeningPass(); FunctionPass *createX86LoadValueInjectionRetHardeningPass(); FunctionPass *createX86SpeculativeLoadHardeningPass(); @@ -159,6 +160,7 @@ void initializeX86ExecutionDomainFixPass(PassRegistry &); void initializeX86ExpandPseudoPass(PassRegistry &); void initializeX86FlagsCopyLoweringPassPass(PassRegistry &); +void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &); void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &); void initializeX86OptimizeLEAPassPass(PassRegistry &); void initializeX86PartialReductionPass(PassRegistry &); Index: llvm/lib/Target/X86/ImmutableGraph.h =================================================================== --- /dev/null +++ llvm/lib/Target/X86/ImmutableGraph.h @@ -0,0 +1,446 @@ +//==========-- ImmutableGraph.h - A fast DAG implementation ---------=========// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Description: ImmutableGraph is a fast DAG implementation that cannot be +/// modified, except by creating a new ImmutableGraph. ImmutableGraph is +/// implemented as two arrays: one containing nodes, and one containing edges. +/// The advantages to this implementation are two-fold: +/// 1. Iteration and traversal operations benefit from cache locality. +/// 2. Operations on sets of nodes/edges are efficient, and representations of +/// those sets in memory are compact. For instance, a set of edges is +/// implemented as a bit vector, wherein each bit corresponds to one edge in +/// the edge array. This implies a lower bound of 64x spatial improvement +/// over, e.g., an llvm::DenseSet or llvm::SmallSet. It also means that +/// insert/erase/contains operations complete in negligible constant time: +/// insert and erase require one load and one store, and contains requires +/// just one load. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_X86_IMMUTABLEGRAPH_H +#define LLVM_LIB_TARGET_X86_IMMUTABLEGRAPH_H + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <iterator> +#include <utility> +#include <vector> + +namespace llvm { + +template <typename NodeValueT, typename EdgeValueT> class ImmutableGraph { + using Traits = GraphTraits<ImmutableGraph<NodeValueT, EdgeValueT> *>; + template <typename> friend class ImmutableGraphBuilder; + +public: + using node_value_type = NodeValueT; + using edge_value_type = EdgeValueT; + using size_type = int; + class Node; + class Edge { + friend class ImmutableGraph; + template <typename> friend class ImmutableGraphBuilder; + + const Node *Dest; + edge_value_type Value; + + public: + const Node *getDest() const { return Dest; }; + const edge_value_type &getValue() const { return Value; } + }; + class Node { + friend class ImmutableGraph; + template <typename> friend class ImmutableGraphBuilder; + + const Edge *Edges; + node_value_type Value; + + public: + const node_value_type &getValue() const { return Value; } + + const Edge *edges_begin() const { return Edges; } + // Nodes are allocated sequentially. Edges for a node are stored together. + // The end of this Node's edges is the beginning of the next node's edges. + // An extra node was allocated to hold the end pointer for the last real + // node. + const Edge *edges_end() const { return (this + 1)->Edges; } + ArrayRef<Edge> edges() const { + return makeArrayRef(edges_begin(), edges_end()); + } + }; + +protected: + ImmutableGraph(std::unique_ptr<Node[]> Nodes, std::unique_ptr<Edge[]> Edges, + size_type NodesSize, size_type EdgesSize) + : Nodes(std::move(Nodes)), Edges(std::move(Edges)), NodesSize(NodesSize), + EdgesSize(EdgesSize) {} + ImmutableGraph(const ImmutableGraph &) = delete; + ImmutableGraph(ImmutableGraph &&) = delete; + ImmutableGraph &operator=(const ImmutableGraph &) = delete; + ImmutableGraph &operator=(ImmutableGraph &&) = delete; + +public: + ArrayRef<Node> nodes() const { return makeArrayRef(Nodes.get(), NodesSize); } + const Node *nodes_begin() const { return nodes().begin(); } + const Node *nodes_end() const { return nodes().end(); } + + ArrayRef<Edge> edges() const { return makeArrayRef(Edges.get(), EdgesSize); } + const Edge *edges_begin() const { return edges().begin(); } + const Edge *edges_end() const { return edges().end(); } + + size_type nodes_size() const { return NodesSize; } + size_type edges_size() const { return EdgesSize; } + + // Node N must belong to this ImmutableGraph. + size_type getNodeIndex(const Node &N) const { + return std::distance(nodes_begin(), &N); + } + // Edge E must belong to this ImmutableGraph. + size_type getEdgeIndex(const Edge &E) const { + return std::distance(edges_begin(), &E); + } + + // FIXME: Could NodeSet and EdgeSet be templated to share code? + class NodeSet { + const ImmutableGraph &G; + BitVector V; + + public: + NodeSet(const ImmutableGraph &G, bool ContainsAll = false) + : G{G}, V{static_cast<unsigned>(G.nodes_size()), ContainsAll} {} + bool insert(const Node &N) { + size_type Idx = G.getNodeIndex(N); + bool AlreadyExists = V.test(Idx); + V.set(Idx); + return !AlreadyExists; + } + void erase(const Node &N) { + size_type Idx = G.getNodeIndex(N); + V.reset(Idx); + } + bool contains(const Node &N) const { + size_type Idx = G.getNodeIndex(N); + return V.test(Idx); + } + void clear() { V.reset(); } + size_type empty() const { return V.none(); } + /// Return the number of elements in the set + size_type count() const { return V.count(); } + /// Return the size of the set's domain + size_type size() const { return V.size(); } + /// Set union + NodeSet &operator|=(const NodeSet &RHS) { + assert(&this->G == &RHS.G); + V |= RHS.V; + return *this; + } + /// Set intersection + NodeSet &operator&=(const NodeSet &RHS) { + assert(&this->G == &RHS.G); + V &= RHS.V; + return *this; + } + /// Set disjoint union + NodeSet &operator^=(const NodeSet &RHS) { + assert(&this->G == &RHS.G); + V ^= RHS.V; + return *this; + } + + using index_iterator = typename BitVector::const_set_bits_iterator; + index_iterator index_begin() const { return V.set_bits_begin(); } + index_iterator index_end() const { return V.set_bits_end(); } + void set(size_type Idx) { V.set(Idx); } + void reset(size_type Idx) { V.reset(Idx); } + + class iterator { + const NodeSet &Set; + size_type Current; + + void advance() { + assert(Current != -1); + Current = Set.V.find_next(Current); + } + + public: + iterator(const NodeSet &Set, size_type Begin) + : Set{Set}, Current{Begin} {} + iterator operator++(int) { + iterator Tmp = *this; + advance(); + return Tmp; + } + iterator &operator++() { + advance(); + return *this; + } + Node *operator*() const { + assert(Current != -1); + return Set.G.nodes_begin() + Current; + } + bool operator==(const iterator &other) const { + assert(&this->Set == &other.Set); + return this->Current == other.Current; + } + bool operator!=(const iterator &other) const { return !(*this == other); } + }; + + iterator begin() const { return iterator{*this, V.find_first()}; } + iterator end() const { return iterator{*this, -1}; } + }; + + class EdgeSet { + const ImmutableGraph &G; + BitVector V; + + public: + EdgeSet(const ImmutableGraph &G, bool ContainsAll = false) + : G{G}, V{static_cast<unsigned>(G.edges_size()), ContainsAll} {} + bool insert(const Edge &E) { + size_type Idx = G.getEdgeIndex(E); + bool AlreadyExists = V.test(Idx); + V.set(Idx); + return !AlreadyExists; + } + void erase(const Edge &E) { + size_type Idx = G.getEdgeIndex(E); + V.reset(Idx); + } + bool contains(const Edge &E) const { + size_type Idx = G.getEdgeIndex(E); + return V.test(Idx); + } + void clear() { V.reset(); } + bool empty() const { return V.none(); } + /// Return the number of elements in the set + size_type count() const { return V.count(); } + /// Return the size of the set's domain + size_type size() const { return V.size(); } + /// Set union + EdgeSet &operator|=(const EdgeSet &RHS) { + assert(&this->G == &RHS.G); + V |= RHS.V; + return *this; + } + /// Set intersection + EdgeSet &operator&=(const EdgeSet &RHS) { + assert(&this->G == &RHS.G); + V &= RHS.V; + return *this; + } + /// Set disjoint union + EdgeSet &operator^=(const EdgeSet &RHS) { + assert(&this->G == &RHS.G); + V ^= RHS.V; + return *this; + } + + using index_iterator = typename BitVector::const_set_bits_iterator; + index_iterator index_begin() const { return V.set_bits_begin(); } + index_iterator index_end() const { return V.set_bits_end(); } + void set(size_type Idx) { V.set(Idx); } + void reset(size_type Idx) { V.reset(Idx); } + + class iterator { + const EdgeSet &Set; + size_type Current; + + void advance() { + assert(Current != -1); + Current = Set.V.find_next(Current); + } + + public: + iterator(const EdgeSet &Set, size_type Begin) + : Set{Set}, Current{Begin} {} + iterator operator++(int) { + iterator Tmp = *this; + advance(); + return Tmp; + } + iterator &operator++() { + advance(); + return *this; + } + Edge *operator*() const { + assert(Current != -1); + return Set.G.edges_begin() + Current; + } + bool operator==(const iterator &other) const { + assert(&this->Set == &other.Set); + return this->Current == other.Current; + } + bool operator!=(const iterator &other) const { return !(*this == other); } + }; + + iterator begin() const { return iterator{*this, V.find_first()}; } + iterator end() const { return iterator{*this, -1}; } + }; + +private: + std::unique_ptr<Node[]> Nodes; + std::unique_ptr<Edge[]> Edges; + size_type NodesSize; + size_type EdgesSize; +}; + +template <typename GraphT> class ImmutableGraphBuilder { + using node_value_type = typename GraphT::node_value_type; + using edge_value_type = typename GraphT::edge_value_type; + static_assert( + std::is_base_of<ImmutableGraph<node_value_type, edge_value_type>, + GraphT>::value, + "Template argument to ImmutableGraphBuilder must derive from " + "ImmutableGraph<>"); + using size_type = typename GraphT::size_type; + using NodeSet = typename GraphT::NodeSet; + using Node = typename GraphT::Node; + using EdgeSet = typename GraphT::EdgeSet; + using Edge = typename GraphT::Edge; + using BuilderEdge = std::pair<edge_value_type, size_type>; + using EdgeList = std::vector<BuilderEdge>; + using BuilderVertex = std::pair<node_value_type, EdgeList>; + using VertexVec = std::vector<BuilderVertex>; + +public: + using BuilderNodeRef = size_type; + + BuilderNodeRef addVertex(const node_value_type &V) { + auto I = AdjList.emplace(AdjList.end(), V, EdgeList{}); + return std::distance(AdjList.begin(), I); + } + + void addEdge(const edge_value_type &E, BuilderNodeRef From, + BuilderNodeRef To) { + AdjList[From].second.emplace_back(E, To); + } + + bool empty() const { return AdjList.empty(); } + + template <typename... ArgT> std::unique_ptr<GraphT> get(ArgT &&... Args) { + size_type VertexSize = AdjList.size(), EdgeSize = 0; + for (const auto &V : AdjList) { + EdgeSize += V.second.size(); + } + auto VertexArray = + std::make_unique<Node[]>(VertexSize + 1 /* terminator node */); + auto EdgeArray = std::make_unique<Edge[]>(EdgeSize); + size_type VI = 0, EI = 0; + for (; VI < VertexSize; ++VI) { + VertexArray[VI].Value = std::move(AdjList[VI].first); + VertexArray[VI].Edges = &EdgeArray[EI]; + auto NumEdges = static_cast<size_type>(AdjList[VI].second.size()); + for (size_type VEI = 0; VEI < NumEdges; ++VEI, ++EI) { + auto &E = AdjList[VI].second[VEI]; + EdgeArray[EI].Value = std::move(E.first); + EdgeArray[EI].Dest = &VertexArray[E.second]; + } + } + assert(VI == VertexSize && EI == EdgeSize && "ImmutableGraph malformed"); + VertexArray[VI].Edges = &EdgeArray[EdgeSize]; // terminator node + return std::make_unique<GraphT>(std::move(VertexArray), + std::move(EdgeArray), VertexSize, EdgeSize, + std::forward<ArgT>(Args)...); + } + + template <typename... ArgT> + static std::unique_ptr<GraphT> trim(const GraphT &G, const NodeSet &TrimNodes, + const EdgeSet &TrimEdges, + ArgT &&... Args) { + size_type NewVertexSize = G.nodes_size() - TrimNodes.count(); + size_type NewEdgeSize = G.edges_size() - TrimEdges.count(); + auto NewVertexArray = + std::make_unique<Node[]>(NewVertexSize + 1 /* terminator node */); + auto NewEdgeArray = std::make_unique<Edge[]>(NewEdgeSize); + + // Walk the nodes and determine the new index for each node. + size_type NewNodeIndex = 0; + std::vector<size_type> RemappedNodeIndex(G.nodes_size()); + for (const Node &N : G.nodes()) { + if (TrimNodes.contains(N)) + continue; + RemappedNodeIndex[G.getNodeIndex(N)] = NewNodeIndex++; + } + assert(NewNodeIndex == NewVertexSize && + "Should have assigned NewVertexSize indices"); + + size_type VertexI = 0, EdgeI = 0; + for (const Node &N : G.nodes()) { + if (TrimNodes.contains(N)) + continue; + NewVertexArray[VertexI].Value = N.getValue(); + NewVertexArray[VertexI].Edges = &NewEdgeArray[EdgeI]; + for (const Edge &E : N.edges()) { + if (TrimEdges.contains(E)) + continue; + NewEdgeArray[EdgeI].Value = E.getValue(); + size_type DestIdx = G.getNodeIndex(*E.getDest()); + size_type NewIdx = RemappedNodeIndex[DestIdx]; + assert(NewIdx < NewVertexSize); + NewEdgeArray[EdgeI].Dest = &NewVertexArray[NewIdx]; + ++EdgeI; + } + ++VertexI; + } + assert(VertexI == NewVertexSize && EdgeI == NewEdgeSize && + "Gadget graph malformed"); + NewVertexArray[VertexI].Edges = &NewEdgeArray[NewEdgeSize]; // terminator + return std::make_unique<GraphT>(std::move(NewVertexArray), + std::move(NewEdgeArray), NewVertexSize, + NewEdgeSize, std::forward<ArgT>(Args)...); + } + +private: + VertexVec AdjList; +}; + +template <typename NodeValueT, typename EdgeValueT> +struct GraphTraits<ImmutableGraph<NodeValueT, EdgeValueT> *> { + using GraphT = ImmutableGraph<NodeValueT, EdgeValueT>; + using NodeRef = typename GraphT::Node const *; + using EdgeRef = typename GraphT::Edge const &; + + static NodeRef edge_dest(EdgeRef E) { return E.getDest(); } + using ChildIteratorType = + mapped_iterator<typename GraphT::Edge const *, decltype(&edge_dest)>; + + static NodeRef getEntryNode(GraphT *G) { return G->nodes_begin(); } + static ChildIteratorType child_begin(NodeRef N) { + return {N->edges_begin(), &edge_dest}; + } + static ChildIteratorType child_end(NodeRef N) { + return {N->edges_end(), &edge_dest}; + } + + static NodeRef getNode(typename GraphT::Node const &N) { return NodeRef{&N}; } + using nodes_iterator = + mapped_iterator<typename GraphT::Node const *, decltype(&getNode)>; + static nodes_iterator nodes_begin(GraphT *G) { + return {G->nodes_begin(), &getNode}; + } + static nodes_iterator nodes_end(GraphT *G) { + return {G->nodes_end(), &getNode}; + } + + using ChildEdgeIteratorType = typename GraphT::Edge const *; + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->edges_begin(); + } + static ChildEdgeIteratorType child_edge_end(NodeRef N) { + return N->edges_end(); + } + static typename GraphT::size_type size(GraphT *G) { return G->nodes_size(); } +}; + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_X86_IMMUTABLEGRAPH_H Index: llvm/lib/Target/X86/CMakeLists.txt =================================================================== --- llvm/lib/Target/X86/CMakeLists.txt +++ llvm/lib/Target/X86/CMakeLists.txt @@ -52,6 +52,7 @@ X86InstrInfo.cpp X86EvexToVex.cpp X86LegalizerInfo.cpp + X86LoadValueInjectionLoadHardening.cpp X86LoadValueInjectionRetHardening.cpp X86MCInstLower.cpp X86MachineFunctionInfo.cpp Index: clang/test/Driver/x86-target-features.c =================================================================== --- clang/test/Driver/x86-target-features.c +++ clang/test/Driver/x86-target-features.c @@ -166,6 +166,11 @@ // RUN: %clang -target i386-linux-gnu -mlvi-cfi -mretpoline-external-thunk %s -### -o %t.o 2>&1 | FileCheck -check-prefix=LVICFI-RETPOLINE-EXTERNAL-THUNK %s // LVICFI-RETPOLINE-EXTERNAL-THUNK: error: invalid argument 'mretpoline-external-thunk' not allowed with 'mlvi-cfi' +// RUN: %clang -target i386-linux-gnu -mlvi-hardening %s -### -o %t.o 2>&1 | FileCheck -check-prefix=LVIHARDENING %s +// RUN: %clang -target i386-linux-gnu -mno-lvi-hardening %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-LVIHARDENING %s +// LVIHARDENING: "-target-feature" "+lvi-load-hardening" "-target-feature" "+lvi-cfi" +// NO-LVIHARDENING-NOT: lvi + // RUN: %clang -target i386-linux-gnu -mwaitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=WAITPKG %s // RUN: %clang -target i386-linux-gnu -mno-waitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-WAITPKG %s // WAITPKG: "-target-feature" "+waitpkg" Index: clang/lib/Driver/ToolChains/Arch/X86.cpp =================================================================== --- clang/lib/Driver/ToolChains/Arch/X86.cpp +++ clang/lib/Driver/ToolChains/Arch/X86.cpp @@ -173,7 +173,13 @@ } auto LVIOpt = clang::driver::options::ID::OPT_INVALID; - if (Args.hasFlag(options::OPT_mlvi_cfi, options::OPT_mno_lvi_cfi, false)) { + if (Args.hasFlag(options::OPT_mlvi_hardening, options::OPT_mno_lvi_hardening, + false)) { + Features.push_back("+lvi-load-hardening"); + Features.push_back("+lvi-cfi"); // load hardening implies CFI protection + LVIOpt = options::OPT_mlvi_hardening; + } else if (Args.hasFlag(options::OPT_mlvi_cfi, options::OPT_mno_lvi_cfi, + false)) { Features.push_back("+lvi-cfi"); LVIOpt = options::OPT_mlvi_cfi; } Index: clang/include/clang/Driver/Options.td =================================================================== --- clang/include/clang/Driver/Options.td +++ clang/include/clang/Driver/Options.td @@ -2318,6 +2318,10 @@ Group<m_Group>, Flags<[CoreOption,CC1Option]>; def mno_speculative_load_hardening : Flag<["-"], "mno-speculative-load-hardening">, Group<m_Group>, Flags<[CoreOption]>; +def mlvi_hardening : Flag<["-"], "mlvi-hardening">, Group<m_Group>, Flags<[CoreOption,DriverOption]>, + HelpText<"Enable all mitigations for Load Value Injection (LVI)">; +def mno_lvi_hardening : Flag<["-"], "mno-lvi-hardening">, Group<m_Group>, Flags<[CoreOption,DriverOption]>, + HelpText<"Disable mitigations for Load Value Injection (LVI)">; def mlvi_cfi : Flag<["-"], "mlvi-cfi">, Group<m_Group>, Flags<[CoreOption,DriverOption]>, HelpText<"Enable only control-flow mitigations for Load Value Injection (LVI)">; def mno_lvi_cfi : Flag<["-"], "mno-lvi-cfi">, Group<m_Group>, Flags<[CoreOption,DriverOption]>,
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits