https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/235635
>From 2796e585d3eb7a9cefa6a4f5db5c3121144c5fb3 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Mon, 26 Oct 2026 00:21:36 -0700 Subject: [PATCH] [CodeGen] Don't fold a load into an inline asm that may overwrite it The peephole optimizer folds a load from a stack slot into a foldable inline asm operand, which then reads the slot itself. An asm that writes the slot first (through an early-clobber "=&m" output) then read what it wrote, so a load isn't folded into an asm that stores, unless the slot is invariant. Nor is a load that doesn't fill the register, e.g. a 32-bit load into an XMM register, since the asm would read the whole register's worth of memory. Assisted-by: Claude Opus 5.5 --- llvm/lib/CodeGen/TargetInstrInfo.cpp | 14 ++- .../CodeGen/X86/peephole-inline-asm-fold.mir | 100 ++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index 81cd2cfc936932..81b61333fce48c 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -929,6 +929,7 @@ TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, // Ask the target to do the actual folding. MachineInstr *NewMI = nullptr; int FrameIndex = 0; + TypeSize MemBytes = TypeSize::getZero(); if ((MI.getOpcode() == TargetOpcode::STACKMAP || MI.getOpcode() == TargetOpcode::PATCHPOINT || @@ -938,7 +939,18 @@ TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this); if (NewMI) NewMI = &*MBB.insert(MI, NewMI); - } else if (MI.isInlineAsm() && isLoadFromStackSlot(LoadMI, FrameIndex)) { + } else if (MI.isInlineAsm() && + isLoadFromStackSlot(LoadMI, FrameIndex, MemBytes)) { + // The asm reads the stack slot in place of the loaded register, so the + // load must fill the register, and the slot must still hold the value when + // the asm reads it. An asm that stores might write the slot first (e.g. + // through an early-clobber "=&m" output), unless it's invariant. + if (Ops.empty() || + MemBytes * 8 != TRI.getRegSizeInBits(MI.getOperand(Ops[0]).getReg(), + MF.getRegInfo()) || + (MI.mayStore() && !LoadMI.isDereferenceableInvariantLoad())) + return nullptr; + return foldInlineAsmMemOperand(MI, Ops, FrameIndex, *this); } else { // Ask the target to do the actual folding. diff --git a/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir b/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir new file mode 100644 index 00000000000000..6e2d22c75ad376 --- /dev/null +++ b/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir @@ -0,0 +1,100 @@ +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=peephole-opt %s -o - \ +# RUN: | FileCheck %s + +# The peephole optimizer folds a load from a stack slot into a foldable ("rm") +# inline asm operand that reads the loaded value, so the asm reads the slot +# itself. + +--- | + define void @fold_load() { ret void } + define void @asm_may_store() { %x = alloca i64, align 8 + ret void } + define void @invariant_load() { ret void } + define void @narrow_load() { ret void } + define void @undef_uses() { ret void } +... +--- +# CHECK-LABEL: name: fold_load +# CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) +name: fold_load +tracksRegLiveness: true +stack: + - { id: 0, size: 8, alignment: 8 } +body: | + bb.0: + %0:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, %0 + RET 0 +... +--- +# The asm writes x through an early-clobber output before it reads v, so v +# can't be read from x. +# +# long x; +# long v = x; +# asm volatile("movq $0, %0; movq %1, %%rax" : "=&m"(x) : "rm"(v) : "rax"); +# +# CHECK-LABEL: name: asm_may_store +# CHECK: %0:gr64 = MOV64rm %stack.0.x +# CHECK-NEXT: INLINEASM {{.*}}, reguse:GR64 foldable, %0, +name: asm_may_store +tracksRegLiveness: true +stack: + - { id: 0, name: x, size: 8, alignment: 8 } +body: | + bb.0: + %0:gr64 = MOV64rm %stack.0.x, 1, $noreg, 0, $noreg :: (dereferenceable load (s64) from %ir.x) + INLINEASM &"movq $$0, $0\0A\09movq $1, %rax", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0.x, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0, clobber, implicit-def early-clobber $rax + RET 0 +... +--- +# An asm that stores can't write an invariant slot, such as an incoming +# argument's. +# +# CHECK-LABEL: name: invariant_load +# CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg :: (store unknown-size, align 1), (load (s64) from %fixed-stack.0, align 16) +name: invariant_load +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +stack: + - { id: 0, size: 8, alignment: 8 } +body: | + bb.0: + %0:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0 + RET 0 +... +--- +# A load that fills only part of the register (here the low 4 bytes of an +# XMM register, with the rest zeroed) isn't folded, since the asm would read +# the whole register's width from the slot. +# +# CHECK-LABEL: name: narrow_load +# CHECK: %0:vr128 = MOVSSrm %fixed-stack.0 +# CHECK-NEXT: INLINEASM &"movaps $0, %xmm1", sideeffect attdialect, reguse:VR128 foldable, %0 +name: narrow_load +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 4, alignment: 16, isImmutable: true } +body: | + bb.0: + %0:vr128 = MOVSSrm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0, align 16) + INLINEASM &"movaps $0, %xmm1", sideeffect attdialect, reguse:VR128 foldable, %0 + RET 0 +... +--- +# An asm that doesn't read the loaded value has nothing to fold. +# +# CHECK-LABEL: name: undef_uses +# CHECK: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, undef %0 +name: undef_uses +tracksRegLiveness: true +stack: + - { id: 0, size: 8, alignment: 8 } +body: | + bb.0: + %0:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, undef %0 + RET 0 +... _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
