https://github.com/lzto updated https://github.com/llvm/llvm-project/pull/206222
>From 9c0b774922323eb9f194c63a1c170b4dcd6e9ba8 Mon Sep 17 00:00:00 2001 From: Tong Zhang <[email protected]> Date: Fri, 26 Jun 2026 22:52:01 -0700 Subject: [PATCH] [CodeGen] Fix inline asm clobber-list registers incorrectly marked EarlyClobber MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clobber-list registers specified via `~{reg}` in inline asm constraints are modified *during* the asm body, not before inputs are read. As such they must NOT carry the EarlyClobber flag — inputs are permitted to use them, matching GCC's behaviour. Before this fix, both the SelectionDAG path (InstrEmitter::EmitSpecialNode) and the GlobalISel path (InlineAsmLowering) used RegState::EarlyClobber for InlineAsm::Kind::Clobber registers. This prevented the register allocator from ever assigning a clobbered register to any input operand. With a heavy clobber list (e.g. rax, rbx, rcx, rdx, r8-r11, r13-r15 — 11 registers), `-fno-omit-frame-pointer` (rbp reserved), and a gcov counter increment between two inlined asm calls (adding register pressure), the allocator ran out of registers and emitted: "inline assembly requires more registers than available" The bug was first observed in the Linux kernel's Curve25519 field arithmetic: `fsqr()`/`fsqr2()` in `arch/x86/crypto/curve25519-x86_64.c` use this exact pattern — 11-register clobber lists with `+&r` early-clobber outputs — and fail to compile with clang under CONFIG_GCOV_KERNEL + frame pointer builds. The bug is reproducible with clang 17 through 22 but not with clang 10 or GCC. Fix: give Kind::Clobber its own case in both InstrEmitter and InlineAsmLowering that emits RegState::Define only, without RegState::EarlyClobber. Update affected tests to reflect the corrected (non-early-clobber) MIR output. --- clang/test/CodeGen/2010-06-17-asmcrash.c | 2 +- clang/test/CodeGen/msp430-register-names.c | 32 ++--- .../CodeGen/GlobalISel/InlineAsmLowering.cpp | 8 +- .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 11 +- .../GlobalISel/irtranslator-inline-asm.ll | 2 +- llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 128 +++++++++--------- .../CodeGen/AArch64/aarch64-za-clobber.ll | 4 +- llvm/test/CodeGen/AArch64/arm64-inline-asm.ll | 3 +- .../sign-return-address-pauth-lr-mir.ll | 10 +- .../GlobalISel/irtranslator-inline-asm.ll | 6 +- llvm/test/CodeGen/PowerPC/pr47155-47156.ll | 6 +- .../GlobalISel/irtranslator-inline-asm.ll | 2 +- llvm/test/CodeGen/X86/cfi-xmm.ll | 2 +- .../X86/inline-asm-avx512f-x-constraint.ll | 6 +- .../inline-asm-clobber-not-early-clobber.ll | 50 +++++++ .../X86/inline-asm-default-clobbers.ll | 2 +- llvm/test/CodeGen/X86/tail-dup-asm-goto.ll | 2 +- 17 files changed, 169 insertions(+), 107 deletions(-) create mode 100644 llvm/test/CodeGen/X86/inline-asm-clobber-not-early-clobber.ll diff --git a/clang/test/CodeGen/2010-06-17-asmcrash.c b/clang/test/CodeGen/2010-06-17-asmcrash.c index 048e42966127e..a4bcec134bfc3 100644 --- a/clang/test/CodeGen/2010-06-17-asmcrash.c +++ b/clang/test/CodeGen/2010-06-17-asmcrash.c @@ -12,5 +12,5 @@ void avg_pixels8_mmx2(uint8_t *block, const uint8_t *pixels, int line_size, int :"+g"(h), "+S"(pixels), "+D"(block) :"r" ((x86_reg)line_size) :"%""rax", "memory"); -// CHECK: # %ecx %rsi %rdi %rdx +// CHECK: # %ecx %rsi %rdi %rax } diff --git a/clang/test/CodeGen/msp430-register-names.c b/clang/test/CodeGen/msp430-register-names.c index 106b6c933a49b..5be512934779b 100644 --- a/clang/test/CodeGen/msp430-register-names.c +++ b/clang/test/CodeGen/msp430-register-names.c @@ -86,20 +86,20 @@ void test_function(void) { // CHECK: call void asm sideeffect "", "~{r13}"() // CHECK: call void asm sideeffect "", "~{r14}"() // CHECK: call void asm sideeffect "", "~{r15}"() - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $pc - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $sp - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $sr - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $cg - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r4 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r5 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r6 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r7 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r8 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r9 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r10 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r11 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r12 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r13 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r14 - // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r15 + // CHECK: INLINEASM &"", {{.*}} implicit-def $pc + // CHECK: INLINEASM &"", {{.*}} implicit-def $sp + // CHECK: INLINEASM &"", {{.*}} implicit-def $sr + // CHECK: INLINEASM &"", {{.*}} implicit-def $cg + // CHECK: INLINEASM &"", {{.*}} implicit-def $r4 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r5 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r6 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r7 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r8 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r9 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r10 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r11 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r12 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r13 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r14 + // CHECK: INLINEASM &"", {{.*}} implicit-def $r15 } diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp index 3efa6ecaa9dc0..ac3324f6a1300 100644 --- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp @@ -581,9 +581,13 @@ bool InlineAsmLowering::lowerInlineAsm( unsigned Flag = InlineAsm::Flag(InlineAsm::Kind::Clobber, NumRegs); Inst.addImm(Flag); + // Clobber-list registers (~{reg}) are modified during the asm, not + // before inputs are read. Mark as regular (non-early-clobber) defs so + // the register allocator can use them as input operand registers, + // matching GCC's behaviour. for (Register Reg : OpInfo.Regs) { - Inst.addReg(Reg, RegState::Define | RegState::EarlyClobber | - getImplRegState(Reg.isPhysical())); + Inst.addReg(Reg, + RegState::Define | getImplRegState(Reg.isPhysical())); } } break; diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 8da255cda656d..fadfdb7556a26 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -1390,7 +1390,6 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, } break; case InlineAsm::Kind::RegDefEarlyClobber: - case InlineAsm::Kind::Clobber: for (unsigned j = 0; j != NumVals; ++j, ++i) { Register Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); MIB.addReg(Reg, RegState::Define | RegState::EarlyClobber | @@ -1398,6 +1397,16 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, ECRegs.push_back(Reg); } break; + case InlineAsm::Kind::Clobber: + // Clobber-list registers (~{reg}) are modified during the asm, not + // before inputs are read. Mark as regular (non-early-clobber) defs so + // the register allocator can use them as input operand registers, + // matching GCC's behaviour. + for (unsigned j = 0; j != NumVals; ++j, ++i) { + Register Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); + MIB.addReg(Reg, RegState::Define | getImplRegState(Reg.isPhysical())); + } + break; case InlineAsm::Kind::RegUse: // Use of register. case InlineAsm::Kind::Imm: // Immediate. case InlineAsm::Kind::Mem: // Non-function addressing mode. diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll index d627e11f4aa0c..c1e77dbaaaae5 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll @@ -17,7 +17,7 @@ define void @asm_simple_memory_clobber() { define void @asm_simple_register_clobber() { ; CHECK-LABEL: name: asm_simple_register_clobber ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: INLINEASM &"mov x0, 7", sideeffect attdialect, clobber, implicit-def early-clobber $x0, !0 + ; CHECK-NEXT: INLINEASM &"mov x0, 7", sideeffect attdialect, clobber, implicit-def $x0, !0 ; CHECK-NEXT: RET_ReallyLR call void asm sideeffect "mov x0, 7", "~{x0}"(), !srcloc !0 ret void diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll index d3f5a769e2d23..99dc0ffe21898 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll @@ -47,38 +47,38 @@ define <2 x float> @sme_nosve_nonstreaming(ptr %in) "target-features"="+sme,-sve entry: ; CHECK-LABEL: name: sme_nosve_nonstreaming ; CHECK: INLINEASM &"smstart sm; smstop sm;" -; CHECK-SAME: implicit-def early-clobber $q0 -; CHECK-SAME: implicit-def early-clobber $q1 -; CHECK-SAME: implicit-def early-clobber $q2 -; CHECK-SAME: implicit-def early-clobber $q3 -; CHECK-SAME: implicit-def early-clobber $q4 -; CHECK-SAME: implicit-def early-clobber $q5 -; CHECK-SAME: implicit-def early-clobber $q6 -; CHECK-SAME: implicit-def early-clobber $q7 -; CHECK-SAME: implicit-def early-clobber $q8 -; CHECK-SAME: implicit-def early-clobber $q9 -; CHECK-SAME: implicit-def early-clobber $q10 -; CHECK-SAME: implicit-def early-clobber $q11 -; CHECK-SAME: implicit-def early-clobber $q12 -; CHECK-SAME: implicit-def early-clobber $q13 -; CHECK-SAME: implicit-def early-clobber $q14 -; CHECK-SAME: implicit-def early-clobber $q15 -; CHECK-SAME: implicit-def early-clobber $q16 -; CHECK-SAME: implicit-def early-clobber $q17 -; CHECK-SAME: implicit-def early-clobber $q18 -; CHECK-SAME: implicit-def early-clobber $q19 -; CHECK-SAME: implicit-def early-clobber $q20 -; CHECK-SAME: implicit-def early-clobber $q21 -; CHECK-SAME: implicit-def early-clobber $q22 -; CHECK-SAME: implicit-def early-clobber $q23 -; CHECK-SAME: implicit-def early-clobber $q24 -; CHECK-SAME: implicit-def early-clobber $q25 -; CHECK-SAME: implicit-def early-clobber $q26 -; CHECK-SAME: implicit-def early-clobber $q27 -; CHECK-SAME: implicit-def early-clobber $q28 -; CHECK-SAME: implicit-def early-clobber $q29 -; CHECK-SAME: implicit-def early-clobber $q30 -; CHECK-SAME: implicit-def early-clobber $q31 +; CHECK-SAME: implicit-def $q0 +; CHECK-SAME: implicit-def $q1 +; CHECK-SAME: implicit-def $q2 +; CHECK-SAME: implicit-def $q3 +; CHECK-SAME: implicit-def $q4 +; CHECK-SAME: implicit-def $q5 +; CHECK-SAME: implicit-def $q6 +; CHECK-SAME: implicit-def $q7 +; CHECK-SAME: implicit-def $q8 +; CHECK-SAME: implicit-def $q9 +; CHECK-SAME: implicit-def $q10 +; CHECK-SAME: implicit-def $q11 +; CHECK-SAME: implicit-def $q12 +; CHECK-SAME: implicit-def $q13 +; CHECK-SAME: implicit-def $q14 +; CHECK-SAME: implicit-def $q15 +; CHECK-SAME: implicit-def $q16 +; CHECK-SAME: implicit-def $q17 +; CHECK-SAME: implicit-def $q18 +; CHECK-SAME: implicit-def $q19 +; CHECK-SAME: implicit-def $q20 +; CHECK-SAME: implicit-def $q21 +; CHECK-SAME: implicit-def $q22 +; CHECK-SAME: implicit-def $q23 +; CHECK-SAME: implicit-def $q24 +; CHECK-SAME: implicit-def $q25 +; CHECK-SAME: implicit-def $q26 +; CHECK-SAME: implicit-def $q27 +; CHECK-SAME: implicit-def $q28 +; CHECK-SAME: implicit-def $q29 +; CHECK-SAME: implicit-def $q30 +; CHECK-SAME: implicit-def $q31 %0 = load <2 x float>, ptr %in, align 8 call void asm sideeffect "smstart sm; smstop sm;", "~{z0},~{z1},~{z2},~{z3},~{z4},~{z5},~{z6},~{z7},~{z8},~{z9},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15},~{z16},~{z17},~{z18},~{z19},~{z20},~{z21},~{z22},~{z23},~{z24},~{z25},~{z26},~{z27},~{z28},~{z29},~{z30},~{z31}"() ret <2 x float> %0 @@ -88,38 +88,38 @@ define <2 x float> @sme_nosve_streaming(ptr %in) "target-features"="+sme,-sve" " entry: ; CHECK-LABEL: name: sme_nosve_streaming ; CHECK: INLINEASM &"smstart sm; smstop sm;" -; CHECK-SAME: implicit-def early-clobber $z0 -; CHECK-SAME: implicit-def early-clobber $z1 -; CHECK-SAME: implicit-def early-clobber $z2 -; CHECK-SAME: implicit-def early-clobber $z3 -; CHECK-SAME: implicit-def early-clobber $z4 -; CHECK-SAME: implicit-def early-clobber $z5 -; CHECK-SAME: implicit-def early-clobber $z6 -; CHECK-SAME: implicit-def early-clobber $z7 -; CHECK-SAME: implicit-def early-clobber $z8 -; CHECK-SAME: implicit-def early-clobber $z9 -; CHECK-SAME: implicit-def early-clobber $z10 -; CHECK-SAME: implicit-def early-clobber $z11 -; CHECK-SAME: implicit-def early-clobber $z12 -; CHECK-SAME: implicit-def early-clobber $z13 -; CHECK-SAME: implicit-def early-clobber $z14 -; CHECK-SAME: implicit-def early-clobber $z15 -; CHECK-SAME: implicit-def early-clobber $z16 -; CHECK-SAME: implicit-def early-clobber $z17 -; CHECK-SAME: implicit-def early-clobber $z18 -; CHECK-SAME: implicit-def early-clobber $z19 -; CHECK-SAME: implicit-def early-clobber $z20 -; CHECK-SAME: implicit-def early-clobber $z21 -; CHECK-SAME: implicit-def early-clobber $z22 -; CHECK-SAME: implicit-def early-clobber $z23 -; CHECK-SAME: implicit-def early-clobber $z24 -; CHECK-SAME: implicit-def early-clobber $z25 -; CHECK-SAME: implicit-def early-clobber $z26 -; CHECK-SAME: implicit-def early-clobber $z27 -; CHECK-SAME: implicit-def early-clobber $z28 -; CHECK-SAME: implicit-def early-clobber $z29 -; CHECK-SAME: implicit-def early-clobber $z30 -; CHECK-SAME: implicit-def early-clobber $z31 +; CHECK-SAME: implicit-def $z0 +; CHECK-SAME: implicit-def $z1 +; CHECK-SAME: implicit-def $z2 +; CHECK-SAME: implicit-def $z3 +; CHECK-SAME: implicit-def $z4 +; CHECK-SAME: implicit-def $z5 +; CHECK-SAME: implicit-def $z6 +; CHECK-SAME: implicit-def $z7 +; CHECK-SAME: implicit-def $z8 +; CHECK-SAME: implicit-def $z9 +; CHECK-SAME: implicit-def $z10 +; CHECK-SAME: implicit-def $z11 +; CHECK-SAME: implicit-def $z12 +; CHECK-SAME: implicit-def $z13 +; CHECK-SAME: implicit-def $z14 +; CHECK-SAME: implicit-def $z15 +; CHECK-SAME: implicit-def $z16 +; CHECK-SAME: implicit-def $z17 +; CHECK-SAME: implicit-def $z18 +; CHECK-SAME: implicit-def $z19 +; CHECK-SAME: implicit-def $z20 +; CHECK-SAME: implicit-def $z21 +; CHECK-SAME: implicit-def $z22 +; CHECK-SAME: implicit-def $z23 +; CHECK-SAME: implicit-def $z24 +; CHECK-SAME: implicit-def $z25 +; CHECK-SAME: implicit-def $z26 +; CHECK-SAME: implicit-def $z27 +; CHECK-SAME: implicit-def $z28 +; CHECK-SAME: implicit-def $z29 +; CHECK-SAME: implicit-def $z30 +; CHECK-SAME: implicit-def $z31 %0 = load <2 x float>, ptr %in, align 8 call void asm sideeffect "smstart sm; smstop sm;", "~{z0},~{z1},~{z2},~{z3},~{z4},~{z5},~{z6},~{z7},~{z8},~{z9},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15},~{z16},~{z17},~{z18},~{z19},~{z20},~{z21},~{z22},~{z23},~{z24},~{z25},~{z26},~{z27},~{z28},~{z29},~{z30},~{z31}"() ret <2 x float> %0 diff --git a/llvm/test/CodeGen/AArch64/aarch64-za-clobber.ll b/llvm/test/CodeGen/AArch64/aarch64-za-clobber.ll index 83e2c4f74c0ff..39dd4994d1128 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-za-clobber.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-za-clobber.ll @@ -3,14 +3,14 @@ define void @alpha(<vscale x 4 x i32> %x) local_unnamed_addr { entry: -; CHECK: INLINEASM &"movt zt0[3, mul vl], z0", sideeffect attdialect, clobber, implicit-def early-clobber $za +; CHECK: INLINEASM &"movt zt0[3, mul vl], z0", sideeffect attdialect, clobber, implicit-def $za tail call void asm sideeffect "movt zt0[3, mul vl], z0", "~{za}"() ret void } define void @beta(<vscale x 4 x i32> %x) local_unnamed_addr { entry: -; CHECK: INLINEASM &"movt zt0[3, mul vl], z0", sideeffect attdialect, clobber, implicit-def early-clobber $zt0 +; CHECK: INLINEASM &"movt zt0[3, mul vl], z0", sideeffect attdialect, clobber, implicit-def $zt0 tail call void asm sideeffect "movt zt0[3, mul vl], z0", "~{zt0}"() ret void } diff --git a/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll b/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll index 8947563e31f85..8a614e574b66a 100644 --- a/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll +++ b/llvm/test/CodeGen/AArch64/arm64-inline-asm.ll @@ -44,9 +44,8 @@ entry: define void @t4(i64 %op) nounwind { ; CHECK-LABEL: t4: ; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: mov x8, x0 ; CHECK-NEXT: ; InlineAsm Start -; CHECK-NEXT: mov x0, x8; svc #0; +; CHECK-NEXT: mov x0, x0; svc #0; ; CHECK-NEXT: ; InlineAsm End ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr-mir.ll b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr-mir.ll index 2de549bdcca26..837aae2d540cc 100644 --- a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr-mir.ll +++ b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr-mir.ll @@ -98,7 +98,7 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a ; COMPAT-MIR-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) ; COMPAT-MIR-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16 ; COMPAT-MIR-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16 - ; COMPAT-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead early-clobber $lr + ; COMPAT-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead $lr ; COMPAT-MIR-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) ; COMPAT-MIR-NEXT: $x16 = frame-destroy ADRP target-flags(aarch64-page) <mcsymbol > ; COMPAT-MIR-NEXT: $x16 = frame-destroy ADDXri $x16, target-flags(aarch64-pageoff, aarch64-nc) <mcsymbol >, 0 @@ -116,7 +116,7 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a ; V83A-MIR-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) ; V83A-MIR-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16 ; V83A-MIR-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16 - ; V83A-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead early-clobber $lr + ; V83A-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead $lr ; V83A-MIR-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) ; V83A-MIR-NEXT: $x16 = frame-destroy ADRP target-flags(aarch64-page) <mcsymbol > ; V83A-MIR-NEXT: $x16 = frame-destroy ADDXri $x16, target-flags(aarch64-pageoff, aarch64-nc) <mcsymbol >, 0 @@ -132,7 +132,7 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a ; PAUTHLR-MIR-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) ; PAUTHLR-MIR-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16 ; PAUTHLR-MIR-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16 - ; PAUTHLR-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead early-clobber $lr + ; PAUTHLR-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead $lr ; PAUTHLR-MIR-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) ; PAUTHLR-MIR-NEXT: frame-destroy RETAASPPCi <mcsymbol >, implicit $lr, implicit $sp, implicit-def $lr, implicit killed $lr, implicit $sp call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1 @@ -323,7 +323,7 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "branch-protection-pauth-lr" ; CHECK-MIR-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) ; CHECK-MIR-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16 ; CHECK-MIR-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16 - ; CHECK-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead early-clobber $lr + ; CHECK-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead $lr ; CHECK-MIR-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) ; CHECK-MIR-NEXT: $x16 = frame-destroy ADRP target-flags(aarch64-page) <mcsymbol > ; CHECK-MIR-NEXT: $x16 = frame-destroy ADDXri $x16, target-flags(aarch64-pageoff, aarch64-nc) <mcsymbol >, 0 @@ -340,7 +340,7 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "branch-protection-pauth-lr" ; PAUTHLR-MIR-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) ; PAUTHLR-MIR-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16 ; PAUTHLR-MIR-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16 - ; PAUTHLR-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead early-clobber $lr + ; PAUTHLR-MIR-NEXT: INLINEASM &"mov x30, $0", sideeffect attdialect, reguse:GPR64common, renamable $x0, clobber, implicit-def dead $lr ; PAUTHLR-MIR-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) ; PAUTHLR-MIR-NEXT: frame-destroy AUTIASPPCi <mcsymbol >, implicit-def $lr, implicit $lr, implicit $sp ; PAUTHLR-MIR-NEXT: TCRETURNdi @bar, 0, csr_aarch64_aapcs, implicit $sp, implicit killed $x0 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll index cf58edac990d0..de4c357dca789 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll @@ -33,7 +33,7 @@ define amdgpu_kernel void @asm_simple_vgpr_clobber() { ; CHECK-NEXT: liveins: $sgpr8_sgpr9 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9 - ; CHECK-NEXT: INLINEASM &"v_mov_b32 v0, 7", sideeffect attdialect, clobber, implicit-def early-clobber $vgpr0, !1 + ; CHECK-NEXT: INLINEASM &"v_mov_b32 v0, 7", sideeffect attdialect, clobber, implicit-def $vgpr0, !1 ; CHECK-NEXT: S_ENDPGM 0 call void asm sideeffect "v_mov_b32 v0, 7", "~{v0}"(), !srcloc !0 ret void @@ -45,7 +45,7 @@ define amdgpu_kernel void @asm_simple_sgpr_clobber() { ; CHECK-NEXT: liveins: $sgpr8_sgpr9 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9 - ; CHECK-NEXT: INLINEASM &"s_mov_b32 s0, 7", sideeffect attdialect, clobber, implicit-def early-clobber $sgpr0, !1 + ; CHECK-NEXT: INLINEASM &"s_mov_b32 s0, 7", sideeffect attdialect, clobber, implicit-def $sgpr0, !1 ; CHECK-NEXT: S_ENDPGM 0 call void asm sideeffect "s_mov_b32 s0, 7", "~{s0}"(), !srcloc !0 ret void @@ -57,7 +57,7 @@ define amdgpu_kernel void @asm_simple_agpr_clobber() { ; CHECK-NEXT: liveins: $sgpr8_sgpr9 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9 - ; CHECK-NEXT: INLINEASM &"; def a0", sideeffect attdialect, clobber, implicit-def early-clobber $agpr0, !1 + ; CHECK-NEXT: INLINEASM &"; def a0", sideeffect attdialect, clobber, implicit-def $agpr0, !1 ; CHECK-NEXT: S_ENDPGM 0 call void asm sideeffect "; def a0", "~{a0}"(), !srcloc !0 ret void diff --git a/llvm/test/CodeGen/PowerPC/pr47155-47156.ll b/llvm/test/CodeGen/PowerPC/pr47155-47156.ll index 1fb3607eb05c3..ffba73609fe34 100644 --- a/llvm/test/CodeGen/PowerPC/pr47155-47156.ll +++ b/llvm/test/CodeGen/PowerPC/pr47155-47156.ll @@ -8,11 +8,11 @@ define void @pr47155() { ; CHECK: *** Final schedule for %bb.0 *** ; CHECK: ********** MI Scheduling ********** ; CHECK-NEXT: pr47155:%bb.0 entry -; CHECK: SU(0): INLINEASM &"mtlr 31"{{.*}}implicit-def early-clobber $lr +; CHECK: SU(0): INLINEASM &"mtlr 31"{{.*}}implicit-def $lr ; CHECK: Successors: ; CHECK-NEXT: SU(1): Out Latency=0 ; CHECK-NEXT: SU(1): Ord Latency=0 Barrier -; CHECK-NEXT: SU(1): INLINEASM &"mtlr 31"{{.*}}implicit-def early-clobber $lr8 +; CHECK-NEXT: SU(1): INLINEASM &"mtlr 31"{{.*}}implicit-def $lr8 ; CHECK: Predecessors: ; CHECK-NEXT: SU(0): Out Latency=0 ; CHECK-NEXT: SU(0): Ord Latency=0 Barrier @@ -28,7 +28,7 @@ define void @pr47156(ptr %fn) { ; CHECK: *** Final schedule for %bb.0 *** ; CHECK: ********** MI Scheduling ********** ; CHECK-NEXT: pr47156:%bb.0 entry -; CHECK: SU(0): INLINEASM &"mtctr 31"{{.*}}implicit-def early-clobber $ctr +; CHECK: SU(0): INLINEASM &"mtctr 31"{{.*}}implicit-def $ctr ; CHECK: Successors: ; CHECK-NEXT: SU(1): Out Latency=0 ; CHECK-NEXT: SU(1): MTCTR8 renamable $x3, implicit-def $ctr8 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-inline-asm.ll index bf3a9fb75dc22..73410921da335 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator-inline-asm.ll @@ -15,7 +15,7 @@ define void @test_simple_memory_clobber() { define void @test_simple_register_clobber() { ; CHECK-LABEL: name: test_simple_register_clobber ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def early-clobber $x10 + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def $x10 ; CHECK-NEXT: PseudoRET call void asm sideeffect "", "~{x10}"() ret void diff --git a/llvm/test/CodeGen/X86/cfi-xmm.ll b/llvm/test/CodeGen/X86/cfi-xmm.ll index dde731c5cbdfb..13afdb1312e9d 100644 --- a/llvm/test/CodeGen/X86/cfi-xmm.ll +++ b/llvm/test/CodeGen/X86/cfi-xmm.ll @@ -12,7 +12,7 @@ define void @_Z1fv() { ; PEI-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 48 ; PEI-NEXT: CFI_INSTRUCTION offset $xmm10, -48 ; PEI-NEXT: CFI_INSTRUCTION offset $xmm15, -32 - ; PEI-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $xmm10, clobber, implicit-def dead early-clobber $xmm15, clobber, implicit-def dead early-clobber $df, clobber, implicit-def early-clobber $fpsw, clobber, implicit-def dead early-clobber $eflags + ; PEI-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead $xmm10, clobber, implicit-def dead $xmm15, clobber, implicit-def dead $df, clobber, implicit-def $fpsw, clobber, implicit-def dead $eflags ; PEI-NEXT: $xmm10 = frame-destroy MOVAPSrm $rsp, 1, $noreg, 0, $noreg :: (load (s128) from %fixed-stack.0) ; PEI-NEXT: $xmm15 = frame-destroy MOVAPSrm $rsp, 1, $noreg, 16, $noreg :: (load (s128) from %fixed-stack.1) ; PEI-NEXT: $rsp = frame-destroy ADD64ri32 $rsp, 40, implicit-def dead $eflags diff --git a/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll b/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll index 56950b995ac59..36ec903d8b8cd 100644 --- a/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll +++ b/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll @@ -6,7 +6,7 @@ ; CHECK-LABEL: name: mask_Yk_i8 ; CHECK: %[[REG1:.*]]:vr512_0_15 = COPY %1 ; CHECK: %[[REG2:.*]]:vr512_0_15 = COPY %2 -; CHECK: INLINEASM &"vpaddq\09$3, $2, $0 {$1}", attdialect, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], clobber, implicit-def early-clobber $df, clobber, implicit-def early-clobber $fpsw, clobber, implicit-def early-clobber $eflags +; CHECK: INLINEASM &"vpaddq\09$3, $2, $0 {$1}", attdialect, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], clobber, implicit-def $df, clobber, implicit-def $fpsw, clobber, implicit-def $eflags define <8 x i64> @mask_Yk_i8(i8 signext %msk, <8 x i64> %x, <8 x i64> %y) { entry: @@ -17,7 +17,7 @@ entry: ; FP16-LABEL: name: mask_Yk_f16 ; FP16: %[[REG1:.*]]:vr512_0_15 = COPY %1 ; FP16: %[[REG2:.*]]:vr512_0_15 = COPY %2 -; FP16: INLINEASM &"vaddph\09$3, $2, $0 {$1}", attdialect, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], clobber, implicit-def early-clobber $df, clobber, implicit-def early-clobber $fpsw, clobber, implicit-def early-clobber $eflags +; FP16: INLINEASM &"vaddph\09$3, $2, $0 {$1}", attdialect, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], clobber, implicit-def $df, clobber, implicit-def $fpsw, clobber, implicit-def $eflags ; CHECK-STDERR: could not allocate output register for constraint 'x' define <32 x half> @mask_Yk_f16(i8 signext %msk, <32 x half> %x, <32 x half> %y) { entry: @@ -28,7 +28,7 @@ entry: ; FP16-LABEL: name: mask_Yk_bf16 ; FP16: %[[REG1:.*]]:vr512_0_15 = COPY %1 ; FP16: %[[REG2:.*]]:vr512_0_15 = COPY %2 -; FP16: INLINEASM &"vaddph\09$3, $2, $0 {$1}", attdialect, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], clobber, implicit-def early-clobber $df, clobber, implicit-def early-clobber $fpsw, clobber, implicit-def early-clobber $eflags +; FP16: INLINEASM &"vaddph\09$3, $2, $0 {$1}", attdialect, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], clobber, implicit-def $df, clobber, implicit-def $fpsw, clobber, implicit-def $eflags ; CHECK-STDERR: could not allocate output register for constraint 'x' define <32 x bfloat> @mask_Yk_bf16(i8 signext %msk, <32 x bfloat> %x, <32 x bfloat> %y) { entry: diff --git a/llvm/test/CodeGen/X86/inline-asm-clobber-not-early-clobber.ll b/llvm/test/CodeGen/X86/inline-asm-clobber-not-early-clobber.ll new file mode 100644 index 0000000000000..c34b153e39a8f --- /dev/null +++ b/llvm/test/CodeGen/X86/inline-asm-clobber-not-early-clobber.ll @@ -0,0 +1,50 @@ +; RUN: llc < %s -mtriple=x86_64-linux-gnu 2>&1 | FileCheck %s +; +; Verify that clobber-list registers (~{reg}) are NOT treated as early-clobber +; by the register allocator. +; +; Before the fix, Kind::Clobber fell through into the Kind::RegDefEarlyClobber +; case in InstrEmitter::EmitSpecialNode(), causing every clobbered register to +; receive the EarlyClobber MachineOperand flag. With 11 clobbered registers +; and -frame-pointer=all (rbp reserved), only rsi/rdi/r12 remained usable, but +; the asm needed 3 operand registers plus a live-across pointer — causing: +; "inline assembly requires more registers than available" +; +; GCC semantics: clobber-list registers are destroyed *during* the asm body, +; not before inputs are read, so inputs are allowed to use them. + +@ctr0 = internal global [1 x i64] zeroinitializer +@ctr1 = internal global [1 x i64] zeroinitializer + +; CHECK-NOT: error: +; CHECK-LABEL: test3: +; CHECK: nop +; CHECK: nop +define dso_local void @test3(ptr noundef %q) local_unnamed_addr #0 { + ; Simulate gcov counter increment between the two asm calls — this is what + ; creates register pressure and triggers the bug in practice. + %c0 = load i64, ptr @ctr0, align 8 + %c1 = add i64 %c0, 1 + store i64 %c1, ptr @ctr0, align 8 + + call { ptr, ptr } asm sideeffect "nop", + "=&r,=&r,r,0,1,~{rax},~{rbx},~{rcx},~{rdx},~{r8},~{r9},~{r10},~{r11},~{r13},~{r14},~{r15},~{memory},~{cc},~{dirflag},~{fpsr},~{flags}" + (ptr nonnull inttoptr (i64 4660 to ptr), + ptr nonnull inttoptr (i64 43981 to ptr), + ptr nonnull inttoptr (i64 239 to ptr)) #1 + + %c2 = load i64, ptr @ctr1, align 8 + %c3 = add i64 %c2, 1 + store i64 %c3, ptr @ctr1, align 8 + + call { ptr, ptr } asm sideeffect "nop", + "=&r,=&r,r,0,1,~{rax},~{rbx},~{rcx},~{rdx},~{r8},~{r9},~{r10},~{r11},~{r13},~{r14},~{r15},~{memory},~{cc},~{dirflag},~{fpsr},~{flags}" + (ptr %q, + ptr nonnull inttoptr (i64 6699 to ptr), + ptr nonnull inttoptr (i64 15437 to ptr)) #1 + + ret void +} + +attributes #0 = { nounwind uwtable "frame-pointer"="all" "target-cpu"="x86-64" } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/X86/inline-asm-default-clobbers.ll b/llvm/test/CodeGen/X86/inline-asm-default-clobbers.ll index b0112bd80a259..740d6c8803968 100644 --- a/llvm/test/CodeGen/X86/inline-asm-default-clobbers.ll +++ b/llvm/test/CodeGen/X86/inline-asm-default-clobbers.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=i686 -stop-after=finalize-isel | FileCheck %s -; CHECK: INLINEASM &"", sideeffect attdialect, clobber, implicit-def early-clobber $df, clobber, implicit-def early-clobber $fpsw, clobber, implicit-def early-clobber $eflags +; CHECK: INLINEASM &"", sideeffect attdialect, clobber, implicit-def $df, clobber, implicit-def $fpsw, clobber, implicit-def $eflags define void @foo() { entry: call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"() diff --git a/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll b/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll index 7c22c0e117091..5198fc0b55796 100644 --- a/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll +++ b/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll @@ -36,7 +36,7 @@ define ptr @test1(ptr %arg1, ptr %arg2) { ; CHECK-NEXT: successors: %bb.5(0x80000000), %bb.4(0x00000000) ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr64 = PHI [[COPY]], %bb.2, [[MOV64rm]], %bb.1 - ; CHECK-NEXT: INLINEASM_BR &"#$0 $1 $2", sideeffect mayload attdialect, imm, 42, imm, 0, imm, %bb.4, clobber, implicit-def early-clobber $df, clobber, implicit-def early-clobber $fpsw, clobber, implicit-def early-clobber $eflags + ; CHECK-NEXT: INLINEASM_BR &"#$0 $1 $2", sideeffect mayload attdialect, imm, 42, imm, 0, imm, %bb.4, clobber, implicit-def $df, clobber, implicit-def $fpsw, clobber, implicit-def $eflags ; CHECK-NEXT: JMP_1 %bb.5 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.4.bb17.i.i.i (inlineasm-br-indirect-target): _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
