llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-x86 Author: Bill Wendling (isanbard) <details> <summary>Changes</summary> An INLINEASM_BR isn't a terminator, so it can branch to an indirect target from the middle of its block, before the instructions after it run. The EFLAGS that live into an indirect target are the ones at the asm, not the ones at the end of the block, but X86FlagsCopyLowering assumed the latter in two places. When the pass lowers an EFLAGS copy, it rewrites the copy's uses in its block and then in every successor that EFLAGS live into. For a copy after an INLINEASM_BR, that includes the asm's indirect targets, which the copy never reaches, so the pass removed their live-in and rewrote their flag reads to use SETCCs computed after the asm, which only the fallthrough path runs. The pass could also hoist the test position for a copy in an indirect target into its predecessor, before the terminator that follows the asm, where the indirect path never goes. Don't follow a copy from its block into the indirect targets of an INLINEASM_BR before it, and don't hoist the test position out of an indirect target. Instruction selection doesn't make EFLAGS live into an indirect target yet, but the fix for #<!-- -->60827 does, for a callbr's flag outputs. With that fix alone, at -O0, two flag outputs, or a flag output and a tied output as in the kernel's try_cmpxchg, made the indirect target reload spill slots that only the fallthrough path stored. Part of #<!-- -->60827. Assisted-by: Claude Opus 5.5 --- Full diff: https://github.com/llvm/llvm-project/pull/239114.diff 2 Files Affected: - (modified) llvm/lib/Target/X86/X86FlagsCopyLowering.cpp (+23-3) - (added) llvm/test/CodeGen/X86/flags-copy-lowering-inlineasm-br.mir (+147) ``````````diff diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index c63583ec181a2..89ea09516d315 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -489,7 +489,11 @@ bool X86FlagsCopyLoweringImpl::runOnMachineFunction(MachineFunction &MF) { // No clobber found along a path from the begin to end. return false; }; + // Don't hoist out of an INLINEASM_BR's indirect target. The asm branches + // there from the middle of its block, so a test at the end of that block + // wouldn't run on the way. while (TestMBB->isLiveIn(X86::EFLAGS) && !TestMBB->pred_empty() && + !TestMBB->isInlineAsmBrIndirectTarget() && !HasEFLAGSClobber(TestMBB->begin(), TestPos)) { // Find the nearest common dominator of the predecessors, as // that will be the best candidate to hoist into. @@ -551,9 +555,25 @@ bool X86FlagsCopyLoweringImpl::runOnMachineFunction(MachineFunction &MF) { SmallPtrSet<MachineBasicBlock *, 2> VisitedBlocks; Blocks.push_back(&MBB); + // An INLINEASM_BR before the copy branches to its indirect targets before + // the copy runs, so the copied EFLAGS don't reach them along that edge. + // They read the EFLAGS at the asm instead. + SmallPtrSet<MachineBasicBlock *, 2> IndirectTargetsBeforeCopy; + if (MBB.mayHaveInlineAsmBr()) + for (MachineInstr &MI : + make_range(MBB.instr_begin(), CopyI->getIterator())) + if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) + for (const MachineOperand &MO : MI.operands()) + if (MO.isMBB()) + IndirectTargetsBeforeCopy.insert(MO.getMBB()); + do { MachineBasicBlock &UseMBB = *Blocks.pop_back_val(); + // The first scan of the copy's block starts at the copy. A later one, + // around a cycle, starts at the top of the block. + bool ScanFromCopy = &UseMBB == &MBB && !VisitedBlocks.count(&UseMBB); + // Track when if/when we find a kill of the flags in this block. bool FlagsKilled = false; @@ -566,9 +586,8 @@ bool X86FlagsCopyLoweringImpl::runOnMachineFunction(MachineFunction &MF) { // a symptom of all of this routine needing to be refactored into // documentable components. Once done, there may be a better way to write // this loop. - for (auto MII = (&UseMBB == &MBB && !VisitedBlocks.count(&UseMBB)) - ? std::next(CopyI->getIterator()) - : UseMBB.instr_begin(), + for (auto MII = ScanFromCopy ? std::next(CopyI->getIterator()) + : UseMBB.instr_begin(), MIE = UseMBB.instr_end(); MII != MIE;) { MachineInstr &MI = *MII++; @@ -644,6 +663,7 @@ bool X86FlagsCopyLoweringImpl::runOnMachineFunction(MachineFunction &MF) { // and queue those up for processing. for (MachineBasicBlock *SuccMBB : UseMBB.successors()) if (SuccMBB->isLiveIn(X86::EFLAGS) && + !(ScanFromCopy && IndirectTargetsBeforeCopy.contains(SuccMBB)) && VisitedBlocks.insert(SuccMBB).second) { // We currently don't do any PHI insertion and so we require that the // test basic block dominates all of the use basic blocks. Further, we diff --git a/llvm/test/CodeGen/X86/flags-copy-lowering-inlineasm-br.mir b/llvm/test/CodeGen/X86/flags-copy-lowering-inlineasm-br.mir new file mode 100644 index 0000000000000..fc0264c4e9101 --- /dev/null +++ b/llvm/test/CodeGen/X86/flags-copy-lowering-inlineasm-br.mir @@ -0,0 +1,147 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=x86_64-- -run-pass=x86-flags-copy-lowering -verify-machineinstrs -o - %s | FileCheck %s +# RUN: llc -mtriple=x86_64-- -passes=x86-flags-copy-lowering -verify-machineinstrs -o - %s | FileCheck %s + +# An INLINEASM_BR branches to its indirect targets from the middle of its +# block, so EFLAGS that live into an indirect target are the ones at the asm, +# not the ones at the end of the block. + +# The EFLAGS copy is after the asm, so its value doesn't reach the indirect +# target, which reads the asm's EFLAGS. +--- +name: copy_after_inlineasm_br +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: copy_after_inlineasm_br + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM_BR &"", attdialect, imm, %bb.2, clobber, implicit-def early-clobber $eflags + ; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: $al = COPY [[SETCCr]] + ; CHECK-NEXT: RET 0, $al + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2 (inlineasm-br-indirect-target): + ; CHECK-NEXT: liveins: $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 2, implicit $eflags + ; CHECK-NEXT: $al = COPY [[SETCCr1]] + ; CHECK-NEXT: RET 0, $al + bb.0: + successors: %bb.1, %bb.2 + + INLINEASM_BR &"", attdialect, imm, %bb.2, clobber, implicit-def early-clobber $eflags + %0:gr32 = COPY $eflags + $eflags = COPY %0 + %1:gr8 = SETCCr 4, implicit $eflags + JMP_1 %bb.1 + + bb.1: + $al = COPY %1 + RET 0, $al + + bb.2 (inlineasm-br-indirect-target): + liveins: $eflags + + %2:gr8 = SETCCr 2, implicit $eflags + $al = COPY %2 + RET 0, $al +... + +# The EFLAGS copy is before an asm that leaves EFLAGS alone, so its value +# reaches the indirect target too. +--- +name: copy_before_inlineasm_br +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: copy_before_inlineasm_br + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $edi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi + ; CHECK-NEXT: TEST32rr [[COPY]], [[COPY]], implicit-def $eflags + ; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags + ; CHECK-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 2, implicit $eflags + ; CHECK-NEXT: [[ADD32ri:%[0-9]+]]:gr32 = ADD32ri [[COPY]], 1, implicit-def dead $eflags + ; CHECK-NEXT: INLINEASM_BR &"", attdialect, imm, %bb.2 + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: $al = COPY [[SETCCr]] + ; CHECK-NEXT: RET 0, $al + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2 (inlineasm-br-indirect-target): + ; CHECK-NEXT: $al = COPY [[SETCCr1]] + ; CHECK-NEXT: RET 0, $al + bb.0: + successors: %bb.1, %bb.2 + liveins: $edi + + %0:gr32 = COPY $edi + TEST32rr %0, %0, implicit-def $eflags + %1:gr32 = COPY $eflags + %2:gr32 = ADD32ri %0, 1, implicit-def dead $eflags + $eflags = COPY %1 + INLINEASM_BR &"", attdialect, imm, %bb.2 + %3:gr8 = SETCCr 4, implicit $eflags + JMP_1 %bb.1 + + bb.1: + $al = COPY %3 + RET 0, $al + + bb.2 (inlineasm-br-indirect-target): + liveins: $eflags + + %4:gr8 = SETCCr 2, implicit $eflags + $al = COPY %4 + RET 0, $al +... + +# The EFLAGS copy is in the indirect target, which only the asm reaches, so +# the copy's test can't be hoisted into the asm's block: the end of that block +# only runs when the asm doesn't branch here. +--- +name: copy_in_indirect_target +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: copy_in_indirect_target + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM_BR &"", attdialect, imm, %bb.2, clobber, implicit-def early-clobber $eflags + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: RET 0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2 (inlineasm-br-indirect-target): + ; CHECK-NEXT: liveins: $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags + ; CHECK-NEXT: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags + ; CHECK-NEXT: $al = COPY [[SETCCr]] + ; CHECK-NEXT: RET 0, $al + bb.0: + successors: %bb.1, %bb.2 + + INLINEASM_BR &"", attdialect, imm, %bb.2, clobber, implicit-def early-clobber $eflags + JMP_1 %bb.1 + + bb.1: + RET 0 + + bb.2 (inlineasm-br-indirect-target): + liveins: $eflags + + %0:gr32 = COPY $eflags + %1:gr32 = MOV32r0 implicit-def dead $eflags + $eflags = COPY %0 + %2:gr8 = SETCCr 4, implicit $eflags + $al = COPY %2 + RET 0, $al +... `````````` </details> https://github.com/llvm/llvm-project/pull/239114 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
