https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/239127
>From 01297f9d3490c48a79ecb047c6618ee7bc3f9499 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Wed, 4 Nov 2026 23:31:04 -0800 Subject: [PATCH 1/4] [AArch64] Use 64-bit registers for wide "r" inline asm operands An integer wider than 64 bits, such as an i128, in an "r" operand was given a 32-bit register for each 64 bits of it, so half of its bits were lost. Give it 64-bit registers instead. Fixes #238926. Assisted-by: Claude Opus 5.5 --- .../Target/AArch64/AArch64ISelLowering.cpp | 8 +++- llvm/test/CodeGen/AArch64/build-pair-isel.ll | 7 +--- .../CodeGen/AArch64/inline-asm-wide-gpr.ll | 41 +++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index a51e6bb18cdd8e..741cad2f162a09 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -14484,8 +14484,14 @@ AArch64TargetLowering::getRegForInlineAsmConstraint( return std::make_pair(0U, nullptr); if (Subtarget->hasLS64() && VT.getSizeInBits() == 512) return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass); - if (VT.getFixedSizeInBits() == 64) + + // An integer wider than 64 bits, such as an i128, takes a 64-bit + // register for each 64 bits of it. 32-bit registers would hold only half + // of it. + if (VT.getFixedSizeInBits() == 64 || + (VT.isScalarInteger() && VT.getFixedSizeInBits() > 64)) return std::make_pair(0U, &AArch64::GPR64commonRegClass); + return std::make_pair(0U, &AArch64::GPR32commonRegClass); case 'w': { if (!Subtarget->hasFPARMv8()) diff --git a/llvm/test/CodeGen/AArch64/build-pair-isel.ll b/llvm/test/CodeGen/AArch64/build-pair-isel.ll index cd68bde0f8da73..ee5fc8b7df13cf 100644 --- a/llvm/test/CodeGen/AArch64/build-pair-isel.ll +++ b/llvm/test/CodeGen/AArch64/build-pair-isel.ll @@ -10,11 +10,8 @@ define void @compare_and_swap128() { ; CHECK-NEXT: //APP ; CHECK-NEXT: nop ; CHECK-NEXT: //NO_APP -; CHECK-NEXT: // implicit-def: $x9 -; CHECK-NEXT: mov w9, w10 -; CHECK-NEXT: mov w8, w8 -; CHECK-NEXT: // kill: def $x8 killed $w8 -; CHECK-NEXT: orr x8, x8, x9, lsl #32 +; CHECK-NEXT: // implicit-def: $x10 +; CHECK-NEXT: str x9, [x10] ; CHECK-NEXT: // implicit-def: $x9 ; CHECK-NEXT: str x8, [x9] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll new file mode 100644 index 00000000000000..05144bcdd69714 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll @@ -0,0 +1,41 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s + +; An integer wider than 64 bits in an "r" operand goes in 64-bit registers, +; one for each 64 bits of it. 32-bit registers would hold only half of it. + +define void @test_i128_input(i128 %x) { +; CHECK-LABEL: test_i128_input: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: //APP +; CHECK-NEXT: // x0 +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: ret +entry: + call void asm sideeffect "// $0", "r"(i128 %x) + ret void +} + +define i128 @test_i128_output() { +; CHECK-LABEL: test_i128_output: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: //APP +; CHECK-NEXT: // x0 +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: ret +entry: + %r = call i128 asm sideeffect "// $0", "=r"() + ret i128 %r +} + +; The asm doesn't change its operand, so the function returns its argument. +define i128 @test_i128_tied(i128 %x) { +; CHECK-LABEL: test_i128_tied: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: //APP +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: ret +entry: + %r = call i128 asm sideeffect "", "=&r,0"(i128 %x) + ret i128 %r +} >From a3644124427fd6094aff3e16b552642c22d29fcd Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Wed, 18 Nov 2026 11:13:28 -0800 Subject: [PATCH 2/4] [AArch64] Add the H modifier for the second register of a 128-bit operand A 128-bit integer in an "r" operand takes two X registers, but an asm could name only the first, so it couldn't use the other half. GCC has the H modifier for this, so support it too. Assisted-by: Claude Opus 5.5 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 18 ++++++ .../AArch64/inline-asm-wide-gpr-error.ll | 18 ++++++ .../CodeGen/AArch64/inline-asm-wide-gpr.ll | 55 ++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 0df6c2cae06303..83bc5e3f1f70f0 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -48,6 +48,7 @@ #include "llvm/IR/Analysis.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/InlineAsm.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" @@ -1290,6 +1291,23 @@ bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, } printOperand(MI, OpNum, O); return false; + case 'H': { // Print the second X register of a 128-bit operand. + // A 128-bit integer in an "r" operand takes two X registers, and $N + // prints the first, which holds the doubleword at the lower address. + // 'H' prints the other, as in GCC, where the two are an even/odd pair. + // Here they needn't be. The operand before the first register is the + // flag that says how many registers there are. + const InlineAsm::Flag F(MI->getOperand(OpNum - 1).getImm()); + if (F.getNumOperandRegisters() != 2) + return true; + + const MachineOperand &SecondMO = MI->getOperand(OpNum + 1); + if (!SecondMO.isReg() || + !AArch64::GPR64allRegClass.contains(SecondMO.getReg())) + return true; + + return printAsmMRegister(SecondMO, 'x', O); + } case 'b': // Print B register. case 'h': // Print H register. case 's': // Print S register. diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll new file mode 100644 index 00000000000000..c16a613d52ed99 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll @@ -0,0 +1,18 @@ +; RUN: not llc -mtriple=aarch64-linux-gnu < %s 2>&1 | FileCheck %s + +; %H names the second register of an operand that takes two, such as an i128, +; so it's an error on an operand that takes one, or more than two. + +; CHECK: error: invalid operand in inline asm: '// ${0:H}' +define void @test_one_register(i64 %x) { +entry: + call void asm sideeffect "// ${0:H}", "r"(i64 %x) + ret void +} + +; CHECK: error: invalid operand in inline asm: '// ${0:H} four' +define void @test_four_registers(i256 %x) { +entry: + call void asm sideeffect "// ${0:H} four", "r"(i256 %x) + ret void +} diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll index 05144bcdd69714..78d948acf108fa 100644 --- a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll +++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll @@ -1,8 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 -; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,LE +; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,BE ; An integer wider than 64 bits in an "r" operand goes in 64-bit registers, ; one for each 64 bits of it. 32-bit registers would hold only half of it. +; Big-endian puts the halves of an i128 in the other order (see +; @test_i128_high_half). define void @test_i128_input(i128 %x) { ; CHECK-LABEL: test_i128_input: @@ -39,3 +42,53 @@ entry: %r = call i128 asm sideeffect "", "=&r,0"(i128 %x) ret i128 %r } + +; %H names the second register of an i128, so an asm can use both halves. +define void @test_i128_store(ptr %p, i128 %x) { +; CHECK-LABEL: test_i128_store: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: //APP +; CHECK-NEXT: stp x2, x3, [x0] +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: ret +entry: + call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, i128 %x) + ret void +} + +define i128 @test_i128_load(ptr %p) { +; CHECK-LABEL: test_i128_load: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: //APP +; CHECK-NEXT: ldp x8, x1, [x0] +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: mov x0, x8 +; CHECK-NEXT: ret +entry: + %r = call i128 asm sideeffect "ldp $0, ${0:H}, [$1]", "=&r,r,~{memory}"(ptr %p) + ret i128 %r +} + +; As in GCC, the first register holds the doubleword at the lower address. So +; the high half is in the second register on little-endian, and in the first +; on big-endian. +define i64 @test_i128_high_half() { +; LE-LABEL: test_i128_high_half: +; LE: // %bb.0: // %entry +; LE-NEXT: //APP +; LE-NEXT: // x8 x0 +; LE-NEXT: //NO_APP +; LE-NEXT: ret +; +; BE-LABEL: test_i128_high_half: +; BE: // %bb.0: // %entry +; BE-NEXT: //APP +; BE-NEXT: // x0 x8 +; BE-NEXT: //NO_APP +; BE-NEXT: ret +entry: + %r = call i128 asm sideeffect "// $0 ${0:H}", "=r"() + %hi = lshr i128 %r, 64 + %t = trunc i128 %hi to i64 + ret i64 %t +} >From 8c5060157d05b2ae061a582b7868e8040907a52c Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Wed, 18 Nov 2026 11:13:28 -0800 Subject: [PATCH 3/4] [Clang][AArch64] Don't warn about a 128-bit "r" operand Clang warned that a 128-bit value in an "r" operand didn't fit its register and suggested %w. But the value takes two X registers, and %H names the second, so don't warn about it. Assisted-by: Claude Opus 5.5 --- clang/lib/Basic/Targets/AArch64.cpp | 4 ++++ clang/test/Sema/arm64-inline-asm.c | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp index 7ddf9d87ad163e..7474b2a7649965 100644 --- a/clang/lib/Basic/Targets/AArch64.cpp +++ b/clang/lib/Basic/Targets/AArch64.cpp @@ -1766,6 +1766,10 @@ bool AArch64TargetInfo::validateConstraintModifier( if (Size == 64) return true; + // A 128-bit value takes two 'x' registers, and %H names the second. + if (Size == 128) + return true; + if (Size == 512) return HasLS64; diff --git a/clang/test/Sema/arm64-inline-asm.c b/clang/test/Sema/arm64-inline-asm.c index c804478ea68d55..22757faa3bc770 100644 --- a/clang/test/Sema/arm64-inline-asm.c +++ b/clang/test/Sema/arm64-inline-asm.c @@ -13,3 +13,8 @@ void test_clobber_conflict(void) { asm volatile("nop" :: "r"(x) : "%x1"); // expected-error {{conflicts with asm clobber list}} asm volatile("nop" : "=r"(x) :: "%x1"); // expected-error {{conflicts with asm clobber list}} } + +// A 128-bit value takes two 64-bit registers, and %H names the second. +void test_int128(unsigned __int128 *p, unsigned __int128 x) { + asm volatile("stp %1, %H1, [%0]" :: "r"(p), "r"(x) : "memory"); +} >From 17bd37dfeac9b55b54c2ab96a30aa055252d5fa7 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 19 Nov 2026 21:05:14 -0800 Subject: [PATCH 4/4] [AArch64] Use 64-bit registers for wide FP and vector "r" operands An fp128 or a 128-bit vector, such as a <2 x double>, in an "r" operand was still given 32-bit registers, so half of its bits were lost, as with an i128 before. Give it 64-bit registers too. Assisted-by: Claude Opus 5.5 --- .../Target/AArch64/AArch64ISelLowering.cpp | 9 ++- llvm/test/CodeGen/AArch64/fmlal-loreg.ll | 7 +-- .../CodeGen/AArch64/inline-asm-wide-gpr.ll | 63 ++++++++++++++++++- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index fc01dcfb402619..9f9b4afb31af6a 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -14488,11 +14488,10 @@ AArch64TargetLowering::getRegForInlineAsmConstraint( if (Subtarget->hasLS64() && VT.getSizeInBits() == 512) return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass); - // An integer wider than 64 bits, such as an i128, takes a 64-bit - // register for each 64 bits of it. 32-bit registers would hold only half - // of it. - if (VT.getFixedSizeInBits() == 64 || - (VT.isScalarInteger() && VT.getFixedSizeInBits() > 64)) + // A value wider than 64 bits, such as an i128 or an fp128, takes a + // 64-bit register for each 64 bits of it. 32-bit registers would hold + // only half of it. + if (VT.getFixedSizeInBits() >= 64) return std::make_pair(0U, &AArch64::GPR64commonRegClass); return std::make_pair(0U, &AArch64::GPR32commonRegClass); diff --git a/llvm/test/CodeGen/AArch64/fmlal-loreg.ll b/llvm/test/CodeGen/AArch64/fmlal-loreg.ll index 20737a73183944..18988969684484 100644 --- a/llvm/test/CodeGen/AArch64/fmlal-loreg.ll +++ b/llvm/test/CodeGen/AArch64/fmlal-loreg.ll @@ -10,16 +10,15 @@ define <4 x float> @test(ptr %lhs_panel, ptr %rhs_panel, <4 x float> %a) { ; CHECK-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset b8, -16 -; CHECK-NEXT: fmov x8, d0 +; CHECK-NEXT: mov x8, v0.d[1] +; CHECK-NEXT: fmov x9, d0 ; CHECK-NEXT: ldr q8, [x0] ; CHECK-NEXT: ldr q16, [x1] -; CHECK-NEXT: lsr x9, x8, #32 ; CHECK-NEXT: //APP ; CHECK-NEXT: nop ; CHECK-NEXT: //NO_APP -; CHECK-NEXT: mov w8, w8 -; CHECK-NEXT: orr x8, x8, x9, lsl #32 ; CHECK-NEXT: fmov d0, x8 +; CHECK-NEXT: mov v0.d[1], x9 ; CHECK-NEXT: fmlal v0.4s, v16.4h, v8.h[0] ; CHECK-NEXT: mov v1.16b, v0.16b ; CHECK-NEXT: fmlal2 v1.4s, v16.4h, v8.h[0] diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll index 78d948acf108fa..659e40854903dc 100644 --- a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll +++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll @@ -2,8 +2,9 @@ ; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,LE ; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,BE -; An integer wider than 64 bits in an "r" operand goes in 64-bit registers, -; one for each 64 bits of it. 32-bit registers would hold only half of it. +; A value wider than 64 bits in an "r" operand, such as an i128 or an fp128, +; goes in 64-bit registers, one for each 64 bits of it. 32-bit registers would +; hold only half of it. ; Big-endian puts the halves of an i128 in the other order (see ; @test_i128_high_half). @@ -92,3 +93,61 @@ entry: %t = trunc i128 %hi to i64 ret i64 %t } + +; A floating-point value or a vector goes in X registers the same way, so +; "stp $1, ${1:H}" stores the same bytes that a plain store of it would. +define void @test_fp128_store(ptr %p, fp128 %x) { +; CHECK-LABEL: test_fp128_store: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: str q0, [sp, #-16]! +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: ldp x8, x9, [sp] +; CHECK-NEXT: //APP +; CHECK-NEXT: stp x8, x9, [x0] +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret +entry: + call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, fp128 %x) + ret void +} + +define fp128 @test_fp128_load(ptr %p) { +; CHECK-LABEL: test_fp128_load: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: //APP +; CHECK-NEXT: ldp x8, x9, [x0] +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: stp x8, x9, [sp, #-16]! +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: ldr q0, [sp], #16 +; CHECK-NEXT: ret +entry: + %r = call fp128 asm sideeffect "ldp $0, ${0:H}, [$1]", "=&r,r,~{memory}"(ptr %p) + ret fp128 %r +} + +define void @test_v2f64_store(ptr %p, <2 x double> %x) { +; LE-LABEL: test_v2f64_store: +; LE: // %bb.0: // %entry +; LE-NEXT: mov x8, v0.d[1] +; LE-NEXT: fmov x9, d0 +; LE-NEXT: //APP +; LE-NEXT: stp x9, x8, [x0] +; LE-NEXT: //NO_APP +; LE-NEXT: ret +; +; BE-LABEL: test_v2f64_store: +; BE: // %bb.0: // %entry +; BE-NEXT: str q0, [sp, #-16]! +; BE-NEXT: .cfi_def_cfa_offset 16 +; BE-NEXT: ldp x8, x9, [sp] +; BE-NEXT: //APP +; BE-NEXT: stp x8, x9, [x0] +; BE-NEXT: //NO_APP +; BE-NEXT: add sp, sp, #16 +; BE-NEXT: ret +entry: + call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, <2 x double> %x) + ret void +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
