https://github.com/SavchenkoValeriy updated https://github.com/llvm/llvm-project/pull/232787
>From 32e4b1400ab3ef6d1729ad08e2b01040f0154130 Mon Sep 17 00:00:00 2001 From: Valeriy Savchenko <[email protected]> Date: Fri, 16 Oct 2026 11:40:47 +0100 Subject: [PATCH 1/2] [AArch64][TTI] Add pre-commit tests (NFC) --- llvm/test/Analysis/CostModel/AArch64/bsl.ll | 184 ++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 llvm/test/Analysis/CostModel/AArch64/bsl.ll diff --git a/llvm/test/Analysis/CostModel/AArch64/bsl.ll b/llvm/test/Analysis/CostModel/AArch64/bsl.ll new file mode 100644 index 0000000000000..2101bd9d8f750 --- /dev/null +++ b/llvm/test/Analysis/CostModel/AArch64/bsl.ll @@ -0,0 +1,184 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -mtriple=aarch64 -mattr=+sve -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output %s | FileCheck %s --check-prefix=COST +; RUN: llc -mtriple=aarch64 -mattr=+sve %s -o - | FileCheck %s --check-prefix=CODE + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +; TODO: The Not is folded into BSL and should be free. +define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) { +; COST-LABEL: 'bsl' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1) +; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r +; +; CODE-LABEL: bsl: +; CODE: // %bb.0: +; CODE-NEXT: bsl v0.8b, v1.8b, v2.8b +; CODE-NEXT: ret + %n = xor <8 x i8> %m, splat (i8 -1) + %x = and <8 x i8> %m, %a + %y = and <8 x i8> %n, %b + %r = or <8 x i8> %x, %y + ret <8 x i8> %r +} + +; TODO: The Not should be free here as well +define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d, ptr %p, ptr %q) { +; COST-LABEL: 'bsl_shared_mask' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <16 x i8> %m, splat (i8 -1) +; COST-NEXT: Cost Model: Found costs of 1 for: %x0 = and <16 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y0 = and <16 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 1 for: %r0 = or <16 x i8> %x0, %y0 +; COST-NEXT: Cost Model: Found costs of 1 for: %x1 = and <16 x i8> %m, %c +; COST-NEXT: Cost Model: Found costs of 1 for: %y1 = and <16 x i8> %n, %d +; COST-NEXT: Cost Model: Found costs of 1 for: %r1 = or <16 x i8> %x1, %y1 +; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r0, ptr %p, align 16 +; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r1, ptr %q, align 16 +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void +; +; CODE-LABEL: bsl_shared_mask: +; CODE: // %bb.0: +; CODE-NEXT: bif v1.16b, v2.16b, v0.16b +; CODE-NEXT: bsl v0.16b, v3.16b, v4.16b +; CODE-NEXT: str q1, [x0] +; CODE-NEXT: str q0, [x1] +; CODE-NEXT: ret + %n = xor <16 x i8> %m, splat (i8 -1) + %x0 = and <16 x i8> %m, %a + %y0 = and <16 x i8> %n, %b + %r0 = or <16 x i8> %x0, %y0 + %x1 = and <16 x i8> %m, %c + %y1 = and <16 x i8> %n, %d + %r1 = or <16 x i8> %x1, %y1 + store <16 x i8> %r0, ptr %p + store <16 x i8> %r1, ptr %q + ret void +} + +define <8 x i8> @bsl_not_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) { +; COST-LABEL: 'bsl_not_multiuse' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1) +; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y +; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %n, ptr %p, align 8 +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r +; +; CODE-LABEL: bsl_not_multiuse: +; CODE: // %bb.0: +; CODE-NEXT: bif v1.8b, v2.8b, v0.8b +; CODE-NEXT: mvn v2.8b, v0.8b +; CODE-NEXT: str d2, [x0] +; CODE-NEXT: fmov d0, d1 +; CODE-NEXT: ret + %n = xor <8 x i8> %m, splat (i8 -1) + %x = and <8 x i8> %m, %a + %y = and <8 x i8> %n, %b + %r = or <8 x i8> %x, %y + store <8 x i8> %n, ptr %p + ret <8 x i8> %r +} + +; TODO: BIT is still formed, so %y and the Not should be free. +define <8 x i8> @neg_bsl_and_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) { +; COST-LABEL: 'neg_bsl_and_multiuse' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1) +; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y +; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %x, ptr %p, align 8 +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r +; +; CODE-LABEL: neg_bsl_and_multiuse: +; CODE: // %bb.0: +; CODE-NEXT: bit v2.8b, v1.8b, v0.8b +; CODE-NEXT: and v1.8b, v0.8b, v1.8b +; CODE-NEXT: str d1, [x0] +; CODE-NEXT: fmov d0, d2 +; CODE-NEXT: ret + %n = xor <8 x i8> %m, splat (i8 -1) + %x = and <8 x i8> %m, %a + %y = and <8 x i8> %n, %b + %r = or <8 x i8> %x, %y + store <8 x i8> %x, ptr %p + ret <8 x i8> %r +} + +define <8 x i8> @neg_bsl_other_block(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, i1 %f, ptr %p) { +; COST-LABEL: 'neg_bsl_other_block' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1) +; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %f, label %t, label %e +; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y +; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %r, ptr %p, align 8 +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %a +; +; CODE-LABEL: neg_bsl_other_block: +; CODE: // %bb.0: // %entry +; CODE-NEXT: tbz w0, #0, .LBB4_2 +; CODE-NEXT: // %bb.1: // %t +; CODE-NEXT: and v1.8b, v0.8b, v1.8b +; CODE-NEXT: bic v0.8b, v2.8b, v0.8b +; CODE-NEXT: orr v1.8b, v1.8b, v0.8b +; CODE-NEXT: str d1, [x1] +; CODE-NEXT: .LBB4_2: // %common.ret +; CODE-NEXT: fmov d0, d1 +; CODE-NEXT: ret +entry: + %n = xor <8 x i8> %m, splat (i8 -1) + %x = and <8 x i8> %m, %a + %y = and <8 x i8> %n, %b + br i1 %f, label %t, label %e +t: + %r = or <8 x i8> %x, %y + store <8 x i8> %r, ptr %p + ret <8 x i8> %r +e: + ret <8 x i8> %a +} + +define i32 @neg_bsl_scalar(i32 %m, i32 %a, i32 %b) { +; COST-LABEL: 'neg_bsl_scalar' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor i32 %m, -1 +; COST-NEXT: Cost Model: Found costs of 1 for: %x = and i32 %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y = and i32 %n, %b +; COST-NEXT: Cost Model: Found costs of 1 for: %r = or i32 %x, %y +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %r +; +; CODE-LABEL: neg_bsl_scalar: +; CODE: // %bb.0: +; CODE-NEXT: and w8, w0, w1 +; CODE-NEXT: bic w9, w2, w0 +; CODE-NEXT: orr w0, w8, w9 +; CODE-NEXT: ret + %n = xor i32 %m, -1 + %x = and i32 %m, %a + %y = and i32 %n, %b + %r = or i32 %x, %y + ret i32 %r +} + +define <vscale x 16 x i8> @neg_bsl_scalable(<vscale x 16 x i8> %m, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) { +; COST-LABEL: 'neg_bsl_scalable' +; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <vscale x 16 x i8> %m, splat (i8 -1) +; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <vscale x 16 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <vscale x 16 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <vscale x 16 x i8> %x, %y +; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 16 x i8> %r +; +; CODE-LABEL: neg_bsl_scalable: +; CODE: // %bb.0: +; CODE-NEXT: and z1.d, z0.d, z1.d +; CODE-NEXT: bic z0.d, z2.d, z0.d +; CODE-NEXT: orr z0.d, z1.d, z0.d +; CODE-NEXT: ret + %n = xor <vscale x 16 x i8> %m, splat (i8 -1) + %x = and <vscale x 16 x i8> %m, %a + %y = and <vscale x 16 x i8> %n, %b + %r = or <vscale x 16 x i8> %x, %y + ret <vscale x 16 x i8> %r +} >From 184ba0e40f9009df59b540b7205c747d456b5b49 Mon Sep 17 00:00:00 2001 From: Valeriy Savchenko <[email protected]> Date: Fri, 16 Oct 2026 13:28:13 +0100 Subject: [PATCH 2/2] [AArch64][TTI] Cost the Ands of a NEON bit select as free Or(And(M, A), And(Not(M), B)) selects to a single BSL/BIT/BIF. Charge the Or and treat the single-use Ands in its block as free, using the matcher shared with operand sinking. --- .../AArch64/AArch64TargetTransformInfo.cpp | 10 ++++++++++ llvm/test/Analysis/CostModel/AArch64/bsl.ll | 16 ++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index b631ca5ac9442..561cca222b26f 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -5123,6 +5123,16 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( if (!is_contained({ISD::ADD, ISD::SUB, ISD::MUL}, ISD)) return InstructionCost::getInvalid(); + if (CtxI && CtxI->getOpcode() == Opcode && !CtxI->use_empty() && + useNeonVector(Ty)) { + // The Ands of a bit select are folded into the BSL that the Or becomes. + Instruction *MainAnd, *OtherAnd; + Value *A, *B; + if (Opcode == Instruction::And && + matchBSL(CtxI->user_back(), MainAnd, OtherAnd, A, B)) + return 0; + } + // Legalize the type. std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty); diff --git a/llvm/test/Analysis/CostModel/AArch64/bsl.ll b/llvm/test/Analysis/CostModel/AArch64/bsl.ll index 2101bd9d8f750..80a03e7307ac2 100644 --- a/llvm/test/Analysis/CostModel/AArch64/bsl.ll +++ b/llvm/test/Analysis/CostModel/AArch64/bsl.ll @@ -8,8 +8,8 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) { ; COST-LABEL: 'bsl' ; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1) -; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a -; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 0 for: %x = and <8 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 0 for: %y = and <8 x i8> %n, %b ; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y ; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r ; @@ -28,11 +28,11 @@ define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) { define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d, ptr %p, ptr %q) { ; COST-LABEL: 'bsl_shared_mask' ; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <16 x i8> %m, splat (i8 -1) -; COST-NEXT: Cost Model: Found costs of 1 for: %x0 = and <16 x i8> %m, %a -; COST-NEXT: Cost Model: Found costs of 1 for: %y0 = and <16 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 0 for: %x0 = and <16 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 0 for: %y0 = and <16 x i8> %n, %b ; COST-NEXT: Cost Model: Found costs of 1 for: %r0 = or <16 x i8> %x0, %y0 -; COST-NEXT: Cost Model: Found costs of 1 for: %x1 = and <16 x i8> %m, %c -; COST-NEXT: Cost Model: Found costs of 1 for: %y1 = and <16 x i8> %n, %d +; COST-NEXT: Cost Model: Found costs of 0 for: %x1 = and <16 x i8> %m, %c +; COST-NEXT: Cost Model: Found costs of 0 for: %y1 = and <16 x i8> %n, %d ; COST-NEXT: Cost Model: Found costs of 1 for: %r1 = or <16 x i8> %x1, %y1 ; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r0, ptr %p, align 16 ; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r1, ptr %q, align 16 @@ -60,8 +60,8 @@ define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8> define <8 x i8> @bsl_not_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) { ; COST-LABEL: 'bsl_not_multiuse' ; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1) -; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a -; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b +; COST-NEXT: Cost Model: Found costs of 0 for: %x = and <8 x i8> %m, %a +; COST-NEXT: Cost Model: Found costs of 0 for: %y = and <8 x i8> %n, %b ; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y ; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %n, ptr %p, align 8 ; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
