https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/223393
>From 455326b678880e25cb2ec74a640ec25f555f732d Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Thu, 1 Oct 2026 08:50:11 +0000 Subject: [PATCH 01/10] Pre-commit tests --- .../Inline/AArch64/sme-pstatesm-attrs.ll | 401 +++++++++++++++--- .../AArch64/sme-pstatesm-inline-transitive.ll | 49 +++ 2 files changed, 387 insertions(+), 63 deletions(-) create mode 100644 llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index 077a3aa49fb414..49af107d7e89b4 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline | FileCheck %s +; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sve,+sme -S -passes=inline | FileCheck %s declare i32 @llvm.vscale.i32() @@ -7,7 +7,7 @@ declare i32 @llvm.vscale.i32() ; by the other functions below. If we see the call to one of these functions ; being replaced by 'llvm.vscale()', then we know it has been inlined. -define i32 @normal_callee() #0 { +define i32 @normal_callee() { ; CHECK-LABEL: define i32 @normal_callee ; CHECK-SAME: () #[[ATTR1:[0-9]+]] { ; CHECK-NEXT: entry: @@ -19,7 +19,7 @@ entry: ret i32 %res } -define i32 @streaming_callee() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_callee() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_callee ; CHECK-SAME: () #[[ATTR2:[0-9]+]] { ; CHECK-NEXT: entry: @@ -31,7 +31,7 @@ entry: ret i32 %res } -define i32 @locally_streaming_callee() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_callee() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_callee ; CHECK-SAME: () #[[ATTR3:[0-9]+]] { ; CHECK-NEXT: entry: @@ -43,7 +43,7 @@ entry: ret i32 %res } -define i32 @streaming_compatible_callee() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_callee() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_callee ; CHECK-SAME: () #[[ATTR4:[0-9]+]] { ; CHECK-NEXT: entry: @@ -55,7 +55,7 @@ entry: ret i32 %res } -define i32 @streaming_compatible_locally_streaming_callee() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_callee() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_callee ; CHECK-SAME: () #[[ATTR5:[0-9]+]] { ; CHECK-NEXT: entry: @@ -84,7 +84,7 @@ entry: ; [ ] N -> SC ; [ ] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_normal_callee_inline() #0 { +define i32 @normal_caller_normal_callee_inline() { ; CHECK-LABEL: define i32 @normal_caller_normal_callee_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -101,7 +101,7 @@ entry: ; [ ] N -> SC ; [ ] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_streaming_callee_dont_inline() #0 { +define i32 @normal_caller_streaming_callee_dont_inline() { ; CHECK-LABEL: define i32 @normal_caller_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -118,7 +118,7 @@ entry: ; [x] N -> SC ; [ ] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_streaming_compatible_callee_inline() #0 { +define i32 @normal_caller_streaming_compatible_callee_inline() { ; CHECK-LABEL: define i32 @normal_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -135,7 +135,7 @@ entry: ; [ ] N -> SC ; [x] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_locally_streaming_callee_dont_inline() #0 { +define i32 @normal_caller_locally_streaming_callee_dont_inline() { ; CHECK-LABEL: define i32 @normal_caller_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -152,7 +152,7 @@ entry: ; [ ] N -> SC ; [ ] N -> N + B ; [x] N -> SC + B -define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() #0 { +define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() { ; CHECK-LABEL: define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -169,7 +169,7 @@ entry: ; [ ] S -> SC ; [ ] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -186,7 +186,7 @@ entry: ; [ ] S -> SC ; [ ] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_streaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -203,7 +203,7 @@ entry: ; [x] S -> SC ; [ ] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -220,7 +220,7 @@ entry: ; [ ] S -> SC ; [x] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -237,7 +237,7 @@ entry: ; [ ] S -> SC ; [ ] S -> N + B ; [x] S -> SC + B -define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -254,7 +254,7 @@ entry: ; [ ] N + B -> SC ; [ ] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -271,7 +271,7 @@ entry: ; [ ] N + B -> SC ; [ ] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -288,7 +288,7 @@ entry: ; [x] N + B -> SC ; [ ] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -305,7 +305,7 @@ entry: ; [ ] N + B -> SC ; [x] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -322,7 +322,7 @@ entry: ; [ ] N + B -> SC ; [ ] N + B -> N + B ; [x] N + B -> SC + B -define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -339,7 +339,7 @@ entry: ; [ ] SC -> SC ; [ ] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -356,7 +356,7 @@ entry: ; [ ] SC -> SC ; [ ] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -373,7 +373,7 @@ entry: ; [x] SC -> SC ; [ ] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -390,7 +390,7 @@ entry: ; [ ] SC -> SC ; [x] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -407,7 +407,7 @@ entry: ; [ ] SC -> SC ; [ ] SC -> N + B ; [x] SC -> SC + B -define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -423,7 +423,7 @@ entry: ; [ ] SC + B -> SC ; [ ] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -440,7 +440,7 @@ entry: ; [ ] SC + B -> SC ; [ ] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -457,7 +457,7 @@ entry: ; [x] SC + B -> SC ; [ ] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -474,7 +474,7 @@ entry: ; [ ] SC + B -> SC ; [x] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -491,7 +491,7 @@ entry: ; [ ] SC + B -> SC ; [ ] SC + B -> N + B ; [x] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -503,7 +503,7 @@ entry: ret i32 %res } -define void @normal_callee_with_inlineasm() #0 { +define void @normal_callee_with_inlineasm() { ; CHECK-LABEL: define void @normal_callee_with_inlineasm ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -515,7 +515,7 @@ entry: ret void } -define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_normal_callee_with_inlineasm_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -527,7 +527,7 @@ entry: ret void } -define i64 @normal_callee_with_intrinsic_call() #0 { +define i64 @normal_callee_with_intrinsic_call() { ; CHECK-LABEL: define i64 @normal_callee_with_intrinsic_call ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -539,7 +539,7 @@ entry: ret i64 %res } -define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -553,7 +553,7 @@ entry: declare i64 @llvm.aarch64.sve.cntb(i32) -define i64 @normal_callee_call_sme_state() #0 { +define i64 @normal_callee_call_sme_state() { ; CHECK-LABEL: define i64 @normal_callee_call_sme_state ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -569,7 +569,7 @@ entry: declare {i64, i64} @__arm_sme_state() -define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -585,7 +585,7 @@ entry: declare void @nonstreaming_body() -define void @nonstreaming_caller_single_nonstreaming_callee() #0 { +define void @nonstreaming_caller_single_nonstreaming_callee() { ; CHECK-LABEL: define void @nonstreaming_caller_single_nonstreaming_callee ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: call void @nonstreaming_body() @@ -595,7 +595,7 @@ define void @nonstreaming_caller_single_nonstreaming_callee() #0 { ret void } -define void @nonstreaming_caller_multiple_nonstreaming_callees() #0 { +define void @nonstreaming_caller_multiple_nonstreaming_callees() { ; CHECK-LABEL: define void @nonstreaming_caller_multiple_nonstreaming_callees ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: call void @nonstreaming_body() @@ -608,7 +608,7 @@ define void @nonstreaming_caller_multiple_nonstreaming_callees() #0 { } ; Allow inlining, as inline it would not increase the number of streaming-mode changes. -define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @nonstreaming_body() @@ -619,7 +619,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_ca } ; Prevent inlining, as inlining it would lead to multiple streaming-mode changes. -define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() @@ -631,7 +631,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_ declare void @streaming_compatible_body() "aarch64_pstate_sm_compatible" -define void @nonstreaming_caller_single_streaming_compatible_callee() #0 { +define void @nonstreaming_caller_single_streaming_compatible_callee() { ; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: call void @streaming_compatible_body() @@ -641,42 +641,39 @@ define void @nonstreaming_caller_single_streaming_compatible_callee() #0 { ret void } -define void @nonstreaming_caller_multiple_streaming_compatible_callees() #0 { -; CHECK-LABEL: define void @nonstreaming_caller_multiple_streaming_compatible_callees -; CHECK-SAME: () #[[ATTR1]] { -; CHECK-NEXT: call void @streaming_compatible_body() +; Reject inlining; while inlining would remove a streaming-mode change, it may also change runtime behaviour. +define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline +; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @streaming_compatible_body() ; CHECK-NEXT: ret void ; - call void @streaming_compatible_body() - call void @streaming_compatible_body() + call void @nonstreaming_caller_single_streaming_compatible_callee() ret void } -; Allow inlining, as inline would remove a streaming-mode change. -define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() #0 "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline -; CHECK-SAME: () #[[ATTR2]] { +define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() alwaysinline { +; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline +; CHECK-SAME: () #[[ATTR6:[0-9]+]] { ; CHECK-NEXT: call void @streaming_compatible_body() ; CHECK-NEXT: ret void ; - call void @nonstreaming_caller_single_streaming_compatible_callee() + call void @streaming_compatible_body() ret void } -; Allow inlining, as inline would remove several streaming-mode changes. -define void @streaming_caller_to_nonstreaming_callee_with_multiple_streamingcompatible_callees_inline() #0 "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_multiple_streamingcompatible_callees_inline +; Conseratively disallow inlining when forced; it is unclear what the user's intentions were. +define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @streaming_compatible_body() -; CHECK-NEXT: call void @streaming_compatible_body() ; CHECK-NEXT: ret void ; - call void @nonstreaming_caller_multiple_streaming_compatible_callees() + call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() ret void } -define void @simple_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_enabled" { +define void @simple_streaming_function(ptr %ptr) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @simple_streaming_function ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR2]] { ; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[PTR]], align 16 @@ -687,7 +684,7 @@ define void @simple_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_enabled" } ; Don't allow inlining a streaming function into a non-streaming function. -define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) #0 { +define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) { ; CHECK-LABEL: define void @non_streaming_caller_streaming_callee_dont_inline ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] { ; CHECK-NEXT: call void @simple_streaming_function(ptr [[PTR]]) @@ -697,7 +694,7 @@ define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) #0 { ret void } -define void @simple_locally_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_body" { +define void @simple_locally_streaming_function(ptr %ptr) "aarch64_pstate_sm_body" { ; CHECK-LABEL: define void @simple_locally_streaming_function ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR3]] { ; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[PTR]], align 16 @@ -708,7 +705,7 @@ define void @simple_locally_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_b } ; Don't allow inlining a locally-streaming function into a non-streaming function. -define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) #0 { +define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) { ; CHECK-LABEL: define void @non_streaming_caller_locally_streaming_callee_dont_inline ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] { ; CHECK-NEXT: call void @simple_locally_streaming_function(ptr [[PTR]]) @@ -718,4 +715,282 @@ define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) ret void } -attributes #0 = { "target-features"="+sve,+sme" } +; +; Don't inline when there are vscale-dependent operations +; + +; It is not safe to inline functions that have vscale-dependent operations in their body +; when the streaming modes don't match up, unless the interface of the callee takes a +; vl-dependent argument. +define ptr @vscale_dependent_op(ptr %p, i64 %k) { +; CHECK-LABEL: define ptr @vscale_dependent_op +; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]] +; CHECK-NEXT: ret ptr [[RES]] +; + %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k + ret ptr %res +} + +define ptr @incompatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define ptr @incompatible_vscale_dependent_operation_sm +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 +; CHECK-NEXT: ret ptr [[RES_I]] +; + %res = call ptr @vscale_dependent_op(ptr %p, i64 4) + ret ptr %res +} + +define ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 %k, <vscale x 4 x i32> %other) { +; CHECK-LABEL: define ptr @vscale_dependent_op_vl_dependent_args +; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]], <vscale x 4 x i32> [[OTHER:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]] +; CHECK-NEXT: store <vscale x 4 x i32> [[OTHER]], ptr [[RES]], align 16 +; CHECK-NEXT: ret ptr [[RES]] +; + %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k + store <vscale x 4 x i32> %other, ptr %res + ret ptr %res +} + +define ptr @compatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define ptr @compatible_vscale_dependent_operation_sm +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 +; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[RES_I]], align 16 +; CHECK-NEXT: ret ptr [[RES_I]] +; + %res = call ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 4, <vscale x 4 x i32> zeroinitializer) + ret ptr %res +} + +; functions with scalable alloca's shouldn't be inlined if the streaming properties don't match +; as a scalable alloca is a vl-dependent operation. +define void @scalable_alloca_op(ptr %p) { +; CHECK-LABEL: define void @scalable_alloca_op +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca <vscale x 4 x i32>, align 1 +; CHECK-NEXT: call void [[P]](ptr [[ALLOCA]]) +; CHECK-NEXT: ret void +; + %alloca = alloca <vscale x 4 x i32>, align 1 + call void %p(ptr %alloca) + ret void +} + +define void @incompatible_scalable_alloca_op_caller(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @incompatible_scalable_alloca_op_caller +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[ALLOCA_I:%.*]] = alloca <vscale x 4 x i32>, align 1 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ALLOCA_I]]) +; CHECK-NEXT: call void [[P]](ptr [[ALLOCA_I]]), !inline_history [[META0:![0-9]+]] +; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ALLOCA_I]]) +; CHECK-NEXT: ret void +; + call void @scalable_alloca_op(ptr %p) + ret void +} + +; Generic intrinsics that take/return a scalable type, are vscale-dependent operations. +define i64 @intrinsic_with_scalable_type(ptr %p) { +; CHECK-LABEL: define i64 @intrinsic_with_scalable_type +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %ld = load <vscale x 2 x i64>, ptr %p + %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld) + ret i64 %res +} + +define i64 @compatible_sve_intrinsic_caller(ptr %p) { +; CHECK-LABEL: define i64 @compatible_sve_intrinsic_caller +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) +; CHECK-NEXT: ret i64 [[RES_I]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + +define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_sve_intrinsic_caller +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + +; +; Don't inline inline-asm when streaming-modes are incompatible, as the asm may +; contain vscale-dependent instructions. +; + +define void @inline_asm() { +; CHECK-LABEL: define void @inline_asm +; CHECK-SAME: () #[[ATTR1]] { +; CHECK-NEXT: call void asm sideeffect "", ""() +; CHECK-NEXT: ret void +; + call void asm sideeffect "", ""() + ret void +} + +define void @incompatible_inline_asm_sm() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @incompatible_inline_asm_sm +; CHECK-SAME: () #[[ATTR2]] { +; CHECK-NEXT: call void @inline_asm() +; CHECK-NEXT: ret void +; + call void @inline_asm() + ret void +} + +; +; Don't inline functions that contain calls to SME ABI routines (like __arm_get_current_vg()) +; + +declare i64 @__arm_get_current_vg() + +define i64 @current_vg() { +; CHECK-LABEL: define i64 @current_vg +; CHECK-SAME: () #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @__arm_get_current_vg() + ret i64 %vscale +} + +define i64 @compatible_current_vg_caller() { +; CHECK-LABEL: define i64 @compatible_current_vg_caller +; CHECK-SAME: () #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE_I]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_current_vg_caller +; CHECK-SAME: () #[[ATTR2]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +; +; Don't inline NEON intrinsics into streaming mode. +; + +define i32 @neon_intrinsic(<4 x i32> %in) { +; CHECK-LABEL: define i32 @neon_intrinsic +; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES]] +; + %res = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %in) + ret i32 %res +} + +define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller +; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[VSCALE]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; a bit of a niche case, but if the caller uses ZA but is not in streaming-mode, a NEON intrinsic is safe. +define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_za_enabled" { +; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller_za +; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR7:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES_I]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; +; Don't inline fixed-length vector code, as performance may be affected. +; + +; Functions with fixed-length vectors shouldn't be inlined if the streaming properties don't match +; as performance may be affected. However, when they have the alwaysinline property, they should +; still be inlined. + +define void @fixed_length_vector_operation_alwaysinline(ptr %p) alwaysinline { +; CHECK-LABEL: define void @fixed_length_vector_operation_alwaysinline +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR6]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + store <4 x i32> zeroinitializer, ptr %p + ret void +} + +define void @fixed_length_vector_operation_caller_force_inline(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operation_caller_force_inline +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + call void @fixed_length_vector_operation_alwaysinline(ptr %p) + ret void +} + +define void @fixed_length_vector_operation(ptr %p) { +; CHECK-LABEL: define void @fixed_length_vector_operation +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + store <4 x i32> zeroinitializer, ptr %p + ret void +} + +define void @fixed_length_vector_operation_caller_dont_inline(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operation_caller_dont_inline +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + call void @fixed_length_vector_operation(ptr %p) + ret void +} + +; +; strictfp and streaming-mode +; + +define float @strict_fp(float %in) strictfp { +; CHECK-LABEL: define float @strict_fp +; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR8:[0-9]+]] { +; CHECK-NEXT: [[RES:%.*]] = fadd float [[IN]], 4.200000e+01 +; CHECK-NEXT: ret float [[RES]] +; + %res = fadd float %in, 42.0; + ret float %res +} + +; floating point environment is different in streaming mode, so don't inline. +define float @incompatible_fp_environment_sm(float %in) strictfp "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define float @incompatible_fp_environment_sm +; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR9:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[IN]], float 4.200000e+01, metadata !"round.tonearest", metadata !"fpexcept.ignore") #[[ATTR14:[0-9]+]] +; CHECK-NEXT: ret float [[RES_I]] +; + %res = call float @strict_fp(float %in) + ret float %res +} diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll new file mode 100644 index 00000000000000..21242b304591de --- /dev/null +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll @@ -0,0 +1,49 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -passes=inline -inlinedefault-threshold=1 < %s | FileCheck %s --check-prefixes=CHECK,TH1 +; RUN: opt -S -passes=inline -inlinedefault-threshold=25 < %s | FileCheck %s --check-prefixes=CHECK,TH25 + +target triple = "aarch64" + +declare void @streaming_f() "aarch64_pstate_sm_enabled" + +define void @c() #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @c +; CHECK-SAME: () #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: call void @streaming_f() +; CHECK-NEXT: call void @streaming_f() +; CHECK-NEXT: ret void +; + call void @streaming_f() + call void @streaming_f() + ret void +} + +; Don't inline the call to @c, as inlining a streaming function into a non-streaming function is considered incompatible. +define void @b() #0 { +; CHECK-LABEL: define void @b +; CHECK-SAME: () #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: call void @c() +; CHECK-NEXT: ret void +; + call void @c() + ret void +} + +; Inline the call to @c in @a, as that avoids streaming mode changes. +define void @a() #0 "aarch64_pstate_sm_enabled" { +; TH1-LABEL: define void @a +; TH1-SAME: () #[[ATTR1]] { +; TH1-NEXT: call void @c() +; TH1-NEXT: ret void +; +; TH25-LABEL: define void @a +; TH25-SAME: () #[[ATTR1]] { +; TH25-NEXT: call void @streaming_f() +; TH25-NEXT: call void @streaming_f() +; TH25-NEXT: ret void +; + call void @b() + ret void +} + +attributes #0 = { "target-features"="+sve,+sme" } >From 69d50be077d2ce512edd3a8e72b98ef012e73d3f Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 14 Sep 2026 08:05:19 +0000 Subject: [PATCH 02/10] [AArch64][SME] Allow more inlining when SME attributes are incompatible. At the moment, 'areInlineCompatible' is very strict as it conservatively disallows inlining any callee if they use intrinsics and have incompatible SME attributes. This PR relaxes those constraints by allowing more intrinsics. It also updates the Clang diagnostic to match the 'new' behaviour that a function is no longer inlined despite 'always_inline' when they are not inline compatible. --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 +- .../AArch64/sme-inline-streaming-attrs.c | 6 +- .../AArch64/AArch64TargetTransformInfo.cpp | 161 ++++++++++++++++-- .../sme-pstatesm-attrs-low-threshold.ll | 47 ----- .../Inline/AArch64/sme-pstatesm-attrs.ll | 19 +-- 5 files changed, 158 insertions(+), 77 deletions(-) delete mode 100644 llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 031858610ede2f..e88031a13c7fcd 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -317,7 +317,7 @@ def err_function_always_inline_attribute_mismatch : Error< "always_inline function %1 and its caller %0 have mismatching %2 attributes">; def warn_function_always_inline_attribute_mismatch : Warning< "always_inline function %1 and its caller %0 have mismatching %2 attributes, " - "inlining may change runtime behaviour">, InGroup<AArch64SMEAttributes>; + "%1 may not be inlined">, InGroup<AArch64SMEAttributes>; def err_function_always_inline_new_za : Error< "always_inline function %0 has new za state">; def err_function_always_inline_new_zt0 diff --git a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c index 68102c9ded40c4..ba9f24032d2745 100644 --- a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c +++ b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c @@ -26,7 +26,7 @@ void caller(void) { #ifdef TEST_COMPATIBLE void caller_compatible(void) __arm_streaming_compatible { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); // expected-error {{always_inline function 'inlined_fn_streaming' and its caller 'caller_compatible' have mismatching streaming attributes}} inlined_fn_local(); // expected-error {{always_inline function 'inlined_fn_local' and its caller 'caller_compatible' have mismatching streaming attributes}} @@ -35,7 +35,7 @@ void caller_compatible(void) __arm_streaming_compatible { #ifdef TEST_STREAMING void caller_streaming(void) __arm_streaming { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); @@ -45,7 +45,7 @@ void caller_streaming(void) __arm_streaming { #ifdef TEST_LOCALLY __arm_locally_streaming void caller_local(void) { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index ddea78fa4e8083..19a9b07389c6ae 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -237,23 +237,100 @@ static bool isSMEABIRoutineCall(const CallInst &CI, SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine(); } +/// Returns true if \p I is an intrinsic that may not be compatible with a +/// different streaming mode (because it depends on vscale). +static bool isPossiblyIncompatibleIntrinsic(const Instruction *I) { + if (I->isDebugOrPseudoInst()) + return false; + + if (auto *II = dyn_cast<IntrinsicInst>(I)) { + unsigned IID = II->getIntrinsicID(); + switch (IID) { + default: + return Intrinsic::isTargetIntrinsic(IID); + case Intrinsic::vscale: + case Intrinsic::masked_gather: + case Intrinsic::masked_scatter: + return true; + } + } + + return false; +} + /// Returns true if the function has explicit operations that can only be /// lowered using incompatible instructions for the selected mode. This also /// returns true if the function F may use or modify ZA state. static bool hasPossibleIncompatibleOps(const Function *F, - const AArch64TargetLowering &TLI) { + const AArch64TargetLowering &TLI, + bool ConsiderZA, bool ConsiderZT, + bool ConsiderSM) { + assert((ConsiderZA || ConsiderZT || ConsiderSM) && + "No SME state to consider"); + + bool IsAlwaysInline = F->hasFnAttribute(Attribute::AlwaysInline); + bool HasVLDependentArgsOrRet = + F->getReturnType()->isScalableTy() || + any_of(F->getFunctionType()->params(), + [](const Type *T) { return T->isScalableTy(); }); + for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { - // Be conservative for now and assume that any call to inline asm or to - // intrinsics could could result in non-streaming ops (e.g. calls to - // @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that - // all native LLVM instructions can be lowered to compatible instructions. - if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() && - (cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) || - isSMEABIRoutineCall(cast<CallInst>(I), TLI))) + // Inlining operations on fixed-length vectors when the streaming + // mode does not match, is rejected because performance may be impacted. + // This decision should eventually be moved the cost-model. + if (!IsAlwaysInline && ConsiderSM && + (isa<FixedVectorType>(I.getType()) || + any_of(I.operand_values(), [](const Value *V) { + return isa<FixedVectorType>(V->getType()); + }))) + return true; + + // Inlining operations on scalable vectors is rejected because it is + // a vscale-dependent operation. The only exception is when the interface + // already has vscale-dependent arguments/return value, as the ACLE + // describes that in order for the program to have defined behaviour is + // for vscale to match in both modes. + if (ConsiderSM && !HasVLDependentArgsOrRet) { + if (I.getType()->isScalableTy() || + any_of( + I.operand_values(), + [](const Value *V) { return V->getType()->isScalableTy(); }) || + (isa<GetElementPtrInst>(I) && cast<GetElementPtrInst>(I) + .getSourceElementType() + ->isScalableTy()) || + (isa<AllocaInst>(I) && cast<AllocaInst>(I).isScalable())) + return true; + } + + auto *CB = dyn_cast<CallBase>(&I); + if (!CB) + continue; + + // Inline asm must be rejected as it could use SME state. + if (CB->isInlineAsm()) return true; + + if (auto *CI = dyn_cast<CallInst>(&I)) { + // If the callee has calls to streaming compatible functions, then those + // may have vl-dependent statements. Be cautious about inlining such + // calls, as the streaming-compatible calls would otherwise be executed + // in a different streaming mode. + if (ConsiderSM && CI->getCalledFunction()) { + SMEAttrs CalleeAttrs(*CI->getCalledFunction()); + if (CalleeAttrs.hasStreamingCompatibleInterface()) + return true; + } + + if (isSMEABIRoutineCall(*CI, TLI)) + return true; + + if (ConsiderSM && isPossiblyIncompatibleIntrinsic(&I)) + return true; + } } } + return false; } @@ -281,6 +358,51 @@ bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const { return F.hasFnAttribute("fmv-features"); } +/// The compiler must not inline when that may alter the behavior of the +/// program. This is especially relevant around SME which implements different +/// runtime modes and maintains external state through attributes. +/// +/// The compiler must not inline when: +// +/// * The module is compiled with 'strict-fp' and the called function +/// implements a different FP environment than the caller. +/// +/// * The called function has operations that are incompatible in the mode +/// of the caller, e.g.: +/// * inlining non-streaming-only instructions into a streaming function. +/// * inlining streaming-only instructions into a non-streaming function. +/// +/// Inline asm blocks must be entered and exited in the [streaming] mode of +/// the parent function. There is no language-level mechanism to inform the +/// compiler that a particular inline asm block is streaming compatible, so +/// the compiler must reject this as a candidate for inlining. +/// +/// * The called function contains vscale-dependent operations but otherwise +/// does not take/return VL-dependent arguments (see definition in the +/// ACLE (Arm C/C++ Language Extensions)). +/// +/// * The called function sets up new ZA/ZT state into a function that already +/// has ZA or ZT state, as that is not valid as per the ACLE. +/// +/// If the called function has an `alwaysinline` attribute and any of the above +/// conditions is true, then an error should be reported. +/// +/// The compiler should not inline when: +// +/// * The called function has fixed-length vectors and the caller is in +/// streaming mode, as this may cause performance regressions. This should +/// never result in an error to be reported. +/// +/// * The called function sets up new ZA/ZT state into a function that has no +/// ZA and no ZT state, as the compiler currently cannot transfer the +/// attribute to the caller. It may also impact performance, but that should +/// be covered by AArch64TTIImpl::getInlineCallPenalty(). +/// +/// If a callee is incompatible with its caller, then the inliner should prevent +/// inlining. LLVM's `alwaysinline` attribute has no impact on whether a callee +/// is inline compatible. This is especially relevant with streaming-mode +/// attributes, where `alwaysinline`ing a function into a function with a +/// different streaming mode may otherwise impact runtime behaviour. bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { SMECallAttrs CallAttrs(*Caller, *Callee); @@ -292,6 +414,9 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().hasStreamingInterfaceOrBody()) return false; + if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + return false; + // When inlining, we should consider the body of the function, not the // interface. if (CallAttrs.callee().hasStreamingBody()) { @@ -299,15 +424,21 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().set(SMEAttrs::SM_Enabled, true); } - if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + bool ConsiderZA = CallAttrs.requiresZASave(); + bool ConsiderZT = CallAttrs.requiresPreservingZT0() || + CallAttrs.requiresPreservingAllZAState(); + bool ConsiderSM = CallAttrs.requiresSMChange(); + + // FP environment is interpreted differently between streaming mode and + // non-streaming mode, so are not inline-compatible. + // FIXME: Analyze whether the callee actually has any FP operations. + if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP())) return false; - if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() || - CallAttrs.requiresPreservingZT0() || - CallAttrs.requiresPreservingAllZAState()) { - if (hasPossibleIncompatibleOps(Callee, *getTLI())) - return false; - } + if ((ConsiderZA || ConsiderZT || ConsiderSM) && + hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderZT, + ConsiderSM)) + return false; return BaseT::areInlineCompatible(Caller, Callee); } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll deleted file mode 100644 index 8a608a1b8e156c..00000000000000 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll +++ /dev/null @@ -1,47 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline -inlinedefault-threshold=1 | FileCheck %s - -; This test sets the inline-threshold to 1 such that by default the call to @streaming_callee is not inlined. -; However, if the call to @streaming_callee requires a streaming-mode change, it should always inline the call because the streaming-mode change is more expensive. -target triple = "aarch64" - -declare void @streaming_compatible_f() #0 "aarch64_pstate_sm_compatible" - -; Function @non_streaming_callee doesn't contain any operations that may use ZA -; state and therefore can be legally inlined into a normal function. -define void @non_streaming_callee() #0 { -; CHECK-LABEL: define void @non_streaming_callee -; CHECK-SAME: () #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: ret void -; - call void @streaming_compatible_f() - call void @streaming_compatible_f() - ret void -} - -; Inline call to @non_streaming_callee to remove a streaming mode change. -define void @streaming_caller_inline() #0 "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_inline -; CHECK-SAME: () #[[ATTR2:[0-9]+]] { -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: ret void -; - call void @non_streaming_callee() - ret void -} - -; Don't inline call to @non_streaming_callee when the inline-threshold is set to 1, because it does not eliminate a streaming-mode change. -define void @non_streaming_caller_dont_inline() #0 { -; CHECK-LABEL: define void @non_streaming_caller_dont_inline -; CHECK-SAME: () #[[ATTR1]] { -; CHECK-NEXT: call void @non_streaming_callee() -; CHECK-NEXT: ret void -; - call void @non_streaming_callee() - ret void -} - -attributes #0 = { "target-features"="+sve,+sme" } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index 49af107d7e89b4..a55d52b970e178 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -645,7 +645,7 @@ define void @nonstreaming_caller_single_streaming_compatible_callee() { define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline ; CHECK-SAME: () #[[ATTR2]] { -; CHECK-NEXT: call void @streaming_compatible_body() +; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee() ; CHECK-NEXT: ret void ; call void @nonstreaming_caller_single_streaming_compatible_callee() @@ -666,7 +666,7 @@ define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline ; CHECK-SAME: () #[[ATTR2]] { -; CHECK-NEXT: call void @streaming_compatible_body() +; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() ; CHECK-NEXT: ret void ; call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() @@ -735,8 +735,8 @@ define ptr @vscale_dependent_op(ptr %p, i64 %k) { define ptr @incompatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define ptr @incompatible_vscale_dependent_operation_sm ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 -; CHECK-NEXT: ret ptr [[RES_I]] +; CHECK-NEXT: [[RES:%.*]] = call ptr @vscale_dependent_op(ptr [[P]], i64 4) +; CHECK-NEXT: ret ptr [[RES]] ; %res = call ptr @vscale_dependent_op(ptr %p, i64 4) ret ptr %res @@ -782,10 +782,7 @@ define void @scalable_alloca_op(ptr %p) { define void @incompatible_scalable_alloca_op_caller(ptr %p) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @incompatible_scalable_alloca_op_caller ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[ALLOCA_I:%.*]] = alloca <vscale x 4 x i32>, align 1 -; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ALLOCA_I]]) -; CHECK-NEXT: call void [[P]](ptr [[ALLOCA_I]]), !inline_history [[META0:![0-9]+]] -; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ALLOCA_I]]) +; CHECK-NEXT: call void @scalable_alloca_op(ptr [[P]]) ; CHECK-NEXT: ret void ; call void @scalable_alloca_op(ptr %p) @@ -963,7 +960,7 @@ define void @fixed_length_vector_operation(ptr %p) { define void @fixed_length_vector_operation_caller_dont_inline(ptr %p) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @fixed_length_vector_operation_caller_dont_inline ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: call void @fixed_length_vector_operation(ptr [[P]]) ; CHECK-NEXT: ret void ; call void @fixed_length_vector_operation(ptr %p) @@ -988,8 +985,8 @@ define float @strict_fp(float %in) strictfp { define float @incompatible_fp_environment_sm(float %in) strictfp "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define float @incompatible_fp_environment_sm ; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR9:[0-9]+]] { -; CHECK-NEXT: [[RES_I:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[IN]], float 4.200000e+01, metadata !"round.tonearest", metadata !"fpexcept.ignore") #[[ATTR14:[0-9]+]] -; CHECK-NEXT: ret float [[RES_I]] +; CHECK-NEXT: [[RES:%.*]] = call float @strict_fp(float [[IN]]) +; CHECK-NEXT: ret float [[RES]] ; %res = call float @strict_fp(float %in) ret float %res >From be5f2012f9b5bbdd6919748fd2d8488356070879 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 5 Oct 2026 10:28:36 +0000 Subject: [PATCH 03/10] Address comments --- .../AArch64/AArch64TargetTransformInfo.cpp | 26 +++++------- .../Inline/AArch64/sme-pstatesm-attrs.ll | 40 ++++++++++++++++++- .../AArch64/sme-pstatesm-inline-transitive.ll | 12 +++--- 3 files changed, 54 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 19a9b07389c6ae..a97cd638750944 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -311,23 +311,19 @@ static bool hasPossibleIncompatibleOps(const Function *F, if (CB->isInlineAsm()) return true; - if (auto *CI = dyn_cast<CallInst>(&I)) { - // If the callee has calls to streaming compatible functions, then those - // may have vl-dependent statements. Be cautious about inlining such - // calls, as the streaming-compatible calls would otherwise be executed - // in a different streaming mode. - if (ConsiderSM && CI->getCalledFunction()) { - SMEAttrs CalleeAttrs(*CI->getCalledFunction()); - if (CalleeAttrs.hasStreamingCompatibleInterface()) - return true; - } + SMECallAttrs CallAttrs(*CB, &TLI.getRuntimeLibcallsInfo()); + // If the callee has calls to streaming compatible functions, then those + // may have vl-dependent statements. Be cautious about inlining such + // calls, as the streaming-compatible calls would otherwise be executed + // in a different streaming mode. + if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface()) + return true; - if (isSMEABIRoutineCall(*CI, TLI)) - return true; + if (CallAttrs.callee().isSMEABIRoutine()) + return true; - if (ConsiderSM && isPossiblyIncompatibleIntrinsic(&I)) - return true; - } + if (ConsiderSM && isPossiblyIncompatibleIntrinsic(&I)) + return true; } } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index a55d52b970e178..a8fcea6164e520 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -663,8 +663,8 @@ define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline } ; Conseratively disallow inlining when forced; it is unclear what the user's intentions were. -define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline +define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_streamingcompatible_callee_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() ; CHECK-NEXT: ret void @@ -673,6 +673,42 @@ define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompat ret void } +define void @invoke_opaque_fptr(ptr %fptr) alwaysinline personality ptr null { +; CHECK-LABEL: define void @invoke_opaque_fptr +; CHECK-SAME: (ptr [[FPTR:%.*]]) #[[ATTR6]] personality ptr null { +; CHECK-NEXT: invoke void [[FPTR]]() #[[ATTR12:[0-9]+]] +; CHECK-NEXT: to label [[NORMAL_RETURN:%.*]] unwind label [[UNWIND_CLEANUP:%.*]] +; CHECK: normal_return: +; CHECK-NEXT: ret void +; CHECK: unwind_cleanup: +; CHECK-NEXT: [[EH_INFO:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: resume { ptr, i32 } [[EH_INFO]] +; + invoke void %fptr() "aarch64_pstate_sm_compatible" + to label %normal_return unwind label %unwind_cleanup + +normal_return: + ret void + +unwind_cleanup: + %eh_info = landingpad { ptr, i32 } + cleanup + resume { ptr, i32 } %eh_info +} + +; Test that we don't inline '@invoke_opaque_fptr_callee' as it contains a streaming compatible callee, +; but this time the callee is opaque pointer and called via 'invoke' rather than 'call'. +define void @callee_has_streaming_compatible_invoke_call_dont_inline(ptr %fptr) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @callee_has_streaming_compatible_invoke_call_dont_inline +; CHECK-SAME: (ptr [[FPTR:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: call void @invoke_opaque_fptr(ptr [[FPTR]]) +; CHECK-NEXT: ret void +; + call void @invoke_opaque_fptr(ptr %fptr) + ret void +} + define void @simple_streaming_function(ptr %ptr) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @simple_streaming_function ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR2]] { diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll index 21242b304591de..808749251ce0ab 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll @@ -1,12 +1,12 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt -S -passes=inline -inlinedefault-threshold=1 < %s | FileCheck %s --check-prefixes=CHECK,TH1 -; RUN: opt -S -passes=inline -inlinedefault-threshold=25 < %s | FileCheck %s --check-prefixes=CHECK,TH25 +; RUN: opt -mattr=+sve,+sme -S -passes=inline -inlinedefault-threshold=1 < %s | FileCheck %s --check-prefixes=CHECK,TH1 +; RUN: opt -mattr=+sve,+sme -S -passes=inline -inlinedefault-threshold=25 < %s | FileCheck %s --check-prefixes=CHECK,TH25 target triple = "aarch64" declare void @streaming_f() "aarch64_pstate_sm_enabled" -define void @c() #0 "aarch64_pstate_sm_enabled" { +define void @c() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @c ; CHECK-SAME: () #[[ATTR1:[0-9]+]] { ; CHECK-NEXT: call void @streaming_f() @@ -19,7 +19,7 @@ define void @c() #0 "aarch64_pstate_sm_enabled" { } ; Don't inline the call to @c, as inlining a streaming function into a non-streaming function is considered incompatible. -define void @b() #0 { +define void @b() { ; CHECK-LABEL: define void @b ; CHECK-SAME: () #[[ATTR2:[0-9]+]] { ; CHECK-NEXT: call void @c() @@ -30,7 +30,7 @@ define void @b() #0 { } ; Inline the call to @c in @a, as that avoids streaming mode changes. -define void @a() #0 "aarch64_pstate_sm_enabled" { +define void @a() "aarch64_pstate_sm_enabled" { ; TH1-LABEL: define void @a ; TH1-SAME: () #[[ATTR1]] { ; TH1-NEXT: call void @c() @@ -45,5 +45,3 @@ define void @a() #0 "aarch64_pstate_sm_enabled" { call void @b() ret void } - -attributes #0 = { "target-features"="+sve,+sme" } >From e9ae3a29af7da2d2b9d8b3fc69e6a65e522af716 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 5 Oct 2026 12:19:06 +0000 Subject: [PATCH 04/10] Remove unused isSMEABIRoutineCall --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index a97cd638750944..f431e420bc2831 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -230,13 +230,6 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode( static cl::opt<bool> EnableScalableAutovecInStreamingMode( "enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden); -static bool isSMEABIRoutineCall(const CallInst &CI, - const AArch64TargetLowering &TLI) { - const auto *F = CI.getCalledFunction(); - return F && - SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine(); -} - /// Returns true if \p I is an intrinsic that may not be compatible with a /// different streaming mode (because it depends on vscale). static bool isPossiblyIncompatibleIntrinsic(const Instruction *I) { >From 4ff95f3b26f6c4d506d6eecda03b957c32f99ea2 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 5 Oct 2026 12:26:01 +0000 Subject: [PATCH 05/10] Remove stale comment --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index f431e420bc2831..10713b6adf6516 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -373,9 +373,6 @@ bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const { /// * The called function sets up new ZA/ZT state into a function that already /// has ZA or ZT state, as that is not valid as per the ACLE. /// -/// If the called function has an `alwaysinline` attribute and any of the above -/// conditions is true, then an error should be reported. -/// /// The compiler should not inline when: // /// * The called function has fixed-length vectors and the caller is in >From 68b7b799ee7a9127c50fb74710014830f9c137d6 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 6 Oct 2026 08:35:43 +0000 Subject: [PATCH 06/10] Address comments --- .../AArch64/AArch64TargetTransformInfo.cpp | 83 +++++---- .../Inline/AArch64/sme-pstatesm-attrs.ll | 160 +++++++++++++----- 2 files changed, 168 insertions(+), 75 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 10713b6adf6516..95358319cf9534 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -230,25 +230,37 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode( static cl::opt<bool> EnableScalableAutovecInStreamingMode( "enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden); -/// Returns true if \p I is an intrinsic that may not be compatible with a -/// different streaming mode (because it depends on vscale). -static bool isPossiblyIncompatibleIntrinsic(const Instruction *I) { - if (I->isDebugOrPseudoInst()) +/// Returns true if \p I is an intrinsic that may not be compatible in the mode +/// of the caller. If \p AssumeVScaleIsEquivalent is true, then we can allow +/// operations that dependent on vscale as long as they are available in both +/// streaming/non-streaming mode. +static bool +isPossiblyIncompatibleIntrinsicInStreamingMode(const Instruction *I, + bool AssumeVScaleIsEquivalent) { + auto *II = dyn_cast<IntrinsicInst>(I); + if (!II) return false; - if (auto *II = dyn_cast<IntrinsicInst>(I)) { - unsigned IID = II->getIntrinsicID(); - switch (IID) { - default: - return Intrinsic::isTargetIntrinsic(IID); - case Intrinsic::vscale: - case Intrinsic::masked_gather: - case Intrinsic::masked_scatter: - return true; - } + unsigned IID = II->getIntrinsicID(); + switch (IID) { + default: + // Conservatively disallow target intrinsics in other modes. + return Intrinsic::isTargetIntrinsic(IID); + case Intrinsic::vscale: + return !AssumeVScaleIsEquivalent; + case Intrinsic::masked_gather: + // Instructions that are not available in streaming mode. + // Fixed-length operations can still be code-generated by scalarizing. + // FIXME: Consider FEAT_FA64. + return I->getType()->isScalableTy(); + case Intrinsic::masked_scatter: + case Intrinsic::masked_compressstore: + case Intrinsic::experimental_vector_histogram_add: + // Instructions that are not available in streaming mode. + // Fixed-length operations can still be code-generated by scalarizing. + // FIXME: Consider FEAT_FA64. + return I->getOperand(0)->getType()->isScalableTy(); } - - return false; } /// Returns true if the function has explicit operations that can only be @@ -258,11 +270,17 @@ static bool hasPossibleIncompatibleOps(const Function *F, const AArch64TargetLowering &TLI, bool ConsiderZA, bool ConsiderZT, bool ConsiderSM) { - assert((ConsiderZA || ConsiderZT || ConsiderSM) && - "No SME state to consider"); + if (!ConsiderZA && !ConsiderZT && !ConsiderSM) + return false; bool IsAlwaysInline = F->hasFnAttribute(Attribute::AlwaysInline); - bool HasVLDependentArgsOrRet = + // If the interface has vscale-dependent arguments/return value, then the ACLE + // describes that the function can only have defined behaviour if vscale is + // the same for both streaming/non-streaming mode, meaning that we can assume + // vscale is equivalent. + // FIXME: Handle the case where vscale_range is known and equivalent for + // caller and callee. + bool AssumeVScaleIsEquivalent = F->getReturnType()->isScalableTy() || any_of(F->getFunctionType()->params(), [](const Type *T) { return T->isScalableTy(); }); @@ -270,8 +288,8 @@ static bool hasPossibleIncompatibleOps(const Function *F, for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { // Inlining operations on fixed-length vectors when the streaming - // mode does not match, is rejected because performance may be impacted. - // This decision should eventually be moved the cost-model. + // mode does not match is rejected because performance may be impacted. + // This decision should eventually be moved to the cost-model. if (!IsAlwaysInline && ConsiderSM && (isa<FixedVectorType>(I.getType()) || any_of(I.operand_values(), [](const Value *V) { @@ -279,12 +297,9 @@ static bool hasPossibleIncompatibleOps(const Function *F, }))) return true; - // Inlining operations on scalable vectors is rejected because it is - // a vscale-dependent operation. The only exception is when the interface - // already has vscale-dependent arguments/return value, as the ACLE - // describes that in order for the program to have defined behaviour is - // for vscale to match in both modes. - if (ConsiderSM && !HasVLDependentArgsOrRet) { + // Inlining operations on scalable vectors is rejected when + // vscale-dependent operations cannot be safely inlined. + if (ConsiderSM && !AssumeVScaleIsEquivalent) { if (I.getType()->isScalableTy() || any_of( I.operand_values(), @@ -306,16 +321,19 @@ static bool hasPossibleIncompatibleOps(const Function *F, SMECallAttrs CallAttrs(*CB, &TLI.getRuntimeLibcallsInfo()); // If the callee has calls to streaming compatible functions, then those - // may have vl-dependent statements. Be cautious about inlining such - // calls, as the streaming-compatible calls would otherwise be executed - // in a different streaming mode. + // may have statements which dependent on SME state (e.g. vscale or + // explicit reads of PSTATE.SM). If we were to inline those calls, + // behaviour may change after inlining because the streaming-compatible + // function would be executed in a different streaming mode. + // Conservatively avoid inlining for now. if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface()) return true; if (CallAttrs.callee().isSMEABIRoutine()) return true; - if (ConsiderSM && isPossiblyIncompatibleIntrinsic(&I)) + if (ConsiderSM && isPossiblyIncompatibleIntrinsicInStreamingMode( + &I, AssumeVScaleIsEquivalent)) return true; } } @@ -421,8 +439,7 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP())) return false; - if ((ConsiderZA || ConsiderZT || ConsiderSM) && - hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderZT, + if (hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderZT, ConsiderSM)) return false; diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index a8fcea6164e520..33257a58ecd27f 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -676,7 +676,7 @@ define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_st define void @invoke_opaque_fptr(ptr %fptr) alwaysinline personality ptr null { ; CHECK-LABEL: define void @invoke_opaque_fptr ; CHECK-SAME: (ptr [[FPTR:%.*]]) #[[ATTR6]] personality ptr null { -; CHECK-NEXT: invoke void [[FPTR]]() #[[ATTR12:[0-9]+]] +; CHECK-NEXT: invoke void [[FPTR]]() #[[ATTR13:[0-9]+]] ; CHECK-NEXT: to label [[NORMAL_RETURN:%.*]] unwind label [[UNWIND_CLEANUP:%.*]] ; CHECK: normal_return: ; CHECK-NEXT: ret void @@ -768,16 +768,6 @@ define ptr @vscale_dependent_op(ptr %p, i64 %k) { ret ptr %res } -define ptr @incompatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define ptr @incompatible_vscale_dependent_operation_sm -; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[RES:%.*]] = call ptr @vscale_dependent_op(ptr [[P]], i64 4) -; CHECK-NEXT: ret ptr [[RES]] -; - %res = call ptr @vscale_dependent_op(ptr %p, i64 4) - ret ptr %res -} - define ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 %k, <vscale x 4 x i32> %other) { ; CHECK-LABEL: define ptr @vscale_dependent_op_vl_dependent_args ; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]], <vscale x 4 x i32> [[OTHER:%.*]]) #[[ATTR1]] { @@ -790,15 +780,21 @@ define ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 %k, <vscale x 4 x ret ptr %res } -define ptr @compatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define ptr @compatible_vscale_dependent_operation_sm -; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +define void @vscale_dependent_operations(ptr %p, ptr %res1ptr, ptr %res2ptr) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @vscale_dependent_operations +; CHECK-SAME: (ptr [[P:%.*]], ptr [[RES1PTR:%.*]], ptr [[RES2PTR:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES1:%.*]] = call ptr @vscale_dependent_op(ptr [[P]], i64 4) ; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 ; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[RES_I]], align 16 -; CHECK-NEXT: ret ptr [[RES_I]] +; CHECK-NEXT: store ptr [[RES1]], ptr [[RES1PTR]], align 8 +; CHECK-NEXT: store ptr [[RES_I]], ptr [[RES2PTR]], align 8 +; CHECK-NEXT: ret void ; - %res = call ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 4, <vscale x 4 x i32> zeroinitializer) - ret ptr %res + %res1 = call ptr @vscale_dependent_op(ptr %p, i64 4) + %res2 = call ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 4, <vscale x 4 x i32> zeroinitializer) + store ptr %res1, ptr %res1ptr + store ptr %res2, ptr %res2ptr + ret void } ; functions with scalable alloca's shouldn't be inlined if the streaming properties don't match @@ -838,30 +834,37 @@ define i64 @intrinsic_with_scalable_type(ptr %p) { ret i64 %res } -define i64 @compatible_sve_intrinsic_caller(ptr %p) { -; CHECK-LABEL: define i64 @compatible_sve_intrinsic_caller -; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 -; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) -; CHECK-NEXT: ret i64 [[RES_I]] +define i64 @intrinsic_with_scalable_type_vscale_must_match(ptr %p, <vscale x 4 x i32> %unused) { +; CHECK-LABEL: define i64 @intrinsic_with_scalable_type_vscale_must_match +; CHECK-SAME: (ptr [[P:%.*]], <vscale x 4 x i32> [[UNUSED:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]]) +; CHECK-NEXT: ret i64 [[RES]] ; - %res = call i64 @intrinsic_with_scalable_type(ptr %p) + %ld = load <vscale x 2 x i64>, ptr %p + %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld) ret i64 %res } -define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define i64 @incompatible_sve_intrinsic_caller +define i64 @generic_intrinsic_with_scalable_types(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @generic_intrinsic_with_scalable_types ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[RES:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) +; CHECK-NEXT: [[RES1:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) +; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) +; CHECK-NEXT: [[RES:%.*]] = add i64 [[RES1]], [[RES_I]] ; CHECK-NEXT: ret i64 [[RES]] ; - %res = call i64 @intrinsic_with_scalable_type(ptr %p) + %res1 = call i64 @intrinsic_with_scalable_type(ptr %p) + %res2 = call i64 @intrinsic_with_scalable_type_vscale_must_match(ptr %p, <vscale x 4 x i32> zeroinitializer) + %res = add i64 %res1, %res2 ret i64 %res } ; ; Don't inline inline-asm when streaming-modes are incompatible, as the asm may -; contain vscale-dependent instructions. +; contain instructions that are invalid or may behave differently in the mode of +; the caller. ; define void @inline_asm() { @@ -921,7 +924,7 @@ define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { } ; -; Don't inline NEON intrinsics into streaming mode. +; Don't inline NEON intrinsics into a streaming mode functions. ; define i32 @neon_intrinsic(<4 x i32> %in) { @@ -973,16 +976,6 @@ define void @fixed_length_vector_operation_alwaysinline(ptr %p) alwaysinline { ret void } -define void @fixed_length_vector_operation_caller_force_inline(ptr %p) "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @fixed_length_vector_operation_caller_force_inline -; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 -; CHECK-NEXT: ret void -; - call void @fixed_length_vector_operation_alwaysinline(ptr %p) - ret void -} - define void @fixed_length_vector_operation(ptr %p) { ; CHECK-LABEL: define void @fixed_length_vector_operation ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { @@ -993,12 +986,14 @@ define void @fixed_length_vector_operation(ptr %p) { ret void } -define void @fixed_length_vector_operation_caller_dont_inline(ptr %p) "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @fixed_length_vector_operation_caller_dont_inline +define void @fixed_length_vector_operations(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operations ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 ; CHECK-NEXT: call void @fixed_length_vector_operation(ptr [[P]]) ; CHECK-NEXT: ret void ; + call void @fixed_length_vector_operation_alwaysinline(ptr %p) call void @fixed_length_vector_operation(ptr %p) ret void } @@ -1027,3 +1022,84 @@ define float @incompatible_fp_environment_sm(float %in) strictfp "aarch64_pstate %res = call float @strict_fp(float %in) ret float %res } + +; +; Test we can inline fixed-length gather/scatter operations into streaming functions, +; as those can be scalarized by the code-generator. +; + +define void @scatter_fixed_length(ptr %ptr_to_ptrs, ptr %ptr_to_vals) alwaysinline { +; CHECK-LABEL: define void @scatter_fixed_length +; CHECK-SAME: (ptr [[PTR_TO_PTRS:%.*]], ptr [[PTR_TO_VALS:%.*]]) #[[ATTR6]] { +; CHECK-NEXT: [[PTRS:%.*]] = load <2 x ptr>, ptr [[PTR_TO_PTRS]], align 16 +; CHECK-NEXT: [[VALS:%.*]] = load <2 x i64>, ptr [[PTR_TO_VALS]], align 16 +; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VALS]], <2 x ptr> align 8 [[PTRS]], <2 x i1> splat (i1 true)) +; CHECK-NEXT: ret void +; + %ptrs = load <2 x ptr>, ptr %ptr_to_ptrs + %vals = load <2 x i64>, ptr %ptr_to_vals + call void @llvm.masked.scatter(<2 x i64> %vals, <2 x ptr> %ptrs, i32 8, <2 x i1> splat(i1 true)) + ret void +} + +define void @scatter_fixed_length_caller(ptr %ptr_to_ptrs, ptr %ptr_to_vals) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @scatter_fixed_length_caller +; CHECK-SAME: (ptr [[PTR_TO_PTRS:%.*]], ptr [[PTR_TO_VALS:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[PTRS_I:%.*]] = load <2 x ptr>, ptr [[PTR_TO_PTRS]], align 16 +; CHECK-NEXT: [[VALS_I:%.*]] = load <2 x i64>, ptr [[PTR_TO_VALS]], align 16 +; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VALS_I]], <2 x ptr> align 8 [[PTRS_I]], <2 x i1> splat (i1 true)) +; CHECK-NEXT: ret void +; + call void @scatter_fixed_length(ptr %ptr_to_ptrs, ptr %ptr_to_vals) + ret void +} + +; +; Test that we never inline a scalable gather/scatter operations into a streaming function. +; + +define void @scatter_scalable_length(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals) alwaysinline { +; CHECK-LABEL: define void @scatter_scalable_length +; CHECK-SAME: (<vscale x 2 x ptr> [[PTRS:%.*]], <vscale x 2 x i64> [[VALS:%.*]]) #[[ATTR6]] { +; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VALS]], <vscale x 2 x ptr> align 8 [[PTRS]], <vscale x 2 x i1> splat (i1 true)) +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter(<vscale x 2 x i64> %vals, <vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> splat(i1 true)) + ret void +} + +define void @scatter_scalable_length_caller(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @scatter_scalable_length_caller +; CHECK-SAME: (<vscale x 2 x ptr> [[PTRS:%.*]], <vscale x 2 x i64> [[VALS:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: call void @scatter_scalable_length(<vscale x 2 x ptr> [[PTRS]], <vscale x 2 x i64> [[VALS]]) +; CHECK-NEXT: ret void +; + call void @scatter_scalable_length(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals) + ret void +} + +; +; Test that we inline a function with a call to llvm.vscale() (which takes/returns +; no scalable vectors) when the function has vl-dependent arguments. The case for +; non-vl-dependent arguments is already tested above. +; + +define i64 @llvm_vscale_vl_dependent_args(<vscale x 2 x i64> %in) { +; CHECK-LABEL: define i64 @llvm_vscale_vl_dependent_args +; CHECK-SAME: (<vscale x 2 x i64> [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: ret i64 [[RES]] +; + %res = call i64 @llvm.vscale.i64() + ret i64 %res +} + +define i64 @llvm_vscale_vl_dependent_args_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @llvm_vscale_vl_dependent_args_caller +; CHECK-SAME: () #[[ATTR2]] { +; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: ret i64 [[RES_I]] +; + %res = call i64 @llvm_vscale_vl_dependent_args(<vscale x 2 x i64> zeroinitializer) + ret i64 %res +} >From 6b416b43b35409bc19c6da19d155df001b333592 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 6 Oct 2026 12:43:55 +0000 Subject: [PATCH 07/10] NFC refactor, to separate 'vscale dependent' and 'incompatible instruction in streaming mode' concerns --- .../AArch64/AArch64TargetTransformInfo.cpp | 152 ++++++++++-------- 1 file changed, 87 insertions(+), 65 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 95358319cf9534..db2e36eefe5e0f 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -230,37 +230,70 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode( static cl::opt<bool> EnableScalableAutovecInStreamingMode( "enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden); +// Utility class to tell if and how an operation is compatible +// in a different streaming mode. +class StreamingCompatibility { + uint8_t V; + +public: + enum StreamingCompatibilityEnum : uint8_t { + Compatible = 0, + VScaleDependent = 1 << 0, + HasIncompatibleInstruction = 1 << 1, + FullyIncompatible = VScaleDependent | HasIncompatibleInstruction + }; + StreamingCompatibility(StreamingCompatibilityEnum V) : V(V) {} + bool test(StreamingCompatibilityEnum Other) const { + return V == Other || (Other && (V & uint8_t(Other)) != 0); + } +}; + /// Returns true if \p I is an intrinsic that may not be compatible in the mode /// of the caller. If \p AssumeVScaleIsEquivalent is true, then we can allow /// operations that dependent on vscale as long as they are available in both /// streaming/non-streaming mode. -static bool -isPossiblyIncompatibleIntrinsicInStreamingMode(const Instruction *I, - bool AssumeVScaleIsEquivalent) { - auto *II = dyn_cast<IntrinsicInst>(I); - if (!II) - return false; - - unsigned IID = II->getIntrinsicID(); - switch (IID) { - default: - // Conservatively disallow target intrinsics in other modes. - return Intrinsic::isTargetIntrinsic(IID); - case Intrinsic::vscale: - return !AssumeVScaleIsEquivalent; - case Intrinsic::masked_gather: - // Instructions that are not available in streaming mode. - // Fixed-length operations can still be code-generated by scalarizing. - // FIXME: Consider FEAT_FA64. - return I->getType()->isScalableTy(); - case Intrinsic::masked_scatter: - case Intrinsic::masked_compressstore: - case Intrinsic::experimental_vector_histogram_add: - // Instructions that are not available in streaming mode. - // Fixed-length operations can still be code-generated by scalarizing. - // FIXME: Consider FEAT_FA64. - return I->getOperand(0)->getType()->isScalableTy(); +static StreamingCompatibility +getCompatibilityForChangeToStreamingMode(const Instruction *I) { + if (auto *II = dyn_cast<IntrinsicInst>(I)) { + unsigned IID = II->getIntrinsicID(); + switch (IID) { + default: + // Conservatively disallow target intrinsics in other modes. + if (Intrinsic::isTargetIntrinsic(IID)) + return StreamingCompatibility::FullyIncompatible; + break; + case Intrinsic::vscale: + return StreamingCompatibility::VScaleDependent; + case Intrinsic::masked_gather: + // Instructions that are not available in streaming mode. + // Fixed-length operations can still be code-generated by scalarizing. + // FIXME: Consider FEAT_FA64. + if (I->getOperand(0)->getType()->isScalableTy()) + return StreamingCompatibility::FullyIncompatible; + break; + case Intrinsic::masked_scatter: + case Intrinsic::masked_compressstore: + case Intrinsic::experimental_vector_histogram_add: + // Instructions that are not available in streaming mode. + // Fixed-length operations can still be code-generated by scalarizing. + // FIXME: Consider FEAT_FA64. + if (I->getOperand(0)->getType()->isScalableTy()) + return StreamingCompatibility::FullyIncompatible; + break; + } } + + // Inlining operations on scalable vectors is rejected when + // vscale-dependent operations cannot be safely inlined. + if (I->getType()->isScalableTy() || + any_of(I->operand_values(), + [](const Value *V) { return V->getType()->isScalableTy(); }) || + (isa<GetElementPtrInst>(I) && + cast<GetElementPtrInst>(I)->getSourceElementType()->isScalableTy()) || + (isa<AllocaInst>(I) && cast<AllocaInst>(I)->isScalable())) + return StreamingCompatibility::VScaleDependent; + + return StreamingCompatibility::Compatible; } /// Returns true if the function has explicit operations that can only be @@ -287,53 +320,42 @@ static bool hasPossibleIncompatibleOps(const Function *F, for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { - // Inlining operations on fixed-length vectors when the streaming - // mode does not match is rejected because performance may be impacted. - // This decision should eventually be moved to the cost-model. - if (!IsAlwaysInline && ConsiderSM && - (isa<FixedVectorType>(I.getType()) || - any_of(I.operand_values(), [](const Value *V) { - return isa<FixedVectorType>(V->getType()); - }))) - return true; + if (auto *CB = dyn_cast<CallBase>(&I)) { + SMECallAttrs CallAttrs(*CB, &TLI.getRuntimeLibcallsInfo()); - // Inlining operations on scalable vectors is rejected when - // vscale-dependent operations cannot be safely inlined. - if (ConsiderSM && !AssumeVScaleIsEquivalent) { - if (I.getType()->isScalableTy() || - any_of( - I.operand_values(), - [](const Value *V) { return V->getType()->isScalableTy(); }) || - (isa<GetElementPtrInst>(I) && cast<GetElementPtrInst>(I) - .getSourceElementType() - ->isScalableTy()) || - (isa<AllocaInst>(I) && cast<AllocaInst>(I).isScalable())) + // Inline asm must be rejected as it could use SME state. + if (CB->isInlineAsm()) return true; - } - auto *CB = dyn_cast<CallBase>(&I); - if (!CB) - continue; + if (CallAttrs.callee().isSMEABIRoutine()) + return true; - // Inline asm must be rejected as it could use SME state. - if (CB->isInlineAsm()) - return true; + // If the callee has calls to streaming compatible functions, then those + // may have statements which dependent on SME state (e.g. vscale or + // explicit reads of PSTATE.SM). If we were to inline those calls, + // behaviour may change after inlining because the streaming-compatible + // function would be executed in a different streaming mode. + // Conservatively avoid inlining for now. + if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface()) + return true; + } - SMECallAttrs CallAttrs(*CB, &TLI.getRuntimeLibcallsInfo()); - // If the callee has calls to streaming compatible functions, then those - // may have statements which dependent on SME state (e.g. vscale or - // explicit reads of PSTATE.SM). If we were to inline those calls, - // behaviour may change after inlining because the streaming-compatible - // function would be executed in a different streaming mode. - // Conservatively avoid inlining for now. - if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface()) - return true; + if (!ConsiderSM) + continue; - if (CallAttrs.callee().isSMEABIRoutine()) + // Inlining operations on fixed-length vectors when the streaming + // mode does not match is rejected because performance may be impacted. + // This decision should eventually be moved to the cost-model. + if (!IsAlwaysInline && (isa<FixedVectorType>(I.getType()) || + any_of(I.operand_values(), [](const Value *V) { + return isa<FixedVectorType>(V->getType()); + }))) return true; - if (ConsiderSM && isPossiblyIncompatibleIntrinsicInStreamingMode( - &I, AssumeVScaleIsEquivalent)) + StreamingCompatibility C = getCompatibilityForChangeToStreamingMode(&I); + if (C.test(StreamingCompatibility::HasIncompatibleInstruction) || + (!AssumeVScaleIsEquivalent && + C.test(StreamingCompatibility::VScaleDependent))) return true; } } >From 7bb1b75351938bef178282c1bfe4650550795f5d Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 6 Oct 2026 13:23:03 +0000 Subject: [PATCH 08/10] Use LLVM_MARK_AS_BITMASK_ENUM --- .../AArch64/AArch64TargetTransformInfo.cpp | 45 ++++++++----------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index db2e36eefe5e0f..f3f95cf0af99c7 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -11,6 +11,7 @@ #include "AArch64PerfectShuffle.h" #include "AArch64SMEAttributes.h" #include "MCTargetDesc/AArch64AddressingModes.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/bit.h" #include "llvm/Analysis/LoopInfo.h" @@ -230,29 +231,21 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode( static cl::opt<bool> EnableScalableAutovecInStreamingMode( "enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden); -// Utility class to tell if and how an operation is compatible -// in a different streaming mode. -class StreamingCompatibility { - uint8_t V; - -public: - enum StreamingCompatibilityEnum : uint8_t { - Compatible = 0, - VScaleDependent = 1 << 0, - HasIncompatibleInstruction = 1 << 1, - FullyIncompatible = VScaleDependent | HasIncompatibleInstruction - }; - StreamingCompatibility(StreamingCompatibilityEnum V) : V(V) {} - bool test(StreamingCompatibilityEnum Other) const { - return V == Other || (Other && (V & uint8_t(Other)) != 0); - } +// Utility class to tell if and how an operation is compatible in a different +// streaming mode. +enum class StreamingIncompatibility : uint8_t { + Compatible = 0, + VScaleDependent = 1 << 0, + HasIncompatibleInstruction = 1 << 1, + FullyIncompatible = VScaleDependent | HasIncompatibleInstruction, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestFlag=*/FullyIncompatible) }; /// Returns true if \p I is an intrinsic that may not be compatible in the mode /// of the caller. If \p AssumeVScaleIsEquivalent is true, then we can allow /// operations that dependent on vscale as long as they are available in both /// streaming/non-streaming mode. -static StreamingCompatibility +static StreamingIncompatibility getCompatibilityForChangeToStreamingMode(const Instruction *I) { if (auto *II = dyn_cast<IntrinsicInst>(I)) { unsigned IID = II->getIntrinsicID(); @@ -260,16 +253,16 @@ getCompatibilityForChangeToStreamingMode(const Instruction *I) { default: // Conservatively disallow target intrinsics in other modes. if (Intrinsic::isTargetIntrinsic(IID)) - return StreamingCompatibility::FullyIncompatible; + return StreamingIncompatibility::FullyIncompatible; break; case Intrinsic::vscale: - return StreamingCompatibility::VScaleDependent; + return StreamingIncompatibility::VScaleDependent; case Intrinsic::masked_gather: // Instructions that are not available in streaming mode. // Fixed-length operations can still be code-generated by scalarizing. // FIXME: Consider FEAT_FA64. if (I->getOperand(0)->getType()->isScalableTy()) - return StreamingCompatibility::FullyIncompatible; + return StreamingIncompatibility::FullyIncompatible; break; case Intrinsic::masked_scatter: case Intrinsic::masked_compressstore: @@ -278,7 +271,7 @@ getCompatibilityForChangeToStreamingMode(const Instruction *I) { // Fixed-length operations can still be code-generated by scalarizing. // FIXME: Consider FEAT_FA64. if (I->getOperand(0)->getType()->isScalableTy()) - return StreamingCompatibility::FullyIncompatible; + return StreamingIncompatibility::FullyIncompatible; break; } } @@ -291,9 +284,9 @@ getCompatibilityForChangeToStreamingMode(const Instruction *I) { (isa<GetElementPtrInst>(I) && cast<GetElementPtrInst>(I)->getSourceElementType()->isScalableTy()) || (isa<AllocaInst>(I) && cast<AllocaInst>(I)->isScalable())) - return StreamingCompatibility::VScaleDependent; + return StreamingIncompatibility::VScaleDependent; - return StreamingCompatibility::Compatible; + return StreamingIncompatibility::Compatible; } /// Returns true if the function has explicit operations that can only be @@ -352,10 +345,10 @@ static bool hasPossibleIncompatibleOps(const Function *F, }))) return true; - StreamingCompatibility C = getCompatibilityForChangeToStreamingMode(&I); - if (C.test(StreamingCompatibility::HasIncompatibleInstruction) || + StreamingIncompatibility C = getCompatibilityForChangeToStreamingMode(&I); + if (any(C & StreamingIncompatibility::HasIncompatibleInstruction) || (!AssumeVScaleIsEquivalent && - C.test(StreamingCompatibility::VScaleDependent))) + any(C & StreamingIncompatibility::VScaleDependent))) return true; } } >From a7fc1f858f11c41a6b6b2ee39197cbed2f3ed9f5 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 6 Oct 2026 13:36:22 +0000 Subject: [PATCH 09/10] Address comments --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index f3f95cf0af99c7..03a88809c29f00 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -231,10 +231,10 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode( static cl::opt<bool> EnableScalableAutovecInStreamingMode( "enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden); -// Utility class to tell if and how an operation is compatible in a different -// streaming mode. +// Enum to record why an operation is incompatible in a different streaming +// mode. enum class StreamingIncompatibility : uint8_t { - Compatible = 0, + None = 0, VScaleDependent = 1 << 0, HasIncompatibleInstruction = 1 << 1, FullyIncompatible = VScaleDependent | HasIncompatibleInstruction, @@ -286,7 +286,7 @@ getCompatibilityForChangeToStreamingMode(const Instruction *I) { (isa<AllocaInst>(I) && cast<AllocaInst>(I)->isScalable())) return StreamingIncompatibility::VScaleDependent; - return StreamingIncompatibility::Compatible; + return StreamingIncompatibility::None; } /// Returns true if the function has explicit operations that can only be >From 18bab60c3d7f6b330aae4f5deecbcce91f47699a Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 6 Oct 2026 13:49:54 +0000 Subject: [PATCH 10/10] Fixup comments Also fix up the value passed to `LLVM_MARK_AS_BITMASK_ENUM`, as that should be the largest individual value (not combined value). --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 10 ++++------ .../Transforms/Inline/AArch64/sme-pstatesm-attrs.ll | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 03a88809c29f00..3ab029619f5752 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -238,13 +238,11 @@ enum class StreamingIncompatibility : uint8_t { VScaleDependent = 1 << 0, HasIncompatibleInstruction = 1 << 1, FullyIncompatible = VScaleDependent | HasIncompatibleInstruction, - LLVM_MARK_AS_BITMASK_ENUM(/*LargestFlag=*/FullyIncompatible) + LLVM_MARK_AS_BITMASK_ENUM(/*LargestFlag=*/HasIncompatibleInstruction) }; -/// Returns true if \p I is an intrinsic that may not be compatible in the mode -/// of the caller. If \p AssumeVScaleIsEquivalent is true, then we can allow -/// operations that dependent on vscale as long as they are available in both -/// streaming/non-streaming mode. +/// Returns an enum describing whether \p I is an instruction that is +/// incompatible in the mode of the caller. static StreamingIncompatibility getCompatibilityForChangeToStreamingMode(const Instruction *I) { if (auto *II = dyn_cast<IntrinsicInst>(I)) { @@ -324,7 +322,7 @@ static bool hasPossibleIncompatibleOps(const Function *F, return true; // If the callee has calls to streaming compatible functions, then those - // may have statements which dependent on SME state (e.g. vscale or + // may have statements which depend on SME state (e.g. vscale or // explicit reads of PSTATE.SM). If we were to inline those calls, // behaviour may change after inlining because the streaming-compatible // function would be executed in a different streaming mode. diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index 33257a58ecd27f..45c39056266231 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -662,7 +662,7 @@ define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline ret void } -; Conseratively disallow inlining when forced; it is unclear what the user's intentions were. +; Reject inlining even when forced; it is unclear what the user's intentions were. define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_streamingcompatible_callee_dont_inline ; CHECK-SAME: () #[[ATTR2]] { @@ -1055,7 +1055,7 @@ define void @scatter_fixed_length_caller(ptr %ptr_to_ptrs, ptr %ptr_to_vals) "aa } ; -; Test that we never inline a scalable gather/scatter operations into a streaming function. +; Test that we never inline a scalable gather/scatter operation into a streaming function. ; define void @scatter_scalable_length(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals) alwaysinline { _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
