https://github.com/SpencerAbson updated
https://github.com/llvm/llvm-project/pull/119546
>From aab52738f106ae6a6bffdb180daf2bd140850a5a Mon Sep 17 00:00:00 2001
From: Spencer Abson <spencer.ab...@arm.com>
Date: Mon, 9 Dec 2024 14:50:12 +0000
Subject: [PATCH 1/2] [AArch64] Implement intrinsics for FP8 SME FMLAL/FMLALL
(multi)
---
clang/include/clang/Basic/arm_sme.td | 10 ++
.../fp8-intrinsics/acle_sme2_fp8_mla.c | 92 +++++++++++++++++++
.../aarch64-fp8-intrinsics/acle_sme_fp8_mla.c | 20 ++++
llvm/include/llvm/IR/IntrinsicsAArch64.td | 24 ++++-
.../lib/Target/AArch64/AArch64SMEInstrInfo.td | 12 ++-
llvm/lib/Target/AArch64/SMEInstrFormats.td | 31 ++++---
.../AArch64/sme2-fp8-intrinsics-mla.ll | 76 +++++++++++++++
7 files changed, 248 insertions(+), 17 deletions(-)
create mode 100644
clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_mla.c
create mode 100644 clang/test/Sema/aarch64-fp8-intrinsics/acle_sme_fp8_mla.c
create mode 100644 llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-mla.ll
diff --git a/clang/include/clang/Basic/arm_sme.td
b/clang/include/clang/Basic/arm_sme.td
index 71b2c7cdd04f93..da19f9be2830c0 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -827,11 +827,21 @@ let SMETargetGuard = "sme-lutv2" in {
let SMETargetGuard = "sme-f8f32" in {
def SVMOPA_FP8_ZA32 : Inst<"svmopa_za32[_mf8]_m_fpm", "viPPdd>", "m",
MergeNone, "aarch64_sme_fp8_fmopa_za32",
[IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], [ImmCheck<0, ImmCheck0_3>]>;
+ // FMLALL (mutliple)
+ def SVMLA_FP8_MULTI_ZA32_VG4x2 : Inst<"svmla_za32[_mf8]_vg4x2_fpm", "vm22>",
"m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x2",
+ [IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], []>;
+ def SVMLA_FP8_MULTI_ZA32_VG4x4 : Inst<"svmla_za32[_mf8]_vg4x4_fpm", "vm44>",
"m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x4",
+ [IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], []>;
}
let SMETargetGuard = "sme-f8f16" in {
def SVMOPA_FP8_ZA16 : Inst<"svmopa_za16[_mf8]_m_fpm", "viPPdd>", "m",
MergeNone, "aarch64_sme_fp8_fmopa_za16",
[IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], [ImmCheck<0, ImmCheck0_1>]>;
+ // FMLAL (mutliple)
+ def SVMLA_FP8_MULTI_ZA16_VG2x2 : Inst<"svmla_za16[_mf8]_vg2x2_fpm", "vm22>",
"m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x2",
+ [IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], []>;
+ def SVMLA_FP8_MULTI_ZA16_VG2x4 : Inst<"svmla_za16[_mf8]_vg2x4_fpm", "vm44>",
"m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x4",
+ [IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], []>;
}
} // let SVETargetGuard = InvalidMode
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_mla.c
b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_mla.c
new file mode 100644
index 00000000000000..656bbf18554f2b
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_mla.c
@@ -0,0 +1,92 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
UTC_ARGS: --version 5
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme
-target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone
-Werror -Wall -emit-llvm -o - %s | opt -S
-passes=mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme
-target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone
-Werror -Wall -emit-llvm -o - -x c++ %s | opt -S
-passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64-none-linux-gnu
-target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32
-disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S
-passes=mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64-none-linux-gnu
-target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32
-disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S
-passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme
-target-feature +sme-f8f16 -target-feature +sme-f8f32 -S -disable-O0-optnone
-Werror -Wall -o /dev/null %s
+
+#include <arm_sme.h>
+
+#ifdef SME_OVERLOADED_FORMS
+#define SME_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// FMLAL (multi)
+
+// CHECK-LABEL: define dso_local void @test_svmla_multi_za16_vg2x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x2(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]])
+// CHECK-NEXT: ret void
+//
+// CPP-CHECK-LABEL: define dso_local void
@_Z27test_svmla_multi_za16_vg2x2j13svmfloat8x2_tS_m(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CPP-CHECK-NEXT: [[ENTRY:.*:]]
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CPP-CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x2(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]])
+// CPP-CHECK-NEXT: ret void
+//
+void test_svmla_multi_za16_vg2x2(uint32_t slice, svmfloat8x2_t zn,
svmfloat8x2_t zm, fpm_t fpm) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svmla_za16,_mf8,_vg2x2_fpm)(slice, zn, zm, fpm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_multi_za16_vg2x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE2:%.*]], <vscale x 16 x i8> [[ZM_COERCE3:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x4(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZN_COERCE2]], <vscale x 16 x i8> [[ZN_COERCE3]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE2]], <vscale x 16 x i8> [[ZM_COERCE3]])
+// CHECK-NEXT: ret void
+//
+// CPP-CHECK-LABEL: define dso_local void
@_Z27test_svmla_multi_za16_vg2x4j13svmfloat8x4_tS_m(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE2:%.*]], <vscale x 16 x i8> [[ZM_COERCE3:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT: [[ENTRY:.*:]]
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CPP-CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x4(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZN_COERCE2]], <vscale x 16 x i8> [[ZN_COERCE3]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE2]], <vscale x 16 x i8> [[ZM_COERCE3]])
+// CPP-CHECK-NEXT: ret void
+//
+void test_svmla_multi_za16_vg2x4(uint32_t slice, svmfloat8x4_t zn,
svmfloat8x4_t zm, fpm_t fpm) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svmla_za16,_mf8,_vg2x4_fpm)(slice, zn, zm, fpm);
+}
+
+// FMLALL (multi)
+
+// CHECK-LABEL: define dso_local void @test_svmla_multi_za32_vg4x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x2(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]])
+// CHECK-NEXT: ret void
+//
+// CPP-CHECK-LABEL: define dso_local void
@_Z27test_svmla_multi_za32_vg4x2j13svmfloat8x2_tS_m(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT: [[ENTRY:.*:]]
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CPP-CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x2(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]])
+// CPP-CHECK-NEXT: ret void
+//
+void test_svmla_multi_za32_vg4x2(uint32_t slice, svmfloat8x2_t zn,
svmfloat8x2_t zm, fpm_t fpm) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svmla_za32,_mf8,_vg4x2_fpm)(slice, zn, zm, fpm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_multi_za32_vg4x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE2:%.*]], <vscale x 16 x i8> [[ZM_COERCE3:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x4(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZN_COERCE2]], <vscale x 16 x i8> [[ZN_COERCE3]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE2]], <vscale x 16 x i8> [[ZM_COERCE3]])
+// CHECK-NEXT: ret void
+//
+// CPP-CHECK-LABEL: define dso_local void
@_Z27test_svmla_multi_za32_vg4x4j13svmfloat8x4_tS_m(
+// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8>
[[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8>
[[ZN_COERCE2:%.*]], <vscale x 16 x i8> [[ZN_COERCE3:%.*]], <vscale x 16 x i8>
[[ZM_COERCE0:%.*]], <vscale x 16 x i8> [[ZM_COERCE1:%.*]], <vscale x 16 x i8>
[[ZM_COERCE2:%.*]], <vscale x 16 x i8> [[ZM_COERCE3:%.*]], i64 noundef
[[FPM:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT: [[ENTRY:.*:]]
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CPP-CHECK-NEXT: tail call void
@llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x4(i32 [[SLICE]], <vscale x 16 x i8>
[[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8>
[[ZN_COERCE2]], <vscale x 16 x i8> [[ZN_COERCE3]], <vscale x 16 x i8>
[[ZM_COERCE0]], <vscale x 16 x i8> [[ZM_COERCE1]], <vscale x 16 x i8>
[[ZM_COERCE2]], <vscale x 16 x i8> [[ZM_COERCE3]])
+// CPP-CHECK-NEXT: ret void
+//
+void test_svmla_multi_za32_vg4x4(uint32_t slice, svmfloat8x4_t zn,
svmfloat8x4_t zm, fpm_t fpm) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svmla_za32,_mf8,_vg4x4_fpm)(slice, zn, zm, fpm);
+}
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme_fp8_mla.c
b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme_fp8_mla.c
new file mode 100644
index 00000000000000..2c83bb847535f8
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme_fp8_mla.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -verify
-emit-llvm-only %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_svmla(uint32_t slice, svmfloat8_t zn, svmfloat8x2_t znx2,
svmfloat8x4_t znx4,
+ fpm_t fpmr) __arm_streaming __arm_inout("za") {
+ // expected-error@+1 {{'svmla_za16_mf8_vg2x2_fpm' needs target feature
sme,sme-f8f16}}
+ svmla_za16_mf8_vg2x2_fpm(slice, znx2, znx2, fpmr);
+
+ // expected-error@+1 {{'svmla_za16_mf8_vg2x4_fpm' needs target feature
sme,sme-f8f16}}
+ svmla_za16_mf8_vg2x4_fpm(slice, znx4, znx4, fpmr);
+
+ // expected-error@+1 {{'svmla_za32_mf8_vg4x2_fpm' needs target feature
sme,sme-f8f32}}
+ svmla_za32_mf8_vg4x2_fpm(slice, znx2, znx2, fpmr);
+
+ // expected-error@+1 {{'svmla_za32_mf8_vg4x4_fpm' needs target feature
sme,sme-f8f32}}
+ svmla_za32_mf8_vg4x4_fpm(slice, znx4, znx4, fpmr);
+}
\ No newline at end of file
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td
b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index b2f0aa2f7e4d90..53b3215e44dc34 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3888,6 +3888,18 @@ let TargetPrefix = "aarch64" in {
llvm_nxv16i1_ty, llvm_nxv16i1_ty,
llvm_nxv16i8_ty, llvm_nxv16i8_ty],
[ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly, IntrHasSideEffects]>;
+
+ class SME_FP8_ZA_MULTI_VGx2_Intrinsic
+ : DefaultAttrsIntrinsic<[], [llvm_i32_ty,
+ llvm_nxv16i8_ty, llvm_nxv16i8_ty,
+ llvm_nxv16i8_ty, llvm_nxv16i8_ty,],
+ [IntrInaccessibleMemOnly, IntrHasSideEffects]>;
+
+ class SME_FP8_ZA_MULTI_VGx4_Intrinsic
+ : DefaultAttrsIntrinsic<[], [llvm_i32_ty,
+ llvm_nxv16i8_ty, llvm_nxv16i8_ty,
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
+ llvm_nxv16i8_ty, llvm_nxv16i8_ty,
llvm_nxv16i8_ty, llvm_nxv16i8_ty,],
+ [IntrInaccessibleMemOnly, IntrHasSideEffects]>;
//
// CVT from FP8 to half-precision/BFloat16 multi-vector
//
@@ -3914,4 +3926,14 @@ let TargetPrefix = "aarch64" in {
// FP8 outer product
def int_aarch64_sme_fp8_fmopa_za16 : SME_FP8_OuterProduct_Intrinsic;
def int_aarch64_sme_fp8_fmopa_za32 : SME_FP8_OuterProduct_Intrinsic;
-}
+
+ //
+ // ZA multiply-add
+ //
+ // Double-vector groups (F8F16)
+ def int_aarch64_sme_fp8_fmlal_multi_za16_vg2x2 :
SME_FP8_ZA_MULTI_VGx2_Intrinsic;
+ def int_aarch64_sme_fp8_fmlal_multi_za16_vg2x4 :
SME_FP8_ZA_MULTI_VGx4_Intrinsic;
+ // Quad-vector groups (F8F32)
+ def int_aarch64_sme_fp8_fmlall_multi_za32_vg4x2 :
SME_FP8_ZA_MULTI_VGx2_Intrinsic;
+ def int_aarch64_sme_fp8_fmlall_multi_za32_vg4x4 :
SME_FP8_ZA_MULTI_VGx4_Intrinsic;
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index bd36e21d1be46c..7a805ab95110d8 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -1001,8 +1001,10 @@ def FMLAL_VG2_MZZ_BtoH :
sme2_mla_long_array_single_16b<"fmlal">;
defm FMLAL_VG2_M2ZZ_BtoH : sme2_fp_mla_long_array_vg2_single<"fmlal", 0b001,
MatrixOp16, ZZ_b, ZPR4b8, nxv16i8, null_frag>;
defm FMLAL_VG4_M4ZZ_BtoH : sme2_fp_mla_long_array_vg4_single<"fmlal", 0b001,
MatrixOp16, ZZZZ_b, ZPR4b8, nxv16i8, null_frag>;
-defm FMLAL_VG2_M2Z2Z_BtoH : sme2_fp_mla_long_array_vg2_multi<"fmlal", 0b100,
MatrixOp16, ZZ_b_mul_r, nxv16i8, null_frag>;
-defm FMLAL_VG4_M4Z4Z_BtoH : sme2_fp_mla_long_array_vg4_multi<"fmlal", 0b100,
MatrixOp16, ZZZZ_b_mul_r, nxv16i8, null_frag>;
+
+// FP8 FMLALL (multi)
+defm FMLAL_VG2_M2Z2Z_BtoH : sme2_fp_mla_long_array_vg2_multi<"fmlal", 0b100,
MatrixOp16, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_fp8_fmlal_multi_za16_vg2x2,
[FPMR, FPCR]>;
+defm FMLAL_VG4_M4Z4Z_BtoH : sme2_fp_mla_long_array_vg4_multi<"fmlal", 0b100,
MatrixOp16, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_fp8_fmlal_multi_za16_vg2x4,
[FPMR, FPCR]>;
defm FMOPA_MPPZZ_BtoH : sme2_fp8_fmopa_za16<"fmopa",
int_aarch64_sme_fp8_fmopa_za16>;
} //[HasSMEF8F16]
@@ -1027,8 +1029,10 @@ defm FMLALL_VG4_M4ZZI_BtoS :
sme2_mla_ll_array_vg4_index_32b<"fmlall", 0b00, 0b1
defm FMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"fmlall", 0b01000,
MatrixOp32, ZPR8, ZPR4b8, nxv16i8, null_frag>;
defm FMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"fmlall", 0b000001,
MatrixOp32, ZZ_b, ZPR4b8>;
defm FMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"fmlall", 0b010001,
MatrixOp32, ZZZZ_b, ZPR4b8>;
-defm FMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"fmlall", 0b01000,
MatrixOp32, ZZ_b_mul_r, nxv16i8, null_frag>;
-defm FMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"fmlall", 0b01000,
MatrixOp32, ZZZZ_b_mul_r, nxv16i8, null_frag>;
+
+// FP8 FMLALL (multi)
+defm FMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"fmlall", 0b01000,
MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_fp8_fmlall_multi_za32_vg4x2,
[FPMR, FPCR]>;
+defm FMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"fmlall", 0b01000,
MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_fp8_fmlall_multi_za32_vg4x4,
[FPMR, FPCR]>;
defm FMOPA_MPPZZ_BtoS : sme2_fp8_fmopa_za32<"fmopa",
int_aarch64_sme_fp8_fmopa_za32>;
} //[HasSMEF8F32]
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td
b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 27995ca5bb701f..015f7e39c603e1 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -2224,11 +2224,12 @@ class sme2_mla_long_array_vg2_multi<string mnemonic,
bits<2> op0, bits<3> op,
}
multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<3> op,
MatrixOperand matrix_ty,
- RegisterOperand multi_vector_ty,
- ValueType zpr_ty,
SDPatternOperator intrinsic> {
-
+ RegisterOperand multi_vector_ty,
ValueType zpr_ty,
+ SDPatternOperator intrinsic,
list<Register> uses=[]> {
def NAME : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op, matrix_ty,
multi_vector_ty>,
- SMEPseudo2Instr<NAME, 1>;
+ SMEPseudo2Instr<NAME, 1> {
+ let Uses = uses;
+ }
def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range,
multi_vector_ty, SMEMatrixArray>;
@@ -2271,9 +2272,11 @@ class sme2_mla_long_array_vg4_multi<string mnemonic,
bits<2> op0, bits<3> op,
multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<3> op,
MatrixOperand matrix_ty,
RegisterOperand multi_vector_ty,
ValueType zpr_ty,
- SDPatternOperator intrinsic> {
+ SDPatternOperator intrinsic,
list<Register> uses=[]> {
def NAME : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op, matrix_ty,
multi_vector_ty>,
- SMEPseudo2Instr<NAME, 1>;
+ SMEPseudo2Instr<NAME, 1> {
+ let Uses = uses;
+ }
def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range,
multi_vector_ty, SMEMatrixArray>;
@@ -3281,9 +3284,11 @@ class sme2_mla_ll_array_vg2_multi<bits<5> op,
MatrixOperand matrix_ty,
multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<5> op,
MatrixOperand matrix_ty,
- RegisterOperand vector_ty,
- ValueType vt, SDPatternOperator
intrinsic> {
- def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>,
SMEPseudo2Instr<NAME, 1>;
+ RegisterOperand vector_ty, ValueType vt,
+ SDPatternOperator intrinsic,
list<Register> uses=[]> {
+ def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>,
SMEPseudo2Instr<NAME, 1> {
+ let Uses = uses;
+ }
def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range,
vector_ty, SMEMatrixArray>;
@@ -3325,9 +3330,11 @@ class sme2_mla_ll_array_vg4_multi<bits<5>
op,MatrixOperand matrix_ty,
multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<5> op,
MatrixOperand matrix_ty,
- RegisterOperand vector_ty,
- ValueType vt, SDPatternOperator
intrinsic> {
- def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>,
SMEPseudo2Instr<NAME, 1>;
+ RegisterOperand vector_ty, ValueType vt,
+ SDPatternOperator intrinsic,
list<Register> uses=[]> {
+ def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>,
SMEPseudo2Instr<NAME, 1> {
+ let Uses = uses;
+ }
def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range,
vector_ty, SMEMatrixArray>;
diff --git a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-mla.ll
b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-mla.ll
new file mode 100644
index 00000000000000..220043279fba11
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-mla.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
UTC_ARGS: --filter-out "// kill:" --version 4
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme-f8f16,+sme-f8f32
-force-streaming < %s | FileCheck %s
+
+; FMLAL (multi)
+
+define void @test_fmlal_multi_vg2x2(i32 %slice, <vscale x 16 x i8> %zn0,
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zm0, <vscale x 16 x i8> %zm1) {
+; CHECK-LABEL: test_fmlal_multi_vg2x2:
+; CHECK: // %bb.0:
+; CHECK: mov w8, w0
+; CHECK: fmlal za.h[w8, 0:1, vgx2], { z0.b, z1.b }, { z2.b, z3.b }
+; CHECK: fmlal za.h[w8, 6:7, vgx2], { z0.b, z1.b }, { z2.b, z3.b }
+; CHECK: ret
+ call void @llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x2(i32 %slice,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1)
+ %add = add i32 %slice, 6
+ call void @llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x2(i32 %add,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1)
+ ret void
+}
+
+define void @test_fmlal_multi_vg2x4(i32 %slice, <vscale x 16 x i8> %zn0,
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
+; CHECK-LABEL: test_fmlal_multi_vg2x4:
+; CHECK: // %bb.0:
+; CHECK: mov w8, w0
+; CHECK: fmlal za.h[w8, 0:1, vgx4], { z0.b - z3.b }, { z4.b - z7.b }
+; CHECK: fmlal za.h[w8, 6:7, vgx4], { z0.b - z3.b }, { z4.b - z7.b }
+; CHECK: ret
+ <vscale x 16 x i8> %zm0, <vscale x 16 x
i8> %zm1, <vscale x 16 x i8> %zm2, <vscale x 16 x i8> %zm3) {
+ call void @llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x4(i32 %slice,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1, <vscale x 16 x i8> %zm2, <vscale x 16 x i8> %zm3)
+ %add = add i32 %slice, 6
+ call void @llvm.aarch64.sme.fp8.fmlal.multi.za16.vg2x4(i32 %add,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1, <vscale x 16 x i8> %zm2, <vscale x 16 x i8> %zm3)
+ ret void
+}
+
+; FMLALL (multi)
+
+define void @test_fmlal_multi_vg4x2(i32 %slice, <vscale x 16 x i8> %zn0,
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zm0, <vscale x 16 x i8> %zm1) {
+; CHECK-LABEL: test_fmlal_multi_vg4x2:
+; CHECK: // %bb.0:
+; CHECK: mov w8, w0
+; CHECK: fmlall za.s[w8, 0:3, vgx2], { z0.b, z1.b }, { z2.b, z3.b }
+; CHECK: fmlall za.s[w8, 4:7, vgx2], { z0.b, z1.b }, { z2.b, z3.b }
+; CHECK: ret
+ call void @llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x2(i32 %slice,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1)
+ %add = add i32 %slice, 4
+ call void @llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x2(i32 %add,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1)
+ ret void
+}
+
+define void @test_fmlal_multi_vg4x4(i32 %slice, <vscale x 16 x i8> %zn0,
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
+; CHECK-LABEL: test_fmlal_multi_vg4x4:
+; CHECK: // %bb.0:
+; CHECK: mov w8, w0
+; CHECK: fmlall za.s[w8, 0:3, vgx4], { z0.b - z3.b }, { z4.b - z7.b }
+; CHECK: fmlall za.s[w8, 4:7, vgx4], { z0.b - z3.b }, { z4.b - z7.b }
+; CHECK: ret
+ <vscale x 16 x i8> %zm0, <vscale x 16 x
i8> %zm1, <vscale x 16 x i8> %zm2, <vscale x 16 x i8> %zm3) {
+ call void @llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x4(i32 %slice,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1, <vscale x 16 x i8> %zm2, <vscale x 16 x i8> %zm3)
+ %add = add i32 %slice, 4
+ call void @llvm.aarch64.sme.fp8.fmlall.multi.za32.vg4x4(i32 %add,
+ <vscale x 16 x i8>
%zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,
+ <vscale x 16 x i8>
%zm0, <vscale x 16 x i8> %zm1, <vscale x 16 x i8> %zm2, <vscale x 16 x i8> %zm3)
+ ret void
+}
>From 941a5ede64daa5ae8223a63504d00fe41d2a3507 Mon Sep 17 00:00:00 2001
From: Spencer Abson <spencer.ab...@arm.com>
Date: Fri, 13 Dec 2024 17:57:35 +0000
Subject: [PATCH 2/2] [NFC] fix typos
---
clang/include/clang/Basic/arm_sme.td | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/arm_sme.td
b/clang/include/clang/Basic/arm_sme.td
index da19f9be2830c0..9fada0f9605653 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -827,7 +827,7 @@ let SMETargetGuard = "sme-lutv2" in {
let SMETargetGuard = "sme-f8f32" in {
def SVMOPA_FP8_ZA32 : Inst<"svmopa_za32[_mf8]_m_fpm", "viPPdd>", "m",
MergeNone, "aarch64_sme_fp8_fmopa_za32",
[IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], [ImmCheck<0, ImmCheck0_3>]>;
- // FMLALL (mutliple)
+ // FMLALL (multiple)
def SVMLA_FP8_MULTI_ZA32_VG4x2 : Inst<"svmla_za32[_mf8]_vg4x2_fpm", "vm22>",
"m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x2",
[IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], []>;
def SVMLA_FP8_MULTI_ZA32_VG4x4 : Inst<"svmla_za32[_mf8]_vg4x4_fpm", "vm44>",
"m", MergeNone, "aarch64_sme_fp8_fmlall_multi_za32_vg4x4",
@@ -837,7 +837,7 @@ let SMETargetGuard = "sme-f8f32" in {
let SMETargetGuard = "sme-f8f16" in {
def SVMOPA_FP8_ZA16 : Inst<"svmopa_za16[_mf8]_m_fpm", "viPPdd>", "m",
MergeNone, "aarch64_sme_fp8_fmopa_za16",
[IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], [ImmCheck<0, ImmCheck0_1>]>;
- // FMLAL (mutliple)
+ // FMLAL (multiple)
def SVMLA_FP8_MULTI_ZA16_VG2x2 : Inst<"svmla_za16[_mf8]_vg2x2_fpm", "vm22>",
"m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x2",
[IsStreaming, IsInOutZA, SetsFPMR,
IsOverloadNone], []>;
def SVMLA_FP8_MULTI_ZA16_VG2x4 : Inst<"svmla_za16[_mf8]_vg2x4_fpm", "vm44>",
"m", MergeNone, "aarch64_sme_fp8_fmlal_multi_za16_vg2x4",
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
