[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
This revision was automatically updated to reflect the committed changes. Closed by commit rGd5fc592b7c26: [AArch64][SVE] Add bfloat16 support to svext intrinsic (authored by c-rhodes). Changed prior to commit: https://reviews.llvm.org/D82391?vs=273685&id=274052#toc Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 Files: clang/include/clang/Basic/arm_sve.td clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll Index: llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll === --- llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll +++ llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll @@ -516,6 +516,16 @@ ret %out } +define @ext_bf16( %a, %b) #0 { +; CHECK-LABEL: ext_bf16: +; CHECK: ext z0.b, z0.b, z1.b, #6 +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.ext.nxv8bf16( %a, +%b, + i32 3) + ret %out +} + define @ext_f16( %a, %b) { ; CHECK-LABEL: ext_f16: ; CHECK: ext z0.b, z0.b, z1.b, #6 @@ -1885,6 +1895,7 @@ declare @llvm.aarch64.sve.ext.nxv8i16(, , i32) declare @llvm.aarch64.sve.ext.nxv4i32(, , i32) declare @llvm.aarch64.sve.ext.nxv2i64(, , i32) +declare @llvm.aarch64.sve.ext.nxv8bf16(, , i32) declare @llvm.aarch64.sve.ext.nxv8f16(, , i32) declare @llvm.aarch64.sve.ext.nxv4f32(, , i32) declare @llvm.aarch64.sve.ext.nxv2f64(, , i32) Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td === --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1446,7 +1446,6 @@ def : Pat<(nxv8i16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv8f16 ZPR:$src))), (nxv8i16 ZPR:$src)>; -def : Pat<(nxv8i16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv2f64 ZPR:$src))), (nxv8i16 ZPR:$src)>; Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c === --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c @@ -0,0 +1,26 @@ +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t +// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it. +// ASM-NOT: warning +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svext_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svext_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.ext.nxv8bf16( %op1, %op2, i32 127) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svext_bf16'}} + return SVE_ACLE_FUNC(svext,_bf16,,)(op1, op2, 127); +} Index: clang/include/clang/Basic/arm_sve.td === --- clang/include/clang/Basic/arm_sve.td +++ clang/include/clang/Basic/arm_sve.td @@ -1208,6 +1208,7 @@ def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2">; let ArchGuard = "defined(__ARM_FEATURE_SVE_B
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
sdesmalen accepted this revision. sdesmalen added a comment. LGTM CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
c-rhodes updated this revision to Diff 273685. c-rhodes added a comment. Changes: - Add tests for bfloat bitcast patterns. CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 Files: clang/include/clang/Basic/arm_sve.td clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td llvm/test/CodeGen/AArch64/sve-bitcast.ll llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll Index: llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll === --- llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll +++ llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll @@ -516,6 +516,16 @@ ret %out } +define @ext_bf16( %a, %b) #0 { +; CHECK-LABEL: ext_bf16: +; CHECK: ext z0.b, z0.b, z1.b, #6 +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.ext.nxv8bf16( %a, +%b, + i32 3) + ret %out +} + define @ext_f16( %a, %b) { ; CHECK-LABEL: ext_f16: ; CHECK: ext z0.b, z0.b, z1.b, #6 @@ -1885,6 +1895,7 @@ declare @llvm.aarch64.sve.ext.nxv8i16(, , i32) declare @llvm.aarch64.sve.ext.nxv4i32(, , i32) declare @llvm.aarch64.sve.ext.nxv2i64(, , i32) +declare @llvm.aarch64.sve.ext.nxv8bf16(, , i32) declare @llvm.aarch64.sve.ext.nxv8f16(, , i32) declare @llvm.aarch64.sve.ext.nxv4f32(, , i32) declare @llvm.aarch64.sve.ext.nxv2f64(, , i32) Index: llvm/test/CodeGen/AArch64/sve-bitcast.ll === --- llvm/test/CodeGen/AArch64/sve-bitcast.ll +++ llvm/test/CodeGen/AArch64/sve-bitcast.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s -; RUN: not --crash llc -mtriple=aarch64_be -mattr=+sve < %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s 2>%t | FileCheck %s +; RUN: not --crash llc -mtriple=aarch64_be -mattr=+sve,+bf16 < %s ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t ; WARN-NOT: warning @@ -53,6 +53,14 @@ ret %bc } +define @bitcast_bfloat_to_i8( %v) { +; CHECK-LABEL: bitcast_bfloat_to_i8: +; CHECK: // %bb.0: +; CHECK-NEXT:ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_i8_to_i16( %v) { ; CHECK-LABEL: bitcast_i8_to_i16: ; CHECK: // %bb.0: @@ -101,6 +109,14 @@ ret %bc } +define @bitcast_bfloat_to_i16( %v) { +; CHECK-LABEL: bitcast_bfloat_to_i16: +; CHECK: // %bb.0: +; CHECK-NEXT:ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_i8_to_i32( %v) { ; CHECK-LABEL: bitcast_i8_to_i32: ; CHECK: // %bb.0: @@ -340,3 +356,19 @@ %bc = bitcast %v to ret %bc } + +define @bitcast_i8_to_bfloat( %v) { +; CHECK-LABEL: bitcast_i8_to_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT:ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_i16_to_bfloat( %v) { +; CHECK-LABEL: bitcast_i16_to_bfloat: +; CHECK: // %bb.0: +; CHECK-NEXT:ret + %bc = bitcast %v to + ret %bc +} Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td === --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1444,7 +1444,6 @@ def : Pat<(nxv8i16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv8f16 ZPR:$src))), (nxv8i16 ZPR:$src)>; -def : Pat<(nxv8i16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv2f64 ZPR:$src))), (nxv8i16 ZPR:$src)>; @@ -1464,7 +1463,6 @@ def : Pat<(nxv8f16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8f16 ZPR:$src)>; -def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8f16 ZPR:$src)>; @@ -1483,6 +1481,16 @@ def : Pat<(nxv2f64 (bitconvert (nxv2i64 ZPR:$src))), (nxv2f64 ZPR:$src)>; def : Pat<(nxv2f64 (bitconvert (nxv8f16 ZPR:$src))), (nxv2f64 ZPR:$src)>; def : Pat<(nxv2f64 (bitconvert (nxv4f32 ZPR:$src))), (nxv2f64 ZPR:$src)>; + + } + + let Predicates = [IsLE, HasBF16] in { +def : Pat<(nxv16i8 (bitconvert (nxv8bf16 ZPR:$src))), (nxv16i8 ZPR:$src)>; + +def : Pat<(nxv8i16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8i16 ZPR:$src)>; + +def : Pat<(nxv8bf16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8bf16 ZP
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
sdesmalen added inline comments. Comment at: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td:1479 + let Predicates = [IsLE, HasBF16] in { +def : Pat<(nxv16i8 (bitconvert (nxv8bf16 ZPR:$src))), (nxv16i8 ZPR:$src)>; + These patterns are missing tests in llvm/test/CodeGen/AArch64/sve-bitcast.ll CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
fpetrogalli accepted this revision. fpetrogalli added a comment. This revision is now accepted and ready to land. LGTM, thank you! CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
c-rhodes updated this revision to Diff 273365. c-rhodes added a comment. Changes: - Guard patterns on `+bf16`. CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 Files: clang/include/clang/Basic/arm_sve.td clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll Index: llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll === --- llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll +++ llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll @@ -516,6 +516,16 @@ ret %out } +define @ext_bf16( %a, %b) #0 { +; CHECK-LABEL: ext_bf16: +; CHECK: ext z0.b, z0.b, z1.b, #6 +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.ext.nxv8bf16( %a, +%b, + i32 3) + ret %out +} + define @ext_f16( %a, %b) { ; CHECK-LABEL: ext_f16: ; CHECK: ext z0.b, z0.b, z1.b, #6 @@ -1876,6 +1886,7 @@ declare @llvm.aarch64.sve.ext.nxv8i16(, , i32) declare @llvm.aarch64.sve.ext.nxv4i32(, , i32) declare @llvm.aarch64.sve.ext.nxv2i64(, , i32) +declare @llvm.aarch64.sve.ext.nxv8bf16(, , i32) declare @llvm.aarch64.sve.ext.nxv8f16(, , i32) declare @llvm.aarch64.sve.ext.nxv4f32(, , i32) declare @llvm.aarch64.sve.ext.nxv2f64(, , i32) Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td === --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1454,7 +1454,6 @@ def : Pat<(nxv8f16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8f16 ZPR:$src)>; -def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8f16 ZPR:$src)>; @@ -1473,6 +1472,14 @@ def : Pat<(nxv2f64 (bitconvert (nxv2i64 ZPR:$src))), (nxv2f64 ZPR:$src)>; def : Pat<(nxv2f64 (bitconvert (nxv8f16 ZPR:$src))), (nxv2f64 ZPR:$src)>; def : Pat<(nxv2f64 (bitconvert (nxv4f32 ZPR:$src))), (nxv2f64 ZPR:$src)>; + + } + + let Predicates = [IsLE, HasBF16] in { +def : Pat<(nxv16i8 (bitconvert (nxv8bf16 ZPR:$src))), (nxv16i8 ZPR:$src)>; + +def : Pat<(nxv8bf16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8bf16 ZPR:$src)>; +def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>; } def : Pat<(nxv16i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>; Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c === --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c @@ -0,0 +1,26 @@ +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t +// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it. +// ASM-NOT: warning +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svext_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svext_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.ext.nxv8bf16( %op1, %op2, i32 127) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit dec
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
fpetrogalli requested changes to this revision. fpetrogalli added a comment. This revision now requires changes to proceed. Putting it on hold as we need to guard those patterns with `HasBF16`. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
fpetrogalli accepted this revision. fpetrogalli added a comment. This revision is now accepted and ready to land. LGTM! Thank you. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D82391/new/ https://reviews.llvm.org/D82391 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[PATCH] D82391: [AArch64][SVE] Add bfloat16 support to svext intrinsic
c-rhodes created this revision. c-rhodes added reviewers: sdesmalen, kmclaughlin, efriedma, david-arm, fpetrogalli. Herald added subscribers: danielkiss, psnobl, rkruppe, hiraditya, kristof.beyls, tschuett. Herald added projects: clang, LLVM. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D82391 Files: clang/include/clang/Basic/arm_sve.td clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c clang/utils/TableGen/SveEmitter.cpp llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll Index: llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll === --- llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll +++ llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll @@ -516,6 +516,16 @@ ret %out } +define @ext_bf16( %a, %b) { +; CHECK-LABEL: ext_bf16: +; CHECK: ext z0.b, z0.b, z1.b, #6 +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.ext.nxv8bf16( %a, +%b, + i32 3) + ret %out +} + define @ext_f16( %a, %b) { ; CHECK-LABEL: ext_f16: ; CHECK: ext z0.b, z0.b, z1.b, #6 @@ -1876,6 +1886,7 @@ declare @llvm.aarch64.sve.ext.nxv8i16(, , i32) declare @llvm.aarch64.sve.ext.nxv4i32(, , i32) declare @llvm.aarch64.sve.ext.nxv2i64(, , i32) +declare @llvm.aarch64.sve.ext.nxv8bf16(, , i32) declare @llvm.aarch64.sve.ext.nxv8f16(, , i32) declare @llvm.aarch64.sve.ext.nxv4f32(, , i32) declare @llvm.aarch64.sve.ext.nxv2f64(, , i32) Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td === --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1405,12 +1405,13 @@ // constraint that none of the bits change when stored to memory as one // type, and and reloaded as another type. let Predicates = [IsLE] in { -def : Pat<(nxv16i8 (bitconvert (nxv8i16 ZPR:$src))), (nxv16i8 ZPR:$src)>; -def : Pat<(nxv16i8 (bitconvert (nxv4i32 ZPR:$src))), (nxv16i8 ZPR:$src)>; -def : Pat<(nxv16i8 (bitconvert (nxv2i64 ZPR:$src))), (nxv16i8 ZPR:$src)>; -def : Pat<(nxv16i8 (bitconvert (nxv8f16 ZPR:$src))), (nxv16i8 ZPR:$src)>; -def : Pat<(nxv16i8 (bitconvert (nxv4f32 ZPR:$src))), (nxv16i8 ZPR:$src)>; -def : Pat<(nxv16i8 (bitconvert (nxv2f64 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv8i16 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv4i32 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv2i64 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv8f16 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv4f32 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv2f64 ZPR:$src))), (nxv16i8 ZPR:$src)>; +def : Pat<(nxv16i8 (bitconvert (nxv8bf16 ZPR:$src))), (nxv16i8 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8i16 ZPR:$src)>; def : Pat<(nxv8i16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8i16 ZPR:$src)>; @@ -1435,7 +1436,6 @@ def : Pat<(nxv8f16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8f16 ZPR:$src)>; -def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8f16 ZPR:$src)>; def : Pat<(nxv8f16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8f16 ZPR:$src)>; @@ -1454,6 +1454,9 @@ def : Pat<(nxv2f64 (bitconvert (nxv2i64 ZPR:$src))), (nxv2f64 ZPR:$src)>; def : Pat<(nxv2f64 (bitconvert (nxv8f16 ZPR:$src))), (nxv2f64 ZPR:$src)>; def : Pat<(nxv2f64 (bitconvert (nxv4f32 ZPR:$src))), (nxv2f64 ZPR:$src)>; + +def : Pat<(nxv8bf16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8bf16 ZPR:$src)>; +def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>; } def : Pat<(nxv16i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>; Index: clang/utils/TableGen/SveEmitter.cpp === --- clang/utils/TableGen/SveEmitter.cpp +++ clang/utils/TableGen/SveEmitter.cpp @@ -596,6 +596,7 @@ case 'i': Predicate = false; Float = false; +BFloat = false; ElementBitwidth = Bitwidth = 64; NumVectors = 0; Signed = false; Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c === --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c @@ -0,0 +1,26 @@ +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC