================ @@ -0,0 +1,766 @@ +/*===-------------- acev1intrin.h - ACEV1 intrinsics -*- C/C++ -*------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===------------------------------------------------------------------------=== + */ + +#ifndef __IMMINTRIN_H +#error "Never use <acev1intrin.h> directly; include <immintrin.h> instead." +#endif /* __IMMINTRIN_H */ + +#ifndef __ACEV1INTRIN_H +#define __ACEV1INTRIN_H +#ifdef __x86_64__ + +/* Define the default attributes for the functions in this file. */ +#define __DEFAULT_FN_ATTRS_ACE \ + __attribute__((__always_inline__, __nodebug__, __target__("acev1"))) + +/// Load tile configuration from a 64-byte memory location. For ACE +/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1), +/// ACE tiles have fixed dimensions of 16 rows × 64 bytes, so per-tile +/// row/column configuration bytes are ignored. If palette_id is zero, +/// tiles return to init state and are zeroed. Invalid configurations +/// result in #GP fault. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> LDTILECFG </c> instruction. +/// +/// \param __config +/// A pointer to 64-byte tile configuration (use __ace_tile_config). +static __inline__ void __DEFAULT_FN_ATTRS_ACE +_tile_ace_loadconfig(const void *__config) { + __builtin_ia32_tile_loadconfig(__config); +} + +/// Store the current tile configuration to a 64-byte memory location. +/// For ACE (Palette 2), the stored palette_id will be 2 and per-tile +/// configuration bytes reflect the fixed 16×64 dimensions. If tiles +/// are not configured, all zeroes are stored. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> STTILECFG </c> instruction. +/// +/// \param __config +/// A pointer to 64-byte tile configuration buffer. +static __inline__ void __DEFAULT_FN_ATTRS_ACE +_tile_ace_storeconfig(void *__config) { + __builtin_ia32_tile_storeconfig(__config); +} + +/// Release the tile configuration to return to init state, releasing +/// all tile storage. After this, tiles must be reconfigured with +/// _tile_ace_loadconfig before use. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILERELEASE </c> instruction. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) { + __builtin_ia32_tilerelease(); +} + +/// Zero the ACE tile specified by "tile". Sets all 1024 bytes +/// (16 rows × 64 bytes) of the tile register to zero. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEZERO </c> instruction. +/// +/// \param tile +/// Destination tile register ID (0-7). +#define _tile_ace_zero(tile) __builtin_ia32_tilezero((tile)) + +/// Move a 64-byte ZMM vector to a tile column. The 16 doublewords from +/// the ZMM are written as a vertical column in the tile at the specified +/// index. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEMOVCOL </c> instruction. +/// +/// \param dst +/// Destination tile register ID (0-7). +/// \param src +/// Source ZMM register ID (0-31). +/// \param idx +/// Column index (0-15). Immediate or register form selected automatically. +#define _tile_setcol(dst, src, idx) \ + __builtin_ia32_tilesetcol((dst), (src), (idx)) + +/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written +/// as a horizontal row in the tile at the specified index. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction. +/// +/// \param dst +/// Destination tile register ID (0-7). +/// \param src +/// Source ZMM register ID (0-31). +/// \param idx +/// Row index (0-15). Immediate or register form selected automatically. +#define _tile_setrow(dst, src, idx) \ + __builtin_ia32_tilesetrow((dst), (src), (idx)) + +/// Initialize the Block Scale Register (BSR) to zero. The BSR holds +/// 32 scaling factors used by mixed-precision outer product instructions +/// (TOP4MX* variants). Must be called before using BSR-scaled operations. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRINIT </c> instruction. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) { + __builtin_ia32_bsrinit(); +} + +/// Load the full BSR (32 scale factors) from two ZMM registers. The low +/// half (16 factors) comes from __src1, high half from __src2. Each +/// doubleword contains one scale factor. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVF </c> instruction. +/// +/// \param __src1 +/// ZMM with scale factors for BSR low half (factors 0-15). +/// \param __src2 +/// ZMM with scale factors for BSR high half (factors 16-31). +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1, + __m512i __src2) { + __builtin_ia32_bsrmovf((__v16si)__src1, (__v16si)__src2); +} + +/// Load the high half of BSR (scale factors 16-31) from a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction. +/// +/// \param __src +/// ZMM with 16 scale factors to write to BSR high half. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) { + __builtin_ia32_bsrmovh_set((__v16si)__src); +} + +/// Read the high half of BSR (scale factors 16-31) to a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction. +/// +/// \returns +/// ZMM containing 16 scale factors from BSR high half. +static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) { + return (__m512i)__builtin_ia32_bsrmovh_get(); +} + +/// Load the low half of BSR (scale factors 0-15) from a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction. +/// +/// \param __src +/// ZMM with 16 scale factors to write to BSR low half. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) { + __builtin_ia32_bsrmovl_set((__v16si)__src); +} + +/// Read the low half of BSR (scale factors 0-15) to a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction. +/// +/// \returns +/// ZMM containing 16 scale factors from BSR low half. +static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) { + return (__m512i)__builtin_ia32_bsrmovl_get(); +} + +/// Compute 2-way outer product of BF16 pairs, accumulating to FP32. +/// Each BF16 pair from src1 and src2 produces two FP32 products that +/// are accumulated into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP2BF16PS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing 32 BF16 values. +/// \param src2 +/// Second source ZMM register ID (0-31) containing 32 BF16 values. +#define _tile_top2bf16ps(dst, src1, src2) \ ---------------- mahesh-attarde wrote:
This is wrong in spec too. There is clash in convention for what 2 means here. Two source or Rank 2? https://github.com/llvm/llvm-project/pull/208408 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
