1/*===------------ avx512bf16intrin.h - AVX512_BF16 intrinsics --------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead."
11#endif
12
13#ifdef __SSE2__
14
15#ifndef __AVX512BF16INTRIN_H
16#define __AVX512BF16INTRIN_H
17
18typedef __bf16 __v32bf __attribute__((__vector_size__(64), __aligned__(64)));
19typedef __bf16 __m512bh __attribute__((__vector_size__(64), __aligned__(64)));
20typedef __bf16 __bfloat16 __attribute__((deprecated("use __bf16 instead")));
21
22#define __DEFAULT_FN_ATTRS512 \
23 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \
24 __min_vector_width__(512)))
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))
27
28#if defined(__cplusplus) && (__cplusplus >= 201103L)
29#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
30#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
31#else
32#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
33#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
34#endif
35
36/// Convert One BF16 Data to One Single Float Data.
37///
38/// \headerfile <x86intrin.h>
39///
40/// This intrinsic does not correspond to a specific instruction.
41///
42/// \param __A
43/// A bfloat data.
44/// \returns A float data whose sign field and exponent field keep unchanged,
45/// and fraction field is extended to 23 bits.
46static __inline__ float __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvtsbh_ss(__bf16 __A) {
47 return (float)(__A);
48}
49
50/// Convert Two Packed Single Data to One Packed BF16 Data.
51///
52/// \headerfile <x86intrin.h>
53///
54/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
55///
56/// \param __A
57/// A 512-bit vector of [16 x float].
58/// \param __B
59/// A 512-bit vector of [16 x float].
60/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
61/// conversion of __B, and higher 256 bits come from conversion of __A.
62static __inline__ __m512bh __DEFAULT_FN_ATTRS512
63_mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) {
64 return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A,
65 (__v16sf) __B);
66}
67
68/// Convert Two Packed Single Data to One Packed BF16 Data.
69///
70/// \headerfile <x86intrin.h>
71///
72/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
73///
74/// \param __A
75/// A 512-bit vector of [16 x float].
76/// \param __B
77/// A 512-bit vector of [16 x float].
78/// \param __W
79/// A 512-bit vector of [32 x bfloat].
80/// \param __U
81/// A 32-bit mask value specifying what is chosen for each element.
82/// A 1 means conversion of __A or __B. A 0 means element from __W.
83/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
84/// conversion of __B, and higher 256 bits come from conversion of __A.
85static __inline__ __m512bh __DEFAULT_FN_ATTRS512
86_mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) {
87 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,
88 (__v32bf)_mm512_cvtne2ps_pbh(__A, __B),
89 (__v32bf)__W);
90}
91
92/// Convert Two Packed Single Data to One Packed BF16 Data.
93///
94/// \headerfile <x86intrin.h>
95///
96/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
97///
98/// \param __A
99/// A 512-bit vector of [16 x float].
100/// \param __B
101/// A 512-bit vector of [16 x float].
102/// \param __U
103/// A 32-bit mask value specifying what is chosen for each element.
104/// A 1 means conversion of __A or __B. A 0 means element is zero.
105/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
106/// conversion of __B, and higher 256 bits come from conversion of __A.
107static __inline__ __m512bh __DEFAULT_FN_ATTRS512
108_mm512_maskz_cvtne2ps_pbh(__mmask32 __U, __m512 __A, __m512 __B) {
109 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,
110 (__v32bf)_mm512_cvtne2ps_pbh(__A, __B),
111 (__v32bf)_mm512_setzero_si512());
112}
113
114/// Convert Packed Single Data to Packed BF16 Data.
115///
116/// \headerfile <x86intrin.h>
117///
118/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
119///
120/// \param __A
121/// A 512-bit vector of [16 x float].
122/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
123static __inline__ __m256bh __DEFAULT_FN_ATTRS512
124_mm512_cvtneps_pbh(__m512 __A) {
125 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
126 (__v16bf)_mm256_undefined_si256(),
127 (__mmask16)-1);
128}
129
130/// Convert Packed Single Data to Packed BF16 Data.
131///
132/// \headerfile <x86intrin.h>
133///
134/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
135///
136/// \param __A
137/// A 512-bit vector of [16 x float].
138/// \param __W
139/// A 256-bit vector of [16 x bfloat].
140/// \param __U
141/// A 16-bit mask value specifying what is chosen for each element.
142/// A 1 means conversion of __A. A 0 means element from __W.
143/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
144static __inline__ __m256bh __DEFAULT_FN_ATTRS512
145_mm512_mask_cvtneps_pbh(__m256bh __W, __mmask16 __U, __m512 __A) {
146 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
147 (__v16bf)__W,
148 (__mmask16)__U);
149}
150
151/// Convert Packed Single Data to Packed BF16 Data.
152///
153/// \headerfile <x86intrin.h>
154///
155/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
156///
157/// \param __A
158/// A 512-bit vector of [16 x float].
159/// \param __U
160/// A 16-bit mask value specifying what is chosen for each element.
161/// A 1 means conversion of __A. A 0 means element is zero.
162/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
163static __inline__ __m256bh __DEFAULT_FN_ATTRS512
164_mm512_maskz_cvtneps_pbh(__mmask16 __U, __m512 __A) {
165 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
166 (__v16bf)_mm256_setzero_si256(),
167 (__mmask16)__U);
168}
169
170/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
171///
172/// \headerfile <x86intrin.h>
173///
174/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
175///
176/// \param __A
177/// A 512-bit vector of [32 x bfloat].
178/// \param __B
179/// A 512-bit vector of [32 x bfloat].
180/// \param __D
181/// A 512-bit vector of [16 x float].
182/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
183/// __A, __B and __D
184static __inline__ __m512 __DEFAULT_FN_ATTRS512
185_mm512_dpbf16_ps(__m512 __D, __m512bh __A, __m512bh __B) {
186 return (__m512)__builtin_ia32_dpbf16ps_512((__v16sf) __D,
187 (__v32bf) __A,
188 (__v32bf) __B);
189}
190
191/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
192///
193/// \headerfile <x86intrin.h>
194///
195/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
196///
197/// \param __A
198/// A 512-bit vector of [32 x bfloat].
199/// \param __B
200/// A 512-bit vector of [32 x bfloat].
201/// \param __D
202/// A 512-bit vector of [16 x float].
203/// \param __U
204/// A 16-bit mask value specifying what is chosen for each element.
205/// A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
206/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
207/// __A, __B and __D
208static __inline__ __m512 __DEFAULT_FN_ATTRS512
209_mm512_mask_dpbf16_ps(__m512 __D, __mmask16 __U, __m512bh __A, __m512bh __B) {
210 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
211 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
212 (__v16sf)__D);
213}
214
215/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
216///
217/// \headerfile <x86intrin.h>
218///
219/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
220///
221/// \param __A
222/// A 512-bit vector of [32 x bfloat].
223/// \param __B
224/// A 512-bit vector of [32 x bfloat].
225/// \param __D
226/// A 512-bit vector of [16 x float].
227/// \param __U
228/// A 16-bit mask value specifying what is chosen for each element.
229/// A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
230/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
231/// __A, __B and __D
232static __inline__ __m512 __DEFAULT_FN_ATTRS512
233_mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {
234 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
235 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
236 (__v16sf)_mm512_setzero_si512());
237}
238
239/// Convert Packed BF16 Data to Packed float Data.
240///
241/// \headerfile <x86intrin.h>
242///
243/// \param __A
244/// A 256-bit vector of [16 x bfloat].
245/// \returns A 512-bit vector of [16 x float] come from conversion of __A
246static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
247_mm512_cvtpbh_ps(__m256bh __A) {
248 return (__m512) __builtin_convertvector(__A, __v16sf);
249}
250
251/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
252///
253/// \headerfile <x86intrin.h>
254///
255/// \param __U
256/// A 16-bit mask. Elements are zeroed out when the corresponding mask
257/// bit is not set.
258/// \param __A
259/// A 256-bit vector of [16 x bfloat].
260/// \returns A 512-bit vector of [16 x float] come from conversion of __A
261static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
262_mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {
263 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
264 (__v16sf)_mm512_cvtpbh_ps(__A),
265 (__v16sf)_mm512_setzero_ps());
266}
267
268/// Convert Packed BF16 Data to Packed float Data using merging mask.
269///
270/// \headerfile <x86intrin.h>
271///
272/// \param __S
273/// A 512-bit vector of [16 x float]. Elements are copied from __S when
274/// the corresponding mask bit is not set.
275/// \param __U
276/// A 16-bit mask.
277/// \param __A
278/// A 256-bit vector of [16 x bfloat].
279/// \returns A 512-bit vector of [16 x float] come from conversion of __A
280static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
281_mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __U, __m256bh __A) {
282 return (__m512)__builtin_ia32_selectps_512(
283 (__mmask16)__U, (__v16sf)_mm512_cvtpbh_ps(__A), (__v16sf)__S);
284}
285
286#undef __DEFAULT_FN_ATTRS
287#undef __DEFAULT_FN_ATTRS_CONSTEXPR
288#undef __DEFAULT_FN_ATTRS512
289#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
290
291#endif
292#endif