1/*===------------- avx512vbmi2intrin.h - VBMI2 intrinsics ------------------===
2 *
3 *
4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * See https://llvm.org/LICENSE.txt for license information.
6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 *
8 *===-----------------------------------------------------------------------===
9 */
10#ifndef __IMMINTRIN_H
11#error "Never use <avx512vbmi2intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __AVX512VBMI2INTRIN_H
15#define __AVX512VBMI2INTRIN_H
16
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx512vbmi2"), \
20 __min_vector_width__(512)))
21
22#if defined(__cplusplus) && (__cplusplus >= 201103L)
23#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
24#else
25#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
26#endif
27
28static __inline__ __m512i __DEFAULT_FN_ATTRS
29_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D)
30{
31 return (__m512i) __builtin_ia32_compresshi512_mask ((__v32hi) __D,
32 (__v32hi) __S,
33 __U);
34}
35
36static __inline__ __m512i __DEFAULT_FN_ATTRS
37_mm512_maskz_compress_epi16(__mmask32 __U, __m512i __D)
38{
39 return (__m512i) __builtin_ia32_compresshi512_mask ((__v32hi) __D,
40 (__v32hi) _mm512_setzero_si512(),
41 __U);
42}
43
44static __inline__ __m512i __DEFAULT_FN_ATTRS
45_mm512_mask_compress_epi8(__m512i __S, __mmask64 __U, __m512i __D)
46{
47 return (__m512i) __builtin_ia32_compressqi512_mask ((__v64qi) __D,
48 (__v64qi) __S,
49 __U);
50}
51
52static __inline__ __m512i __DEFAULT_FN_ATTRS
53_mm512_maskz_compress_epi8(__mmask64 __U, __m512i __D)
54{
55 return (__m512i) __builtin_ia32_compressqi512_mask ((__v64qi) __D,
56 (__v64qi) _mm512_setzero_si512(),
57 __U);
58}
59
60static __inline__ void __DEFAULT_FN_ATTRS
61_mm512_mask_compressstoreu_epi16(void *__P, __mmask32 __U, __m512i __D)
62{
63 __builtin_ia32_compressstorehi512_mask ((__v32hi *) __P, (__v32hi) __D,
64 __U);
65}
66
67static __inline__ void __DEFAULT_FN_ATTRS
68_mm512_mask_compressstoreu_epi8(void *__P, __mmask64 __U, __m512i __D)
69{
70 __builtin_ia32_compressstoreqi512_mask ((__v64qi *) __P, (__v64qi) __D,
71 __U);
72}
73
74static __inline__ __m512i __DEFAULT_FN_ATTRS
75_mm512_mask_expand_epi16(__m512i __S, __mmask32 __U, __m512i __D)
76{
77 return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __D,
78 (__v32hi) __S,
79 __U);
80}
81
82static __inline__ __m512i __DEFAULT_FN_ATTRS
83_mm512_maskz_expand_epi16(__mmask32 __U, __m512i __D)
84{
85 return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __D,
86 (__v32hi) _mm512_setzero_si512(),
87 __U);
88}
89
90static __inline__ __m512i __DEFAULT_FN_ATTRS
91_mm512_mask_expand_epi8(__m512i __S, __mmask64 __U, __m512i __D)
92{
93 return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __D,
94 (__v64qi) __S,
95 __U);
96}
97
98static __inline__ __m512i __DEFAULT_FN_ATTRS
99_mm512_maskz_expand_epi8(__mmask64 __U, __m512i __D)
100{
101 return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __D,
102 (__v64qi) _mm512_setzero_si512(),
103 __U);
104}
105
106static __inline__ __m512i __DEFAULT_FN_ATTRS
107_mm512_mask_expandloadu_epi16(__m512i __S, __mmask32 __U, void const *__P)
108{
109 return (__m512i) __builtin_ia32_expandloadhi512_mask ((const __v32hi *)__P,
110 (__v32hi) __S,
111 __U);
112}
113
114static __inline__ __m512i __DEFAULT_FN_ATTRS
115_mm512_maskz_expandloadu_epi16(__mmask32 __U, void const *__P)
116{
117 return (__m512i) __builtin_ia32_expandloadhi512_mask ((const __v32hi *)__P,
118 (__v32hi) _mm512_setzero_si512(),
119 __U);
120}
121
122static __inline__ __m512i __DEFAULT_FN_ATTRS
123_mm512_mask_expandloadu_epi8(__m512i __S, __mmask64 __U, void const *__P)
124{
125 return (__m512i) __builtin_ia32_expandloadqi512_mask ((const __v64qi *)__P,
126 (__v64qi) __S,
127 __U);
128}
129
130static __inline__ __m512i __DEFAULT_FN_ATTRS
131_mm512_maskz_expandloadu_epi8(__mmask64 __U, void const *__P)
132{
133 return (__m512i) __builtin_ia32_expandloadqi512_mask ((const __v64qi *)__P,
134 (__v64qi) _mm512_setzero_si512(),
135 __U);
136}
137
138#define _mm512_shldi_epi64(A, B, I) \
139 ((__m512i)__builtin_ia32_vpshldq512((__v8di)(__m512i)(A), \
140 (__v8di)(__m512i)(B), (int)(I)))
141
142#define _mm512_mask_shldi_epi64(S, U, A, B, I) \
143 ((__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
144 (__v8di)_mm512_shldi_epi64((A), (B), (I)), \
145 (__v8di)(__m512i)(S)))
146
147#define _mm512_maskz_shldi_epi64(U, A, B, I) \
148 ((__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
149 (__v8di)_mm512_shldi_epi64((A), (B), (I)), \
150 (__v8di)_mm512_setzero_si512()))
151
152#define _mm512_shldi_epi32(A, B, I) \
153 ((__m512i)__builtin_ia32_vpshldd512((__v16si)(__m512i)(A), \
154 (__v16si)(__m512i)(B), (int)(I)))
155
156#define _mm512_mask_shldi_epi32(S, U, A, B, I) \
157 ((__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
158 (__v16si)_mm512_shldi_epi32((A), (B), (I)), \
159 (__v16si)(__m512i)(S)))
160
161#define _mm512_maskz_shldi_epi32(U, A, B, I) \
162 ((__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
163 (__v16si)_mm512_shldi_epi32((A), (B), (I)), \
164 (__v16si)_mm512_setzero_si512()))
165
166#define _mm512_shldi_epi16(A, B, I) \
167 ((__m512i)__builtin_ia32_vpshldw512((__v32hi)(__m512i)(A), \
168 (__v32hi)(__m512i)(B), (int)(I)))
169
170#define _mm512_mask_shldi_epi16(S, U, A, B, I) \
171 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
172 (__v32hi)_mm512_shldi_epi16((A), (B), (I)), \
173 (__v32hi)(__m512i)(S)))
174
175#define _mm512_maskz_shldi_epi16(U, A, B, I) \
176 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
177 (__v32hi)_mm512_shldi_epi16((A), (B), (I)), \
178 (__v32hi)_mm512_setzero_si512()))
179
180#define _mm512_shrdi_epi64(A, B, I) \
181 ((__m512i)__builtin_ia32_vpshrdq512((__v8di)(__m512i)(A), \
182 (__v8di)(__m512i)(B), (int)(I)))
183
184#define _mm512_mask_shrdi_epi64(S, U, A, B, I) \
185 ((__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
186 (__v8di)_mm512_shrdi_epi64((A), (B), (I)), \
187 (__v8di)(__m512i)(S)))
188
189#define _mm512_maskz_shrdi_epi64(U, A, B, I) \
190 ((__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
191 (__v8di)_mm512_shrdi_epi64((A), (B), (I)), \
192 (__v8di)_mm512_setzero_si512()))
193
194#define _mm512_shrdi_epi32(A, B, I) \
195 ((__m512i)__builtin_ia32_vpshrdd512((__v16si)(__m512i)(A), \
196 (__v16si)(__m512i)(B), (int)(I)))
197
198#define _mm512_mask_shrdi_epi32(S, U, A, B, I) \
199 ((__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
200 (__v16si)_mm512_shrdi_epi32((A), (B), (I)), \
201 (__v16si)(__m512i)(S)))
202
203#define _mm512_maskz_shrdi_epi32(U, A, B, I) \
204 ((__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
205 (__v16si)_mm512_shrdi_epi32((A), (B), (I)), \
206 (__v16si)_mm512_setzero_si512()))
207
208#define _mm512_shrdi_epi16(A, B, I) \
209 ((__m512i)__builtin_ia32_vpshrdw512((__v32hi)(__m512i)(A), \
210 (__v32hi)(__m512i)(B), (int)(I)))
211
212#define _mm512_mask_shrdi_epi16(S, U, A, B, I) \
213 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
214 (__v32hi)_mm512_shrdi_epi16((A), (B), (I)), \
215 (__v32hi)(__m512i)(S)))
216
217#define _mm512_maskz_shrdi_epi16(U, A, B, I) \
218 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
219 (__v32hi)_mm512_shrdi_epi16((A), (B), (I)), \
220 (__v32hi)_mm512_setzero_si512()))
221
222static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
223_mm512_shldv_epi64(__m512i __A, __m512i __B, __m512i __C)
224{
225 return (__m512i)__builtin_elementwise_fshl((__v8du)__A, (__v8du)__B,
226 (__v8du)__C);
227}
228
229static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
230_mm512_mask_shldv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C)
231{
232 return (__m512i)__builtin_ia32_selectq_512(__U,
233 (__v8di)_mm512_shldv_epi64(__A, __B, __C),
234 (__v8di)__A);
235}
236
237static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
238_mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
239{
240 return (__m512i)__builtin_ia32_selectq_512(__U,
241 (__v8di)_mm512_shldv_epi64(__A, __B, __C),
242 (__v8di)_mm512_setzero_si512());
243}
244
245static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
246_mm512_shldv_epi32(__m512i __A, __m512i __B, __m512i __C)
247{
248 return (__m512i)__builtin_elementwise_fshl((__v16su)__A, (__v16su)__B,
249 (__v16su)__C);
250}
251
252static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
253_mm512_mask_shldv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C)
254{
255 return (__m512i)__builtin_ia32_selectd_512(__U,
256 (__v16si)_mm512_shldv_epi32(__A, __B, __C),
257 (__v16si)__A);
258}
259
260static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
261_mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
262{
263 return (__m512i)__builtin_ia32_selectd_512(__U,
264 (__v16si)_mm512_shldv_epi32(__A, __B, __C),
265 (__v16si)_mm512_setzero_si512());
266}
267
268static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
269_mm512_shldv_epi16(__m512i __A, __m512i __B, __m512i __C)
270{
271 return (__m512i)__builtin_elementwise_fshl((__v32hu)__A, (__v32hu)__B,
272 (__v32hu)__C);
273}
274
275static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
276_mm512_mask_shldv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C)
277{
278 return (__m512i)__builtin_ia32_selectw_512(__U,
279 (__v32hi)_mm512_shldv_epi16(__A, __B, __C),
280 (__v32hi)__A);
281}
282
283static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
284_mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
285{
286 return (__m512i)__builtin_ia32_selectw_512(__U,
287 (__v32hi)_mm512_shldv_epi16(__A, __B, __C),
288 (__v32hi)_mm512_setzero_si512());
289}
290
291static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
292_mm512_shrdv_epi64(__m512i __A, __m512i __B, __m512i __C)
293{
294 // Ops __A and __B are swapped.
295 return (__m512i)__builtin_elementwise_fshr((__v8du)__B, (__v8du)__A,
296 (__v8du)__C);
297}
298
299static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
300_mm512_mask_shrdv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C)
301{
302 return (__m512i)__builtin_ia32_selectq_512(__U,
303 (__v8di)_mm512_shrdv_epi64(__A, __B, __C),
304 (__v8di)__A);
305}
306
307static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
308_mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
309{
310 return (__m512i)__builtin_ia32_selectq_512(__U,
311 (__v8di)_mm512_shrdv_epi64(__A, __B, __C),
312 (__v8di)_mm512_setzero_si512());
313}
314
315static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
316_mm512_shrdv_epi32(__m512i __A, __m512i __B, __m512i __C)
317{
318 // Ops __A and __B are swapped.
319 return (__m512i)__builtin_elementwise_fshr((__v16su)__B, (__v16su)__A,
320 (__v16su)__C);
321}
322
323static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
324_mm512_mask_shrdv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C)
325{
326 return (__m512i) __builtin_ia32_selectd_512(__U,
327 (__v16si)_mm512_shrdv_epi32(__A, __B, __C),
328 (__v16si)__A);
329}
330
331static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
332_mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
333{
334 return (__m512i) __builtin_ia32_selectd_512(__U,
335 (__v16si)_mm512_shrdv_epi32(__A, __B, __C),
336 (__v16si)_mm512_setzero_si512());
337}
338
339static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
340_mm512_shrdv_epi16(__m512i __A, __m512i __B, __m512i __C)
341{
342 // Ops __A and __B are swapped.
343 return (__m512i)__builtin_elementwise_fshr((__v32hu)__B, (__v32hu)__A,
344 (__v32hu)__C);
345}
346
347static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
348_mm512_mask_shrdv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C)
349{
350 return (__m512i)__builtin_ia32_selectw_512(__U,
351 (__v32hi)_mm512_shrdv_epi16(__A, __B, __C),
352 (__v32hi)__A);
353}
354
355static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
356_mm512_maskz_shrdv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
357{
358 return (__m512i)__builtin_ia32_selectw_512(__U,
359 (__v32hi)_mm512_shrdv_epi16(__A, __B, __C),
360 (__v32hi)_mm512_setzero_si512());
361}
362
363
364#undef __DEFAULT_FN_ATTRS
365#undef __DEFAULT_FN_ATTRS_CONSTEXPR
366
367#endif
368