authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-02-05 10:23:43+01:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-04-04 06:08:09+02:00
logce754724b31e141688e1203ba7bb0a2538f2b544
tree2989b6fbb7ebb1ba5370ed9a7fc1317ff935f525
parent0181cfe8adfab3e7a682aaec10f932cd3279f40a
signaturebadge-check Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

zig cc: Update intrinsic headers to Clang 20.

80 files changed, 17990 insertions(+), 2925 deletions(-)

lib/include/adcintrin.h+5
...@@ -15,7 +15,12 @@...@@ -15,7 +15,12 @@
15#endif15#endif
1616
17/* Define the default attributes for the functions in this file. */17/* Define the default attributes for the functions in this file. */
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__)) constexpr
21#else
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))22#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
23#endif
1924
20/* Use C++ inline semantics in C++, GNU inline for C mode. */25/* Use C++ inline semantics in C++, GNU inline for C mode. */
21#if defined(__cplusplus)26#if defined(__cplusplus)
lib/include/adxintrin.h+5
...@@ -15,8 +15,13 @@...@@ -15,8 +15,13 @@
15#define __ADXINTRIN_H15#define __ADXINTRIN_H
1616
17/* Define the default attributes for the functions in this file. */17/* Define the default attributes for the functions in this file. */
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("adx"))) constexpr
21#else
18#define __DEFAULT_FN_ATTRS \22#define __DEFAULT_FN_ATTRS \
19 __attribute__((__always_inline__, __nodebug__, __target__("adx")))23 __attribute__((__always_inline__, __nodebug__, __target__("adx")))
24#endif
2025
21/* Use C++ inline semantics in C++, GNU inline for C mode. */26/* Use C++ inline semantics in C++, GNU inline for C mode. */
22#if defined(__cplusplus)27#if defined(__cplusplus)
lib/include/altivec.h+8-8
...@@ -2502,37 +2502,37 @@ vec_cmplt(vector unsigned long long __a, vector unsigned long long __b) {...@@ -2502,37 +2502,37 @@ vec_cmplt(vector unsigned long long __a, vector unsigned long long __b) {
25022502
2503static __inline__ vector unsigned char __ATTRS_o_ai2503static __inline__ vector unsigned char __ATTRS_o_ai
2504vec_popcnt(vector signed char __a) {2504vec_popcnt(vector signed char __a) {
2505 return (vector unsigned char)__builtin_altivec_vpopcntb(2505 return (vector unsigned char)__builtin_elementwise_popcount(
2506 (vector unsigned char)__a);2506 (vector unsigned char)__a);
2507}2507}
2508static __inline__ vector unsigned char __ATTRS_o_ai2508static __inline__ vector unsigned char __ATTRS_o_ai
2509vec_popcnt(vector unsigned char __a) {2509vec_popcnt(vector unsigned char __a) {
2510 return __builtin_altivec_vpopcntb(__a);2510 return __builtin_elementwise_popcount(__a);
2511}2511}
2512static __inline__ vector unsigned short __ATTRS_o_ai2512static __inline__ vector unsigned short __ATTRS_o_ai
2513vec_popcnt(vector signed short __a) {2513vec_popcnt(vector signed short __a) {
2514 return (vector unsigned short)__builtin_altivec_vpopcnth(2514 return (vector unsigned short)__builtin_elementwise_popcount(
2515 (vector unsigned short)__a);2515 (vector unsigned short)__a);
2516}2516}
2517static __inline__ vector unsigned short __ATTRS_o_ai2517static __inline__ vector unsigned short __ATTRS_o_ai
2518vec_popcnt(vector unsigned short __a) {2518vec_popcnt(vector unsigned short __a) {
2519 return __builtin_altivec_vpopcnth(__a);2519 return __builtin_elementwise_popcount(__a);
2520}2520}
2521static __inline__ vector unsigned int __ATTRS_o_ai2521static __inline__ vector unsigned int __ATTRS_o_ai
2522vec_popcnt(vector signed int __a) {2522vec_popcnt(vector signed int __a) {
2523 return __builtin_altivec_vpopcntw((vector unsigned int)__a);2523 return __builtin_elementwise_popcount((vector unsigned int)__a);
2524}2524}
2525static __inline__ vector unsigned int __ATTRS_o_ai2525static __inline__ vector unsigned int __ATTRS_o_ai
2526vec_popcnt(vector unsigned int __a) {2526vec_popcnt(vector unsigned int __a) {
2527 return __builtin_altivec_vpopcntw(__a);2527 return __builtin_elementwise_popcount(__a);
2528}2528}
2529static __inline__ vector unsigned long long __ATTRS_o_ai2529static __inline__ vector unsigned long long __ATTRS_o_ai
2530vec_popcnt(vector signed long long __a) {2530vec_popcnt(vector signed long long __a) {
2531 return __builtin_altivec_vpopcntd((vector unsigned long long)__a);2531 return __builtin_elementwise_popcount((vector unsigned long long)__a);
2532}2532}
2533static __inline__ vector unsigned long long __ATTRS_o_ai2533static __inline__ vector unsigned long long __ATTRS_o_ai
2534vec_popcnt(vector unsigned long long __a) {2534vec_popcnt(vector unsigned long long __a) {
2535 return __builtin_altivec_vpopcntd(__a);2535 return __builtin_elementwise_popcount(__a);
2536}2536}
25372537
2538#define vec_vclz vec_cntlz2538#define vec_vclz vec_cntlz
lib/include/amxavx512intrin.h created+382
...@@ -0,0 +1,382 @@
1/*===--------------------- amxavx512intrin.h - AMXAVX512 --------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <amxavx512intrin.h> directly; include <immintrin.h> instead."
11#endif // __IMMINTRIN_H
12
13#ifndef __AMX_AVX512INTRIN_H
14#define __AMX_AVX512INTRIN_H
15#if defined(__x86_64__) && defined(__SSE2__)
16
17#define __DEFAULT_FN_ATTRS_AVX512 \
18 __attribute__((__always_inline__, __nodebug__, \
19 __target__("amx-avx512,avx10.2-512")))
20
21/// Moves a row from a tile register to a zmm destination register, converting
22/// the int32 source elements to fp32. The row of the tile is selected by a
23/// 32b GPR.
24///
25/// \headerfile <x86intrin.h>
26///
27/// \code
28/// __m512i _tile_cvtrowd2ps(__tile tsrc, unsigned int row);
29/// \endcode
30///
31/// \code{.operation}
32/// VL := 512
33/// VL_bytes := VL >> 3
34/// row_index := row & 0xffff
35/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
36/// FOR i := 0 TO (VL_bytes / 4) - 1
37/// IF i + row_chunk / 4 >= tsrc.colsb / 4
38/// dst.dword[i] := 0
39/// ELSE
40/// dst.f32[i] := CONVERT_INT32_TO_FP32(tsrc.row[row_index].dword[row_chunk/4+i], RNE)
41/// FI
42/// ENDFOR
43/// dst[MAX_VL-1:VL] := 0
44/// zero_tileconfig_start()
45/// \endcode
46///
47/// This intrinsic corresponds to the \c TCVTROWD2PS instruction.
48///
49/// \param tsrc
50/// The source tile. Max size is 1024 Bytes.
51/// \param row
52/// The row of the source tile
53#define _tile_cvtrowd2ps(tsrc, row) __builtin_ia32_tcvtrowd2ps(tsrc, row)
54
55/// Moves a row from a tile register to a zmm destination register, converting
56/// the fp32 source elements to bf16. It places the resulting bf16 elements
57/// in the high 16 bits within each dword. The row of the tile is selected
58/// by a 32b GPR.
59///
60/// \headerfile <x86intrin.h>
61///
62/// \code
63/// __m512i _tile_cvtrowps2bf16h(__tile tsrc, unsigned int row);
64/// \endcode
65///
66/// \code{.operation}
67/// VL := 512
68/// VL_bytes := VL >> 3
69/// row_index := row & 0xffff
70/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
71/// FOR i := 0 TO (VL_bytes / 4) - 1
72/// IF i + row_chunk / 4 >= tsrc.colsb / 4
73/// dst.dword[i] := 0
74/// ELSE
75/// dst.word[2*i+0] := 0
76/// dst.bf16[2*i+1] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
77/// FI
78/// ENDFOR
79/// dst[MAX_VL-1:VL] := 0
80/// zero_tileconfig_start()
81/// \endcode
82///
83/// This intrinsic corresponds to the \c TCVTROWPS2BF16H instruction.
84///
85/// \param tsrc
86/// The source tile. Max size is 1024 Bytes.
87/// \param row
88/// The the row of the source tile.
89#define _tile_cvtrowps2bf16h(tsrc, row) \
90 __builtin_ia32_tcvtrowps2bf16h(tsrc, row)
91
92/// Moves a row from a tile register to a zmm destination register, converting
93/// the fp32 source elements to bf16. It places the resulting bf16 elements
94/// in the low 16 bits within each dword. The row of the tile is selected
95/// by a 32b GPR.
96///
97/// \headerfile <x86intrin.h>
98///
99/// \code
100/// __m512i _tile_cvtrowps2bf16l(__tile tsrc, unsigned int row);
101/// \endcode
102///
103/// \code{.operation}
104/// VL := 512
105/// VL_bytes := VL >> 3
106/// row_index := row & 0xffff
107/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
108/// FOR i := 0 TO (VL_bytes / 4) - 1
109/// IF i + row_chunk / 4 >= tsrc.colsb / 4
110/// dst.dword[i] := 0
111/// ELSE
112/// dst.word[2*i+1] := 0
113/// dst.bf16[2*i+0] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
114/// FI
115/// ENDFOR
116/// dst[MAX_VL-1:VL] := 0
117/// zero_tileconfig_start()
118/// \endcode
119///
120/// This intrinsic corresponds to the \c TCVTROWPS2BF16L instruction.
121///
122/// \param tsrc
123/// The source tile. Max size is 1024 Bytes.
124/// \param row
125/// The the row of the source tile.
126#define _tile_cvtrowps2bf16l(tsrc, row) \
127 __builtin_ia32_tcvtrowps2bf16l(tsrc, row)
128
129/// Moves a row from a tile register to a zmm destination register, converting
130/// the fp32 source elements to fp16. It places the resulting fp16 elements
131/// in the high 16 bits within each dword. The row of the tile is selected
132/// by a 32b GPR.
133///
134/// \headerfile <x86intrin.h>
135///
136/// \code
137/// __m512i _tile_cvtrowps2phh(__tile tsrc, unsigned int row);
138/// \endcode
139///
140/// \code{.operation}
141/// VL := 512
142/// VL_bytes := VL >> 3
143/// row_index := row & 0xffff
144/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
145/// FOR i := 0 TO (VL_bytes / 4) - 1
146/// IF i + row_chunk / 4 >= tsrc.colsb / 4
147/// dst.dword[i] := 0
148/// ELSE
149/// dst.word[2*i+0] := 0
150/// dst.fp16[2*i+1] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
151/// FI
152/// ENDFOR
153/// dst[MAX_VL-1:VL] := 0
154/// zero_tileconfig_start()
155/// \endcode
156///
157/// This intrinsic corresponds to the \c TCVTROWPS2PHH instruction.
158///
159/// \param tsrc
160/// The source tile. Max size is 1024 Bytes.
161/// \param row
162/// The the row of the source tile.
163#define _tile_cvtrowps2phh(tsrc, row) __builtin_ia32_tcvtrowps2phh(tsrc, row)
164
165/// Moves a row from a tile register to a zmm destination register, converting
166/// the fp32 source elements to fp16. It places the resulting fp16 elements
167/// in the low 16 bits within each dword. The row of the tile is selected
168/// by a 32b GPR.
169///
170/// \headerfile <x86intrin.h>
171///
172/// \code
173/// __m512i _tile_cvtrowps2phl(__tile tsrc, unsigned int row);
174/// \endcode
175///
176/// \code{.operation}
177/// VL := 512
178/// VL_bytes := VL >> 3
179/// row_index := row & 0xffff
180/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
181/// FOR i := 0 TO (VL_bytes / 4) - 1
182/// IF i + row_chunk / 4 >= tsrc.colsb / 4
183/// dst.dword[i] := 0
184/// ELSE
185/// dst.word[2*i+1] := 0
186/// dst.fp16[2*i+0] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
187/// FI
188/// ENDFOR
189/// dst[MAX_VL-1:VL] := 0
190/// zero_tileconfig_start()
191/// \endcode
192///
193/// This intrinsic corresponds to the \c TCVTROWPS2PHL instruction.
194///
195/// \param tsrc
196/// The source tile. Max size is 1024 Bytes.
197/// \param row
198/// The the row of the source tile.
199#define _tile_cvtrowps2phl(tsrc, row) __builtin_ia32_tcvtrowps2phl(tsrc, row)
200
201/// Move one row of a tile data to a v16f32 data.
202/// The row of the tile is selected by a 32b GPR.
203///
204/// \headerfile <immintrin.h>
205///
206/// \code
207/// __m512 _tile_movrow(__tile a, unsigned b);
208/// \endcode
209///
210/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
211///
212/// \param a
213/// The 1st source tile. Max size is 1024 Bytes.
214/// \param b
215/// The 2nd source r32. Size is 4 Bytes.
216/// \returns
217/// The destination v16f32 data. Size is 64 Bytes.
218///
219/// \code{.operation}
220/// VL := 512
221/// VL_bytes := VL>>3
222/// row_index := b&0xffff
223/// row_chunk := ((b>>16)&0xffff) * VL_bytes
224/// FOR i := 0 TO (VL_bytes-1)
225/// IF (row_chunk + i >= a.colsb)
226/// dst.byte[i] := 0
227/// ELSE
228/// dst.byte[i] := a.row[row_index].byte[row_chunk+i]
229/// ENDFOR
230/// \endcode
231#define _tile_movrow(a, b) __builtin_ia32_tilemovrow(a, b)
232
233/// This is internal intrinsic. C/C++ user should avoid calling it directly.
234
235static __inline__ __m512 __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowd2ps_internal(
236 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
237 return __builtin_ia32_tcvtrowd2ps_internal(m, n, src, u);
238}
239
240static __inline__ __m512bh __DEFAULT_FN_ATTRS_AVX512
241_tile_cvtrowps2bf16h_internal(unsigned short m, unsigned short n,
242 _tile1024i src, unsigned u) {
243 return __builtin_ia32_tcvtrowps2bf16h_internal(m, n, src, u);
244}
245
246static __inline__ __m512bh __DEFAULT_FN_ATTRS_AVX512
247_tile_cvtrowps2bf16l_internal(unsigned short m, unsigned short n,
248 _tile1024i src, unsigned u) {
249 return __builtin_ia32_tcvtrowps2bf16l_internal(m, n, src, u);
250}
251
252static __inline__ __m512h __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowps2phh_internal(
253 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
254 return __builtin_ia32_tcvtrowps2phh_internal(m, n, src, u);
255}
256
257static __inline__ __m512h __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowps2phl_internal(
258 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
259 return __builtin_ia32_tcvtrowps2phl_internal(m, n, src, u);
260}
261
262static __inline__ __m512i __DEFAULT_FN_ATTRS_AVX512 _tile_movrow_internal(
263 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
264 return (__m512i)__builtin_ia32_tilemovrow_internal(m, n, src, u);
265}
266
267/// Move a row from a tile (src0) to a v16f32 dst, converting the int32 source
268/// elements to fp32. No SIMD exceptions are generated. Rounding is done as if
269/// MXCSR.RC=RNE. Embedded rounding is not supported.
270/// The row and chunk elements of tile is fetched from 32bit src1.
271///
272/// \headerfile <immintrin.h>
273///
274/// This intrinsic corresponds to the <c> TCVTROWD2PS </c> instruction.
275///
276/// \param src0
277/// The 1st source tile. Max size is 1024 Bytes.
278/// \param src1
279/// The 2nd source r32. Size is 4 Bytes.
280/// \returns
281/// The destination v16f32 data. Size is 64 Bytes.
282__DEFAULT_FN_ATTRS_AVX512
283static __m512 __tile_cvtrowd2ps(__tile1024i src0, unsigned src1) {
284 return _tile_cvtrowd2ps_internal(src0.row, src0.col, src0.tile, src1);
285}
286
287/// Move a row from a tile (src0) to a v32bf16 dst, converting the fp32 source
288/// elements to bf16 at high 16-bits of each dword.
289/// The row and chunk elements of tile is fetched from 32bit src1.
290///
291/// \headerfile <immintrin.h>
292///
293/// This intrinsic corresponds to the <c> TCVTROWPS2BF16H </c> instruction.
294///
295/// \param src0
296/// The 1st source tile. Max size is 1024 Bytes.
297/// \param src1
298/// The 2nd source r32. Size is 4 Bytes.
299/// \returns
300/// The destination v32bf16 data. Size is 64 Bytes.
301__DEFAULT_FN_ATTRS_AVX512
302static __m512bh __tile_cvtrowps2bf16h(__tile1024i src0, unsigned src1) {
303 return _tile_cvtrowps2bf16h_internal(src0.row, src0.col, src0.tile, src1);
304}
305
306/// Move a row from a tile (src0) to a v32bf16 dst, converting the fp32 source
307/// elements to bf16 at low 16-bits of each dword.
308/// The row and chunk elements of tile is fetched from 32bit src1.
309///
310/// \headerfile <immintrin.h>
311///
312/// This intrinsic corresponds to the <c> TCVTROWPS2BF16L </c> instruction.
313///
314/// \param src0
315/// The 1st source tile. Max size is 1024 Bytes.
316/// \param src1
317/// The 2nd source r32. Size is 4 Bytes.
318/// \returns
319/// The destination v32bf16 data. Size is 64 Bytes.
320__DEFAULT_FN_ATTRS_AVX512
321static __m512bh __tile_cvtrowps2bf16l(__tile1024i src0, unsigned src1) {
322 return _tile_cvtrowps2bf16l_internal(src0.row, src0.col, src0.tile, src1);
323}
324
325/// Move a row from a tile (src0) to a v32fp16 dst, converting the fp32 source
326/// elements to fp16 at high 16-bits of each dword.
327/// The row and chunk elements of tile is fetched from 32bit src1.
328///
329/// \headerfile <immintrin.h>
330///
331/// This intrinsic corresponds to the <c> TCVTROWPS2PHH </c> instruction.
332///
333/// \param src0
334/// The 1st source tile. Max size is 1024 Bytes.
335/// \param src1
336/// The 2nd source r32. Size is 4 Bytes.
337/// \returns
338/// The destination v32fp16 data. Size is 64 Bytes.
339__DEFAULT_FN_ATTRS_AVX512
340static __m512h __tile_cvtrowps2phh(__tile1024i src0, unsigned src1) {
341 return _tile_cvtrowps2phh_internal(src0.row, src0.col, src0.tile, src1);
342}
343
344/// Move a row from a tile (src0) to a v32fp16 dst, converting the fp32 source
345/// elements to fp16 at low 16-bits of each dword.
346/// The row and chunk elements of tile is fetched from 32bit src1.
347///
348/// \headerfile <immintrin.h>
349///
350/// This intrinsic corresponds to the <c> TCVTROWPS2PHL </c> instruction.
351///
352/// \param src0
353/// The 1st source tile. Max size is 1024 Bytes.
354/// \param src1
355/// The 2nd source r32. Size is 4 Bytes.
356/// \returns
357/// The destination v32fp16 data. Size is 64 Bytes.
358__DEFAULT_FN_ATTRS_AVX512
359static __m512h __tile_cvtrowps2phl(__tile1024i src0, unsigned src1) {
360 return _tile_cvtrowps2phl_internal(src0.row, src0.col, src0.tile, src1);
361}
362
363/// Move one row of a tile data to a v16f32 data.
364/// The row of the tile is selected by a 32b GPR.
365///
366/// \headerfile <immintrin.h>
367///
368/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
369///
370/// \param src0
371/// The 1st source tile. Max size is 1024 Bytes.
372/// \param src1
373/// The 2nd source r32. Size is 4 Bytes.
374/// \returns
375/// The destination v16i32 data. Size is 64 Bytes.
376__DEFAULT_FN_ATTRS_AVX512
377static __m512i __tile_movrow(__tile1024i src0, unsigned src1) {
378 return (__m512i)_tile_movrow_internal(src0.row, src0.col, src0.tile, src1);
379}
380
381#endif // __x86_64__ && __SSE2__
382#endif // __AMX_AVX512INTRIN_H
lib/include/amxbf16transposeintrin.h created+94
...@@ -0,0 +1,94 @@
1/*===----- amxbf16transposeintrin.h - AMX-BF16 and AMX-TRANSPOSE ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxbf16transposeintrin.h> directly; use <immintrin.h> instead."
13#endif /* __IMMINTRIN_H */
14
15#ifndef __AMX_BF16TRANSPOSEINTRIN_H
16#define __AMX_BF16TRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \
22 __target__("amx-bf16,amx-transpose")))
23
24/// Compute transpose and dot-product of BF16 (16-bit) floating-point pairs in
25/// tiles \a a and \a b, accumulating the intermediate single-precision
26/// (32-bit) floating-point elements with elements in \a dst, and store the
27/// 32-bit result back to tile \a dst.
28///
29/// \headerfile <immintrin.h>
30///
31/// \code
32/// void _tile_tdpbf16ps (__tile dst, __tile a, __tile b)
33/// \endcode
34///
35/// \code{.operation}
36/// FOR m := 0 TO dst.rows - 1
37/// tmp := dst.row[m]
38/// FOR k := 0 TO (a.colsb / 4) - 1
39/// FOR n := 0 TO (dst.colsb / 4) - 1
40/// tmp.bf32[n] += FP32(a.row[m].bf16[2*k+0]) *
41/// FP32(b.row[k].bf16[2*n+0])
42/// tmp.bf32[n] += FP32(a.row[m].bf16[2*k+1]) *
43/// FP32(b.row[k].bf16[2*n+1])
44/// ENDFOR
45/// ENDFOR
46/// write_row_and_zero(dst, m, tmp, dst.colsb)
47/// ENDFOR
48/// zero_upper_rows(dst, dst.rows)
49/// zero_tileconfig_start()
50/// \endcode
51///
52/// This intrinsic corresponds to the \c TTDPBF16PS instruction.
53///
54/// \param dst
55/// The destination tile. Max size is 1024 Bytes.
56/// \param a
57/// The 1st source tile. Max size is 1024 Bytes.
58/// \param b
59/// The 2nd source tile. Max size is 1024 Bytes.
60#define _tile_tdpbf16ps(dst, a, b) __builtin_ia32_ttdpbf16ps((dst), (a), (b))
61
62/// This is internal intrinsic. C/C++ user should avoid calling it directly.
63static __inline__ _tile1024i __DEFAULT_FN_ATTRS
64_tile_tdpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
65 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
66 return __builtin_ia32_ttdpbf16ps_internal(m, n, k, dst, src1, src2);
67}
68
69/// Compute transpose and dot-product of BF16 (16-bit) floating-point pairs in
70/// tiles src0 and src1, accumulating the intermediate single-precision
71/// (32-bit) floating-point elements with elements in "dst", and store the
72/// 32-bit result back to tile "dst".
73///
74/// \headerfile <immintrin.h>
75///
76/// This intrinsic corresponds to the <c> TTDPBF16PS </c> instruction.
77///
78/// \param dst
79/// The destination tile. Max size is 1024 Bytes.
80/// \param src0
81/// The 1st source tile. Max size is 1024 Bytes.
82/// \param src1
83/// The 2nd source tile. Max size is 1024 Bytes.
84__DEFAULT_FN_ATTRS
85static __inline__ void __tile_tdpbf16ps(__tile1024i *dst, __tile1024i src0,
86 __tile1024i src1) {
87 dst->tile = _tile_tdpbf16ps_internal(src0.row, src1.col, src0.col, dst->tile,
88 src0.tile, src1.tile);
89}
90
91#undef __DEFAULT_FN_ATTRS
92
93#endif /* __x86_64__ */
94#endif /* __AMX_BF16TRANSPOSEINTRIN_H */
lib/include/amxcomplextransposeintrin.h created+303
...@@ -0,0 +1,303 @@
1/*===----- amxcomplextransposeintrin.h - AMX-COMPLEX and AMX-TRANSPOSE ------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxcomplextransposeintrin.h> directly; include <immintrin.h> instead."
13#endif // __IMMINTRIN_H
14
15#ifndef __AMX_COMPLEXTRANSPOSEINTRIN_H
16#define __AMX_COMPLEXTRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("amx-complex,amx-transpose")))
22
23/// Perform matrix multiplication of two tiles containing complex elements and
24/// accumulate the results into a packed single precision tile. Each dword
25/// element in input tiles \a a and \a b is interpreted as a complex number
26/// with FP16 real part and FP16 imaginary part.
27/// Calculates the imaginary part of the result. For each possible combination
28/// of (transposed column of \a a, column of \a b), it performs a set of
29/// multiplication and accumulations on all corresponding complex numbers
30/// (one from \a a and one from \a b). The imaginary part of the \a a element
31/// is multiplied with the real part of the corresponding \a b element, and
32/// the real part of the \a a element is multiplied with the imaginary part
33/// of the corresponding \a b elements. The two accumulated results are
34/// added, and then accumulated into the corresponding row and column of
35/// \a dst.
36///
37/// \headerfile <x86intrin.h>
38///
39/// \code
40/// void _tile_tcmmimfp16ps(__tile dst, __tile a, __tile b);
41/// \endcode
42///
43/// \code{.operation}
44/// FOR m := 0 TO dst.rows - 1
45/// tmp := dst.row[m]
46/// FOR k := 0 TO a.rows - 1
47/// FOR n := 0 TO (dst.colsb / 4) - 1
48/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+1])
49/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+0])
50/// ENDFOR
51/// ENDFOR
52/// write_row_and_zero(dst, m, tmp, dst.colsb)
53/// ENDFOR
54/// zero_upper_rows(dst, dst.rows)
55/// zero_tileconfig_start()
56/// \endcode
57///
58/// This intrinsic corresponds to the \c TTCMMIMFP16PS instruction.
59///
60/// \param dst
61/// The destination tile. Max size is 1024 Bytes.
62/// \param a
63/// The 1st source tile. Max size is 1024 Bytes.
64/// \param b
65/// The 2nd source tile. Max size is 1024 Bytes.
66#define _tile_tcmmimfp16ps(dst, a, b) \
67 __builtin_ia32_ttcmmimfp16ps((dst), (a), (b))
68
69/// Perform matrix multiplication of two tiles containing complex elements and
70/// accumulate the results into a packed single precision tile. Each dword
71/// element in input tiles \a a and \a b is interpreted as a complex number
72/// with FP16 real part and FP16 imaginary part.
73/// Calculates the real part of the result. For each possible combination
74/// of (rtransposed colum of \a a, column of \a b), it performs a set of
75/// multiplication and accumulations on all corresponding complex numbers
76/// (one from \a a and one from \a b). The real part of the \a a element is
77/// multiplied with the real part of the corresponding \a b element, and the
78/// negated imaginary part of the \a a element is multiplied with the
79/// imaginary part of the corresponding \a b elements. The two accumulated
80/// results are added, and then accumulated into the corresponding row and
81/// column of \a dst.
82///
83/// \headerfile <x86intrin.h>
84///
85/// \code
86/// void _tile_tcmmrlfp16ps(__tile dst, __tile a, __tile b);
87/// \endcode
88///
89/// \code{.operation}
90/// FOR m := 0 TO dst.rows - 1
91/// tmp := dst.row[m]
92/// FOR k := 0 TO a.rows - 1
93/// FOR n := 0 TO (dst.colsb / 4) - 1
94/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+0])
95/// tmp.fp32[n] += FP32(-a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+1])
96/// ENDFOR
97/// ENDFOR
98/// write_row_and_zero(dst, m, tmp, dst.colsb)
99/// ENDFOR
100/// zero_upper_rows(dst, dst.rows)
101/// zero_tileconfig_start()
102/// \endcode
103///
104/// This intrinsic corresponds to the \c TTCMMIMFP16PS instruction.
105///
106/// \param dst
107/// The destination tile. Max size is 1024 Bytes.
108/// \param a
109/// The 1st source tile. Max size is 1024 Bytes.
110/// \param b
111/// The 2nd source tile. Max size is 1024 Bytes.
112#define _tile_tcmmrlfp16ps(dst, a, b) \
113 __builtin_ia32_ttcmmrlfp16ps((dst), (a), (b))
114
115/// Perform matrix conjugate transpose and multiplication of two tiles
116/// containing complex elements and accumulate the results into a packed
117/// single precision tile. Each dword element in input tiles \a a and \a b
118/// is interpreted as a complex number with FP16 real part and FP16 imaginary
119/// part.
120/// Calculates the imaginary part of the result. For each possible combination
121/// of (transposed column of \a a, column of \a b), it performs a set of
122/// multiplication and accumulations on all corresponding complex numbers
123/// (one from \a a and one from \a b). The negated imaginary part of the \a a
124/// element is multiplied with the real part of the corresponding \a b
125/// element, and the real part of the \a a element is multiplied with the
126/// imaginary part of the corresponding \a b elements. The two accumulated
127/// results are added, and then accumulated into the corresponding row and
128/// column of \a dst.
129///
130/// \headerfile <x86intrin.h>
131///
132/// \code
133/// void _tile_conjtcmmimfp16ps(__tile dst, __tile a, __tile b);
134/// \endcode
135///
136/// \code{.operation}
137/// FOR m := 0 TO dst.rows - 1
138/// tmp := dst.row[m]
139/// FOR k := 0 TO a.rows - 1
140/// FOR n := 0 TO (dst.colsb / 4) - 1
141/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+1])
142/// tmp.fp32[n] += FP32(-a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+0])
143/// ENDFOR
144/// ENDFOR
145/// write_row_and_zero(dst, m, tmp, dst.colsb)
146/// ENDFOR
147/// zero_upper_rows(dst, dst.rows)
148/// zero_tileconfig_start()
149/// \endcode
150///
151/// This intrinsic corresponds to the \c TCONJTCMMIMFP16PS instruction.
152///
153/// \param dst
154/// The destination tile. Max size is 1024 Bytes.
155/// \param a
156/// The 1st source tile. Max size is 1024 Bytes.
157/// \param b
158/// The 2nd source tile. Max size is 1024 Bytes.
159#define _tile_conjtcmmimfp16ps(dst, a, b) \
160 __builtin_ia32_tconjtcmmimfp16ps((dst), (a), (b))
161
162/// Perform conjugate transpose of an FP16-pair of complex elements from \a a
163/// and writes the result to \a dst.
164///
165/// \headerfile <x86intrin.h>
166///
167/// \code
168/// void _tile_conjtfp16(__tile dst, __tile a);
169/// \endcode
170///
171/// \code{.operation}
172/// FOR i := 0 TO dst.rows - 1
173/// FOR j := 0 TO (dst.colsb / 4) - 1
174/// tmp.fp16[2*j+0] := a.row[j].fp16[2*i+0]
175/// tmp.fp16[2*j+1] := -a.row[j].fp16[2*i+1]
176/// ENDFOR
177/// write_row_and_zero(dst, i, tmp, dst.colsb)
178/// ENDFOR
179/// zero_upper_rows(dst, dst.rows)
180/// zero_tileconfig_start()
181/// \endcode
182///
183/// This intrinsic corresponds to the \c TCONJTFP16 instruction.
184///
185/// \param dst
186/// The destination tile. Max size is 1024 Bytes.
187/// \param a
188/// The source tile. Max size is 1024 Bytes.
189#define _tile_conjtfp16(dst, a) __builtin_ia32_tconjtfp16((dst), (a))
190
191static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_tcmmimfp16ps_internal(
192 unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
193 _tile1024i src1, _tile1024i src2) {
194 return __builtin_ia32_ttcmmimfp16ps_internal(m, n, k, dst, src1, src2);
195}
196
197static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_tcmmrlfp16ps_internal(
198 unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
199 _tile1024i src1, _tile1024i src2) {
200 return __builtin_ia32_ttcmmrlfp16ps_internal(m, n, k, dst, src1, src2);
201}
202
203static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_conjtcmmimfp16ps_internal(
204 unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
205 _tile1024i src1, _tile1024i src2) {
206 return __builtin_ia32_tconjtcmmimfp16ps_internal(m, n, k, dst, src1, src2);
207}
208
209static __inline__ _tile1024i __DEFAULT_FN_ATTRS
210_tile_conjtfp16_internal(unsigned short m, unsigned short n, _tile1024i src) {
211 return __builtin_ia32_tconjtfp16_internal(m, n, src);
212}
213
214/// Perform matrix multiplication of two tiles containing complex elements and
215/// accumulate the results into a packed single precision tile. Each dword
216/// element in input tiles src0 and src1 is interpreted as a complex number
217/// with FP16 real part and FP16 imaginary part.
218/// This function calculates the imaginary part of the result.
219///
220/// \headerfile <immintrin.h>
221///
222/// This intrinsic corresponds to the <c> TTCMMIMFP16PS </c> instruction.
223///
224/// \param dst
225/// The destination tile. Max size is 1024 Bytes.
226/// \param src0
227/// The 1st source tile. Max size is 1024 Bytes.
228/// \param src1
229/// The 2nd source tile. Max size is 1024 Bytes.
230__DEFAULT_FN_ATTRS
231static void __tile_tcmmimfp16ps(__tile1024i *dst, __tile1024i src0,
232 __tile1024i src1) {
233 dst->tile = _tile_tcmmimfp16ps_internal(src0.row, src1.col, src0.col,
234 dst->tile, src0.tile, src1.tile);
235}
236
237/// Perform matrix multiplication of two tiles containing complex elements and
238/// accumulate the results into a packed single precision tile. Each dword
239/// element in input tiles src0 and src1 is interpreted as a complex number
240/// with FP16 real part and FP16 imaginary part.
241/// This function calculates the real part of the result.
242///
243/// \headerfile <immintrin.h>
244///
245/// This intrinsic corresponds to the <c> TTCMMRLFP16PS </c> instruction.
246///
247/// \param dst
248/// The destination tile. Max size is 1024 Bytes.
249/// \param src0
250/// The 1st source tile. Max size is 1024 Bytes.
251/// \param src1
252/// The 2nd source tile. Max size is 1024 Bytes.
253__DEFAULT_FN_ATTRS
254static void __tile_tcmmrlfp16ps(__tile1024i *dst, __tile1024i src0,
255 __tile1024i src1) {
256 dst->tile = _tile_tcmmrlfp16ps_internal(src0.row, src1.col, src0.col,
257 dst->tile, src0.tile, src1.tile);
258}
259
260/// Perform matrix conjugate transpose and multiplication of two tiles
261/// containing complex elements and accumulate the results into a packed
262/// single precision tile. Each dword element in input tiles src0 and src1
263/// is interpreted as a complex number with FP16 real part and FP16 imaginary
264/// part.
265/// This function calculates the imaginary part of the result.
266///
267/// \headerfile <immintrin.h>
268///
269/// This intrinsic corresponds to the <c> TCONJTCMMIMFP16PS </c> instruction.
270///
271/// \param dst
272/// The destination tile. Max size is 1024 Bytes.
273/// \param src0
274/// The 1st source tile. Max size is 1024 Bytes.
275/// \param src1
276/// The 2nd source tile. Max size is 1024 Bytes.
277__DEFAULT_FN_ATTRS
278static void __tile_conjtcmmimfp16ps(__tile1024i *dst, __tile1024i src0,
279 __tile1024i src1) {
280 dst->tile = _tile_conjtcmmimfp16ps_internal(src0.row, src1.col, src0.col,
281 dst->tile, src0.tile, src1.tile);
282}
283
284/// Perform conjugate transpose of an FP16-pair of complex elements from src and
285/// writes the result to dst.
286///
287/// \headerfile <immintrin.h>
288///
289/// This intrinsic corresponds to the <c> TCONJTFP16 </c> instruction.
290///
291/// \param dst
292/// The destination tile. Max size is 1024 Bytes.
293/// \param src
294/// The source tile. Max size is 1024 Bytes.
295__DEFAULT_FN_ATTRS
296static void __tile_conjtfp16(__tile1024i *dst, __tile1024i src) {
297 dst->tile = _tile_conjtfp16_internal(src.row, src.col, src.tile);
298}
299
300#undef __DEFAULT_FN_ATTRS
301
302#endif // __x86_64__
303#endif // __AMX_COMPLEXTRANSPOSEINTRIN_H
lib/include/amxfp16intrin.h+35
...@@ -15,6 +15,10 @@...@@ -15,6 +15,10 @@
15#define __AMX_FP16INTRIN_H15#define __AMX_FP16INTRIN_H
16#ifdef __x86_64__16#ifdef __x86_64__
1717
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp16")))
21
18/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles \a a22/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles \a a
19/// and \a b, accumulating the intermediate single-precision (32-bit)23/// and \a b, accumulating the intermediate single-precision (32-bit)
20/// floating-point elements with elements in \a dst, and store the 32-bit24/// floating-point elements with elements in \a dst, and store the 32-bit
...@@ -54,5 +58,36 @@...@@ -54,5 +58,36 @@
54#define _tile_dpfp16ps(dst, a, b) \58#define _tile_dpfp16ps(dst, a, b) \
55 __builtin_ia32_tdpfp16ps(dst, a, b)59 __builtin_ia32_tdpfp16ps(dst, a, b)
5660
61/// This is internal intrinsic. C/C++ user should avoid calling it directly.
62static __inline__ _tile1024i __DEFAULT_FN_ATTRS
63_tile_dpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
64 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
65 return __builtin_ia32_tdpfp16ps_internal(m, n, k, dst, src1, src2);
66}
67
68/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles src0 and
69/// src1, accumulating the intermediate single-precision (32-bit) floating-point
70/// elements with elements in "dst", and store the 32-bit result back to tile
71/// "dst".
72///
73/// \headerfile <immintrin.h>
74///
75/// This intrinsic corresponds to the <c> TDPFP16PS </c> instruction.
76///
77/// \param dst
78/// The destination tile. Max size is 1024 Bytes.
79/// \param src0
80/// The 1st source tile. Max size is 1024 Bytes.
81/// \param src1
82/// The 2nd source tile. Max size is 1024 Bytes.
83__DEFAULT_FN_ATTRS
84static __inline__ void __tile_dpfp16ps(__tile1024i *dst, __tile1024i src0,
85 __tile1024i src1) {
86 dst->tile = _tile_dpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
87 src0.tile, src1.tile);
88}
89
90#undef __DEFAULT_FN_ATTRS
91
57#endif /* __x86_64__ */92#endif /* __x86_64__ */
58#endif /* __AMX_FP16INTRIN_H */93#endif /* __AMX_FP16INTRIN_H */
lib/include/amxfp16transposeintrin.h created+94
...@@ -0,0 +1,94 @@
1/*===----- amxfp16transposeintrin.h - AMX-FP16 and AMX-TRANSPOSE ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxfp16transposeintrin.h> directly; use <immintrin.h> instead."
13#endif /* __IMMINTRIN_H */
14
15#ifndef __AMX_FP16TRANSPOSEINTRIN_H
16#define __AMX_FP16TRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \
22 __target__("amx-fp16,amx-transpose")))
23
24/// Compute transpose and dot-product of FP16 (16-bit) floating-point pairs in
25/// tiles \a a and \a b, accumulating the intermediate single-precision
26/// (32-bit) floating-point elements with elements in \a dst, and store the
27/// 32-bit result back to tile \a dst.
28///
29/// \headerfile <immintrin.h>
30///
31/// \code
32/// void _tile_tdpfp16ps (__tile dst, __tile a, __tile b)
33/// \endcode
34///
35/// \code{.operation}
36/// FOR m := 0 TO dst.rows - 1
37/// tmp := dst.row[m]
38/// FOR k := 0 TO (a.colsb / 4) - 1
39/// FOR n := 0 TO (dst.colsb / 4) - 1
40/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) *
41/// FP32(b.row[k].fp16[2*n+0])
42/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+1]) *
43/// FP32(b.row[k].fp16[2*n+1])
44/// ENDFOR
45/// ENDFOR
46/// write_row_and_zero(dst, m, tmp, dst.colsb)
47/// ENDFOR
48/// zero_upper_rows(dst, dst.rows)
49/// zero_tileconfig_start()
50/// \endcode
51///
52/// This intrinsic corresponds to the \c TTDPFP16PS instruction.
53///
54/// \param dst
55/// The destination tile. Max size is 1024 Bytes.
56/// \param a
57/// The 1st source tile. Max size is 1024 Bytes.
58/// \param b
59/// The 2nd source tile. Max size is 1024 Bytes.
60#define _tile_tdpfp16ps(dst, a, b) __builtin_ia32_ttdpfp16ps((dst), (a), (b))
61
62/// This is internal intrinsic. C/C++ user should avoid calling it directly.
63static __inline__ _tile1024i __DEFAULT_FN_ATTRS
64_tile_tdpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
65 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
66 return __builtin_ia32_ttdpfp16ps_internal(m, n, k, dst, src1, src2);
67}
68
69/// Compute transpose and dot-product of FP16 (16-bit) floating-point pairs in
70/// tiles src0 and src1, accumulating the intermediate single-precision
71/// (32-bit) floating-point elements with elements in "dst", and store the
72/// 32-bit result back to tile "dst".
73///
74/// \headerfile <immintrin.h>
75///
76/// This intrinsic corresponds to the <c> TTDPFP16PS </c> instruction.
77///
78/// \param dst
79/// The destination tile. Max size is 1024 Bytes.
80/// \param src0
81/// The 1st source tile. Max size is 1024 Bytes.
82/// \param src1
83/// The 2nd source tile. Max size is 1024 Bytes.
84__DEFAULT_FN_ATTRS
85static __inline__ void __tile_tdpfp16ps(__tile1024i *dst, __tile1024i src0,
86 __tile1024i src1) {
87 dst->tile = _tile_tdpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
88 src0.tile, src1.tile);
89}
90
91#undef __DEFAULT_FN_ATTRS
92
93#endif /* __x86_64__ */
94#endif /* __AMX_FP16TRANSPOSEINTRIN_H */
lib/include/amxfp8intrin.h created+230
...@@ -0,0 +1,230 @@
1/*===------------- amxfp8intrin.h - AMX intrinsics -*- C++ -*----------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <amxfp8intrin.h> directly; include <immintrin.h> instead."
12#endif /* __IMMINTRIN_H */
13
14#ifndef __AMXFP8INTRIN_H
15#define __AMXFP8INTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_FP8 \
19 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp8")))
20
21static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
22_tile_dpbf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
23 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
24 return __builtin_ia32_tdpbf8ps_internal(m, n, k, dst, src1, src2);
25}
26
27/// Perform the dot product of a BF8 value \a src1 by a BF8 value \a src2
28/// accumulating into a Single Precision (FP32) source/dest \a dst.
29///
30/// \headerfile <immintrin.h>
31///
32/// \code
33/// void __tile_dpbf8ps (__tile1024i *dst, __tile1024i src1, __tile1024i src2)
34/// \endcode
35///
36/// \code{.operation}
37/// FOR m := 0 TO dst.rows - 1
38/// temp1[(dst.colsb / 4 - 1) : 0] = 0
39/// FOR k := 0 TO src1.colsb / 4 - 1
40/// FOR n := 0 TO dst.colsb / 4 - 1
41/// temp1[n] +=
42/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
43/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
44/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
45/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
46/// ENDFOR
47/// ENDFOR
48/// FOR n := 0 TO dst.colsb / 4 - 1
49/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
50/// ENDFOR
51/// write_row_and_zero(dst, m, tmp, dst.colsb)
52/// zero_upper_rows(dst, dst.rows)
53/// zero_tileconfig_start()
54/// \endcode
55///
56/// This intrinsic corresponds to the \c TDPBF8PS instruction.
57///
58/// \param dst
59/// The destination tile. Max size is 1024 Bytes.
60/// \param src1
61/// The 1st source tile. Max size is 1024 Bytes.
62/// \param src2
63/// The 2nd source tile. Max size is 1024 Bytes.
64__DEFAULT_FN_ATTRS_FP8 static void
65__tile_dpbf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
66 dst->tile = _tile_dpbf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
67 src1.tile, src2.tile);
68}
69
70static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
71_tile_dpbhf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
72 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
73 return __builtin_ia32_tdpbhf8ps_internal(m, n, k, dst, src1, src2);
74}
75
76/// Perform the dot product of a BF8 value \a src1 by an HF8 value \a src2
77/// accumulating into a Single Precision (FP32) source/dest \a dst.
78///
79/// \headerfile <immintrin.h>
80///
81/// \code
82/// void __tile_dpbhf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)
83/// \endcode
84///
85/// \code{.operation}
86/// FOR m := 0 TO dst.rows - 1
87/// temp1[(dst.colsb / 4 - 1) : 0] = 0
88/// FOR k := 0 TO src1.colsb / 4 - 1
89/// FOR n := 0 TO dst.colsb / 4 - 1
90/// temp1[n] +=
91/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
92/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
93/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
94/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
95/// ENDFOR
96/// ENDFOR
97/// FOR n := 0 TO dst.colsb / 4 - 1
98/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
99/// ENDFOR
100/// write_row_and_zero(dst, m, tmp, dst.colsb)
101/// zero_upper_rows(dst, dst.rows)
102/// zero_tileconfig_start()
103/// \endcode
104///
105/// This intrinsic corresponds to the \c TDPBHF8PS instruction.
106///
107/// \param dst
108/// The destination tile. Max size is 1024 Bytes.
109/// \param src1
110/// The 1st source tile. Max size is 1024 Bytes.
111/// \param src2
112/// The 2nd source tile. Max size is 1024 Bytes.
113__DEFAULT_FN_ATTRS_FP8 static void
114__tile_dpbhf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
115 dst->tile = _tile_dpbhf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
116 src1.tile, src2.tile);
117}
118
119static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
120_tile_dphbf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
121 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
122 return __builtin_ia32_tdphbf8ps_internal(m, n, k, dst, src1, src2);
123}
124
125/// Perform the dot product of an HF8 value \a src1 by a BF8 value \a src2
126/// accumulating into a Single Precision (FP32) source/dest \a dst.
127///
128/// \headerfile <immintrin.h>
129///
130/// \code
131/// void __tile_dphbf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)
132/// \endcode
133///
134/// \code{.operation}
135/// FOR m := 0 TO dst.rows - 1
136/// temp1[(dst.colsb / 4 - 1) : 0] = 0
137/// FOR k := 0 TO src1.colsb / 4 - 1
138/// FOR n := 0 TO dst.colsb / 4 - 1
139/// temp1[n] +=
140/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
141/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
142/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
143/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
144/// ENDFOR
145/// ENDFOR
146/// FOR n := 0 TO dst.colsb / 4 - 1
147/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
148/// ENDFOR
149/// write_row_and_zero(dst, m, tmp, dst.colsb)
150/// zero_upper_rows(dst, dst.rows)
151/// zero_tileconfig_start()
152/// \endcode
153///
154/// This intrinsic corresponds to the \c TDPHBF8PS instruction.
155///
156/// \param dst
157/// The destination tile. Max size is 1024 Bytes.
158/// \param src1
159/// The 1st source tile. Max size is 1024 Bytes.
160/// \param src2
161/// The 2nd source tile. Max size is 1024 Bytes.
162
163__DEFAULT_FN_ATTRS_FP8 static void
164__tile_dphbf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
165 dst->tile = _tile_dphbf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
166 src1.tile, src2.tile);
167}
168
169static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
170_tile_dphf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
171 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
172 return __builtin_ia32_tdphf8ps_internal(m, n, k, dst, src1, src2);
173}
174
175/// Perform the dot product of an HF8 value \a src1 by an HF8 value \a src2
176/// accumulating into a Single Precision (FP32) source/dest \a dst.
177///
178/// \headerfile <immintrin.h>
179///
180/// \code
181/// void __tile_dphf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)
182/// \endcode
183///
184/// \code{.operation}
185/// FOR m := 0 TO dst.rows - 1
186/// temp1[(dst.colsb / 4 - 1) : 0] = 0
187/// FOR k := 0 TO src1.colsb / 4 - 1
188/// FOR n := 0 TO dst.colsb / 4 - 1
189/// temp1[n] +=
190/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
191/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
192/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
193/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
194/// ENDFOR
195/// ENDFOR
196/// FOR n := 0 TO dst.colsb / 4 - 1
197/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
198/// ENDFOR
199/// write_row_and_zero(dst, m, tmp, dst.colsb)
200/// zero_upper_rows(dst, dst.rows)
201/// zero_tileconfig_start()
202/// \endcode
203///
204/// This intrinsic corresponds to the \c TDPHF8PS instruction.
205///
206/// \param dst
207/// The destination tile. Max size is 1024 Bytes.
208/// \param src1
209/// The 1st source tile. Max size is 1024 Bytes.
210/// \param src2
211/// The 2nd source tile. Max size is 1024 Bytes.
212__DEFAULT_FN_ATTRS_FP8 static void
213__tile_dphf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
214 dst->tile = _tile_dphf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
215 src1.tile, src2.tile);
216}
217
218#define _tile_dpbf8ps(dst, src1, src2) \
219 __builtin_ia32_tdpbf8ps((dst), (src1), (src2))
220#define _tile_dpbhf8ps(dst, src1, src2) \
221 __builtin_ia32_tdpbhf8ps((dst), (src1), (src2))
222#define _tile_dphbf8ps(dst, src1, src2) \
223 __builtin_ia32_tdphbf8ps((dst), (src1), (src2))
224#define _tile_dphf8ps(dst, src1, src2) \
225 __builtin_ia32_tdphf8ps((dst), (src1), (src2))
226
227#undef __DEFAULT_FN_ATTRS_FP8
228
229#endif /* __x86_64__ */
230#endif /* __AMXFP8INTRIN_H */
lib/include/amxintrin.h+5-35
...@@ -22,8 +22,6 @@...@@ -22,8 +22,6 @@
22 __attribute__((__always_inline__, __nodebug__, __target__("amx-int8")))22 __attribute__((__always_inline__, __nodebug__, __target__("amx-int8")))
23#define __DEFAULT_FN_ATTRS_BF16 \23#define __DEFAULT_FN_ATTRS_BF16 \
24 __attribute__((__always_inline__, __nodebug__, __target__("amx-bf16")))24 __attribute__((__always_inline__, __nodebug__, __target__("amx-bf16")))
25#define __DEFAULT_FN_ATTRS_FP16 \
26 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp16")))
2725
28/// Load tile configuration from a 64-byte memory location specified by26/// Load tile configuration from a 64-byte memory location specified by
29/// "mem_addr". The tile configuration includes the tile type palette, the27/// "mem_addr". The tile configuration includes the tile type palette, the
...@@ -232,9 +230,11 @@ static __inline__ void __DEFAULT_FN_ATTRS_TILE _tile_release(void) {...@@ -232,9 +230,11 @@ static __inline__ void __DEFAULT_FN_ATTRS_TILE _tile_release(void) {
232/// bytes. Since there is no 2D type in llvm IR, we use vector type to230/// bytes. Since there is no 2D type in llvm IR, we use vector type to
233/// represent 2D tile and the fixed size is maximum amx tile register size.231/// represent 2D tile and the fixed size is maximum amx tile register size.
234typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));232typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));
233typedef int _tile1024i_1024a
234 __attribute__((__vector_size__(1024), __aligned__(1024)));
235235
236/// This is internal intrinsic. C/C++ user should avoid calling it directly.236/// This is internal intrinsic. C/C++ user should avoid calling it directly.
237static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8237static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TILE
238_tile_loadd_internal(unsigned short m, unsigned short n, const void *base,238_tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
239 __SIZE_TYPE__ stride) {239 __SIZE_TYPE__ stride) {
240 return __builtin_ia32_tileloadd64_internal(m, n, base,240 return __builtin_ia32_tileloadd64_internal(m, n, base,
...@@ -242,7 +242,7 @@ _tile_loadd_internal(unsigned short m, unsigned short n, const void *base,...@@ -242,7 +242,7 @@ _tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
242}242}
243243
244/// This is internal intrinsic. C/C++ user should avoid calling it directly.244/// This is internal intrinsic. C/C++ user should avoid calling it directly.
245static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8245static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TILE
246_tile_loaddt1_internal(unsigned short m, unsigned short n, const void *base,246_tile_loaddt1_internal(unsigned short m, unsigned short n, const void *base,
247 __SIZE_TYPE__ stride) {247 __SIZE_TYPE__ stride) {
248 return __builtin_ia32_tileloaddt164_internal(m, n, base,248 return __builtin_ia32_tileloaddt164_internal(m, n, base,
...@@ -278,7 +278,7 @@ _tile_dpbuud_internal(unsigned short m, unsigned short n, unsigned short k,...@@ -278,7 +278,7 @@ _tile_dpbuud_internal(unsigned short m, unsigned short n, unsigned short k,
278}278}
279279
280/// This is internal intrinsic. C/C++ user should avoid calling it directly.280/// This is internal intrinsic. C/C++ user should avoid calling it directly.
281static __inline__ void __DEFAULT_FN_ATTRS_INT8281static __inline__ void __DEFAULT_FN_ATTRS_TILE
282_tile_stored_internal(unsigned short m, unsigned short n, void *base,282_tile_stored_internal(unsigned short m, unsigned short n, void *base,
283 __SIZE_TYPE__ stride, _tile1024i tile) {283 __SIZE_TYPE__ stride, _tile1024i tile) {
284 return __builtin_ia32_tilestored64_internal(m, n, base,284 return __builtin_ia32_tilestored64_internal(m, n, base,
...@@ -292,13 +292,6 @@ _tile_dpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,...@@ -292,13 +292,6 @@ _tile_dpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
292 return __builtin_ia32_tdpbf16ps_internal(m, n, k, dst, src1, src2);292 return __builtin_ia32_tdpbf16ps_internal(m, n, k, dst, src1, src2);
293}293}
294294
295/// This is internal intrinsic. C/C++ user should avoid calling it directly.
296static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP16
297_tile_dpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
298 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
299 return __builtin_ia32_tdpfp16ps_internal(m, n, k, dst, src1, src2);
300}
301
302/// This struct pack the shape and tile data together for user. We suggest295/// This struct pack the shape and tile data together for user. We suggest
303/// initializing the struct as early as possible, because compiler depends296/// initializing the struct as early as possible, because compiler depends
304/// on the shape information to do configure. The constant value is preferred297/// on the shape information to do configure. The constant value is preferred
...@@ -493,32 +486,9 @@ static __inline__ void __tile_dpbf16ps(__tile1024i *dst, __tile1024i src0,...@@ -493,32 +486,9 @@ static __inline__ void __tile_dpbf16ps(__tile1024i *dst, __tile1024i src0,
493 src0.tile, src1.tile);486 src0.tile, src1.tile);
494}487}
495488
496/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles src0 and
497/// src1, accumulating the intermediate single-precision (32-bit) floating-point
498/// elements with elements in "dst", and store the 32-bit result back to tile
499/// "dst".
500///
501/// \headerfile <immintrin.h>
502///
503/// This intrinsic corresponds to the <c> TDPFP16PS </c> instruction.
504///
505/// \param dst
506/// The destination tile. Max size is 1024 Bytes.
507/// \param src0
508/// The 1st source tile. Max size is 1024 Bytes.
509/// \param src1
510/// The 2nd source tile. Max size is 1024 Bytes.
511__DEFAULT_FN_ATTRS_FP16
512static __inline__ void __tile_dpfp16ps(__tile1024i *dst, __tile1024i src0,
513 __tile1024i src1) {
514 dst->tile = _tile_dpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
515 src0.tile, src1.tile);
516}
517
518#undef __DEFAULT_FN_ATTRS_TILE489#undef __DEFAULT_FN_ATTRS_TILE
519#undef __DEFAULT_FN_ATTRS_INT8490#undef __DEFAULT_FN_ATTRS_INT8
520#undef __DEFAULT_FN_ATTRS_BF16491#undef __DEFAULT_FN_ATTRS_BF16
521#undef __DEFAULT_FN_ATTRS_FP16
522492
523#endif /* __x86_64__ */493#endif /* __x86_64__ */
524#endif /* __AMXINTRIN_H */494#endif /* __AMXINTRIN_H */
lib/include/amxmovrsintrin.h created+48
...@@ -0,0 +1,48 @@
1/*===-------- amxmovrsintrin.h - AMX MOVRS intrinsics -*- C++ -*---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 * ===-------------------------------------------------------------------=== */
8
9#ifndef __IMMINTRIN_H
10#error "Never use <amxmovrsintrin.h> directly; include <immintrin.h> instead."
11#endif /* __IMMINTRIN_H */
12
13#ifndef __AMXMOVRSINTRIN_H
14#define __AMXMOVRSINTRIN_H
15#ifdef __x86_64__
16
17#define __DEFAULT_FN_ATTRS_MOVRS \
18 __attribute__((__always_inline__, __nodebug__, __target__("amx-movrs")))
19
20#define _tile_loaddrs(dst, base, stride) \
21 __builtin_ia32_tileloaddrs64((dst), ((const void *)(base)), \
22 (__SIZE_TYPE__)(stride))
23#define _tile_stream_loaddrs(dst, base, stride) \
24 __builtin_ia32_tileloaddrst164((dst), ((const void *)(base)), \
25 (__SIZE_TYPE__)(stride))
26static __inline__ _tile1024i __DEFAULT_FN_ATTRS_MOVRS
27_tile_loaddrs_internal(unsigned short m, unsigned short n, const void *base,
28 __SIZE_TYPE__ stride) {
29 return __builtin_ia32_tileloaddrs64_internal(m, n, base,
30 (__SIZE_TYPE__)(stride));
31}
32static __inline__ _tile1024i __DEFAULT_FN_ATTRS_MOVRS
33_tile_loaddrst1_internal(unsigned short m, unsigned short n, const void *base,
34 __SIZE_TYPE__ stride) {
35 return __builtin_ia32_tileloaddrst164_internal(m, n, base,
36 (__SIZE_TYPE__)(stride));
37}
38static __inline__ void __DEFAULT_FN_ATTRS_MOVRS
39__tile_loaddrs(__tile1024i *dst, const void *base, __SIZE_TYPE__ stride) {
40 dst->tile = _tile_loaddrs_internal(dst->row, dst->col, base, stride);
41}
42static __inline__ void __DEFAULT_FN_ATTRS_MOVRS __tile_stream_loaddrs(
43 __tile1024i *dst, const void *base, __SIZE_TYPE__ stride) {
44 dst->tile = _tile_loaddrst1_internal(dst->row, dst->col, base, stride);
45}
46#undef __DEFAULT_FN_ATTRS_MOVRS
47#endif /* __x86_64__ */
48#endif /* __AMXMOVRSINTRIN_H */
lib/include/amxmovrstransposeintrin.h created+200
...@@ -0,0 +1,200 @@
1/* ===--- amxmovrstransposeintrin.h - AMX_MOVRS_TRANSPOSE intrinsics --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 * ===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxmovrstransposeintrin.h> directly; use <immintrin.h> instead."
13#endif /* __IMMINTRIN_H */
14
15#ifndef __AMX_MOVRS_TRANSPOSEINTRIN_H
16#define __AMX_MOVRS_TRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("amx-transpose,amx-movrs")))
22
23#define _tile_2rpntlvwz0rs(tdst, base, stride) \
24 __builtin_ia32_t2rpntlvwz0rs(tdst, base, stride)
25#define _tile_2rpntlvwz0rst1(tdst, base, stride) \
26 __builtin_ia32_t2rpntlvwz0rst1(tdst, base, stride)
27#define _tile_2rpntlvwz1rs(tdst, base, stride) \
28 __builtin_ia32_t2rpntlvwz1rs(tdst, base, stride)
29#define _tile_2rpntlvwz1rst1(tdst, base, stride) \
30 __builtin_ia32_t2rpntlvwz1rst1(tdst, base, stride)
31
32static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rs_internal(
33 unsigned short row, unsigned short col0, unsigned short col1,
34 _tile1024i *dst0, _tile1024i *dst1, const void *base,
35 __SIZE_TYPE__ stride) {
36 // Use __tile1024i_1024a* to escape the alignment check in
37 // clang/test/Headers/x86-intrinsics-headers-clean.cpp
38 __builtin_ia32_t2rpntlvwz0rs_internal(
39 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
40 (__SIZE_TYPE__)(stride));
41}
42
43static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rst1_internal(
44 unsigned short row, unsigned short col0, unsigned short col1,
45 _tile1024i *dst0, _tile1024i *dst1, const void *base,
46 __SIZE_TYPE__ stride) {
47 __builtin_ia32_t2rpntlvwz0rst1_internal(
48 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
49 (__SIZE_TYPE__)(stride));
50}
51
52static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rs_internal(
53 unsigned short row, unsigned short col0, unsigned short col1,
54 _tile1024i *dst0, _tile1024i *dst1, const void *base,
55 __SIZE_TYPE__ stride) {
56 __builtin_ia32_t2rpntlvwz1rs_internal(
57 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
58 (__SIZE_TYPE__)(stride));
59}
60
61static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rst1_internal(
62 unsigned short row, unsigned short col0, unsigned short col1,
63 _tile1024i *dst0, _tile1024i *dst1, const void *base,
64 __SIZE_TYPE__ stride) {
65 __builtin_ia32_t2rpntlvwz1rst1_internal(
66 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
67 (__SIZE_TYPE__)(stride));
68}
69
70/// Converts a pair of tiles from memory into VNNI format, and places the
71/// results in a pair of destinations specified by dst. The pair of tiles
72/// in memory is specified via a tsib; the second tile is after the first
73/// one, separated by the same stride that separates each row.
74/// The tile configuration for the destination tiles indicates the amount
75/// of data to read from memory. The instruction will load a number of rows
76/// that is equal to twice the number of rows in tmm1. The size of each row
77/// is equal to the average width of the destination tiles. If the second
78/// tile is configured with zero rows and columns, only the first tile will
79/// be written.
80/// Provides a hint to the implementation that the data will likely become
81/// read shared in the near future and the data caching can be optimized.
82///
83/// \headerfile <immintrin.h>
84///
85/// This intrinsic corresponds to the <c> T2RPNTLVWZ0RS </c> instruction.
86///
87/// \param dst0
88/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
89/// \param dst1
90/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
91/// \param base
92/// A pointer to base address.
93/// \param stride
94/// The stride between the rows' data to be loaded in memory.
95__DEFAULT_FN_ATTRS
96static void __tile_2rpntlvwz0rs(__tile1024i *dst0, __tile1024i *dst1,
97 const void *base, __SIZE_TYPE__ stride) {
98 _tile_2rpntlvwz0rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
99 &dst1->tile, base, stride);
100}
101
102/// Converts a pair of tiles from memory into VNNI format, and places the
103/// results in a pair of destinations specified by dst. The pair of tiles
104/// in memory is specified via a tsib; the second tile is after the first
105/// one, separated by the same stride that separates each row.
106/// The tile configuration for the destination tiles indicates the amount
107/// of data to read from memory. The instruction will load a number of rows
108/// that is equal to twice the number of rows in tmm1. The size of each row
109/// is equal to the average width of the destination tiles. If the second
110/// tile is configured with zero rows and columns, only the first tile will
111/// be written.
112///
113/// \headerfile <immintrin.h>
114///
115/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1RS </c> instruction.
116///
117/// \param dst0
118/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
119/// \param dst1
120/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
121/// \param base
122/// A pointer to base address.
123/// \param stride
124/// The stride between the rows' data to be loaded in memory.
125__DEFAULT_FN_ATTRS
126static void __tile_2rpntlvwz0rst1(__tile1024i *dst0, __tile1024i *dst1,
127 const void *base, __SIZE_TYPE__ stride) {
128 _tile_2rpntlvwz0rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
129 &dst1->tile, base, stride);
130}
131
132/// Converts a pair of tiles from memory into VNNI format, and places the
133/// results in a pair of destinations specified by dst. The pair of tiles
134/// in memory is specified via a tsib; the second tile is after the first
135/// one, separated by the same stride that separates each row.
136/// The tile configuration for the destination tiles indicates the amount
137/// of data to read from memory. The instruction will load a number of rows
138/// that is equal to twice the number of rows in tmm1. The size of each row
139/// is equal to the average width of the destination tiles. If the second
140/// tile is configured with zero rows and columns, only the first tile will
141/// be written. The last row will be not be read from memory but instead
142/// filled with zeros.
143/// Provides a hint to the implementation that the data will likely become
144/// read shared in the near future and the data caching can be optimized.
145///
146/// \headerfile <immintrin.h>
147///
148/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
149///
150/// \param dst0
151/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
152/// \param dst1
153/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
154/// \param base
155/// A pointer to base address.
156/// \param stride
157/// The stride between the rows' data to be loaded in memory.
158__DEFAULT_FN_ATTRS
159static void __tile_2rpntlvwz1rs(__tile1024i *dst0, __tile1024i *dst1,
160 const void *base, __SIZE_TYPE__ stride) {
161 _tile_2rpntlvwz1rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
162 &dst1->tile, base, stride);
163}
164
165/// Converts a pair of tiles from memory into VNNI format, and places the
166/// results in a pair of destinations specified by dst. The pair of tiles
167/// in memory is specified via a tsib; the second tile is after the first
168/// one, separated by the same stride that separates each row.
169/// The tile configuration for the destination tiles indicates the amount
170/// of data to read from memory. The instruction will load a number of rows
171/// that is equal to twice the number of rows in tmm1. The size of each row
172/// is equal to the average width of the destination tiles. If the second
173/// tile is configured with zero rows and columns, only the first tile will
174/// be written. The last row will be not be read from memory but instead
175/// filled with zeros.
176/// Provides a hint to the implementation that the data will likely become
177/// read shared in the near future and the data caching can be optimized.
178///
179/// \headerfile <immintrin.h>
180///
181/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1RS </c> instruction.
182///
183/// \param dst0
184/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
185/// \param dst1
186/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
187/// \param base
188/// A pointer to base address.
189/// \param stride
190/// The stride between the rows' data to be loaded in memory.
191__DEFAULT_FN_ATTRS
192static void __tile_2rpntlvwz1rst1(__tile1024i *dst0, __tile1024i *dst1,
193 const void *base, __SIZE_TYPE__ stride) {
194 _tile_2rpntlvwz1rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
195 &dst1->tile, base, stride);
196}
197
198#undef __DEFAULT_FN_ATTRS
199#endif /* __x86_64__ */
200#endif /* __AMX_MOVRS_TRANSPOSEINTRIN_H */
\ No newline at end of file
lib/include/amxtf32intrin.h created+108
...@@ -0,0 +1,108 @@
1/*===------------- amxtf32intrin.h - AMX_TF32 intrinsics -*- C++ -*---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <amxtf32intrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AMX_TF32INTRIN_H
15#define __AMX_TF32INTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_TF32 \
19 __attribute__((__always_inline__, __nodebug__, __target__("amx-tf32")))
20
21/// Do Matrix Multiplication of \a a and \a b, and then do Matrix Plus
22/// with \a srcdst.
23/// All the calculation is base on float32 but with the lower 13-bit set to 0.
24///
25/// \headerfile <immintrin.h>
26///
27/// \code
28/// void _tile_mmultf32ps(constexpr int srcdst, constexpr int a, \
29/// constexpr int b);
30/// \endcode
31///
32/// This intrinsic corresponds to the <c> TMMULTF32PS </c> instruction.
33///
34/// \param srcdst
35/// The destination tile. Max size is 1024 Bytes.
36/// \param a
37/// The 1st source tile. Max size is 1024 Bytes.
38/// \param b
39/// The 2nd source tile. Max size is 1024 Bytes.
40///
41/// \code{.operation}
42/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) {
43/// dword[12:0] := 0
44/// dword[31:13] := x[31:13]
45/// return dword
46/// }
47///
48/// DEFINE silence_snan_fp32(x[31:0]) {
49/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0)
50/// x.fraction[22] := 1
51/// return x
52/// }
53///
54/// elements_a := a.colsb / 4
55/// elements_dest := srcdst.colsb / 4
56///
57/// FOR m = 0 TO (srcdst.rows-1)
58/// tmp[511:0] := 0
59/// FOR k = 0 TO (elements_a-1)
60/// FOR n = 0 TO (elements_dest-1)
61/// af := silence_snan_fp32(a.row[m].fp32[k])
62/// bf := silence_snan_fp32(b.row[k].fp32[n])
63/// tmp.fp32[n] += zero_lower_mantissa_bits_fp32(af)
64/// * zero_lower_mantissa_bits_fp32(bf)
65/// ENDFOR
66/// ENDFOR
67///
68/// FOR n = 0 TO (elements_dest-1)
69/// tmp.fp32[n] += srcdst.row[m].fp32[n]
70/// ENDFOR
71/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb)
72///
73/// ENDFOR
74///
75/// zero_upper_rows(srcdst, srcdst.rows)
76/// zero_tileconfig_start()
77/// \endcode
78#define _tile_mmultf32ps(srcdst, a, b) \
79 __builtin_ia32_tmmultf32ps((srcdst), (a), (b))
80
81static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32
82_tile_mmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k,
83 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
84 return __builtin_ia32_tmmultf32ps_internal(m, n, k, dst, src1, src2);
85}
86
87/// Do Matrix Multiplication of src0 and src1, and then do Matrix Plus with dst.
88/// All the calculation is base on float32 but with the lower 13-bit set to 0.
89///
90/// \headerfile <immintrin.h>
91///
92/// This intrinsic corresponds to the <c> TMMULTF32PS </c> instruction.
93///
94/// \param dst
95/// The destination tile. Max size is 1024 Bytes.
96/// \param src0
97/// The 1st source tile. Max size is 1024 Bytes.
98/// \param src1
99/// The 2nd source tile. Max size is 1024 Bytes.
100__DEFAULT_FN_ATTRS_TF32
101static void __tile_mmultf32ps(__tile1024i *dst, __tile1024i src0,
102 __tile1024i src1) {
103 dst->tile = _tile_mmultf32ps_internal(src0.row, src1.col, src0.col, dst->tile,
104 src0.tile, src1.tile);
105}
106
107#endif // __x86_64__
108#endif // __AMX_TF32INTRIN_H
lib/include/amxtf32transposeintrin.h created+105
...@@ -0,0 +1,105 @@
1/*===--------- amxtf32transposeintrin.h - AMX-TF32 and AMX-TRANSPOSE --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <amxtf32tranposeintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AMX_TF32TRANSPOSEINTRIN_H
15#define __AMX_TF32TRANSPOSEINTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_TF32_TRANSPOSE \
19 __attribute__((__always_inline__, __nodebug__, \
20 __target__("amx-tf32,amx-transpose")))
21
22/// \code
23/// void _tile_tmmultf32ps(constexpr int srcdst, constexpr int a, \
24/// constexpr int b);
25/// \endcode
26///
27/// This intrinsic corresponds to the <c> TTMMULTF32PS </c> instruction.
28///
29/// \param srcdst
30/// The destination tile. Max size is 1024 Bytes.
31/// \param a
32/// The 1st source tile. Max size is 1024 Bytes.
33/// \param b
34/// The 2nd source tile. Max size is 1024 Bytes.
35///
36/// \code{.operation}
37/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) {
38/// dword[12:0] := 0
39/// dword[31:13] := x[31:13]
40/// return dword
41/// }
42///
43/// DEFINE silence_snan_fp32(x[31:0]) {
44/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0)
45/// x.fraction[22] := 1
46/// return x
47/// }
48///
49/// elements_dest:= srcdst.colsb/4
50///
51/// FOR m := 0 TO (srcdst.rows-1)
52/// tmp[511:0] := 0
53/// FOR k := 0 TO (a.rows-1)
54/// FOR n := 0 TO (elements_dest-1)
55/// a1e := silence_snan_fp32(a.row[k].fp32[m])
56/// a2e := silence_snan_fp32(b.row[k].fp32[n])
57/// s1e := zero_lower_mantissa_bits_fp32(a1e)
58/// s2e := zero_lower_mantissa_bits_fp32(a2e)
59/// tmp.fp32[n] += s1e * s2e
60/// ENDFOR
61/// ENDFOR
62///
63/// FOR n := 0 TO (elements_dest-1)
64/// tmp.fp32[n] += srcdst.row[m].fp32[n]
65/// ENDFOR
66/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb)
67///
68/// ENDFOR
69///
70/// zero_upper_rows(srcdst, srcdst.rows)
71/// zero_tileconfig_start()
72/// \endcode
73#define _tile_tmmultf32ps(srcdst, a, b) \
74 __builtin_ia32_ttmmultf32ps((srcdst), (a), (b))
75
76// dst = m x n (srcdest), src1 = k x m, src2 = k x n
77static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32_TRANSPOSE
78_tile_tmmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k,
79 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
80 return __builtin_ia32_ttmmultf32ps_internal(m, n, k, dst, src1, src2);
81}
82
83/// Compute transpose and do Matrix Multiplication of src0 and src1, and then do
84/// Matrix Plus with dst. All the calculation is base on float32 but with the
85/// lower 13-bit set to 0.
86///
87/// \headerfile <immintrin.h>
88///
89/// This intrinsic corresponds to the <c> TTMMULTF32PS </c> instruction.
90///
91/// \param dst
92/// The destination tile. Max size is 1024 Bytes.
93/// \param src0
94/// The 1st source tile. Max size is 1024 Bytes.
95/// \param src1
96/// The 2nd source tile. Max size is 1024 Bytes.
97__DEFAULT_FN_ATTRS_TF32_TRANSPOSE
98static void __tile_tmmultf32ps(__tile1024i *dst, __tile1024i src0,
99 __tile1024i src1) {
100 dst->tile = _tile_tmmultf32ps_internal(src0.row, src1.col, src0.col,
101 dst->tile, src0.tile, src1.tile);
102}
103
104#endif // __x86_64__
105#endif // __AMX_TF32TRANSPOSEINTRIN_H
lib/include/amxtransposeintrin.h created+248
...@@ -0,0 +1,248 @@
1/* ===--- amxtransposeintrin.h - AMX_TRANSPOSE intrinsics -*- C++ -*---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 * ===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <amxtransposeintrin.h> directly; use <immintrin.h> instead."
12#endif /* __IMMINTRIN_H */
13
14#ifndef __AMX_TRANSPOSEINTRIN_H
15#define __AMX_TRANSPOSEINTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_TRANSPOSE \
19 __attribute__((__always_inline__, __nodebug__, __target__("amx-transpose")))
20
21#define _tile_2rpntlvwz0(tdst, base, stride) \
22 __builtin_ia32_t2rpntlvwz0(tdst, base, stride)
23#define _tile_2rpntlvwz0t1(tdst, base, stride) \
24 __builtin_ia32_t2rpntlvwz0t1(tdst, base, stride)
25#define _tile_2rpntlvwz1(tdst, base, stride) \
26 __builtin_ia32_t2rpntlvwz1(tdst, base, stride)
27#define _tile_2rpntlvwz1t1(tdst, base, stride) \
28 __builtin_ia32_t2rpntlvwz1t1(tdst, base, stride)
29
30/// Transpose 32-bit elements from \a src and write the result to \a dst.
31///
32/// \headerfile <immintrin.h>
33///
34/// \code
35/// void _tile_transposed(__tile dst, __tile src);
36/// \endcode
37///
38/// This intrinsic corresponds to the <c> TTRANSPOSED </c> instruction.
39///
40/// \param dst
41/// The destination tile. Max size is 1024 Bytes.
42/// \param src
43/// The source tile. Max size is 1024 Bytes.
44///
45/// \code{.operation}
46///
47/// FOR i := 0 TO (dst.rows-1)
48/// tmp[511:0] := 0
49/// FOR j := 0 TO (dst.colsb/4-1)
50/// tmp.dword[j] := src.row[j].dword[i]
51/// ENDFOR
52/// dst.row[i] := tmp
53/// ENDFOR
54///
55/// zero_upper_rows(dst, dst.rows)
56/// zero_tileconfig_start()
57/// \endcode
58#define _tile_transposed(dst, src) __builtin_ia32_ttransposed(dst, src)
59
60static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz0_internal(
61 unsigned short row, unsigned short col0, unsigned short col1,
62 _tile1024i *dst0, _tile1024i *dst1, const void *base,
63 __SIZE_TYPE__ stride) {
64 // Use __tile1024i_1024a* to escape the alignment check in
65 // clang/test/Headers/x86-intrinsics-headers-clean.cpp
66 __builtin_ia32_t2rpntlvwz0_internal(row, col0, col1, (_tile1024i_1024a *)dst0,
67 (_tile1024i_1024a *)dst1, base,
68 (__SIZE_TYPE__)(stride));
69}
70
71static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz0t1_internal(
72 unsigned short row, unsigned short col0, unsigned short col1,
73 _tile1024i *dst0, _tile1024i *dst1, const void *base,
74 __SIZE_TYPE__ stride) {
75 __builtin_ia32_t2rpntlvwz0t1_internal(
76 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
77 (__SIZE_TYPE__)(stride));
78}
79
80static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz1_internal(
81 unsigned short row, unsigned short col0, unsigned short col1,
82 _tile1024i *dst0, _tile1024i *dst1, const void *base,
83 __SIZE_TYPE__ stride) {
84 __builtin_ia32_t2rpntlvwz1_internal(row, col0, col1, (_tile1024i_1024a *)dst0,
85 (_tile1024i_1024a *)dst1, base,
86 (__SIZE_TYPE__)(stride));
87}
88
89static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz1t1_internal(
90 unsigned short row, unsigned short col0, unsigned short col1,
91 _tile1024i *dst0, _tile1024i *dst1, const void *base,
92 __SIZE_TYPE__ stride) {
93 __builtin_ia32_t2rpntlvwz1t1_internal(
94 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
95 (__SIZE_TYPE__)(stride));
96}
97
98// This is internal intrinsic. C/C++ user should avoid calling it directly.
99static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TRANSPOSE
100_tile_transposed_internal(unsigned short m, unsigned short n, _tile1024i src) {
101 return __builtin_ia32_ttransposed_internal(m, n, src);
102}
103
104/// Converts a pair of tiles from memory into VNNI format, and places the
105/// results in a pair of destinations specified by dst. The pair of tiles
106/// in memory is specified via a tsib; the second tile is after the first
107/// one, separated by the same stride that separates each row.
108/// The tile configuration for the destination tiles indicates the amount
109/// of data to read from memory. The instruction will load a number of rows
110/// that is equal to twice the number of rows in tmm1. The size of each row
111/// is equal to the average width of the destination tiles. If the second
112/// tile is configured with zero rows and columns, only the first tile will
113/// be written.
114/// Provides a hint to the implementation that the data will likely not be
115/// reused in the near future and the data caching can be optimized.
116///
117/// \headerfile <immintrin.h>
118///
119/// This intrinsic corresponds to the <c> T2RPNTLVWZ0 </c> instruction.
120///
121/// \param dst0
122/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
123/// \param dst1
124/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
125/// \param base
126/// A pointer to base address.
127/// \param stride
128/// The stride between the rows' data to be loaded in memory.
129__DEFAULT_FN_ATTRS_TRANSPOSE
130static void __tile_2rpntlvwz0(__tile1024i *dst0, __tile1024i *dst1,
131 const void *base, __SIZE_TYPE__ stride) {
132 _tile_2rpntlvwz0_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
133 &dst1->tile, base, stride);
134}
135
136/// Converts a pair of tiles from memory into VNNI format, and places the
137/// results in a pair of destinations specified by dst. The pair of tiles
138/// in memory is specified via a tsib; the second tile is after the first
139/// one, separated by the same stride that separates each row.
140/// The tile configuration for the destination tiles indicates the amount
141/// of data to read from memory. The instruction will load a number of rows
142/// that is equal to twice the number of rows in tmm1. The size of each row
143/// is equal to the average width of the destination tiles. If the second
144/// tile is configured with zero rows and columns, only the first tile will
145/// be written.
146///
147/// \headerfile <immintrin.h>
148///
149/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1 </c> instruction.
150///
151/// \param dst0
152/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
153/// \param dst1
154/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
155/// \param base
156/// A pointer to base address.
157/// \param stride
158/// The stride between the rows' data to be loaded in memory.
159__DEFAULT_FN_ATTRS_TRANSPOSE
160static void __tile_2rpntlvwz0t1(__tile1024i *dst0, __tile1024i *dst1,
161 const void *base, __SIZE_TYPE__ stride) {
162 _tile_2rpntlvwz0t1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
163 &dst1->tile, base, stride);
164}
165
166/// Converts a pair of tiles from memory into VNNI format, and places the
167/// results in a pair of destinations specified by dst. The pair of tiles
168/// in memory is specified via a tsib; the second tile is after the first
169/// one, separated by the same stride that separates each row.
170/// The tile configuration for the destination tiles indicates the amount
171/// of data to read from memory. The instruction will load a number of rows
172/// that is equal to twice the number of rows in tmm1. The size of each row
173/// is equal to the average width of the destination tiles. If the second
174/// tile is configured with zero rows and columns, only the first tile will
175/// be written. The last row will be not be read from memory but instead
176/// filled with zeros.
177/// Provides a hint to the implementation that the data will likely not be
178/// reused in the near future and the data caching can be optimized.
179///
180/// \headerfile <immintrin.h>
181///
182/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
183///
184/// \param dst0
185/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
186/// \param dst1
187/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
188/// \param base
189/// A pointer to base address.
190/// \param stride
191/// The stride between the rows' data to be loaded in memory.
192__DEFAULT_FN_ATTRS_TRANSPOSE
193static void __tile_2rpntlvwz1(__tile1024i *dst0, __tile1024i *dst1,
194 const void *base, __SIZE_TYPE__ stride) {
195 _tile_2rpntlvwz1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
196 &dst1->tile, base, stride);
197}
198
199/// Converts a pair of tiles from memory into VNNI format, and places the
200/// results in a pair of destinations specified by dst. The pair of tiles
201/// in memory is specified via a tsib; the second tile is after the first
202/// one, separated by the same stride that separates each row.
203/// The tile configuration for the destination tiles indicates the amount
204/// of data to read from memory. The instruction will load a number of rows
205/// that is equal to twice the number of rows in tmm1. The size of each row
206/// is equal to the average width of the destination tiles. If the second
207/// tile is configured with zero rows and columns, only the first tile will
208/// be written. The last row will be not be read from memory but instead
209/// filled with zeros.
210/// Provides a hint to the implementation that the data will likely not be
211/// reused in the near future and the data caching can be optimized.
212///
213/// \headerfile <immintrin.h>
214///
215/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1 </c> instruction.
216///
217/// \param dst0
218/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
219/// \param dst1
220/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
221/// \param base
222/// A pointer to base address.
223/// \param stride
224/// The stride between the rows' data to be loaded in memory.
225__DEFAULT_FN_ATTRS_TRANSPOSE
226static void __tile_2rpntlvwz1t1(__tile1024i *dst0, __tile1024i *dst1,
227 const void *base, __SIZE_TYPE__ stride) {
228 _tile_2rpntlvwz1t1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
229 &dst1->tile, base, stride);
230}
231
232/// Transpose 32-bit elements from src and write the result to dst.
233///
234/// \headerfile <immintrin.h>
235///
236/// This intrinsic corresponds to the <c> TTRANSPOSED </c> instruction.
237///
238/// \param dst
239/// The destination tile. Max size is 1024 Bytes.
240/// \param src
241/// The source tile. Max size is 1024 Bytes.
242__DEFAULT_FN_ATTRS_TRANSPOSE
243static void __tile_transposed(__tile1024i *dst, __tile1024i src) {
244 dst->tile = _tile_transposed_internal(dst->row, dst->col, src.tile);
245}
246
247#endif /* __x86_64__ */
248#endif /* __AMX_TRANSPOSEINTRIN_H */
lib/include/arm_acle.h+18-21
...@@ -264,28 +264,28 @@ __rbitl(unsigned long __t) {...@@ -264,28 +264,28 @@ __rbitl(unsigned long __t) {
264}264}
265265
266/* 8.3 16-bit multiplications */266/* 8.3 16-bit multiplications */
267#if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP267#if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
268static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))268static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
269__smulbb(int32_t __a, int32_t __b) {269__smulbb(int32_t __a, int32_t __b) {
270 return __builtin_arm_smulbb(__a, __b);270 return __builtin_arm_smulbb(__a, __b);
271}271}
272static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))272static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
273__smulbt(int32_t __a, int32_t __b) {273__smulbt(int32_t __a, int32_t __b) {
274 return __builtin_arm_smulbt(__a, __b);274 return __builtin_arm_smulbt(__a, __b);
275}275}
276static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))276static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
277__smultb(int32_t __a, int32_t __b) {277__smultb(int32_t __a, int32_t __b) {
278 return __builtin_arm_smultb(__a, __b);278 return __builtin_arm_smultb(__a, __b);
279}279}
280static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))280static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
281__smultt(int32_t __a, int32_t __b) {281__smultt(int32_t __a, int32_t __b) {
282 return __builtin_arm_smultt(__a, __b);282 return __builtin_arm_smultt(__a, __b);
283}283}
284static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))284static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
285__smulwb(int32_t __a, int32_t __b) {285__smulwb(int32_t __a, int32_t __b) {
286 return __builtin_arm_smulwb(__a, __b);286 return __builtin_arm_smulwb(__a, __b);
287}287}
288static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))288static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
289__smulwt(int32_t __a, int32_t __b) {289__smulwt(int32_t __a, int32_t __b) {
290 return __builtin_arm_smulwt(__a, __b);290 return __builtin_arm_smulwt(__a, __b);
291}291}
...@@ -304,46 +304,46 @@ __smulwt(int32_t __a, int32_t __b) {...@@ -304,46 +304,46 @@ __smulwt(int32_t __a, int32_t __b) {
304#endif304#endif
305305
306/* 8.4.2 Saturating addition and subtraction intrinsics */306/* 8.4.2 Saturating addition and subtraction intrinsics */
307#if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP307#if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
308static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))308static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
309__qadd(int32_t __t, int32_t __v) {309__qadd(int32_t __t, int32_t __v) {
310 return __builtin_arm_qadd(__t, __v);310 return __builtin_arm_qadd(__t, __v);
311}311}
312312
313static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))313static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
314__qsub(int32_t __t, int32_t __v) {314__qsub(int32_t __t, int32_t __v) {
315 return __builtin_arm_qsub(__t, __v);315 return __builtin_arm_qsub(__t, __v);
316}316}
317317
318static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))318static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
319__qdbl(int32_t __t) {319__qdbl(int32_t __t) {
320 return __builtin_arm_qadd(__t, __t);320 return __builtin_arm_qadd(__t, __t);
321}321}
322#endif322#endif
323323
324/* 8.4.3 Accumulating multiplications */324/* 8.4.3 Accumulating multiplications */
325#if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP325#if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
326static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))326static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
327__smlabb(int32_t __a, int32_t __b, int32_t __c) {327__smlabb(int32_t __a, int32_t __b, int32_t __c) {
328 return __builtin_arm_smlabb(__a, __b, __c);328 return __builtin_arm_smlabb(__a, __b, __c);
329}329}
330static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))330static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
331__smlabt(int32_t __a, int32_t __b, int32_t __c) {331__smlabt(int32_t __a, int32_t __b, int32_t __c) {
332 return __builtin_arm_smlabt(__a, __b, __c);332 return __builtin_arm_smlabt(__a, __b, __c);
333}333}
334static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))334static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
335__smlatb(int32_t __a, int32_t __b, int32_t __c) {335__smlatb(int32_t __a, int32_t __b, int32_t __c) {
336 return __builtin_arm_smlatb(__a, __b, __c);336 return __builtin_arm_smlatb(__a, __b, __c);
337}337}
338static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))338static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
339__smlatt(int32_t __a, int32_t __b, int32_t __c) {339__smlatt(int32_t __a, int32_t __b, int32_t __c) {
340 return __builtin_arm_smlatt(__a, __b, __c);340 return __builtin_arm_smlatt(__a, __b, __c);
341}341}
342static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))342static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
343__smlawb(int32_t __a, int32_t __b, int32_t __c) {343__smlawb(int32_t __a, int32_t __b, int32_t __c) {
344 return __builtin_arm_smlawb(__a, __b, __c);344 return __builtin_arm_smlawb(__a, __b, __c);
345}345}
346static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))346static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
347__smlawt(int32_t __a, int32_t __b, int32_t __c) {347__smlawt(int32_t __a, int32_t __b, int32_t __c) {
348 return __builtin_arm_smlawt(__a, __b, __c);348 return __builtin_arm_smlawt(__a, __b, __c);
349}349}
...@@ -621,8 +621,6 @@ __rintnf(float __a) {...@@ -621,8 +621,6 @@ __rintnf(float __a) {
621#endif621#endif
622622
623/* 8.8 CRC32 intrinsics */623/* 8.8 CRC32 intrinsics */
624#if (defined(__ARM_FEATURE_CRC32) && __ARM_FEATURE_CRC32) || \
625 (defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE)
626static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target("crc")))624static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target("crc")))
627__crc32b(uint32_t __a, uint8_t __b) {625__crc32b(uint32_t __a, uint8_t __b) {
628 return __builtin_arm_crc32b(__a, __b);626 return __builtin_arm_crc32b(__a, __b);
...@@ -662,7 +660,6 @@ static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target...@@ -662,7 +660,6 @@ static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target
662__crc32cd(uint32_t __a, uint64_t __b) {660__crc32cd(uint32_t __a, uint64_t __b) {
663 return __builtin_arm_crc32cd(__a, __b);661 return __builtin_arm_crc32cd(__a, __b);
664}662}
665#endif
666663
667/* 8.6 Floating-point data-processing intrinsics */664/* 8.6 Floating-point data-processing intrinsics */
668/* Armv8.3-A Javascript conversion intrinsic */665/* Armv8.3-A Javascript conversion intrinsic */
lib/include/arm_neon.h+3587-1696
...@@ -359,9 +359,7 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap_vcombine_bf16(bf...@@ -359,9 +359,7 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap_vcombine_bf16(bf
359})359})
360__ai __attribute__((target("bf16,neon"))) float32_t vcvtah_f32_bf16(bfloat16_t __p0) {360__ai __attribute__((target("bf16,neon"))) float32_t vcvtah_f32_bf16(bfloat16_t __p0) {
361 float32_t __ret;361 float32_t __ret;
362bfloat16_t __reint = __p0;362 __ret = __builtin_bit_cast(float32_t, (uint32_t)(__builtin_bit_cast(uint16_t, __p0)) << 16);
363int32_t __reint1 = (int32_t)(*(int16_t *) &__reint) << 16;
364 __ret = *(float32_t *) &__reint1;
365 return __ret;363 return __ret;
366}364}
367__ai __attribute__((target("bf16,neon"))) bfloat16_t vcvth_bf16_f32(float32_t __p0) {365__ai __attribute__((target("bf16,neon"))) bfloat16_t vcvth_bf16_f32(float32_t __p0) {
...@@ -35841,9 +35839,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_f...@@ -35841,9 +35839,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_f
35841 float16x4_t __s0_150 = __p0_150; \35839 float16x4_t __s0_150 = __p0_150; \
35842 float16x4_t __s1_150 = __p1_150; \35840 float16x4_t __s1_150 = __p1_150; \
35843 float16x4_t __s2_150 = __p2_150; \35841 float16x4_t __s2_150 = __p2_150; \
35844float16x4_t __reint_150 = __s2_150; \35842 __ret_150 = vcmla_f16(__s0_150, __s1_150, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_150), __p3_150), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_150), __p3_150)})); \
35845uint32x2_t __reint1_150 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_150, __p3_150), vget_lane_u32(*(uint32x2_t *) &__reint_150, __p3_150)}; \
35846 __ret_150 = vcmla_f16(__s0_150, __s1_150, *(float16x4_t *) &__reint1_150); \
35847 __ret_150; \35843 __ret_150; \
35848})35844})
35849#else35845#else
...@@ -35855,9 +35851,7 @@ uint32x2_t __reint1_150 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -35855,9 +35851,7 @@ uint32x2_t __reint1_150 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
35855 float16x4_t __rev0_151; __rev0_151 = __builtin_shufflevector(__s0_151, __s0_151, 3, 2, 1, 0); \35851 float16x4_t __rev0_151; __rev0_151 = __builtin_shufflevector(__s0_151, __s0_151, 3, 2, 1, 0); \
35856 float16x4_t __rev1_151; __rev1_151 = __builtin_shufflevector(__s1_151, __s1_151, 3, 2, 1, 0); \35852 float16x4_t __rev1_151; __rev1_151 = __builtin_shufflevector(__s1_151, __s1_151, 3, 2, 1, 0); \
35857 float16x4_t __rev2_151; __rev2_151 = __builtin_shufflevector(__s2_151, __s2_151, 3, 2, 1, 0); \35853 float16x4_t __rev2_151; __rev2_151 = __builtin_shufflevector(__s2_151, __s2_151, 3, 2, 1, 0); \
35858float16x4_t __reint_151 = __rev2_151; \35854 __ret_151 = __noswap_vcmla_f16(__rev0_151, __rev1_151, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_151), __p3_151), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_151), __p3_151)})); \
35859uint32x2_t __reint1_151 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_151, __p3_151), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_151, __p3_151)}; \
35860 __ret_151 = __noswap_vcmla_f16(__rev0_151, __rev1_151, *(float16x4_t *) &__reint1_151); \
35861 __ret_151 = __builtin_shufflevector(__ret_151, __ret_151, 3, 2, 1, 0); \35855 __ret_151 = __builtin_shufflevector(__ret_151, __ret_151, 3, 2, 1, 0); \
35862 __ret_151; \35856 __ret_151; \
35863})35857})
...@@ -35869,9 +35863,7 @@ uint32x2_t __reint1_151 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -35869,9 +35863,7 @@ uint32x2_t __reint1_151 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
35869 float16x8_t __s0_152 = __p0_152; \35863 float16x8_t __s0_152 = __p0_152; \
35870 float16x8_t __s1_152 = __p1_152; \35864 float16x8_t __s1_152 = __p1_152; \
35871 float16x4_t __s2_152 = __p2_152; \35865 float16x4_t __s2_152 = __p2_152; \
35872float16x4_t __reint_152 = __s2_152; \35866 __ret_152 = vcmlaq_f16(__s0_152, __s1_152, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152)})); \
35873uint32x4_t __reint1_152 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152), vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152), vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152), vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152)}; \
35874 __ret_152 = vcmlaq_f16(__s0_152, __s1_152, *(float16x8_t *) &__reint1_152); \
35875 __ret_152; \35867 __ret_152; \
35876})35868})
35877#else35869#else
...@@ -35883,9 +35875,7 @@ uint32x4_t __reint1_152 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -35883,9 +35875,7 @@ uint32x4_t __reint1_152 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
35883 float16x8_t __rev0_153; __rev0_153 = __builtin_shufflevector(__s0_153, __s0_153, 7, 6, 5, 4, 3, 2, 1, 0); \35875 float16x8_t __rev0_153; __rev0_153 = __builtin_shufflevector(__s0_153, __s0_153, 7, 6, 5, 4, 3, 2, 1, 0); \
35884 float16x8_t __rev1_153; __rev1_153 = __builtin_shufflevector(__s1_153, __s1_153, 7, 6, 5, 4, 3, 2, 1, 0); \35876 float16x8_t __rev1_153; __rev1_153 = __builtin_shufflevector(__s1_153, __s1_153, 7, 6, 5, 4, 3, 2, 1, 0); \
35885 float16x4_t __rev2_153; __rev2_153 = __builtin_shufflevector(__s2_153, __s2_153, 3, 2, 1, 0); \35877 float16x4_t __rev2_153; __rev2_153 = __builtin_shufflevector(__s2_153, __s2_153, 3, 2, 1, 0); \
35886float16x4_t __reint_153 = __rev2_153; \35878 __ret_153 = __noswap_vcmlaq_f16(__rev0_153, __rev1_153, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153)})); \
35887uint32x4_t __reint1_153 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153)}; \
35888 __ret_153 = __noswap_vcmlaq_f16(__rev0_153, __rev1_153, *(float16x8_t *) &__reint1_153); \
35889 __ret_153 = __builtin_shufflevector(__ret_153, __ret_153, 7, 6, 5, 4, 3, 2, 1, 0); \35879 __ret_153 = __builtin_shufflevector(__ret_153, __ret_153, 7, 6, 5, 4, 3, 2, 1, 0); \
35890 __ret_153; \35880 __ret_153; \
35891})35881})
...@@ -35897,9 +35887,7 @@ uint32x4_t __reint1_153 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -35897,9 +35887,7 @@ uint32x4_t __reint1_153 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
35897 float16x4_t __s0_154 = __p0_154; \35887 float16x4_t __s0_154 = __p0_154; \
35898 float16x4_t __s1_154 = __p1_154; \35888 float16x4_t __s1_154 = __p1_154; \
35899 float16x8_t __s2_154 = __p2_154; \35889 float16x8_t __s2_154 = __p2_154; \
35900float16x8_t __reint_154 = __s2_154; \35890 __ret_154 = vcmla_f16(__s0_154, __s1_154, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_154), __p3_154), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_154), __p3_154)})); \
35901uint32x2_t __reint1_154 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_154, __p3_154), vgetq_lane_u32(*(uint32x4_t *) &__reint_154, __p3_154)}; \
35902 __ret_154 = vcmla_f16(__s0_154, __s1_154, *(float16x4_t *) &__reint1_154); \
35903 __ret_154; \35891 __ret_154; \
35904})35892})
35905#else35893#else
...@@ -35911,9 +35899,7 @@ uint32x2_t __reint1_154 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -35911,9 +35899,7 @@ uint32x2_t __reint1_154 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
35911 float16x4_t __rev0_155; __rev0_155 = __builtin_shufflevector(__s0_155, __s0_155, 3, 2, 1, 0); \35899 float16x4_t __rev0_155; __rev0_155 = __builtin_shufflevector(__s0_155, __s0_155, 3, 2, 1, 0); \
35912 float16x4_t __rev1_155; __rev1_155 = __builtin_shufflevector(__s1_155, __s1_155, 3, 2, 1, 0); \35900 float16x4_t __rev1_155; __rev1_155 = __builtin_shufflevector(__s1_155, __s1_155, 3, 2, 1, 0); \
35913 float16x8_t __rev2_155; __rev2_155 = __builtin_shufflevector(__s2_155, __s2_155, 7, 6, 5, 4, 3, 2, 1, 0); \35901 float16x8_t __rev2_155; __rev2_155 = __builtin_shufflevector(__s2_155, __s2_155, 7, 6, 5, 4, 3, 2, 1, 0); \
35914float16x8_t __reint_155 = __rev2_155; \35902 __ret_155 = __noswap_vcmla_f16(__rev0_155, __rev1_155, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_155), __p3_155), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_155), __p3_155)})); \
35915uint32x2_t __reint1_155 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_155, __p3_155), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_155, __p3_155)}; \
35916 __ret_155 = __noswap_vcmla_f16(__rev0_155, __rev1_155, *(float16x4_t *) &__reint1_155); \
35917 __ret_155 = __builtin_shufflevector(__ret_155, __ret_155, 3, 2, 1, 0); \35903 __ret_155 = __builtin_shufflevector(__ret_155, __ret_155, 3, 2, 1, 0); \
35918 __ret_155; \35904 __ret_155; \
35919})35905})
...@@ -35925,9 +35911,7 @@ uint32x2_t __reint1_155 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)...@@ -35925,9 +35911,7 @@ uint32x2_t __reint1_155 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
35925 float16x8_t __s0_156 = __p0_156; \35911 float16x8_t __s0_156 = __p0_156; \
35926 float16x8_t __s1_156 = __p1_156; \35912 float16x8_t __s1_156 = __p1_156; \
35927 float16x8_t __s2_156 = __p2_156; \35913 float16x8_t __s2_156 = __p2_156; \
35928float16x8_t __reint_156 = __s2_156; \35914 __ret_156 = vcmlaq_f16(__s0_156, __s1_156, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156)})); \
35929uint32x4_t __reint1_156 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156), vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156), vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156), vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156)}; \
35930 __ret_156 = vcmlaq_f16(__s0_156, __s1_156, *(float16x8_t *) &__reint1_156); \
35931 __ret_156; \35915 __ret_156; \
35932})35916})
35933#else35917#else
...@@ -35939,9 +35923,7 @@ uint32x4_t __reint1_156 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -35939,9 +35923,7 @@ uint32x4_t __reint1_156 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
35939 float16x8_t __rev0_157; __rev0_157 = __builtin_shufflevector(__s0_157, __s0_157, 7, 6, 5, 4, 3, 2, 1, 0); \35923 float16x8_t __rev0_157; __rev0_157 = __builtin_shufflevector(__s0_157, __s0_157, 7, 6, 5, 4, 3, 2, 1, 0); \
35940 float16x8_t __rev1_157; __rev1_157 = __builtin_shufflevector(__s1_157, __s1_157, 7, 6, 5, 4, 3, 2, 1, 0); \35924 float16x8_t __rev1_157; __rev1_157 = __builtin_shufflevector(__s1_157, __s1_157, 7, 6, 5, 4, 3, 2, 1, 0); \
35941 float16x8_t __rev2_157; __rev2_157 = __builtin_shufflevector(__s2_157, __s2_157, 7, 6, 5, 4, 3, 2, 1, 0); \35925 float16x8_t __rev2_157; __rev2_157 = __builtin_shufflevector(__s2_157, __s2_157, 7, 6, 5, 4, 3, 2, 1, 0); \
35942float16x8_t __reint_157 = __rev2_157; \35926 __ret_157 = __noswap_vcmlaq_f16(__rev0_157, __rev1_157, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157)})); \
35943uint32x4_t __reint1_157 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157)}; \
35944 __ret_157 = __noswap_vcmlaq_f16(__rev0_157, __rev1_157, *(float16x8_t *) &__reint1_157); \
35945 __ret_157 = __builtin_shufflevector(__ret_157, __ret_157, 7, 6, 5, 4, 3, 2, 1, 0); \35927 __ret_157 = __builtin_shufflevector(__ret_157, __ret_157, 7, 6, 5, 4, 3, 2, 1, 0); \
35946 __ret_157; \35928 __ret_157; \
35947})35929})
...@@ -35999,9 +35981,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r...@@ -35999,9 +35981,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r
35999 float16x4_t __s0_158 = __p0_158; \35981 float16x4_t __s0_158 = __p0_158; \
36000 float16x4_t __s1_158 = __p1_158; \35982 float16x4_t __s1_158 = __p1_158; \
36001 float16x4_t __s2_158 = __p2_158; \35983 float16x4_t __s2_158 = __p2_158; \
36002float16x4_t __reint_158 = __s2_158; \35984 __ret_158 = vcmla_rot180_f16(__s0_158, __s1_158, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_158), __p3_158), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_158), __p3_158)})); \
36003uint32x2_t __reint1_158 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_158, __p3_158), vget_lane_u32(*(uint32x2_t *) &__reint_158, __p3_158)}; \
36004 __ret_158 = vcmla_rot180_f16(__s0_158, __s1_158, *(float16x4_t *) &__reint1_158); \
36005 __ret_158; \35985 __ret_158; \
36006})35986})
36007#else35987#else
...@@ -36013,9 +35993,7 @@ uint32x2_t __reint1_158 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -36013,9 +35993,7 @@ uint32x2_t __reint1_158 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
36013 float16x4_t __rev0_159; __rev0_159 = __builtin_shufflevector(__s0_159, __s0_159, 3, 2, 1, 0); \35993 float16x4_t __rev0_159; __rev0_159 = __builtin_shufflevector(__s0_159, __s0_159, 3, 2, 1, 0); \
36014 float16x4_t __rev1_159; __rev1_159 = __builtin_shufflevector(__s1_159, __s1_159, 3, 2, 1, 0); \35994 float16x4_t __rev1_159; __rev1_159 = __builtin_shufflevector(__s1_159, __s1_159, 3, 2, 1, 0); \
36015 float16x4_t __rev2_159; __rev2_159 = __builtin_shufflevector(__s2_159, __s2_159, 3, 2, 1, 0); \35995 float16x4_t __rev2_159; __rev2_159 = __builtin_shufflevector(__s2_159, __s2_159, 3, 2, 1, 0); \
36016float16x4_t __reint_159 = __rev2_159; \35996 __ret_159 = __noswap_vcmla_rot180_f16(__rev0_159, __rev1_159, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_159), __p3_159), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_159), __p3_159)})); \
36017uint32x2_t __reint1_159 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_159, __p3_159), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_159, __p3_159)}; \
36018 __ret_159 = __noswap_vcmla_rot180_f16(__rev0_159, __rev1_159, *(float16x4_t *) &__reint1_159); \
36019 __ret_159 = __builtin_shufflevector(__ret_159, __ret_159, 3, 2, 1, 0); \35997 __ret_159 = __builtin_shufflevector(__ret_159, __ret_159, 3, 2, 1, 0); \
36020 __ret_159; \35998 __ret_159; \
36021})35999})
...@@ -36027,9 +36005,7 @@ uint32x2_t __reint1_159 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -36027,9 +36005,7 @@ uint32x2_t __reint1_159 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
36027 float16x8_t __s0_160 = __p0_160; \36005 float16x8_t __s0_160 = __p0_160; \
36028 float16x8_t __s1_160 = __p1_160; \36006 float16x8_t __s1_160 = __p1_160; \
36029 float16x4_t __s2_160 = __p2_160; \36007 float16x4_t __s2_160 = __p2_160; \
36030float16x4_t __reint_160 = __s2_160; \36008 __ret_160 = vcmlaq_rot180_f16(__s0_160, __s1_160, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160)})); \
36031uint32x4_t __reint1_160 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160), vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160), vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160), vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160)}; \
36032 __ret_160 = vcmlaq_rot180_f16(__s0_160, __s1_160, *(float16x8_t *) &__reint1_160); \
36033 __ret_160; \36009 __ret_160; \
36034})36010})
36035#else36011#else
...@@ -36041,9 +36017,7 @@ uint32x4_t __reint1_160 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -36041,9 +36017,7 @@ uint32x4_t __reint1_160 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
36041 float16x8_t __rev0_161; __rev0_161 = __builtin_shufflevector(__s0_161, __s0_161, 7, 6, 5, 4, 3, 2, 1, 0); \36017 float16x8_t __rev0_161; __rev0_161 = __builtin_shufflevector(__s0_161, __s0_161, 7, 6, 5, 4, 3, 2, 1, 0); \
36042 float16x8_t __rev1_161; __rev1_161 = __builtin_shufflevector(__s1_161, __s1_161, 7, 6, 5, 4, 3, 2, 1, 0); \36018 float16x8_t __rev1_161; __rev1_161 = __builtin_shufflevector(__s1_161, __s1_161, 7, 6, 5, 4, 3, 2, 1, 0); \
36043 float16x4_t __rev2_161; __rev2_161 = __builtin_shufflevector(__s2_161, __s2_161, 3, 2, 1, 0); \36019 float16x4_t __rev2_161; __rev2_161 = __builtin_shufflevector(__s2_161, __s2_161, 3, 2, 1, 0); \
36044float16x4_t __reint_161 = __rev2_161; \36020 __ret_161 = __noswap_vcmlaq_rot180_f16(__rev0_161, __rev1_161, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161)})); \
36045uint32x4_t __reint1_161 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161)}; \
36046 __ret_161 = __noswap_vcmlaq_rot180_f16(__rev0_161, __rev1_161, *(float16x8_t *) &__reint1_161); \
36047 __ret_161 = __builtin_shufflevector(__ret_161, __ret_161, 7, 6, 5, 4, 3, 2, 1, 0); \36021 __ret_161 = __builtin_shufflevector(__ret_161, __ret_161, 7, 6, 5, 4, 3, 2, 1, 0); \
36048 __ret_161; \36022 __ret_161; \
36049})36023})
...@@ -36055,9 +36029,7 @@ uint32x4_t __reint1_161 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -36055,9 +36029,7 @@ uint32x4_t __reint1_161 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
36055 float16x4_t __s0_162 = __p0_162; \36029 float16x4_t __s0_162 = __p0_162; \
36056 float16x4_t __s1_162 = __p1_162; \36030 float16x4_t __s1_162 = __p1_162; \
36057 float16x8_t __s2_162 = __p2_162; \36031 float16x8_t __s2_162 = __p2_162; \
36058float16x8_t __reint_162 = __s2_162; \36032 __ret_162 = vcmla_rot180_f16(__s0_162, __s1_162, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_162), __p3_162), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_162), __p3_162)})); \
36059uint32x2_t __reint1_162 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_162, __p3_162), vgetq_lane_u32(*(uint32x4_t *) &__reint_162, __p3_162)}; \
36060 __ret_162 = vcmla_rot180_f16(__s0_162, __s1_162, *(float16x4_t *) &__reint1_162); \
36061 __ret_162; \36033 __ret_162; \
36062})36034})
36063#else36035#else
...@@ -36069,9 +36041,7 @@ uint32x2_t __reint1_162 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -36069,9 +36041,7 @@ uint32x2_t __reint1_162 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
36069 float16x4_t __rev0_163; __rev0_163 = __builtin_shufflevector(__s0_163, __s0_163, 3, 2, 1, 0); \36041 float16x4_t __rev0_163; __rev0_163 = __builtin_shufflevector(__s0_163, __s0_163, 3, 2, 1, 0); \
36070 float16x4_t __rev1_163; __rev1_163 = __builtin_shufflevector(__s1_163, __s1_163, 3, 2, 1, 0); \36042 float16x4_t __rev1_163; __rev1_163 = __builtin_shufflevector(__s1_163, __s1_163, 3, 2, 1, 0); \
36071 float16x8_t __rev2_163; __rev2_163 = __builtin_shufflevector(__s2_163, __s2_163, 7, 6, 5, 4, 3, 2, 1, 0); \36043 float16x8_t __rev2_163; __rev2_163 = __builtin_shufflevector(__s2_163, __s2_163, 7, 6, 5, 4, 3, 2, 1, 0); \
36072float16x8_t __reint_163 = __rev2_163; \36044 __ret_163 = __noswap_vcmla_rot180_f16(__rev0_163, __rev1_163, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_163), __p3_163), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_163), __p3_163)})); \
36073uint32x2_t __reint1_163 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_163, __p3_163), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_163, __p3_163)}; \
36074 __ret_163 = __noswap_vcmla_rot180_f16(__rev0_163, __rev1_163, *(float16x4_t *) &__reint1_163); \
36075 __ret_163 = __builtin_shufflevector(__ret_163, __ret_163, 3, 2, 1, 0); \36045 __ret_163 = __builtin_shufflevector(__ret_163, __ret_163, 3, 2, 1, 0); \
36076 __ret_163; \36046 __ret_163; \
36077})36047})
...@@ -36083,9 +36053,7 @@ uint32x2_t __reint1_163 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)...@@ -36083,9 +36053,7 @@ uint32x2_t __reint1_163 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
36083 float16x8_t __s0_164 = __p0_164; \36053 float16x8_t __s0_164 = __p0_164; \
36084 float16x8_t __s1_164 = __p1_164; \36054 float16x8_t __s1_164 = __p1_164; \
36085 float16x8_t __s2_164 = __p2_164; \36055 float16x8_t __s2_164 = __p2_164; \
36086float16x8_t __reint_164 = __s2_164; \36056 __ret_164 = vcmlaq_rot180_f16(__s0_164, __s1_164, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164)})); \
36087uint32x4_t __reint1_164 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164), vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164), vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164), vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164)}; \
36088 __ret_164 = vcmlaq_rot180_f16(__s0_164, __s1_164, *(float16x8_t *) &__reint1_164); \
36089 __ret_164; \36057 __ret_164; \
36090})36058})
36091#else36059#else
...@@ -36097,9 +36065,7 @@ uint32x4_t __reint1_164 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -36097,9 +36065,7 @@ uint32x4_t __reint1_164 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
36097 float16x8_t __rev0_165; __rev0_165 = __builtin_shufflevector(__s0_165, __s0_165, 7, 6, 5, 4, 3, 2, 1, 0); \36065 float16x8_t __rev0_165; __rev0_165 = __builtin_shufflevector(__s0_165, __s0_165, 7, 6, 5, 4, 3, 2, 1, 0); \
36098 float16x8_t __rev1_165; __rev1_165 = __builtin_shufflevector(__s1_165, __s1_165, 7, 6, 5, 4, 3, 2, 1, 0); \36066 float16x8_t __rev1_165; __rev1_165 = __builtin_shufflevector(__s1_165, __s1_165, 7, 6, 5, 4, 3, 2, 1, 0); \
36099 float16x8_t __rev2_165; __rev2_165 = __builtin_shufflevector(__s2_165, __s2_165, 7, 6, 5, 4, 3, 2, 1, 0); \36067 float16x8_t __rev2_165; __rev2_165 = __builtin_shufflevector(__s2_165, __s2_165, 7, 6, 5, 4, 3, 2, 1, 0); \
36100float16x8_t __reint_165 = __rev2_165; \36068 __ret_165 = __noswap_vcmlaq_rot180_f16(__rev0_165, __rev1_165, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165)})); \
36101uint32x4_t __reint1_165 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165)}; \
36102 __ret_165 = __noswap_vcmlaq_rot180_f16(__rev0_165, __rev1_165, *(float16x8_t *) &__reint1_165); \
36103 __ret_165 = __builtin_shufflevector(__ret_165, __ret_165, 7, 6, 5, 4, 3, 2, 1, 0); \36069 __ret_165 = __builtin_shufflevector(__ret_165, __ret_165, 7, 6, 5, 4, 3, 2, 1, 0); \
36104 __ret_165; \36070 __ret_165; \
36105})36071})
...@@ -36157,9 +36123,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r...@@ -36157,9 +36123,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r
36157 float16x4_t __s0_166 = __p0_166; \36123 float16x4_t __s0_166 = __p0_166; \
36158 float16x4_t __s1_166 = __p1_166; \36124 float16x4_t __s1_166 = __p1_166; \
36159 float16x4_t __s2_166 = __p2_166; \36125 float16x4_t __s2_166 = __p2_166; \
36160float16x4_t __reint_166 = __s2_166; \36126 __ret_166 = vcmla_rot270_f16(__s0_166, __s1_166, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_166), __p3_166), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_166), __p3_166)})); \
36161uint32x2_t __reint1_166 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_166, __p3_166), vget_lane_u32(*(uint32x2_t *) &__reint_166, __p3_166)}; \
36162 __ret_166 = vcmla_rot270_f16(__s0_166, __s1_166, *(float16x4_t *) &__reint1_166); \
36163 __ret_166; \36127 __ret_166; \
36164})36128})
36165#else36129#else
...@@ -36171,9 +36135,7 @@ uint32x2_t __reint1_166 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -36171,9 +36135,7 @@ uint32x2_t __reint1_166 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
36171 float16x4_t __rev0_167; __rev0_167 = __builtin_shufflevector(__s0_167, __s0_167, 3, 2, 1, 0); \36135 float16x4_t __rev0_167; __rev0_167 = __builtin_shufflevector(__s0_167, __s0_167, 3, 2, 1, 0); \
36172 float16x4_t __rev1_167; __rev1_167 = __builtin_shufflevector(__s1_167, __s1_167, 3, 2, 1, 0); \36136 float16x4_t __rev1_167; __rev1_167 = __builtin_shufflevector(__s1_167, __s1_167, 3, 2, 1, 0); \
36173 float16x4_t __rev2_167; __rev2_167 = __builtin_shufflevector(__s2_167, __s2_167, 3, 2, 1, 0); \36137 float16x4_t __rev2_167; __rev2_167 = __builtin_shufflevector(__s2_167, __s2_167, 3, 2, 1, 0); \
36174float16x4_t __reint_167 = __rev2_167; \36138 __ret_167 = __noswap_vcmla_rot270_f16(__rev0_167, __rev1_167, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_167), __p3_167), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_167), __p3_167)})); \
36175uint32x2_t __reint1_167 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_167, __p3_167), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_167, __p3_167)}; \
36176 __ret_167 = __noswap_vcmla_rot270_f16(__rev0_167, __rev1_167, *(float16x4_t *) &__reint1_167); \
36177 __ret_167 = __builtin_shufflevector(__ret_167, __ret_167, 3, 2, 1, 0); \36139 __ret_167 = __builtin_shufflevector(__ret_167, __ret_167, 3, 2, 1, 0); \
36178 __ret_167; \36140 __ret_167; \
36179})36141})
...@@ -36185,9 +36147,7 @@ uint32x2_t __reint1_167 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -36185,9 +36147,7 @@ uint32x2_t __reint1_167 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
36185 float16x8_t __s0_168 = __p0_168; \36147 float16x8_t __s0_168 = __p0_168; \
36186 float16x8_t __s1_168 = __p1_168; \36148 float16x8_t __s1_168 = __p1_168; \
36187 float16x4_t __s2_168 = __p2_168; \36149 float16x4_t __s2_168 = __p2_168; \
36188float16x4_t __reint_168 = __s2_168; \36150 __ret_168 = vcmlaq_rot270_f16(__s0_168, __s1_168, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168)})); \
36189uint32x4_t __reint1_168 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168), vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168), vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168), vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168)}; \
36190 __ret_168 = vcmlaq_rot270_f16(__s0_168, __s1_168, *(float16x8_t *) &__reint1_168); \
36191 __ret_168; \36151 __ret_168; \
36192})36152})
36193#else36153#else
...@@ -36199,9 +36159,7 @@ uint32x4_t __reint1_168 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -36199,9 +36159,7 @@ uint32x4_t __reint1_168 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
36199 float16x8_t __rev0_169; __rev0_169 = __builtin_shufflevector(__s0_169, __s0_169, 7, 6, 5, 4, 3, 2, 1, 0); \36159 float16x8_t __rev0_169; __rev0_169 = __builtin_shufflevector(__s0_169, __s0_169, 7, 6, 5, 4, 3, 2, 1, 0); \
36200 float16x8_t __rev1_169; __rev1_169 = __builtin_shufflevector(__s1_169, __s1_169, 7, 6, 5, 4, 3, 2, 1, 0); \36160 float16x8_t __rev1_169; __rev1_169 = __builtin_shufflevector(__s1_169, __s1_169, 7, 6, 5, 4, 3, 2, 1, 0); \
36201 float16x4_t __rev2_169; __rev2_169 = __builtin_shufflevector(__s2_169, __s2_169, 3, 2, 1, 0); \36161 float16x4_t __rev2_169; __rev2_169 = __builtin_shufflevector(__s2_169, __s2_169, 3, 2, 1, 0); \
36202float16x4_t __reint_169 = __rev2_169; \36162 __ret_169 = __noswap_vcmlaq_rot270_f16(__rev0_169, __rev1_169, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169)})); \
36203uint32x4_t __reint1_169 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169)}; \
36204 __ret_169 = __noswap_vcmlaq_rot270_f16(__rev0_169, __rev1_169, *(float16x8_t *) &__reint1_169); \
36205 __ret_169 = __builtin_shufflevector(__ret_169, __ret_169, 7, 6, 5, 4, 3, 2, 1, 0); \36163 __ret_169 = __builtin_shufflevector(__ret_169, __ret_169, 7, 6, 5, 4, 3, 2, 1, 0); \
36206 __ret_169; \36164 __ret_169; \
36207})36165})
...@@ -36213,9 +36171,7 @@ uint32x4_t __reint1_169 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -36213,9 +36171,7 @@ uint32x4_t __reint1_169 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
36213 float16x4_t __s0_170 = __p0_170; \36171 float16x4_t __s0_170 = __p0_170; \
36214 float16x4_t __s1_170 = __p1_170; \36172 float16x4_t __s1_170 = __p1_170; \
36215 float16x8_t __s2_170 = __p2_170; \36173 float16x8_t __s2_170 = __p2_170; \
36216float16x8_t __reint_170 = __s2_170; \36174 __ret_170 = vcmla_rot270_f16(__s0_170, __s1_170, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_170), __p3_170), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_170), __p3_170)})); \
36217uint32x2_t __reint1_170 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_170, __p3_170), vgetq_lane_u32(*(uint32x4_t *) &__reint_170, __p3_170)}; \
36218 __ret_170 = vcmla_rot270_f16(__s0_170, __s1_170, *(float16x4_t *) &__reint1_170); \
36219 __ret_170; \36175 __ret_170; \
36220})36176})
36221#else36177#else
...@@ -36227,9 +36183,7 @@ uint32x2_t __reint1_170 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -36227,9 +36183,7 @@ uint32x2_t __reint1_170 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
36227 float16x4_t __rev0_171; __rev0_171 = __builtin_shufflevector(__s0_171, __s0_171, 3, 2, 1, 0); \36183 float16x4_t __rev0_171; __rev0_171 = __builtin_shufflevector(__s0_171, __s0_171, 3, 2, 1, 0); \
36228 float16x4_t __rev1_171; __rev1_171 = __builtin_shufflevector(__s1_171, __s1_171, 3, 2, 1, 0); \36184 float16x4_t __rev1_171; __rev1_171 = __builtin_shufflevector(__s1_171, __s1_171, 3, 2, 1, 0); \
36229 float16x8_t __rev2_171; __rev2_171 = __builtin_shufflevector(__s2_171, __s2_171, 7, 6, 5, 4, 3, 2, 1, 0); \36185 float16x8_t __rev2_171; __rev2_171 = __builtin_shufflevector(__s2_171, __s2_171, 7, 6, 5, 4, 3, 2, 1, 0); \
36230float16x8_t __reint_171 = __rev2_171; \36186 __ret_171 = __noswap_vcmla_rot270_f16(__rev0_171, __rev1_171, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_171), __p3_171), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_171), __p3_171)})); \
36231uint32x2_t __reint1_171 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_171, __p3_171), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_171, __p3_171)}; \
36232 __ret_171 = __noswap_vcmla_rot270_f16(__rev0_171, __rev1_171, *(float16x4_t *) &__reint1_171); \
36233 __ret_171 = __builtin_shufflevector(__ret_171, __ret_171, 3, 2, 1, 0); \36187 __ret_171 = __builtin_shufflevector(__ret_171, __ret_171, 3, 2, 1, 0); \
36234 __ret_171; \36188 __ret_171; \
36235})36189})
...@@ -36241,9 +36195,7 @@ uint32x2_t __reint1_171 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)...@@ -36241,9 +36195,7 @@ uint32x2_t __reint1_171 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
36241 float16x8_t __s0_172 = __p0_172; \36195 float16x8_t __s0_172 = __p0_172; \
36242 float16x8_t __s1_172 = __p1_172; \36196 float16x8_t __s1_172 = __p1_172; \
36243 float16x8_t __s2_172 = __p2_172; \36197 float16x8_t __s2_172 = __p2_172; \
36244float16x8_t __reint_172 = __s2_172; \36198 __ret_172 = vcmlaq_rot270_f16(__s0_172, __s1_172, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172)})); \
36245uint32x4_t __reint1_172 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172), vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172), vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172), vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172)}; \
36246 __ret_172 = vcmlaq_rot270_f16(__s0_172, __s1_172, *(float16x8_t *) &__reint1_172); \
36247 __ret_172; \36199 __ret_172; \
36248})36200})
36249#else36201#else
...@@ -36255,9 +36207,7 @@ uint32x4_t __reint1_172 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -36255,9 +36207,7 @@ uint32x4_t __reint1_172 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
36255 float16x8_t __rev0_173; __rev0_173 = __builtin_shufflevector(__s0_173, __s0_173, 7, 6, 5, 4, 3, 2, 1, 0); \36207 float16x8_t __rev0_173; __rev0_173 = __builtin_shufflevector(__s0_173, __s0_173, 7, 6, 5, 4, 3, 2, 1, 0); \
36256 float16x8_t __rev1_173; __rev1_173 = __builtin_shufflevector(__s1_173, __s1_173, 7, 6, 5, 4, 3, 2, 1, 0); \36208 float16x8_t __rev1_173; __rev1_173 = __builtin_shufflevector(__s1_173, __s1_173, 7, 6, 5, 4, 3, 2, 1, 0); \
36257 float16x8_t __rev2_173; __rev2_173 = __builtin_shufflevector(__s2_173, __s2_173, 7, 6, 5, 4, 3, 2, 1, 0); \36209 float16x8_t __rev2_173; __rev2_173 = __builtin_shufflevector(__s2_173, __s2_173, 7, 6, 5, 4, 3, 2, 1, 0); \
36258float16x8_t __reint_173 = __rev2_173; \36210 __ret_173 = __noswap_vcmlaq_rot270_f16(__rev0_173, __rev1_173, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173)})); \
36259uint32x4_t __reint1_173 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173)}; \
36260 __ret_173 = __noswap_vcmlaq_rot270_f16(__rev0_173, __rev1_173, *(float16x8_t *) &__reint1_173); \
36261 __ret_173 = __builtin_shufflevector(__ret_173, __ret_173, 7, 6, 5, 4, 3, 2, 1, 0); \36211 __ret_173 = __builtin_shufflevector(__ret_173, __ret_173, 7, 6, 5, 4, 3, 2, 1, 0); \
36262 __ret_173; \36212 __ret_173; \
36263})36213})
...@@ -36315,9 +36265,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r...@@ -36315,9 +36265,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r
36315 float16x4_t __s0_174 = __p0_174; \36265 float16x4_t __s0_174 = __p0_174; \
36316 float16x4_t __s1_174 = __p1_174; \36266 float16x4_t __s1_174 = __p1_174; \
36317 float16x4_t __s2_174 = __p2_174; \36267 float16x4_t __s2_174 = __p2_174; \
36318float16x4_t __reint_174 = __s2_174; \36268 __ret_174 = vcmla_rot90_f16(__s0_174, __s1_174, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_174), __p3_174), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_174), __p3_174)})); \
36319uint32x2_t __reint1_174 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_174, __p3_174), vget_lane_u32(*(uint32x2_t *) &__reint_174, __p3_174)}; \
36320 __ret_174 = vcmla_rot90_f16(__s0_174, __s1_174, *(float16x4_t *) &__reint1_174); \
36321 __ret_174; \36269 __ret_174; \
36322})36270})
36323#else36271#else
...@@ -36329,9 +36277,7 @@ uint32x2_t __reint1_174 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -36329,9 +36277,7 @@ uint32x2_t __reint1_174 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
36329 float16x4_t __rev0_175; __rev0_175 = __builtin_shufflevector(__s0_175, __s0_175, 3, 2, 1, 0); \36277 float16x4_t __rev0_175; __rev0_175 = __builtin_shufflevector(__s0_175, __s0_175, 3, 2, 1, 0); \
36330 float16x4_t __rev1_175; __rev1_175 = __builtin_shufflevector(__s1_175, __s1_175, 3, 2, 1, 0); \36278 float16x4_t __rev1_175; __rev1_175 = __builtin_shufflevector(__s1_175, __s1_175, 3, 2, 1, 0); \
36331 float16x4_t __rev2_175; __rev2_175 = __builtin_shufflevector(__s2_175, __s2_175, 3, 2, 1, 0); \36279 float16x4_t __rev2_175; __rev2_175 = __builtin_shufflevector(__s2_175, __s2_175, 3, 2, 1, 0); \
36332float16x4_t __reint_175 = __rev2_175; \36280 __ret_175 = __noswap_vcmla_rot90_f16(__rev0_175, __rev1_175, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_175), __p3_175), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_175), __p3_175)})); \
36333uint32x2_t __reint1_175 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_175, __p3_175), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_175, __p3_175)}; \
36334 __ret_175 = __noswap_vcmla_rot90_f16(__rev0_175, __rev1_175, *(float16x4_t *) &__reint1_175); \
36335 __ret_175 = __builtin_shufflevector(__ret_175, __ret_175, 3, 2, 1, 0); \36281 __ret_175 = __builtin_shufflevector(__ret_175, __ret_175, 3, 2, 1, 0); \
36336 __ret_175; \36282 __ret_175; \
36337})36283})
...@@ -36343,9 +36289,7 @@ uint32x2_t __reint1_175 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -36343,9 +36289,7 @@ uint32x2_t __reint1_175 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
36343 float16x8_t __s0_176 = __p0_176; \36289 float16x8_t __s0_176 = __p0_176; \
36344 float16x8_t __s1_176 = __p1_176; \36290 float16x8_t __s1_176 = __p1_176; \
36345 float16x4_t __s2_176 = __p2_176; \36291 float16x4_t __s2_176 = __p2_176; \
36346float16x4_t __reint_176 = __s2_176; \36292 __ret_176 = vcmlaq_rot90_f16(__s0_176, __s1_176, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176)})); \
36347uint32x4_t __reint1_176 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176), vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176), vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176), vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176)}; \
36348 __ret_176 = vcmlaq_rot90_f16(__s0_176, __s1_176, *(float16x8_t *) &__reint1_176); \
36349 __ret_176; \36293 __ret_176; \
36350})36294})
36351#else36295#else
...@@ -36357,9 +36301,7 @@ uint32x4_t __reint1_176 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1...@@ -36357,9 +36301,7 @@ uint32x4_t __reint1_176 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
36357 float16x8_t __rev0_177; __rev0_177 = __builtin_shufflevector(__s0_177, __s0_177, 7, 6, 5, 4, 3, 2, 1, 0); \36301 float16x8_t __rev0_177; __rev0_177 = __builtin_shufflevector(__s0_177, __s0_177, 7, 6, 5, 4, 3, 2, 1, 0); \
36358 float16x8_t __rev1_177; __rev1_177 = __builtin_shufflevector(__s1_177, __s1_177, 7, 6, 5, 4, 3, 2, 1, 0); \36302 float16x8_t __rev1_177; __rev1_177 = __builtin_shufflevector(__s1_177, __s1_177, 7, 6, 5, 4, 3, 2, 1, 0); \
36359 float16x4_t __rev2_177; __rev2_177 = __builtin_shufflevector(__s2_177, __s2_177, 3, 2, 1, 0); \36303 float16x4_t __rev2_177; __rev2_177 = __builtin_shufflevector(__s2_177, __s2_177, 3, 2, 1, 0); \
36360float16x4_t __reint_177 = __rev2_177; \36304 __ret_177 = __noswap_vcmlaq_rot90_f16(__rev0_177, __rev1_177, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177)})); \
36361uint32x4_t __reint1_177 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177)}; \
36362 __ret_177 = __noswap_vcmlaq_rot90_f16(__rev0_177, __rev1_177, *(float16x8_t *) &__reint1_177); \
36363 __ret_177 = __builtin_shufflevector(__ret_177, __ret_177, 7, 6, 5, 4, 3, 2, 1, 0); \36305 __ret_177 = __builtin_shufflevector(__ret_177, __ret_177, 7, 6, 5, 4, 3, 2, 1, 0); \
36364 __ret_177; \36306 __ret_177; \
36365})36307})
...@@ -36371,9 +36313,7 @@ uint32x4_t __reint1_177 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &...@@ -36371,9 +36313,7 @@ uint32x4_t __reint1_177 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
36371 float16x4_t __s0_178 = __p0_178; \36313 float16x4_t __s0_178 = __p0_178; \
36372 float16x4_t __s1_178 = __p1_178; \36314 float16x4_t __s1_178 = __p1_178; \
36373 float16x8_t __s2_178 = __p2_178; \36315 float16x8_t __s2_178 = __p2_178; \
36374float16x8_t __reint_178 = __s2_178; \36316 __ret_178 = vcmla_rot90_f16(__s0_178, __s1_178, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_178), __p3_178), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_178), __p3_178)})); \
36375uint32x2_t __reint1_178 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_178, __p3_178), vgetq_lane_u32(*(uint32x4_t *) &__reint_178, __p3_178)}; \
36376 __ret_178 = vcmla_rot90_f16(__s0_178, __s1_178, *(float16x4_t *) &__reint1_178); \
36377 __ret_178; \36317 __ret_178; \
36378})36318})
36379#else36319#else
...@@ -36385,9 +36325,7 @@ uint32x2_t __reint1_178 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -36385,9 +36325,7 @@ uint32x2_t __reint1_178 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
36385 float16x4_t __rev0_179; __rev0_179 = __builtin_shufflevector(__s0_179, __s0_179, 3, 2, 1, 0); \36325 float16x4_t __rev0_179; __rev0_179 = __builtin_shufflevector(__s0_179, __s0_179, 3, 2, 1, 0); \
36386 float16x4_t __rev1_179; __rev1_179 = __builtin_shufflevector(__s1_179, __s1_179, 3, 2, 1, 0); \36326 float16x4_t __rev1_179; __rev1_179 = __builtin_shufflevector(__s1_179, __s1_179, 3, 2, 1, 0); \
36387 float16x8_t __rev2_179; __rev2_179 = __builtin_shufflevector(__s2_179, __s2_179, 7, 6, 5, 4, 3, 2, 1, 0); \36327 float16x8_t __rev2_179; __rev2_179 = __builtin_shufflevector(__s2_179, __s2_179, 7, 6, 5, 4, 3, 2, 1, 0); \
36388float16x8_t __reint_179 = __rev2_179; \36328 __ret_179 = __noswap_vcmla_rot90_f16(__rev0_179, __rev1_179, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_179), __p3_179), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_179), __p3_179)})); \
36389uint32x2_t __reint1_179 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_179, __p3_179), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_179, __p3_179)}; \
36390 __ret_179 = __noswap_vcmla_rot90_f16(__rev0_179, __rev1_179, *(float16x4_t *) &__reint1_179); \
36391 __ret_179 = __builtin_shufflevector(__ret_179, __ret_179, 3, 2, 1, 0); \36329 __ret_179 = __builtin_shufflevector(__ret_179, __ret_179, 3, 2, 1, 0); \
36392 __ret_179; \36330 __ret_179; \
36393})36331})
...@@ -36399,9 +36337,7 @@ uint32x2_t __reint1_179 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)...@@ -36399,9 +36337,7 @@ uint32x2_t __reint1_179 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
36399 float16x8_t __s0_180 = __p0_180; \36337 float16x8_t __s0_180 = __p0_180; \
36400 float16x8_t __s1_180 = __p1_180; \36338 float16x8_t __s1_180 = __p1_180; \
36401 float16x8_t __s2_180 = __p2_180; \36339 float16x8_t __s2_180 = __p2_180; \
36402float16x8_t __reint_180 = __s2_180; \36340 __ret_180 = vcmlaq_rot90_f16(__s0_180, __s1_180, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180)})); \
36403uint32x4_t __reint1_180 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180), vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180), vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180), vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180)}; \
36404 __ret_180 = vcmlaq_rot90_f16(__s0_180, __s1_180, *(float16x8_t *) &__reint1_180); \
36405 __ret_180; \36341 __ret_180; \
36406})36342})
36407#else36343#else
...@@ -36413,9 +36349,7 @@ uint32x4_t __reint1_180 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_...@@ -36413,9 +36349,7 @@ uint32x4_t __reint1_180 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
36413 float16x8_t __rev0_181; __rev0_181 = __builtin_shufflevector(__s0_181, __s0_181, 7, 6, 5, 4, 3, 2, 1, 0); \36349 float16x8_t __rev0_181; __rev0_181 = __builtin_shufflevector(__s0_181, __s0_181, 7, 6, 5, 4, 3, 2, 1, 0); \
36414 float16x8_t __rev1_181; __rev1_181 = __builtin_shufflevector(__s1_181, __s1_181, 7, 6, 5, 4, 3, 2, 1, 0); \36350 float16x8_t __rev1_181; __rev1_181 = __builtin_shufflevector(__s1_181, __s1_181, 7, 6, 5, 4, 3, 2, 1, 0); \
36415 float16x8_t __rev2_181; __rev2_181 = __builtin_shufflevector(__s2_181, __s2_181, 7, 6, 5, 4, 3, 2, 1, 0); \36351 float16x8_t __rev2_181; __rev2_181 = __builtin_shufflevector(__s2_181, __s2_181, 7, 6, 5, 4, 3, 2, 1, 0); \
36416float16x8_t __reint_181 = __rev2_181; \36352 __ret_181 = __noswap_vcmlaq_rot90_f16(__rev0_181, __rev1_181, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181)})); \
36417uint32x4_t __reint1_181 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181)}; \
36418 __ret_181 = __noswap_vcmlaq_rot90_f16(__rev0_181, __rev1_181, *(float16x8_t *) &__reint1_181); \
36419 __ret_181 = __builtin_shufflevector(__ret_181, __ret_181, 7, 6, 5, 4, 3, 2, 1, 0); \36353 __ret_181 = __builtin_shufflevector(__ret_181, __ret_181, 7, 6, 5, 4, 3, 2, 1, 0); \
36420 __ret_181; \36354 __ret_181; \
36421})36355})
...@@ -36541,9 +36475,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_f32(float3...@@ -36541,9 +36475,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_f32(float3
36541 float32x2_t __s0_182 = __p0_182; \36475 float32x2_t __s0_182 = __p0_182; \
36542 float32x2_t __s1_182 = __p1_182; \36476 float32x2_t __s1_182 = __p1_182; \
36543 float32x2_t __s2_182 = __p2_182; \36477 float32x2_t __s2_182 = __p2_182; \
36544float32x2_t __reint_182 = __s2_182; \36478 __ret_182 = vcmla_f32(__s0_182, __s1_182, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_182), __p3_182)})); \
36545uint64x1_t __reint1_182 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_182, __p3_182)}; \
36546 __ret_182 = vcmla_f32(__s0_182, __s1_182, *(float32x2_t *) &__reint1_182); \
36547 __ret_182; \36479 __ret_182; \
36548})36480})
36549#else36481#else
...@@ -36555,9 +36487,7 @@ uint64x1_t __reint1_182 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36555,9 +36487,7 @@ uint64x1_t __reint1_182 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36555 float32x2_t __rev0_183; __rev0_183 = __builtin_shufflevector(__s0_183, __s0_183, 1, 0); \36487 float32x2_t __rev0_183; __rev0_183 = __builtin_shufflevector(__s0_183, __s0_183, 1, 0); \
36556 float32x2_t __rev1_183; __rev1_183 = __builtin_shufflevector(__s1_183, __s1_183, 1, 0); \36488 float32x2_t __rev1_183; __rev1_183 = __builtin_shufflevector(__s1_183, __s1_183, 1, 0); \
36557 float32x2_t __rev2_183; __rev2_183 = __builtin_shufflevector(__s2_183, __s2_183, 1, 0); \36489 float32x2_t __rev2_183; __rev2_183 = __builtin_shufflevector(__s2_183, __s2_183, 1, 0); \
36558float32x2_t __reint_183 = __rev2_183; \36490 __ret_183 = __noswap_vcmla_f32(__rev0_183, __rev1_183, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_183), __p3_183)})); \
36559uint64x1_t __reint1_183 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_183, __p3_183)}; \
36560 __ret_183 = __noswap_vcmla_f32(__rev0_183, __rev1_183, *(float32x2_t *) &__reint1_183); \
36561 __ret_183 = __builtin_shufflevector(__ret_183, __ret_183, 1, 0); \36491 __ret_183 = __builtin_shufflevector(__ret_183, __ret_183, 1, 0); \
36562 __ret_183; \36492 __ret_183; \
36563})36493})
...@@ -36569,9 +36499,7 @@ uint64x1_t __reint1_183 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36569,9 +36499,7 @@ uint64x1_t __reint1_183 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36569 float32x4_t __s0_184 = __p0_184; \36499 float32x4_t __s0_184 = __p0_184; \
36570 float32x4_t __s1_184 = __p1_184; \36500 float32x4_t __s1_184 = __p1_184; \
36571 float32x2_t __s2_184 = __p2_184; \36501 float32x2_t __s2_184 = __p2_184; \
36572float32x2_t __reint_184 = __s2_184; \36502 __ret_184 = vcmlaq_f32(__s0_184, __s1_184, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_184), __p3_184), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_184), __p3_184)})); \
36573uint64x2_t __reint1_184 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_184, __p3_184), vget_lane_u64(*(uint64x1_t *) &__reint_184, __p3_184)}; \
36574 __ret_184 = vcmlaq_f32(__s0_184, __s1_184, *(float32x4_t *) &__reint1_184); \
36575 __ret_184; \36503 __ret_184; \
36576})36504})
36577#else36505#else
...@@ -36583,9 +36511,7 @@ uint64x2_t __reint1_184 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36583,9 +36511,7 @@ uint64x2_t __reint1_184 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36583 float32x4_t __rev0_185; __rev0_185 = __builtin_shufflevector(__s0_185, __s0_185, 3, 2, 1, 0); \36511 float32x4_t __rev0_185; __rev0_185 = __builtin_shufflevector(__s0_185, __s0_185, 3, 2, 1, 0); \
36584 float32x4_t __rev1_185; __rev1_185 = __builtin_shufflevector(__s1_185, __s1_185, 3, 2, 1, 0); \36512 float32x4_t __rev1_185; __rev1_185 = __builtin_shufflevector(__s1_185, __s1_185, 3, 2, 1, 0); \
36585 float32x2_t __rev2_185; __rev2_185 = __builtin_shufflevector(__s2_185, __s2_185, 1, 0); \36513 float32x2_t __rev2_185; __rev2_185 = __builtin_shufflevector(__s2_185, __s2_185, 1, 0); \
36586float32x2_t __reint_185 = __rev2_185; \36514 __ret_185 = __noswap_vcmlaq_f32(__rev0_185, __rev1_185, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_185), __p3_185), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_185), __p3_185)})); \
36587uint64x2_t __reint1_185 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_185, __p3_185), vget_lane_u64(*(uint64x1_t *) &__reint_185, __p3_185)}; \
36588 __ret_185 = __noswap_vcmlaq_f32(__rev0_185, __rev1_185, *(float32x4_t *) &__reint1_185); \
36589 __ret_185 = __builtin_shufflevector(__ret_185, __ret_185, 3, 2, 1, 0); \36515 __ret_185 = __builtin_shufflevector(__ret_185, __ret_185, 3, 2, 1, 0); \
36590 __ret_185; \36516 __ret_185; \
36591})36517})
...@@ -36597,9 +36523,7 @@ uint64x2_t __reint1_185 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36597,9 +36523,7 @@ uint64x2_t __reint1_185 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36597 float32x2_t __s0_186 = __p0_186; \36523 float32x2_t __s0_186 = __p0_186; \
36598 float32x2_t __s1_186 = __p1_186; \36524 float32x2_t __s1_186 = __p1_186; \
36599 float32x4_t __s2_186 = __p2_186; \36525 float32x4_t __s2_186 = __p2_186; \
36600float32x4_t __reint_186 = __s2_186; \36526 __ret_186 = vcmla_f32(__s0_186, __s1_186, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_186), __p3_186)})); \
36601uint64x1_t __reint1_186 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_186, __p3_186)}; \
36602 __ret_186 = vcmla_f32(__s0_186, __s1_186, *(float32x2_t *) &__reint1_186); \
36603 __ret_186; \36527 __ret_186; \
36604})36528})
36605#else36529#else
...@@ -36611,9 +36535,7 @@ uint64x1_t __reint1_186 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -36611,9 +36535,7 @@ uint64x1_t __reint1_186 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
36611 float32x2_t __rev0_187; __rev0_187 = __builtin_shufflevector(__s0_187, __s0_187, 1, 0); \36535 float32x2_t __rev0_187; __rev0_187 = __builtin_shufflevector(__s0_187, __s0_187, 1, 0); \
36612 float32x2_t __rev1_187; __rev1_187 = __builtin_shufflevector(__s1_187, __s1_187, 1, 0); \36536 float32x2_t __rev1_187; __rev1_187 = __builtin_shufflevector(__s1_187, __s1_187, 1, 0); \
36613 float32x4_t __rev2_187; __rev2_187 = __builtin_shufflevector(__s2_187, __s2_187, 3, 2, 1, 0); \36537 float32x4_t __rev2_187; __rev2_187 = __builtin_shufflevector(__s2_187, __s2_187, 3, 2, 1, 0); \
36614float32x4_t __reint_187 = __rev2_187; \36538 __ret_187 = __noswap_vcmla_f32(__rev0_187, __rev1_187, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_187), __p3_187)})); \
36615uint64x1_t __reint1_187 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_187, __p3_187)}; \
36616 __ret_187 = __noswap_vcmla_f32(__rev0_187, __rev1_187, *(float32x2_t *) &__reint1_187); \
36617 __ret_187 = __builtin_shufflevector(__ret_187, __ret_187, 1, 0); \36539 __ret_187 = __builtin_shufflevector(__ret_187, __ret_187, 1, 0); \
36618 __ret_187; \36540 __ret_187; \
36619})36541})
...@@ -36625,9 +36547,7 @@ uint64x1_t __reint1_187 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)...@@ -36625,9 +36547,7 @@ uint64x1_t __reint1_187 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
36625 float32x4_t __s0_188 = __p0_188; \36547 float32x4_t __s0_188 = __p0_188; \
36626 float32x4_t __s1_188 = __p1_188; \36548 float32x4_t __s1_188 = __p1_188; \
36627 float32x4_t __s2_188 = __p2_188; \36549 float32x4_t __s2_188 = __p2_188; \
36628float32x4_t __reint_188 = __s2_188; \36550 __ret_188 = vcmlaq_f32(__s0_188, __s1_188, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_188), __p3_188), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_188), __p3_188)})); \
36629uint64x2_t __reint1_188 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_188, __p3_188), vgetq_lane_u64(*(uint64x2_t *) &__reint_188, __p3_188)}; \
36630 __ret_188 = vcmlaq_f32(__s0_188, __s1_188, *(float32x4_t *) &__reint1_188); \
36631 __ret_188; \36551 __ret_188; \
36632})36552})
36633#else36553#else
...@@ -36639,9 +36559,7 @@ uint64x2_t __reint1_188 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -36639,9 +36559,7 @@ uint64x2_t __reint1_188 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
36639 float32x4_t __rev0_189; __rev0_189 = __builtin_shufflevector(__s0_189, __s0_189, 3, 2, 1, 0); \36559 float32x4_t __rev0_189; __rev0_189 = __builtin_shufflevector(__s0_189, __s0_189, 3, 2, 1, 0); \
36640 float32x4_t __rev1_189; __rev1_189 = __builtin_shufflevector(__s1_189, __s1_189, 3, 2, 1, 0); \36560 float32x4_t __rev1_189; __rev1_189 = __builtin_shufflevector(__s1_189, __s1_189, 3, 2, 1, 0); \
36641 float32x4_t __rev2_189; __rev2_189 = __builtin_shufflevector(__s2_189, __s2_189, 3, 2, 1, 0); \36561 float32x4_t __rev2_189; __rev2_189 = __builtin_shufflevector(__s2_189, __s2_189, 3, 2, 1, 0); \
36642float32x4_t __reint_189 = __rev2_189; \36562 __ret_189 = __noswap_vcmlaq_f32(__rev0_189, __rev1_189, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_189), __p3_189), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_189), __p3_189)})); \
36643uint64x2_t __reint1_189 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_189, __p3_189), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_189, __p3_189)}; \
36644 __ret_189 = __noswap_vcmlaq_f32(__rev0_189, __rev1_189, *(float32x4_t *) &__reint1_189); \
36645 __ret_189 = __builtin_shufflevector(__ret_189, __ret_189, 3, 2, 1, 0); \36563 __ret_189 = __builtin_shufflevector(__ret_189, __ret_189, 3, 2, 1, 0); \
36646 __ret_189; \36564 __ret_189; \
36647})36565})
...@@ -36699,9 +36617,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot180_f32...@@ -36699,9 +36617,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot180_f32
36699 float32x2_t __s0_190 = __p0_190; \36617 float32x2_t __s0_190 = __p0_190; \
36700 float32x2_t __s1_190 = __p1_190; \36618 float32x2_t __s1_190 = __p1_190; \
36701 float32x2_t __s2_190 = __p2_190; \36619 float32x2_t __s2_190 = __p2_190; \
36702float32x2_t __reint_190 = __s2_190; \36620 __ret_190 = vcmla_rot180_f32(__s0_190, __s1_190, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_190), __p3_190)})); \
36703uint64x1_t __reint1_190 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_190, __p3_190)}; \
36704 __ret_190 = vcmla_rot180_f32(__s0_190, __s1_190, *(float32x2_t *) &__reint1_190); \
36705 __ret_190; \36621 __ret_190; \
36706})36622})
36707#else36623#else
...@@ -36713,9 +36629,7 @@ uint64x1_t __reint1_190 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36713,9 +36629,7 @@ uint64x1_t __reint1_190 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36713 float32x2_t __rev0_191; __rev0_191 = __builtin_shufflevector(__s0_191, __s0_191, 1, 0); \36629 float32x2_t __rev0_191; __rev0_191 = __builtin_shufflevector(__s0_191, __s0_191, 1, 0); \
36714 float32x2_t __rev1_191; __rev1_191 = __builtin_shufflevector(__s1_191, __s1_191, 1, 0); \36630 float32x2_t __rev1_191; __rev1_191 = __builtin_shufflevector(__s1_191, __s1_191, 1, 0); \
36715 float32x2_t __rev2_191; __rev2_191 = __builtin_shufflevector(__s2_191, __s2_191, 1, 0); \36631 float32x2_t __rev2_191; __rev2_191 = __builtin_shufflevector(__s2_191, __s2_191, 1, 0); \
36716float32x2_t __reint_191 = __rev2_191; \36632 __ret_191 = __noswap_vcmla_rot180_f32(__rev0_191, __rev1_191, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_191), __p3_191)})); \
36717uint64x1_t __reint1_191 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_191, __p3_191)}; \
36718 __ret_191 = __noswap_vcmla_rot180_f32(__rev0_191, __rev1_191, *(float32x2_t *) &__reint1_191); \
36719 __ret_191 = __builtin_shufflevector(__ret_191, __ret_191, 1, 0); \36633 __ret_191 = __builtin_shufflevector(__ret_191, __ret_191, 1, 0); \
36720 __ret_191; \36634 __ret_191; \
36721})36635})
...@@ -36727,9 +36641,7 @@ uint64x1_t __reint1_191 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36727,9 +36641,7 @@ uint64x1_t __reint1_191 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36727 float32x4_t __s0_192 = __p0_192; \36641 float32x4_t __s0_192 = __p0_192; \
36728 float32x4_t __s1_192 = __p1_192; \36642 float32x4_t __s1_192 = __p1_192; \
36729 float32x2_t __s2_192 = __p2_192; \36643 float32x2_t __s2_192 = __p2_192; \
36730float32x2_t __reint_192 = __s2_192; \36644 __ret_192 = vcmlaq_rot180_f32(__s0_192, __s1_192, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_192), __p3_192), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_192), __p3_192)})); \
36731uint64x2_t __reint1_192 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_192, __p3_192), vget_lane_u64(*(uint64x1_t *) &__reint_192, __p3_192)}; \
36732 __ret_192 = vcmlaq_rot180_f32(__s0_192, __s1_192, *(float32x4_t *) &__reint1_192); \
36733 __ret_192; \36645 __ret_192; \
36734})36646})
36735#else36647#else
...@@ -36741,9 +36653,7 @@ uint64x2_t __reint1_192 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36741,9 +36653,7 @@ uint64x2_t __reint1_192 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36741 float32x4_t __rev0_193; __rev0_193 = __builtin_shufflevector(__s0_193, __s0_193, 3, 2, 1, 0); \36653 float32x4_t __rev0_193; __rev0_193 = __builtin_shufflevector(__s0_193, __s0_193, 3, 2, 1, 0); \
36742 float32x4_t __rev1_193; __rev1_193 = __builtin_shufflevector(__s1_193, __s1_193, 3, 2, 1, 0); \36654 float32x4_t __rev1_193; __rev1_193 = __builtin_shufflevector(__s1_193, __s1_193, 3, 2, 1, 0); \
36743 float32x2_t __rev2_193; __rev2_193 = __builtin_shufflevector(__s2_193, __s2_193, 1, 0); \36655 float32x2_t __rev2_193; __rev2_193 = __builtin_shufflevector(__s2_193, __s2_193, 1, 0); \
36744float32x2_t __reint_193 = __rev2_193; \36656 __ret_193 = __noswap_vcmlaq_rot180_f32(__rev0_193, __rev1_193, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_193), __p3_193), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_193), __p3_193)})); \
36745uint64x2_t __reint1_193 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_193, __p3_193), vget_lane_u64(*(uint64x1_t *) &__reint_193, __p3_193)}; \
36746 __ret_193 = __noswap_vcmlaq_rot180_f32(__rev0_193, __rev1_193, *(float32x4_t *) &__reint1_193); \
36747 __ret_193 = __builtin_shufflevector(__ret_193, __ret_193, 3, 2, 1, 0); \36657 __ret_193 = __builtin_shufflevector(__ret_193, __ret_193, 3, 2, 1, 0); \
36748 __ret_193; \36658 __ret_193; \
36749})36659})
...@@ -36755,9 +36665,7 @@ uint64x2_t __reint1_193 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36755,9 +36665,7 @@ uint64x2_t __reint1_193 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36755 float32x2_t __s0_194 = __p0_194; \36665 float32x2_t __s0_194 = __p0_194; \
36756 float32x2_t __s1_194 = __p1_194; \36666 float32x2_t __s1_194 = __p1_194; \
36757 float32x4_t __s2_194 = __p2_194; \36667 float32x4_t __s2_194 = __p2_194; \
36758float32x4_t __reint_194 = __s2_194; \36668 __ret_194 = vcmla_rot180_f32(__s0_194, __s1_194, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_194), __p3_194)})); \
36759uint64x1_t __reint1_194 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_194, __p3_194)}; \
36760 __ret_194 = vcmla_rot180_f32(__s0_194, __s1_194, *(float32x2_t *) &__reint1_194); \
36761 __ret_194; \36669 __ret_194; \
36762})36670})
36763#else36671#else
...@@ -36769,9 +36677,7 @@ uint64x1_t __reint1_194 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -36769,9 +36677,7 @@ uint64x1_t __reint1_194 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
36769 float32x2_t __rev0_195; __rev0_195 = __builtin_shufflevector(__s0_195, __s0_195, 1, 0); \36677 float32x2_t __rev0_195; __rev0_195 = __builtin_shufflevector(__s0_195, __s0_195, 1, 0); \
36770 float32x2_t __rev1_195; __rev1_195 = __builtin_shufflevector(__s1_195, __s1_195, 1, 0); \36678 float32x2_t __rev1_195; __rev1_195 = __builtin_shufflevector(__s1_195, __s1_195, 1, 0); \
36771 float32x4_t __rev2_195; __rev2_195 = __builtin_shufflevector(__s2_195, __s2_195, 3, 2, 1, 0); \36679 float32x4_t __rev2_195; __rev2_195 = __builtin_shufflevector(__s2_195, __s2_195, 3, 2, 1, 0); \
36772float32x4_t __reint_195 = __rev2_195; \36680 __ret_195 = __noswap_vcmla_rot180_f32(__rev0_195, __rev1_195, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_195), __p3_195)})); \
36773uint64x1_t __reint1_195 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_195, __p3_195)}; \
36774 __ret_195 = __noswap_vcmla_rot180_f32(__rev0_195, __rev1_195, *(float32x2_t *) &__reint1_195); \
36775 __ret_195 = __builtin_shufflevector(__ret_195, __ret_195, 1, 0); \36681 __ret_195 = __builtin_shufflevector(__ret_195, __ret_195, 1, 0); \
36776 __ret_195; \36682 __ret_195; \
36777})36683})
...@@ -36783,9 +36689,7 @@ uint64x1_t __reint1_195 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)...@@ -36783,9 +36689,7 @@ uint64x1_t __reint1_195 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
36783 float32x4_t __s0_196 = __p0_196; \36689 float32x4_t __s0_196 = __p0_196; \
36784 float32x4_t __s1_196 = __p1_196; \36690 float32x4_t __s1_196 = __p1_196; \
36785 float32x4_t __s2_196 = __p2_196; \36691 float32x4_t __s2_196 = __p2_196; \
36786float32x4_t __reint_196 = __s2_196; \36692 __ret_196 = vcmlaq_rot180_f32(__s0_196, __s1_196, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_196), __p3_196), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_196), __p3_196)})); \
36787uint64x2_t __reint1_196 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_196, __p3_196), vgetq_lane_u64(*(uint64x2_t *) &__reint_196, __p3_196)}; \
36788 __ret_196 = vcmlaq_rot180_f32(__s0_196, __s1_196, *(float32x4_t *) &__reint1_196); \
36789 __ret_196; \36693 __ret_196; \
36790})36694})
36791#else36695#else
...@@ -36797,9 +36701,7 @@ uint64x2_t __reint1_196 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -36797,9 +36701,7 @@ uint64x2_t __reint1_196 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
36797 float32x4_t __rev0_197; __rev0_197 = __builtin_shufflevector(__s0_197, __s0_197, 3, 2, 1, 0); \36701 float32x4_t __rev0_197; __rev0_197 = __builtin_shufflevector(__s0_197, __s0_197, 3, 2, 1, 0); \
36798 float32x4_t __rev1_197; __rev1_197 = __builtin_shufflevector(__s1_197, __s1_197, 3, 2, 1, 0); \36702 float32x4_t __rev1_197; __rev1_197 = __builtin_shufflevector(__s1_197, __s1_197, 3, 2, 1, 0); \
36799 float32x4_t __rev2_197; __rev2_197 = __builtin_shufflevector(__s2_197, __s2_197, 3, 2, 1, 0); \36703 float32x4_t __rev2_197; __rev2_197 = __builtin_shufflevector(__s2_197, __s2_197, 3, 2, 1, 0); \
36800float32x4_t __reint_197 = __rev2_197; \36704 __ret_197 = __noswap_vcmlaq_rot180_f32(__rev0_197, __rev1_197, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_197), __p3_197), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_197), __p3_197)})); \
36801uint64x2_t __reint1_197 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_197, __p3_197), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_197, __p3_197)}; \
36802 __ret_197 = __noswap_vcmlaq_rot180_f32(__rev0_197, __rev1_197, *(float32x4_t *) &__reint1_197); \
36803 __ret_197 = __builtin_shufflevector(__ret_197, __ret_197, 3, 2, 1, 0); \36705 __ret_197 = __builtin_shufflevector(__ret_197, __ret_197, 3, 2, 1, 0); \
36804 __ret_197; \36706 __ret_197; \
36805})36707})
...@@ -36857,9 +36759,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot270_f32...@@ -36857,9 +36759,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot270_f32
36857 float32x2_t __s0_198 = __p0_198; \36759 float32x2_t __s0_198 = __p0_198; \
36858 float32x2_t __s1_198 = __p1_198; \36760 float32x2_t __s1_198 = __p1_198; \
36859 float32x2_t __s2_198 = __p2_198; \36761 float32x2_t __s2_198 = __p2_198; \
36860float32x2_t __reint_198 = __s2_198; \36762 __ret_198 = vcmla_rot270_f32(__s0_198, __s1_198, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_198), __p3_198)})); \
36861uint64x1_t __reint1_198 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_198, __p3_198)}; \
36862 __ret_198 = vcmla_rot270_f32(__s0_198, __s1_198, *(float32x2_t *) &__reint1_198); \
36863 __ret_198; \36763 __ret_198; \
36864})36764})
36865#else36765#else
...@@ -36871,9 +36771,7 @@ uint64x1_t __reint1_198 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36871,9 +36771,7 @@ uint64x1_t __reint1_198 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36871 float32x2_t __rev0_199; __rev0_199 = __builtin_shufflevector(__s0_199, __s0_199, 1, 0); \36771 float32x2_t __rev0_199; __rev0_199 = __builtin_shufflevector(__s0_199, __s0_199, 1, 0); \
36872 float32x2_t __rev1_199; __rev1_199 = __builtin_shufflevector(__s1_199, __s1_199, 1, 0); \36772 float32x2_t __rev1_199; __rev1_199 = __builtin_shufflevector(__s1_199, __s1_199, 1, 0); \
36873 float32x2_t __rev2_199; __rev2_199 = __builtin_shufflevector(__s2_199, __s2_199, 1, 0); \36773 float32x2_t __rev2_199; __rev2_199 = __builtin_shufflevector(__s2_199, __s2_199, 1, 0); \
36874float32x2_t __reint_199 = __rev2_199; \36774 __ret_199 = __noswap_vcmla_rot270_f32(__rev0_199, __rev1_199, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_199), __p3_199)})); \
36875uint64x1_t __reint1_199 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_199, __p3_199)}; \
36876 __ret_199 = __noswap_vcmla_rot270_f32(__rev0_199, __rev1_199, *(float32x2_t *) &__reint1_199); \
36877 __ret_199 = __builtin_shufflevector(__ret_199, __ret_199, 1, 0); \36775 __ret_199 = __builtin_shufflevector(__ret_199, __ret_199, 1, 0); \
36878 __ret_199; \36776 __ret_199; \
36879})36777})
...@@ -36885,9 +36783,7 @@ uint64x1_t __reint1_199 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1...@@ -36885,9 +36783,7 @@ uint64x1_t __reint1_199 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
36885 float32x4_t __s0_200 = __p0_200; \36783 float32x4_t __s0_200 = __p0_200; \
36886 float32x4_t __s1_200 = __p1_200; \36784 float32x4_t __s1_200 = __p1_200; \
36887 float32x2_t __s2_200 = __p2_200; \36785 float32x2_t __s2_200 = __p2_200; \
36888float32x2_t __reint_200 = __s2_200; \36786 __ret_200 = vcmlaq_rot270_f32(__s0_200, __s1_200, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_200), __p3_200), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_200), __p3_200)})); \
36889uint64x2_t __reint1_200 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_200, __p3_200), vget_lane_u64(*(uint64x1_t *) &__reint_200, __p3_200)}; \
36890 __ret_200 = vcmlaq_rot270_f32(__s0_200, __s1_200, *(float32x4_t *) &__reint1_200); \
36891 __ret_200; \36787 __ret_200; \
36892})36788})
36893#else36789#else
...@@ -36899,9 +36795,7 @@ uint64x2_t __reint1_200 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2...@@ -36899,9 +36795,7 @@ uint64x2_t __reint1_200 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
36899 float32x4_t __rev0_201; __rev0_201 = __builtin_shufflevector(__s0_201, __s0_201, 3, 2, 1, 0); \36795 float32x4_t __rev0_201; __rev0_201 = __builtin_shufflevector(__s0_201, __s0_201, 3, 2, 1, 0); \
36900 float32x4_t __rev1_201; __rev1_201 = __builtin_shufflevector(__s1_201, __s1_201, 3, 2, 1, 0); \36796 float32x4_t __rev1_201; __rev1_201 = __builtin_shufflevector(__s1_201, __s1_201, 3, 2, 1, 0); \
36901 float32x2_t __rev2_201; __rev2_201 = __builtin_shufflevector(__s2_201, __s2_201, 1, 0); \36797 float32x2_t __rev2_201; __rev2_201 = __builtin_shufflevector(__s2_201, __s2_201, 1, 0); \
36902float32x2_t __reint_201 = __rev2_201; \36798 __ret_201 = __noswap_vcmlaq_rot270_f32(__rev0_201, __rev1_201, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_201), __p3_201), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_201), __p3_201)})); \
36903uint64x2_t __reint1_201 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_201, __p3_201), vget_lane_u64(*(uint64x1_t *) &__reint_201, __p3_201)}; \
36904 __ret_201 = __noswap_vcmlaq_rot270_f32(__rev0_201, __rev1_201, *(float32x4_t *) &__reint1_201); \
36905 __ret_201 = __builtin_shufflevector(__ret_201, __ret_201, 3, 2, 1, 0); \36799 __ret_201 = __builtin_shufflevector(__ret_201, __ret_201, 3, 2, 1, 0); \
36906 __ret_201; \36800 __ret_201; \
36907})36801})
...@@ -36913,9 +36807,7 @@ uint64x2_t __reint1_201 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2...@@ -36913,9 +36807,7 @@ uint64x2_t __reint1_201 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
36913 float32x2_t __s0_202 = __p0_202; \36807 float32x2_t __s0_202 = __p0_202; \
36914 float32x2_t __s1_202 = __p1_202; \36808 float32x2_t __s1_202 = __p1_202; \
36915 float32x4_t __s2_202 = __p2_202; \36809 float32x4_t __s2_202 = __p2_202; \
36916float32x4_t __reint_202 = __s2_202; \36810 __ret_202 = vcmla_rot270_f32(__s0_202, __s1_202, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_202), __p3_202)})); \
36917uint64x1_t __reint1_202 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_202, __p3_202)}; \
36918 __ret_202 = vcmla_rot270_f32(__s0_202, __s1_202, *(float32x2_t *) &__reint1_202); \
36919 __ret_202; \36811 __ret_202; \
36920})36812})
36921#else36813#else
...@@ -36927,9 +36819,7 @@ uint64x1_t __reint1_202 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -36927,9 +36819,7 @@ uint64x1_t __reint1_202 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
36927 float32x2_t __rev0_203; __rev0_203 = __builtin_shufflevector(__s0_203, __s0_203, 1, 0); \36819 float32x2_t __rev0_203; __rev0_203 = __builtin_shufflevector(__s0_203, __s0_203, 1, 0); \
36928 float32x2_t __rev1_203; __rev1_203 = __builtin_shufflevector(__s1_203, __s1_203, 1, 0); \36820 float32x2_t __rev1_203; __rev1_203 = __builtin_shufflevector(__s1_203, __s1_203, 1, 0); \
36929 float32x4_t __rev2_203; __rev2_203 = __builtin_shufflevector(__s2_203, __s2_203, 3, 2, 1, 0); \36821 float32x4_t __rev2_203; __rev2_203 = __builtin_shufflevector(__s2_203, __s2_203, 3, 2, 1, 0); \
36930float32x4_t __reint_203 = __rev2_203; \36822 __ret_203 = __noswap_vcmla_rot270_f32(__rev0_203, __rev1_203, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_203), __p3_203)})); \
36931uint64x1_t __reint1_203 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_203, __p3_203)}; \
36932 __ret_203 = __noswap_vcmla_rot270_f32(__rev0_203, __rev1_203, *(float32x2_t *) &__reint1_203); \
36933 __ret_203 = __builtin_shufflevector(__ret_203, __ret_203, 1, 0); \36823 __ret_203 = __builtin_shufflevector(__ret_203, __ret_203, 1, 0); \
36934 __ret_203; \36824 __ret_203; \
36935})36825})
...@@ -36941,9 +36831,7 @@ uint64x1_t __reint1_203 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)...@@ -36941,9 +36831,7 @@ uint64x1_t __reint1_203 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
36941 float32x4_t __s0_204 = __p0_204; \36831 float32x4_t __s0_204 = __p0_204; \
36942 float32x4_t __s1_204 = __p1_204; \36832 float32x4_t __s1_204 = __p1_204; \
36943 float32x4_t __s2_204 = __p2_204; \36833 float32x4_t __s2_204 = __p2_204; \
36944float32x4_t __reint_204 = __s2_204; \36834 __ret_204 = vcmlaq_rot270_f32(__s0_204, __s1_204, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_204), __p3_204), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_204), __p3_204)})); \
36945uint64x2_t __reint1_204 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_204, __p3_204), vgetq_lane_u64(*(uint64x2_t *) &__reint_204, __p3_204)}; \
36946 __ret_204 = vcmlaq_rot270_f32(__s0_204, __s1_204, *(float32x4_t *) &__reint1_204); \
36947 __ret_204; \36835 __ret_204; \
36948})36836})
36949#else36837#else
...@@ -36955,9 +36843,7 @@ uint64x2_t __reint1_204 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -36955,9 +36843,7 @@ uint64x2_t __reint1_204 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
36955 float32x4_t __rev0_205; __rev0_205 = __builtin_shufflevector(__s0_205, __s0_205, 3, 2, 1, 0); \36843 float32x4_t __rev0_205; __rev0_205 = __builtin_shufflevector(__s0_205, __s0_205, 3, 2, 1, 0); \
36956 float32x4_t __rev1_205; __rev1_205 = __builtin_shufflevector(__s1_205, __s1_205, 3, 2, 1, 0); \36844 float32x4_t __rev1_205; __rev1_205 = __builtin_shufflevector(__s1_205, __s1_205, 3, 2, 1, 0); \
36957 float32x4_t __rev2_205; __rev2_205 = __builtin_shufflevector(__s2_205, __s2_205, 3, 2, 1, 0); \36845 float32x4_t __rev2_205; __rev2_205 = __builtin_shufflevector(__s2_205, __s2_205, 3, 2, 1, 0); \
36958float32x4_t __reint_205 = __rev2_205; \36846 __ret_205 = __noswap_vcmlaq_rot270_f32(__rev0_205, __rev1_205, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_205), __p3_205), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_205), __p3_205)})); \
36959uint64x2_t __reint1_205 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_205, __p3_205), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_205, __p3_205)}; \
36960 __ret_205 = __noswap_vcmlaq_rot270_f32(__rev0_205, __rev1_205, *(float32x4_t *) &__reint1_205); \
36961 __ret_205 = __builtin_shufflevector(__ret_205, __ret_205, 3, 2, 1, 0); \36847 __ret_205 = __builtin_shufflevector(__ret_205, __ret_205, 3, 2, 1, 0); \
36962 __ret_205; \36848 __ret_205; \
36963})36849})
...@@ -37015,9 +36901,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot90_f32(...@@ -37015,9 +36901,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot90_f32(
37015 float32x2_t __s0_206 = __p0_206; \36901 float32x2_t __s0_206 = __p0_206; \
37016 float32x2_t __s1_206 = __p1_206; \36902 float32x2_t __s1_206 = __p1_206; \
37017 float32x2_t __s2_206 = __p2_206; \36903 float32x2_t __s2_206 = __p2_206; \
37018float32x2_t __reint_206 = __s2_206; \36904 __ret_206 = vcmla_rot90_f32(__s0_206, __s1_206, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_206), __p3_206)})); \
37019uint64x1_t __reint1_206 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_206, __p3_206)}; \
37020 __ret_206 = vcmla_rot90_f32(__s0_206, __s1_206, *(float32x2_t *) &__reint1_206); \
37021 __ret_206; \36905 __ret_206; \
37022})36906})
37023#else36907#else
...@@ -37029,9 +36913,7 @@ uint64x1_t __reint1_206 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2...@@ -37029,9 +36913,7 @@ uint64x1_t __reint1_206 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
37029 float32x2_t __rev0_207; __rev0_207 = __builtin_shufflevector(__s0_207, __s0_207, 1, 0); \36913 float32x2_t __rev0_207; __rev0_207 = __builtin_shufflevector(__s0_207, __s0_207, 1, 0); \
37030 float32x2_t __rev1_207; __rev1_207 = __builtin_shufflevector(__s1_207, __s1_207, 1, 0); \36914 float32x2_t __rev1_207; __rev1_207 = __builtin_shufflevector(__s1_207, __s1_207, 1, 0); \
37031 float32x2_t __rev2_207; __rev2_207 = __builtin_shufflevector(__s2_207, __s2_207, 1, 0); \36915 float32x2_t __rev2_207; __rev2_207 = __builtin_shufflevector(__s2_207, __s2_207, 1, 0); \
37032float32x2_t __reint_207 = __rev2_207; \36916 __ret_207 = __noswap_vcmla_rot90_f32(__rev0_207, __rev1_207, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_207), __p3_207)})); \
37033uint64x1_t __reint1_207 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_207, __p3_207)}; \
37034 __ret_207 = __noswap_vcmla_rot90_f32(__rev0_207, __rev1_207, *(float32x2_t *) &__reint1_207); \
37035 __ret_207 = __builtin_shufflevector(__ret_207, __ret_207, 1, 0); \36917 __ret_207 = __builtin_shufflevector(__ret_207, __ret_207, 1, 0); \
37036 __ret_207; \36918 __ret_207; \
37037})36919})
...@@ -37043,9 +36925,7 @@ uint64x1_t __reint1_207 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2...@@ -37043,9 +36925,7 @@ uint64x1_t __reint1_207 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
37043 float32x4_t __s0_208 = __p0_208; \36925 float32x4_t __s0_208 = __p0_208; \
37044 float32x4_t __s1_208 = __p1_208; \36926 float32x4_t __s1_208 = __p1_208; \
37045 float32x2_t __s2_208 = __p2_208; \36927 float32x2_t __s2_208 = __p2_208; \
37046float32x2_t __reint_208 = __s2_208; \36928 __ret_208 = vcmlaq_rot90_f32(__s0_208, __s1_208, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_208), __p3_208), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_208), __p3_208)})); \
37047uint64x2_t __reint1_208 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_208, __p3_208), vget_lane_u64(*(uint64x1_t *) &__reint_208, __p3_208)}; \
37048 __ret_208 = vcmlaq_rot90_f32(__s0_208, __s1_208, *(float32x4_t *) &__reint1_208); \
37049 __ret_208; \36929 __ret_208; \
37050})36930})
37051#else36931#else
...@@ -37057,9 +36937,7 @@ uint64x2_t __reint1_208 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2...@@ -37057,9 +36937,7 @@ uint64x2_t __reint1_208 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
37057 float32x4_t __rev0_209; __rev0_209 = __builtin_shufflevector(__s0_209, __s0_209, 3, 2, 1, 0); \36937 float32x4_t __rev0_209; __rev0_209 = __builtin_shufflevector(__s0_209, __s0_209, 3, 2, 1, 0); \
37058 float32x4_t __rev1_209; __rev1_209 = __builtin_shufflevector(__s1_209, __s1_209, 3, 2, 1, 0); \36938 float32x4_t __rev1_209; __rev1_209 = __builtin_shufflevector(__s1_209, __s1_209, 3, 2, 1, 0); \
37059 float32x2_t __rev2_209; __rev2_209 = __builtin_shufflevector(__s2_209, __s2_209, 1, 0); \36939 float32x2_t __rev2_209; __rev2_209 = __builtin_shufflevector(__s2_209, __s2_209, 1, 0); \
37060float32x2_t __reint_209 = __rev2_209; \36940 __ret_209 = __noswap_vcmlaq_rot90_f32(__rev0_209, __rev1_209, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_209), __p3_209), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_209), __p3_209)})); \
37061uint64x2_t __reint1_209 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_209, __p3_209), vget_lane_u64(*(uint64x1_t *) &__reint_209, __p3_209)}; \
37062 __ret_209 = __noswap_vcmlaq_rot90_f32(__rev0_209, __rev1_209, *(float32x4_t *) &__reint1_209); \
37063 __ret_209 = __builtin_shufflevector(__ret_209, __ret_209, 3, 2, 1, 0); \36941 __ret_209 = __builtin_shufflevector(__ret_209, __ret_209, 3, 2, 1, 0); \
37064 __ret_209; \36942 __ret_209; \
37065})36943})
...@@ -37071,9 +36949,7 @@ uint64x2_t __reint1_209 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2...@@ -37071,9 +36949,7 @@ uint64x2_t __reint1_209 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
37071 float32x2_t __s0_210 = __p0_210; \36949 float32x2_t __s0_210 = __p0_210; \
37072 float32x2_t __s1_210 = __p1_210; \36950 float32x2_t __s1_210 = __p1_210; \
37073 float32x4_t __s2_210 = __p2_210; \36951 float32x4_t __s2_210 = __p2_210; \
37074float32x4_t __reint_210 = __s2_210; \36952 __ret_210 = vcmla_rot90_f32(__s0_210, __s1_210, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_210), __p3_210)})); \
37075uint64x1_t __reint1_210 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_210, __p3_210)}; \
37076 __ret_210 = vcmla_rot90_f32(__s0_210, __s1_210, *(float32x2_t *) &__reint1_210); \
37077 __ret_210; \36953 __ret_210; \
37078})36954})
37079#else36955#else
...@@ -37085,9 +36961,7 @@ uint64x1_t __reint1_210 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -37085,9 +36961,7 @@ uint64x1_t __reint1_210 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
37085 float32x2_t __rev0_211; __rev0_211 = __builtin_shufflevector(__s0_211, __s0_211, 1, 0); \36961 float32x2_t __rev0_211; __rev0_211 = __builtin_shufflevector(__s0_211, __s0_211, 1, 0); \
37086 float32x2_t __rev1_211; __rev1_211 = __builtin_shufflevector(__s1_211, __s1_211, 1, 0); \36962 float32x2_t __rev1_211; __rev1_211 = __builtin_shufflevector(__s1_211, __s1_211, 1, 0); \
37087 float32x4_t __rev2_211; __rev2_211 = __builtin_shufflevector(__s2_211, __s2_211, 3, 2, 1, 0); \36963 float32x4_t __rev2_211; __rev2_211 = __builtin_shufflevector(__s2_211, __s2_211, 3, 2, 1, 0); \
37088float32x4_t __reint_211 = __rev2_211; \36964 __ret_211 = __noswap_vcmla_rot90_f32(__rev0_211, __rev1_211, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_211), __p3_211)})); \
37089uint64x1_t __reint1_211 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_211, __p3_211)}; \
37090 __ret_211 = __noswap_vcmla_rot90_f32(__rev0_211, __rev1_211, *(float32x2_t *) &__reint1_211); \
37091 __ret_211 = __builtin_shufflevector(__ret_211, __ret_211, 1, 0); \36965 __ret_211 = __builtin_shufflevector(__ret_211, __ret_211, 1, 0); \
37092 __ret_211; \36966 __ret_211; \
37093})36967})
...@@ -37099,9 +36973,7 @@ uint64x1_t __reint1_211 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)...@@ -37099,9 +36973,7 @@ uint64x1_t __reint1_211 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
37099 float32x4_t __s0_212 = __p0_212; \36973 float32x4_t __s0_212 = __p0_212; \
37100 float32x4_t __s1_212 = __p1_212; \36974 float32x4_t __s1_212 = __p1_212; \
37101 float32x4_t __s2_212 = __p2_212; \36975 float32x4_t __s2_212 = __p2_212; \
37102float32x4_t __reint_212 = __s2_212; \36976 __ret_212 = vcmlaq_rot90_f32(__s0_212, __s1_212, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_212), __p3_212), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_212), __p3_212)})); \
37103uint64x2_t __reint1_212 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_212, __p3_212), vgetq_lane_u64(*(uint64x2_t *) &__reint_212, __p3_212)}; \
37104 __ret_212 = vcmlaq_rot90_f32(__s0_212, __s1_212, *(float32x4_t *) &__reint1_212); \
37105 __ret_212; \36977 __ret_212; \
37106})36978})
37107#else36979#else
...@@ -37113,9 +36985,7 @@ uint64x2_t __reint1_212 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_...@@ -37113,9 +36985,7 @@ uint64x2_t __reint1_212 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
37113 float32x4_t __rev0_213; __rev0_213 = __builtin_shufflevector(__s0_213, __s0_213, 3, 2, 1, 0); \36985 float32x4_t __rev0_213; __rev0_213 = __builtin_shufflevector(__s0_213, __s0_213, 3, 2, 1, 0); \
37114 float32x4_t __rev1_213; __rev1_213 = __builtin_shufflevector(__s1_213, __s1_213, 3, 2, 1, 0); \36986 float32x4_t __rev1_213; __rev1_213 = __builtin_shufflevector(__s1_213, __s1_213, 3, 2, 1, 0); \
37115 float32x4_t __rev2_213; __rev2_213 = __builtin_shufflevector(__s2_213, __s2_213, 3, 2, 1, 0); \36987 float32x4_t __rev2_213; __rev2_213 = __builtin_shufflevector(__s2_213, __s2_213, 3, 2, 1, 0); \
37116float32x4_t __reint_213 = __rev2_213; \36988 __ret_213 = __noswap_vcmlaq_rot90_f32(__rev0_213, __rev1_213, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_213), __p3_213), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_213), __p3_213)})); \
37117uint64x2_t __reint1_213 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_213, __p3_213), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_213, __p3_213)}; \
37118 __ret_213 = __noswap_vcmlaq_rot90_f32(__rev0_213, __rev1_213, *(float32x4_t *) &__reint1_213); \
37119 __ret_213 = __builtin_shufflevector(__ret_213, __ret_213, 3, 2, 1, 0); \36989 __ret_213 = __builtin_shufflevector(__ret_213, __ret_213, 3, 2, 1, 0); \
37120 __ret_213; \36990 __ret_213; \
37121})36991})
...@@ -41249,33 +41119,2258 @@ __ai __attribute__((target("neon"))) float32x2_t vfms_f32(float32x2_t __p0, floa...@@ -41249,33 +41119,2258 @@ __ai __attribute__((target("neon"))) float32x2_t vfms_f32(float32x2_t __p0, floa
41249#endif41119#endif
4125041120
41251#endif41121#endif
41252#if defined(__aarch64__) || defined(__arm64ec__)41122#if defined(__aarch64__)
41253__ai __attribute__((target("aes,neon"))) poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) {41123#ifdef __LITTLE_ENDIAN__
41254 poly128_t __ret;41124__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41255 __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1);41125 bfloat16x8_t __ret;
41126 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_bf16_mf8_fpm(__p0, __p1);
41127 return __ret;
41128}
41129#else
41130__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41131 bfloat16x8_t __ret;
41132 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41133 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_bf16_mf8_fpm(__rev0, __p1);
41134 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41135 return __ret;
41136}
41137#endif
41138
41139#ifdef __LITTLE_ENDIAN__
41140__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41141 float16x8_t __ret;
41142 __ret = (float16x8_t) __builtin_neon_vcvt1_f16_mf8_fpm(__p0, __p1);
41143 return __ret;
41144}
41145#else
41146__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41147 float16x8_t __ret;
41148 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41149 __ret = (float16x8_t) __builtin_neon_vcvt1_f16_mf8_fpm(__rev0, __p1);
41150 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41151 return __ret;
41152}
41153#endif
41154
41155#ifdef __LITTLE_ENDIAN__
41156__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41157 bfloat16x8_t __ret;
41158 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_high_bf16_mf8_fpm(__p0, __p1);
41159 return __ret;
41160}
41161#else
41162__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41163 bfloat16x8_t __ret;
41164 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41165 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_high_bf16_mf8_fpm(__rev0, __p1);
41166 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41167 return __ret;
41168}
41169#endif
41170
41171#ifdef __LITTLE_ENDIAN__
41172__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41173 float16x8_t __ret;
41174 __ret = (float16x8_t) __builtin_neon_vcvt1_high_f16_mf8_fpm(__p0, __p1);
41175 return __ret;
41176}
41177#else
41178__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41179 float16x8_t __ret;
41180 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41181 __ret = (float16x8_t) __builtin_neon_vcvt1_high_f16_mf8_fpm(__rev0, __p1);
41182 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41183 return __ret;
41184}
41185#endif
41186
41187#ifdef __LITTLE_ENDIAN__
41188__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41189 bfloat16x8_t __ret;
41190 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_low_bf16_mf8_fpm(__p0, __p1);
41191 return __ret;
41192}
41193#else
41194__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41195 bfloat16x8_t __ret;
41196 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41197 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_low_bf16_mf8_fpm(__rev0, __p1);
41198 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41199 return __ret;
41200}
41201#endif
41202
41203#ifdef __LITTLE_ENDIAN__
41204__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41205 float16x8_t __ret;
41206 __ret = (float16x8_t) __builtin_neon_vcvt1_low_f16_mf8_fpm(__p0, __p1);
41207 return __ret;
41208}
41209#else
41210__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41211 float16x8_t __ret;
41212 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41213 __ret = (float16x8_t) __builtin_neon_vcvt1_low_f16_mf8_fpm(__rev0, __p1);
41214 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41215 return __ret;
41216}
41217#endif
41218
41219#ifdef __LITTLE_ENDIAN__
41220__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41221 bfloat16x8_t __ret;
41222 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_bf16_mf8_fpm(__p0, __p1);
41223 return __ret;
41224}
41225#else
41226__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41227 bfloat16x8_t __ret;
41228 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41229 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_bf16_mf8_fpm(__rev0, __p1);
41230 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41231 return __ret;
41232}
41233#endif
41234
41235#ifdef __LITTLE_ENDIAN__
41236__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41237 float16x8_t __ret;
41238 __ret = (float16x8_t) __builtin_neon_vcvt2_f16_mf8_fpm(__p0, __p1);
41239 return __ret;
41240}
41241#else
41242__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41243 float16x8_t __ret;
41244 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41245 __ret = (float16x8_t) __builtin_neon_vcvt2_f16_mf8_fpm(__rev0, __p1);
41246 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41247 return __ret;
41248}
41249#endif
41250
41251#ifdef __LITTLE_ENDIAN__
41252__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41253 bfloat16x8_t __ret;
41254 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_high_bf16_mf8_fpm(__p0, __p1);
41255 return __ret;
41256}
41257#else
41258__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41259 bfloat16x8_t __ret;
41260 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41261 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_high_bf16_mf8_fpm(__rev0, __p1);
41262 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41263 return __ret;
41264}
41265#endif
41266
41267#ifdef __LITTLE_ENDIAN__
41268__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41269 float16x8_t __ret;
41270 __ret = (float16x8_t) __builtin_neon_vcvt2_high_f16_mf8_fpm(__p0, __p1);
41271 return __ret;
41272}
41273#else
41274__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41275 float16x8_t __ret;
41276 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41277 __ret = (float16x8_t) __builtin_neon_vcvt2_high_f16_mf8_fpm(__rev0, __p1);
41278 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41279 return __ret;
41280}
41281#endif
41282
41283#ifdef __LITTLE_ENDIAN__
41284__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41285 bfloat16x8_t __ret;
41286 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_low_bf16_mf8_fpm(__p0, __p1);
41287 return __ret;
41288}
41289#else
41290__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41291 bfloat16x8_t __ret;
41292 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41293 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_low_bf16_mf8_fpm(__rev0, __p1);
41294 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41295 return __ret;
41296}
41297#endif
41298
41299#ifdef __LITTLE_ENDIAN__
41300__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41301 float16x8_t __ret;
41302 __ret = (float16x8_t) __builtin_neon_vcvt2_low_f16_mf8_fpm(__p0, __p1);
41303 return __ret;
41304}
41305#else
41306__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41307 float16x8_t __ret;
41308 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41309 __ret = (float16x8_t) __builtin_neon_vcvt2_low_f16_mf8_fpm(__rev0, __p1);
41310 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41311 return __ret;
41312}
41313#endif
41314
41315#ifdef __LITTLE_ENDIAN__
41316__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvt_high_mf8_f32_fpm(mfloat8x8_t __p0, float32x4_t __p1, float32x4_t __p2, fpm_t __p3) {
41317 mfloat8x16_t __ret;
41318 __ret = (mfloat8x16_t) __builtin_neon_vcvt_high_mf8_f32_fpm(__p0, __p1, __p2, __p3);
41319 return __ret;
41320}
41321#else
41322__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvt_high_mf8_f32_fpm(mfloat8x8_t __p0, float32x4_t __p1, float32x4_t __p2, fpm_t __p3) {
41323 mfloat8x16_t __ret;
41324 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41325 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41326 float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
41327 __ret = (mfloat8x16_t) __builtin_neon_vcvt_high_mf8_f32_fpm(__rev0, __rev1, __rev2, __p3);
41328 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41329 return __ret;
41330}
41331#endif
41332
41333#ifdef __LITTLE_ENDIAN__
41334__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvtq_mf8_f16_fpm(float16x8_t __p0, float16x8_t __p1, fpm_t __p2) {
41335 mfloat8x16_t __ret;
41336 __ret = (mfloat8x16_t) __builtin_neon_vcvtq_mf8_f16_fpm((int8x16_t)__p0, (int8x16_t)__p1, __p2);
41337 return __ret;
41338}
41339#else
41340__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvtq_mf8_f16_fpm(float16x8_t __p0, float16x8_t __p1, fpm_t __p2) {
41341 mfloat8x16_t __ret;
41342 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41343 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41344 __ret = (mfloat8x16_t) __builtin_neon_vcvtq_mf8_f16_fpm((int8x16_t)__rev0, (int8x16_t)__rev1, __p2);
41345 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41346 return __ret;
41347}
41348#endif
41349
41350#ifdef __LITTLE_ENDIAN__
41351__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f16_fpm(float16x4_t __p0, float16x4_t __p1, fpm_t __p2) {
41352 mfloat8x8_t __ret;
41353 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f16_fpm((int8x8_t)__p0, (int8x8_t)__p1, __p2);
41354 return __ret;
41355}
41356#else
41357__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f16_fpm(float16x4_t __p0, float16x4_t __p1, fpm_t __p2) {
41358 mfloat8x8_t __ret;
41359 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41360 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41361 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f16_fpm((int8x8_t)__rev0, (int8x8_t)__rev1, __p2);
41362 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41363 return __ret;
41364}
41365#endif
41366
41367#ifdef __LITTLE_ENDIAN__
41368__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f32_fpm(float32x4_t __p0, float32x4_t __p1, fpm_t __p2) {
41369 mfloat8x8_t __ret;
41370 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f32_fpm(__p0, __p1, __p2);
41371 return __ret;
41372}
41373#else
41374__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f32_fpm(float32x4_t __p0, float32x4_t __p1, fpm_t __p2) {
41375 mfloat8x8_t __ret;
41376 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41377 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41378 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f32_fpm(__rev0, __rev1, __p2);
41379 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41380 return __ret;
41381}
41382#endif
41383
41384#ifdef __LITTLE_ENDIAN__
41385__ai __attribute__((target("fp8,neon"))) float32x2_t vscale_f32(float32x2_t __p0, int32x2_t __p1) {
41386 float32x2_t __ret;
41387 __ret = (float32x2_t) __builtin_neon_vscale_f32((int8x8_t)__p0, (int8x8_t)__p1, 9);
41388 return __ret;
41389}
41390#else
41391__ai __attribute__((target("fp8,neon"))) float32x2_t vscale_f32(float32x2_t __p0, int32x2_t __p1) {
41392 float32x2_t __ret;
41393 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
41394 int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
41395 __ret = (float32x2_t) __builtin_neon_vscale_f32((int8x8_t)__rev0, (int8x8_t)__rev1, 9);
41396 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41397 return __ret;
41398}
41399#endif
41400
41401#ifdef __LITTLE_ENDIAN__
41402__ai __attribute__((target("fp8,neon"))) float16x4_t vscale_f16(float16x4_t __p0, int16x4_t __p1) {
41403 float16x4_t __ret;
41404 __ret = (float16x4_t) __builtin_neon_vscale_f16((int8x8_t)__p0, (int8x8_t)__p1, 8);
41405 return __ret;
41406}
41407#else
41408__ai __attribute__((target("fp8,neon"))) float16x4_t vscale_f16(float16x4_t __p0, int16x4_t __p1) {
41409 float16x4_t __ret;
41410 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41411 int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41412 __ret = (float16x4_t) __builtin_neon_vscale_f16((int8x8_t)__rev0, (int8x8_t)__rev1, 8);
41413 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41414 return __ret;
41415}
41416#endif
41417
41418#ifdef __LITTLE_ENDIAN__
41419__ai __attribute__((target("fp8,neon"))) float64x2_t vscaleq_f64(float64x2_t __p0, int64x2_t __p1) {
41420 float64x2_t __ret;
41421 __ret = (float64x2_t) __builtin_neon_vscaleq_f64((int8x16_t)__p0, (int8x16_t)__p1, 42);
41422 return __ret;
41423}
41424#else
41425__ai __attribute__((target("fp8,neon"))) float64x2_t vscaleq_f64(float64x2_t __p0, int64x2_t __p1) {
41426 float64x2_t __ret;
41427 float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
41428 int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
41429 __ret = (float64x2_t) __builtin_neon_vscaleq_f64((int8x16_t)__rev0, (int8x16_t)__rev1, 42);
41430 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41431 return __ret;
41432}
41433#endif
41434
41435#ifdef __LITTLE_ENDIAN__
41436__ai __attribute__((target("fp8,neon"))) float32x4_t vscaleq_f32(float32x4_t __p0, int32x4_t __p1) {
41437 float32x4_t __ret;
41438 __ret = (float32x4_t) __builtin_neon_vscaleq_f32((int8x16_t)__p0, (int8x16_t)__p1, 41);
41439 return __ret;
41440}
41441#else
41442__ai __attribute__((target("fp8,neon"))) float32x4_t vscaleq_f32(float32x4_t __p0, int32x4_t __p1) {
41443 float32x4_t __ret;
41444 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41445 int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41446 __ret = (float32x4_t) __builtin_neon_vscaleq_f32((int8x16_t)__rev0, (int8x16_t)__rev1, 41);
41447 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41448 return __ret;
41449}
41450#endif
41451
41452#ifdef __LITTLE_ENDIAN__
41453__ai __attribute__((target("fp8,neon"))) float16x8_t vscaleq_f16(float16x8_t __p0, int16x8_t __p1) {
41454 float16x8_t __ret;
41455 __ret = (float16x8_t) __builtin_neon_vscaleq_f16((int8x16_t)__p0, (int8x16_t)__p1, 40);
41456 return __ret;
41457}
41458#else
41459__ai __attribute__((target("fp8,neon"))) float16x8_t vscaleq_f16(float16x8_t __p0, int16x8_t __p1) {
41460 float16x8_t __ret;
41461 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41462 int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41463 __ret = (float16x8_t) __builtin_neon_vscaleq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, 40);
41464 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41465 return __ret;
41466}
41467#endif
41468
41469#ifdef __LITTLE_ENDIAN__
41470__ai __attribute__((target("fp8dot2,neon"))) float16x8_t vdotq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41471 float16x8_t __ret;
41472 __ret = (float16x8_t) __builtin_neon_vdotq_f16_mf8_fpm((int8x16_t)__p0, __p1, __p2, __p3);
41473 return __ret;
41474}
41475#else
41476__ai __attribute__((target("fp8dot2,neon"))) float16x8_t vdotq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41477 float16x8_t __ret;
41478 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41479 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41480 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41481 __ret = (float16x8_t) __builtin_neon_vdotq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3);
41482 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41483 return __ret;
41484}
41485#endif
41486
41487#ifdef __LITTLE_ENDIAN__
41488__ai __attribute__((target("fp8dot2,neon"))) float16x4_t vdot_f16_mf8_fpm(float16x4_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41489 float16x4_t __ret;
41490 __ret = (float16x4_t) __builtin_neon_vdot_f16_mf8_fpm((int8x8_t)__p0, __p1, __p2, __p3);
41491 return __ret;
41492}
41493#else
41494__ai __attribute__((target("fp8dot2,neon"))) float16x4_t vdot_f16_mf8_fpm(float16x4_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41495 float16x4_t __ret;
41496 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41497 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41498 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
41499 __ret = (float16x4_t) __builtin_neon_vdot_f16_mf8_fpm((int8x8_t)__rev0, __rev1, __rev2, __p3);
41500 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41501 return __ret;
41502}
41503#endif
41504
41505#ifdef __LITTLE_ENDIAN__
41506#define vdotq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41507 float16x8_t __ret; \
41508 float16x8_t __s0 = __p0; \
41509 mfloat8x16_t __s1 = __p1; \
41510 mfloat8x8_t __s2 = __p2; \
41511 fpm_t __s4 = __p4; \
41512 __ret = (float16x8_t) __builtin_neon_vdotq_lane_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41513 __ret; \
41514})
41515#else
41516#define vdotq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41517 float16x8_t __ret; \
41518 float16x8_t __s0 = __p0; \
41519 mfloat8x16_t __s1 = __p1; \
41520 mfloat8x8_t __s2 = __p2; \
41521 fpm_t __s4 = __p4; \
41522 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41523 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41524 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41525 __ret = (float16x8_t) __builtin_neon_vdotq_lane_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41526 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41527 __ret; \
41528})
41529#endif
41530
41531#ifdef __LITTLE_ENDIAN__
41532#define vdot_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41533 float16x4_t __ret; \
41534 float16x4_t __s0 = __p0; \
41535 mfloat8x8_t __s1 = __p1; \
41536 mfloat8x8_t __s2 = __p2; \
41537 fpm_t __s4 = __p4; \
41538 __ret = (float16x4_t) __builtin_neon_vdot_lane_f16_mf8_fpm((int8x8_t)__s0, __s1, __s2, __p3, __s4); \
41539 __ret; \
41540})
41541#else
41542#define vdot_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41543 float16x4_t __ret; \
41544 float16x4_t __s0 = __p0; \
41545 mfloat8x8_t __s1 = __p1; \
41546 mfloat8x8_t __s2 = __p2; \
41547 fpm_t __s4 = __p4; \
41548 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41549 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41550 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41551 __ret = (float16x4_t) __builtin_neon_vdot_lane_f16_mf8_fpm((int8x8_t)__rev0, __rev1, __rev2, __p3, __s4); \
41552 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41553 __ret; \
41554})
41555#endif
41556
41557#ifdef __LITTLE_ENDIAN__
41558#define vdotq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41559 float16x8_t __ret; \
41560 float16x8_t __s0 = __p0; \
41561 mfloat8x16_t __s1 = __p1; \
41562 mfloat8x16_t __s2 = __p2; \
41563 fpm_t __s4 = __p4; \
41564 __ret = (float16x8_t) __builtin_neon_vdotq_laneq_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41565 __ret; \
41566})
41567#else
41568#define vdotq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41569 float16x8_t __ret; \
41570 float16x8_t __s0 = __p0; \
41571 mfloat8x16_t __s1 = __p1; \
41572 mfloat8x16_t __s2 = __p2; \
41573 fpm_t __s4 = __p4; \
41574 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41575 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41576 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41577 __ret = (float16x8_t) __builtin_neon_vdotq_laneq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41578 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41579 __ret; \
41580})
41581#endif
41582
41583#ifdef __LITTLE_ENDIAN__
41584#define vdot_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41585 float16x4_t __ret; \
41586 float16x4_t __s0 = __p0; \
41587 mfloat8x8_t __s1 = __p1; \
41588 mfloat8x16_t __s2 = __p2; \
41589 fpm_t __s4 = __p4; \
41590 __ret = (float16x4_t) __builtin_neon_vdot_laneq_f16_mf8_fpm((int8x8_t)__s0, __s1, __s2, __p3, __s4); \
41591 __ret; \
41592})
41593#else
41594#define vdot_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41595 float16x4_t __ret; \
41596 float16x4_t __s0 = __p0; \
41597 mfloat8x8_t __s1 = __p1; \
41598 mfloat8x16_t __s2 = __p2; \
41599 fpm_t __s4 = __p4; \
41600 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41601 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41602 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41603 __ret = (float16x4_t) __builtin_neon_vdot_laneq_f16_mf8_fpm((int8x8_t)__rev0, __rev1, __rev2, __p3, __s4); \
41604 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41605 __ret; \
41606})
41607#endif
41608
41609#ifdef __LITTLE_ENDIAN__
41610__ai __attribute__((target("fp8dot4,neon"))) float32x4_t vdotq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41611 float32x4_t __ret;
41612 __ret = (float32x4_t) __builtin_neon_vdotq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41613 return __ret;
41614}
41615#else
41616__ai __attribute__((target("fp8dot4,neon"))) float32x4_t vdotq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41617 float32x4_t __ret;
41618 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41619 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41620 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41621 __ret = (float32x4_t) __builtin_neon_vdotq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41622 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41623 return __ret;
41624}
41625#endif
41626
41627#ifdef __LITTLE_ENDIAN__
41628__ai __attribute__((target("fp8dot4,neon"))) float32x2_t vdot_f32_mf8_fpm(float32x2_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41629 float32x2_t __ret;
41630 __ret = (float32x2_t) __builtin_neon_vdot_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41631 return __ret;
41632}
41633#else
41634__ai __attribute__((target("fp8dot4,neon"))) float32x2_t vdot_f32_mf8_fpm(float32x2_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41635 float32x2_t __ret;
41636 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
41637 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41638 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
41639 __ret = (float32x2_t) __builtin_neon_vdot_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41640 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41641 return __ret;
41642}
41643#endif
41644
41645#ifdef __LITTLE_ENDIAN__
41646#define vdotq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41647 float32x4_t __ret; \
41648 float32x4_t __s0 = __p0; \
41649 mfloat8x16_t __s1 = __p1; \
41650 mfloat8x8_t __s2 = __p2; \
41651 fpm_t __s4 = __p4; \
41652 __ret = (float32x4_t) __builtin_neon_vdotq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41653 __ret; \
41654})
41655#else
41656#define vdotq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41657 float32x4_t __ret; \
41658 float32x4_t __s0 = __p0; \
41659 mfloat8x16_t __s1 = __p1; \
41660 mfloat8x8_t __s2 = __p2; \
41661 fpm_t __s4 = __p4; \
41662 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41663 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41664 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41665 __ret = (float32x4_t) __builtin_neon_vdotq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41666 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41667 __ret; \
41668})
41669#endif
41670
41671#ifdef __LITTLE_ENDIAN__
41672#define vdot_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41673 float32x2_t __ret; \
41674 float32x2_t __s0 = __p0; \
41675 mfloat8x8_t __s1 = __p1; \
41676 mfloat8x8_t __s2 = __p2; \
41677 fpm_t __s4 = __p4; \
41678 __ret = (float32x2_t) __builtin_neon_vdot_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41679 __ret; \
41680})
41681#else
41682#define vdot_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41683 float32x2_t __ret; \
41684 float32x2_t __s0 = __p0; \
41685 mfloat8x8_t __s1 = __p1; \
41686 mfloat8x8_t __s2 = __p2; \
41687 fpm_t __s4 = __p4; \
41688 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \
41689 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41690 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41691 __ret = (float32x2_t) __builtin_neon_vdot_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41692 __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
41693 __ret; \
41694})
41695#endif
41696
41697#ifdef __LITTLE_ENDIAN__
41698#define vdotq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41699 float32x4_t __ret; \
41700 float32x4_t __s0 = __p0; \
41701 mfloat8x16_t __s1 = __p1; \
41702 mfloat8x16_t __s2 = __p2; \
41703 fpm_t __s4 = __p4; \
41704 __ret = (float32x4_t) __builtin_neon_vdotq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41705 __ret; \
41706})
41707#else
41708#define vdotq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41709 float32x4_t __ret; \
41710 float32x4_t __s0 = __p0; \
41711 mfloat8x16_t __s1 = __p1; \
41712 mfloat8x16_t __s2 = __p2; \
41713 fpm_t __s4 = __p4; \
41714 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41715 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41716 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41717 __ret = (float32x4_t) __builtin_neon_vdotq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41718 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41719 __ret; \
41720})
41721#endif
41722
41723#ifdef __LITTLE_ENDIAN__
41724#define vdot_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41725 float32x2_t __ret; \
41726 float32x2_t __s0 = __p0; \
41727 mfloat8x8_t __s1 = __p1; \
41728 mfloat8x16_t __s2 = __p2; \
41729 fpm_t __s4 = __p4; \
41730 __ret = (float32x2_t) __builtin_neon_vdot_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41731 __ret; \
41732})
41733#else
41734#define vdot_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41735 float32x2_t __ret; \
41736 float32x2_t __s0 = __p0; \
41737 mfloat8x8_t __s1 = __p1; \
41738 mfloat8x16_t __s2 = __p2; \
41739 fpm_t __s4 = __p4; \
41740 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \
41741 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41742 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41743 __ret = (float32x2_t) __builtin_neon_vdot_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41744 __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
41745 __ret; \
41746})
41747#endif
41748
41749#ifdef __LITTLE_ENDIAN__
41750__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlalbq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41751 float16x8_t __ret;
41752 __ret = (float16x8_t) __builtin_neon_vmlalbq_f16_mf8_fpm((int8x16_t)__p0, __p1, __p2, __p3);
41753 return __ret;
41754}
41755#else
41756__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlalbq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41757 float16x8_t __ret;
41758 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41759 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41760 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41761 __ret = (float16x8_t) __builtin_neon_vmlalbq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3);
41762 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41763 return __ret;
41764}
41765#endif
41766
41767#ifdef __LITTLE_ENDIAN__
41768#define vmlalbq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41769 float16x8_t __ret; \
41770 float16x8_t __s0 = __p0; \
41771 mfloat8x16_t __s1 = __p1; \
41772 mfloat8x8_t __s2 = __p2; \
41773 fpm_t __s4 = __p4; \
41774 __ret = (float16x8_t) __builtin_neon_vmlalbq_lane_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41775 __ret; \
41776})
41777#else
41778#define vmlalbq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41779 float16x8_t __ret; \
41780 float16x8_t __s0 = __p0; \
41781 mfloat8x16_t __s1 = __p1; \
41782 mfloat8x8_t __s2 = __p2; \
41783 fpm_t __s4 = __p4; \
41784 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41785 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41786 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41787 __ret = (float16x8_t) __builtin_neon_vmlalbq_lane_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41788 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41789 __ret; \
41790})
41791#endif
41792
41793#ifdef __LITTLE_ENDIAN__
41794#define vmlalbq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41795 float16x8_t __ret; \
41796 float16x8_t __s0 = __p0; \
41797 mfloat8x16_t __s1 = __p1; \
41798 mfloat8x16_t __s2 = __p2; \
41799 fpm_t __s4 = __p4; \
41800 __ret = (float16x8_t) __builtin_neon_vmlalbq_laneq_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41801 __ret; \
41802})
41803#else
41804#define vmlalbq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41805 float16x8_t __ret; \
41806 float16x8_t __s0 = __p0; \
41807 mfloat8x16_t __s1 = __p1; \
41808 mfloat8x16_t __s2 = __p2; \
41809 fpm_t __s4 = __p4; \
41810 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41811 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41812 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41813 __ret = (float16x8_t) __builtin_neon_vmlalbq_laneq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41814 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41815 __ret; \
41816})
41817#endif
41818
41819#ifdef __LITTLE_ENDIAN__
41820__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41821 float32x4_t __ret;
41822 __ret = (float32x4_t) __builtin_neon_vmlallbbq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41823 return __ret;
41824}
41825#else
41826__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41827 float32x4_t __ret;
41828 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41829 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41830 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41831 __ret = (float32x4_t) __builtin_neon_vmlallbbq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41832 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41833 return __ret;
41834}
41835#endif
41836
41837#ifdef __LITTLE_ENDIAN__
41838#define vmlallbbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41839 float32x4_t __ret; \
41840 float32x4_t __s0 = __p0; \
41841 mfloat8x16_t __s1 = __p1; \
41842 mfloat8x8_t __s2 = __p2; \
41843 fpm_t __s4 = __p4; \
41844 __ret = (float32x4_t) __builtin_neon_vmlallbbq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41845 __ret; \
41846})
41847#else
41848#define vmlallbbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41849 float32x4_t __ret; \
41850 float32x4_t __s0 = __p0; \
41851 mfloat8x16_t __s1 = __p1; \
41852 mfloat8x8_t __s2 = __p2; \
41853 fpm_t __s4 = __p4; \
41854 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41855 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41856 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41857 __ret = (float32x4_t) __builtin_neon_vmlallbbq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41858 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41859 __ret; \
41860})
41861#endif
41862
41863#ifdef __LITTLE_ENDIAN__
41864#define vmlallbbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41865 float32x4_t __ret; \
41866 float32x4_t __s0 = __p0; \
41867 mfloat8x16_t __s1 = __p1; \
41868 mfloat8x16_t __s2 = __p2; \
41869 fpm_t __s4 = __p4; \
41870 __ret = (float32x4_t) __builtin_neon_vmlallbbq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41871 __ret; \
41872})
41873#else
41874#define vmlallbbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41875 float32x4_t __ret; \
41876 float32x4_t __s0 = __p0; \
41877 mfloat8x16_t __s1 = __p1; \
41878 mfloat8x16_t __s2 = __p2; \
41879 fpm_t __s4 = __p4; \
41880 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41881 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41882 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41883 __ret = (float32x4_t) __builtin_neon_vmlallbbq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41884 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41885 __ret; \
41886})
41887#endif
41888
41889#ifdef __LITTLE_ENDIAN__
41890__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbtq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41891 float32x4_t __ret;
41892 __ret = (float32x4_t) __builtin_neon_vmlallbtq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41893 return __ret;
41894}
41895#else
41896__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbtq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41897 float32x4_t __ret;
41898 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41899 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41900 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41901 __ret = (float32x4_t) __builtin_neon_vmlallbtq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41902 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41903 return __ret;
41904}
41905#endif
41906
41907#ifdef __LITTLE_ENDIAN__
41908#define vmlallbtq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41909 float32x4_t __ret; \
41910 float32x4_t __s0 = __p0; \
41911 mfloat8x16_t __s1 = __p1; \
41912 mfloat8x8_t __s2 = __p2; \
41913 fpm_t __s4 = __p4; \
41914 __ret = (float32x4_t) __builtin_neon_vmlallbtq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41915 __ret; \
41916})
41917#else
41918#define vmlallbtq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41919 float32x4_t __ret; \
41920 float32x4_t __s0 = __p0; \
41921 mfloat8x16_t __s1 = __p1; \
41922 mfloat8x8_t __s2 = __p2; \
41923 fpm_t __s4 = __p4; \
41924 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41925 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41926 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41927 __ret = (float32x4_t) __builtin_neon_vmlallbtq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41928 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41929 __ret; \
41930})
41931#endif
41932
41933#ifdef __LITTLE_ENDIAN__
41934#define vmlallbtq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41935 float32x4_t __ret; \
41936 float32x4_t __s0 = __p0; \
41937 mfloat8x16_t __s1 = __p1; \
41938 mfloat8x16_t __s2 = __p2; \
41939 fpm_t __s4 = __p4; \
41940 __ret = (float32x4_t) __builtin_neon_vmlallbtq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41941 __ret; \
41942})
41943#else
41944#define vmlallbtq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41945 float32x4_t __ret; \
41946 float32x4_t __s0 = __p0; \
41947 mfloat8x16_t __s1 = __p1; \
41948 mfloat8x16_t __s2 = __p2; \
41949 fpm_t __s4 = __p4; \
41950 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41951 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41952 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41953 __ret = (float32x4_t) __builtin_neon_vmlallbtq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41954 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41955 __ret; \
41956})
41957#endif
41958
41959#ifdef __LITTLE_ENDIAN__
41960__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlalltbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41961 float32x4_t __ret;
41962 __ret = (float32x4_t) __builtin_neon_vmlalltbq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41963 return __ret;
41964}
41965#else
41966__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlalltbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41967 float32x4_t __ret;
41968 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41969 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41970 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41971 __ret = (float32x4_t) __builtin_neon_vmlalltbq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41972 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41973 return __ret;
41974}
41975#endif
41976
41977#ifdef __LITTLE_ENDIAN__
41978#define vmlalltbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41979 float32x4_t __ret; \
41980 float32x4_t __s0 = __p0; \
41981 mfloat8x16_t __s1 = __p1; \
41982 mfloat8x8_t __s2 = __p2; \
41983 fpm_t __s4 = __p4; \
41984 __ret = (float32x4_t) __builtin_neon_vmlalltbq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41985 __ret; \
41986})
41987#else
41988#define vmlalltbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41989 float32x4_t __ret; \
41990 float32x4_t __s0 = __p0; \
41991 mfloat8x16_t __s1 = __p1; \
41992 mfloat8x8_t __s2 = __p2; \
41993 fpm_t __s4 = __p4; \
41994 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41995 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41996 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41997 __ret = (float32x4_t) __builtin_neon_vmlalltbq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41998 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41999 __ret; \
42000})
42001#endif
42002
42003#ifdef __LITTLE_ENDIAN__
42004#define vmlalltbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42005 float32x4_t __ret; \
42006 float32x4_t __s0 = __p0; \
42007 mfloat8x16_t __s1 = __p1; \
42008 mfloat8x16_t __s2 = __p2; \
42009 fpm_t __s4 = __p4; \
42010 __ret = (float32x4_t) __builtin_neon_vmlalltbq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
42011 __ret; \
42012})
42013#else
42014#define vmlalltbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42015 float32x4_t __ret; \
42016 float32x4_t __s0 = __p0; \
42017 mfloat8x16_t __s1 = __p1; \
42018 mfloat8x16_t __s2 = __p2; \
42019 fpm_t __s4 = __p4; \
42020 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42021 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42022 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42023 __ret = (float32x4_t) __builtin_neon_vmlalltbq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
42024 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
42025 __ret; \
42026})
42027#endif
42028
42029#ifdef __LITTLE_ENDIAN__
42030__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallttq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42031 float32x4_t __ret;
42032 __ret = (float32x4_t) __builtin_neon_vmlallttq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
42033 return __ret;
42034}
42035#else
42036__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallttq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42037 float32x4_t __ret;
42038 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
42039 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42040 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42041 __ret = (float32x4_t) __builtin_neon_vmlallttq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
42042 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
42043 return __ret;
42044}
42045#endif
42046
42047#ifdef __LITTLE_ENDIAN__
42048#define vmlallttq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42049 float32x4_t __ret; \
42050 float32x4_t __s0 = __p0; \
42051 mfloat8x16_t __s1 = __p1; \
42052 mfloat8x8_t __s2 = __p2; \
42053 fpm_t __s4 = __p4; \
42054 __ret = (float32x4_t) __builtin_neon_vmlallttq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
42055 __ret; \
42056})
42057#else
42058#define vmlallttq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42059 float32x4_t __ret; \
42060 float32x4_t __s0 = __p0; \
42061 mfloat8x16_t __s1 = __p1; \
42062 mfloat8x8_t __s2 = __p2; \
42063 fpm_t __s4 = __p4; \
42064 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42065 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42066 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
42067 __ret = (float32x4_t) __builtin_neon_vmlallttq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
42068 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
42069 __ret; \
42070})
42071#endif
42072
42073#ifdef __LITTLE_ENDIAN__
42074#define vmlallttq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42075 float32x4_t __ret; \
42076 float32x4_t __s0 = __p0; \
42077 mfloat8x16_t __s1 = __p1; \
42078 mfloat8x16_t __s2 = __p2; \
42079 fpm_t __s4 = __p4; \
42080 __ret = (float32x4_t) __builtin_neon_vmlallttq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
42081 __ret; \
42082})
42083#else
42084#define vmlallttq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42085 float32x4_t __ret; \
42086 float32x4_t __s0 = __p0; \
42087 mfloat8x16_t __s1 = __p1; \
42088 mfloat8x16_t __s2 = __p2; \
42089 fpm_t __s4 = __p4; \
42090 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42091 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42092 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42093 __ret = (float32x4_t) __builtin_neon_vmlallttq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
42094 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
42095 __ret; \
42096})
42097#endif
42098
42099#ifdef __LITTLE_ENDIAN__
42100__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42101 float16x8_t __ret;
42102 __ret = (float16x8_t) __builtin_neon_vmlaltq_f16_mf8_fpm((int8x16_t)__p0, __p1, __p2, __p3);
42103 return __ret;
42104}
42105#else
42106__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42107 float16x8_t __ret;
42108 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
42109 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42110 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42111 __ret = (float16x8_t) __builtin_neon_vmlaltq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3);
42112 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
42113 return __ret;
42114}
42115#endif
42116
42117#ifdef __LITTLE_ENDIAN__
42118#define vmlaltq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42119 float16x8_t __ret; \
42120 float16x8_t __s0 = __p0; \
42121 mfloat8x16_t __s1 = __p1; \
42122 mfloat8x8_t __s2 = __p2; \
42123 fpm_t __s4 = __p4; \
42124 __ret = (float16x8_t) __builtin_neon_vmlaltq_lane_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
42125 __ret; \
42126})
42127#else
42128#define vmlaltq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42129 float16x8_t __ret; \
42130 float16x8_t __s0 = __p0; \
42131 mfloat8x16_t __s1 = __p1; \
42132 mfloat8x8_t __s2 = __p2; \
42133 fpm_t __s4 = __p4; \
42134 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42135 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42136 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
42137 __ret = (float16x8_t) __builtin_neon_vmlaltq_lane_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
42138 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42139 __ret; \
42140})
42141#endif
42142
42143#ifdef __LITTLE_ENDIAN__
42144#define vmlaltq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42145 float16x8_t __ret; \
42146 float16x8_t __s0 = __p0; \
42147 mfloat8x16_t __s1 = __p1; \
42148 mfloat8x16_t __s2 = __p2; \
42149 fpm_t __s4 = __p4; \
42150 __ret = (float16x8_t) __builtin_neon_vmlaltq_laneq_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
42151 __ret; \
42152})
42153#else
42154#define vmlaltq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42155 float16x8_t __ret; \
42156 float16x8_t __s0 = __p0; \
42157 mfloat8x16_t __s1 = __p1; \
42158 mfloat8x16_t __s2 = __p2; \
42159 fpm_t __s4 = __p4; \
42160 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42161 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42162 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42163 __ret = (float16x8_t) __builtin_neon_vmlaltq_laneq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
42164 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42165 __ret; \
42166})
42167#endif
42168
42169#ifdef __LITTLE_ENDIAN__
42170#define vluti2_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42171 poly8x16_t __ret; \
42172 poly8x8_t __s0 = __p0; \
42173 uint8x8_t __s1 = __p1; \
42174 __ret = (poly8x16_t) __builtin_neon_vluti2_lane_p8((int8x8_t)__s0, (int8x8_t)__s1, __p2, 36); \
42175 __ret; \
42176})
42177#else
42178#define vluti2_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42179 poly8x16_t __ret; \
42180 poly8x8_t __s0 = __p0; \
42181 uint8x8_t __s1 = __p1; \
42182 poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42183 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42184 __ret = (poly8x16_t) __builtin_neon_vluti2_lane_p8((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 36); \
42185 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42186 __ret; \
42187})
42188#endif
42189
42190#ifdef __LITTLE_ENDIAN__
42191#define vluti2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42192 poly8x16_t __ret; \
42193 poly8x16_t __s0 = __p0; \
42194 uint8x8_t __s1 = __p1; \
42195 __ret = (poly8x16_t) __builtin_neon_vluti2q_lane_p8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 36); \
42196 __ret; \
42197})
42198#else
42199#define vluti2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42200 poly8x16_t __ret; \
42201 poly8x16_t __s0 = __p0; \
42202 uint8x8_t __s1 = __p1; \
42203 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42204 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42205 __ret = (poly8x16_t) __builtin_neon_vluti2q_lane_p8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 36); \
42206 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42207 __ret; \
42208})
42209#endif
42210
42211#ifdef __LITTLE_ENDIAN__
42212#define vluti2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42213 uint8x16_t __ret; \
42214 uint8x16_t __s0 = __p0; \
42215 uint8x8_t __s1 = __p1; \
42216 __ret = (uint8x16_t) __builtin_neon_vluti2q_lane_u8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 48); \
42217 __ret; \
42218})
42219#else
42220#define vluti2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42221 uint8x16_t __ret; \
42222 uint8x16_t __s0 = __p0; \
42223 uint8x8_t __s1 = __p1; \
42224 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42225 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42226 __ret = (uint8x16_t) __builtin_neon_vluti2q_lane_u8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 48); \
42227 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42228 __ret; \
42229})
42230#endif
42231
42232#ifdef __LITTLE_ENDIAN__
42233#define vluti2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42234 int8x16_t __ret; \
42235 int8x16_t __s0 = __p0; \
42236 uint8x8_t __s1 = __p1; \
42237 __ret = (int8x16_t) __builtin_neon_vluti2q_lane_s8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 32); \
42238 __ret; \
42239})
42240#else
42241#define vluti2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42242 int8x16_t __ret; \
42243 int8x16_t __s0 = __p0; \
42244 uint8x8_t __s1 = __p1; \
42245 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42246 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42247 __ret = (int8x16_t) __builtin_neon_vluti2q_lane_s8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 32); \
42248 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42249 __ret; \
42250})
42251#endif
42252
42253#ifdef __LITTLE_ENDIAN__
42254#define vluti2_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42255 uint8x16_t __ret; \
42256 uint8x8_t __s0 = __p0; \
42257 uint8x8_t __s1 = __p1; \
42258 __ret = (uint8x16_t) __builtin_neon_vluti2_lane_u8((int8x8_t)__s0, (int8x8_t)__s1, __p2, 48); \
42259 __ret; \
42260})
42261#else
42262#define vluti2_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42263 uint8x16_t __ret; \
42264 uint8x8_t __s0 = __p0; \
42265 uint8x8_t __s1 = __p1; \
42266 uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42267 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42268 __ret = (uint8x16_t) __builtin_neon_vluti2_lane_u8((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 48); \
42269 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42270 __ret; \
42271})
42272#endif
42273
42274#ifdef __LITTLE_ENDIAN__
42275#define vluti2_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42276 int8x16_t __ret; \
42277 int8x8_t __s0 = __p0; \
42278 uint8x8_t __s1 = __p1; \
42279 __ret = (int8x16_t) __builtin_neon_vluti2_lane_s8((int8x8_t)__s0, (int8x8_t)__s1, __p2, 32); \
42280 __ret; \
42281})
42282#else
42283#define vluti2_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42284 int8x16_t __ret; \
42285 int8x8_t __s0 = __p0; \
42286 uint8x8_t __s1 = __p1; \
42287 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42288 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42289 __ret = (int8x16_t) __builtin_neon_vluti2_lane_s8((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 32); \
42290 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42291 __ret; \
42292})
42293#endif
42294
42295#ifdef __LITTLE_ENDIAN__
42296#define vluti2_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42297 poly16x8_t __ret; \
42298 poly16x4_t __s0 = __p0; \
42299 uint8x8_t __s1 = __p1; \
42300 __ret = (poly16x8_t) __builtin_neon_vluti2_lane_p16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 37); \
42301 __ret; \
42302})
42303#else
42304#define vluti2_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42305 poly16x8_t __ret; \
42306 poly16x4_t __s0 = __p0; \
42307 uint8x8_t __s1 = __p1; \
42308 poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42309 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42310 __ret = (poly16x8_t) __builtin_neon_vluti2_lane_p16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 37); \
42311 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42312 __ret; \
42313})
42314#endif
42315
42316#ifdef __LITTLE_ENDIAN__
42317#define vluti2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42318 poly16x8_t __ret; \
42319 poly16x8_t __s0 = __p0; \
42320 uint8x8_t __s1 = __p1; \
42321 __ret = (poly16x8_t) __builtin_neon_vluti2q_lane_p16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 37); \
42322 __ret; \
42323})
42324#else
42325#define vluti2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42326 poly16x8_t __ret; \
42327 poly16x8_t __s0 = __p0; \
42328 uint8x8_t __s1 = __p1; \
42329 poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42330 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42331 __ret = (poly16x8_t) __builtin_neon_vluti2q_lane_p16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 37); \
42332 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42333 __ret; \
42334})
42335#endif
42336
42337#ifdef __LITTLE_ENDIAN__
42338#define vluti2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42339 uint16x8_t __ret; \
42340 uint16x8_t __s0 = __p0; \
42341 uint8x8_t __s1 = __p1; \
42342 __ret = (uint16x8_t) __builtin_neon_vluti2q_lane_u16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 49); \
42343 __ret; \
42344})
42345#else
42346#define vluti2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42347 uint16x8_t __ret; \
42348 uint16x8_t __s0 = __p0; \
42349 uint8x8_t __s1 = __p1; \
42350 uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42351 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42352 __ret = (uint16x8_t) __builtin_neon_vluti2q_lane_u16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 49); \
42353 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42354 __ret; \
42355})
42356#endif
42357
42358#ifdef __LITTLE_ENDIAN__
42359#define vluti2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42360 float16x8_t __ret; \
42361 float16x8_t __s0 = __p0; \
42362 uint8x8_t __s1 = __p1; \
42363 __ret = (float16x8_t) __builtin_neon_vluti2q_lane_f16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 40); \
42364 __ret; \
42365})
42366#else
42367#define vluti2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42368 float16x8_t __ret; \
42369 float16x8_t __s0 = __p0; \
42370 uint8x8_t __s1 = __p1; \
42371 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42372 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42373 __ret = (float16x8_t) __builtin_neon_vluti2q_lane_f16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 40); \
42374 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42375 __ret; \
42376})
42377#endif
42378
42379#ifdef __LITTLE_ENDIAN__
42380#define vluti2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42381 int16x8_t __ret; \
42382 int16x8_t __s0 = __p0; \
42383 uint8x8_t __s1 = __p1; \
42384 __ret = (int16x8_t) __builtin_neon_vluti2q_lane_s16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 33); \
42385 __ret; \
42386})
42387#else
42388#define vluti2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42389 int16x8_t __ret; \
42390 int16x8_t __s0 = __p0; \
42391 uint8x8_t __s1 = __p1; \
42392 int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42393 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42394 __ret = (int16x8_t) __builtin_neon_vluti2q_lane_s16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 33); \
42395 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42396 __ret; \
42397})
42398#endif
42399
42400#ifdef __LITTLE_ENDIAN__
42401#define vluti2_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42402 uint16x8_t __ret; \
42403 uint16x4_t __s0 = __p0; \
42404 uint8x8_t __s1 = __p1; \
42405 __ret = (uint16x8_t) __builtin_neon_vluti2_lane_u16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 49); \
42406 __ret; \
42407})
42408#else
42409#define vluti2_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42410 uint16x8_t __ret; \
42411 uint16x4_t __s0 = __p0; \
42412 uint8x8_t __s1 = __p1; \
42413 uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42414 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42415 __ret = (uint16x8_t) __builtin_neon_vluti2_lane_u16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 49); \
42416 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42417 __ret; \
42418})
42419#endif
42420
42421#ifdef __LITTLE_ENDIAN__
42422#define vluti2_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42423 float16x8_t __ret; \
42424 float16x4_t __s0 = __p0; \
42425 uint8x8_t __s1 = __p1; \
42426 __ret = (float16x8_t) __builtin_neon_vluti2_lane_f16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 40); \
42427 __ret; \
42428})
42429#else
42430#define vluti2_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42431 float16x8_t __ret; \
42432 float16x4_t __s0 = __p0; \
42433 uint8x8_t __s1 = __p1; \
42434 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42435 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42436 __ret = (float16x8_t) __builtin_neon_vluti2_lane_f16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 40); \
42437 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42438 __ret; \
42439})
42440#endif
42441
42442#ifdef __LITTLE_ENDIAN__
42443#define vluti2_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42444 int16x8_t __ret; \
42445 int16x4_t __s0 = __p0; \
42446 uint8x8_t __s1 = __p1; \
42447 __ret = (int16x8_t) __builtin_neon_vluti2_lane_s16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 33); \
42448 __ret; \
42449})
42450#else
42451#define vluti2_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42452 int16x8_t __ret; \
42453 int16x4_t __s0 = __p0; \
42454 uint8x8_t __s1 = __p1; \
42455 int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42456 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42457 __ret = (int16x8_t) __builtin_neon_vluti2_lane_s16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 33); \
42458 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42459 __ret; \
42460})
42461#endif
42462
42463#ifdef __LITTLE_ENDIAN__
42464#define vluti2_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42465 poly8x16_t __ret; \
42466 poly8x8_t __s0 = __p0; \
42467 uint8x16_t __s1 = __p1; \
42468 __ret = (poly8x16_t) __builtin_neon_vluti2_laneq_p8((int8x8_t)__s0, (int8x16_t)__s1, __p2, 36); \
42469 __ret; \
42470})
42471#else
42472#define vluti2_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42473 poly8x16_t __ret; \
42474 poly8x8_t __s0 = __p0; \
42475 uint8x16_t __s1 = __p1; \
42476 poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42477 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42478 __ret = (poly8x16_t) __builtin_neon_vluti2_laneq_p8((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 36); \
42479 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42480 __ret; \
42481})
42482#endif
42483
42484#ifdef __LITTLE_ENDIAN__
42485#define vluti2q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42486 poly8x16_t __ret; \
42487 poly8x16_t __s0 = __p0; \
42488 uint8x16_t __s1 = __p1; \
42489 __ret = (poly8x16_t) __builtin_neon_vluti2q_laneq_p8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \
42490 __ret; \
42491})
42492#else
42493#define vluti2q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42494 poly8x16_t __ret; \
42495 poly8x16_t __s0 = __p0; \
42496 uint8x16_t __s1 = __p1; \
42497 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42498 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42499 __ret = (poly8x16_t) __builtin_neon_vluti2q_laneq_p8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \
42500 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42501 __ret; \
42502})
42503#endif
42504
42505#ifdef __LITTLE_ENDIAN__
42506#define vluti2q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42507 uint8x16_t __ret; \
42508 uint8x16_t __s0 = __p0; \
42509 uint8x16_t __s1 = __p1; \
42510 __ret = (uint8x16_t) __builtin_neon_vluti2q_laneq_u8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \
42511 __ret; \
42512})
42513#else
42514#define vluti2q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42515 uint8x16_t __ret; \
42516 uint8x16_t __s0 = __p0; \
42517 uint8x16_t __s1 = __p1; \
42518 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42519 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42520 __ret = (uint8x16_t) __builtin_neon_vluti2q_laneq_u8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \
42521 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42522 __ret; \
42523})
42524#endif
42525
42526#ifdef __LITTLE_ENDIAN__
42527#define vluti2q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42528 int8x16_t __ret; \
42529 int8x16_t __s0 = __p0; \
42530 uint8x16_t __s1 = __p1; \
42531 __ret = (int8x16_t) __builtin_neon_vluti2q_laneq_s8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \
42532 __ret; \
42533})
42534#else
42535#define vluti2q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42536 int8x16_t __ret; \
42537 int8x16_t __s0 = __p0; \
42538 uint8x16_t __s1 = __p1; \
42539 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42540 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42541 __ret = (int8x16_t) __builtin_neon_vluti2q_laneq_s8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \
42542 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42543 __ret; \
42544})
42545#endif
42546
42547#ifdef __LITTLE_ENDIAN__
42548#define vluti2_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42549 uint8x16_t __ret; \
42550 uint8x8_t __s0 = __p0; \
42551 uint8x16_t __s1 = __p1; \
42552 __ret = (uint8x16_t) __builtin_neon_vluti2_laneq_u8((int8x8_t)__s0, (int8x16_t)__s1, __p2, 48); \
42553 __ret; \
42554})
42555#else
42556#define vluti2_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42557 uint8x16_t __ret; \
42558 uint8x8_t __s0 = __p0; \
42559 uint8x16_t __s1 = __p1; \
42560 uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42561 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42562 __ret = (uint8x16_t) __builtin_neon_vluti2_laneq_u8((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 48); \
42563 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42564 __ret; \
42565})
42566#endif
42567
42568#ifdef __LITTLE_ENDIAN__
42569#define vluti2_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42570 int8x16_t __ret; \
42571 int8x8_t __s0 = __p0; \
42572 uint8x16_t __s1 = __p1; \
42573 __ret = (int8x16_t) __builtin_neon_vluti2_laneq_s8((int8x8_t)__s0, (int8x16_t)__s1, __p2, 32); \
42574 __ret; \
42575})
42576#else
42577#define vluti2_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42578 int8x16_t __ret; \
42579 int8x8_t __s0 = __p0; \
42580 uint8x16_t __s1 = __p1; \
42581 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42582 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42583 __ret = (int8x16_t) __builtin_neon_vluti2_laneq_s8((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 32); \
42584 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42585 __ret; \
42586})
42587#endif
42588
42589#ifdef __LITTLE_ENDIAN__
42590#define vluti2_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42591 poly16x8_t __ret; \
42592 poly16x4_t __s0 = __p0; \
42593 uint8x16_t __s1 = __p1; \
42594 __ret = (poly16x8_t) __builtin_neon_vluti2_laneq_p16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 37); \
42595 __ret; \
42596})
42597#else
42598#define vluti2_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42599 poly16x8_t __ret; \
42600 poly16x4_t __s0 = __p0; \
42601 uint8x16_t __s1 = __p1; \
42602 poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42603 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42604 __ret = (poly16x8_t) __builtin_neon_vluti2_laneq_p16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 37); \
42605 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42606 __ret; \
42607})
42608#endif
42609
42610#ifdef __LITTLE_ENDIAN__
42611#define vluti2q_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42612 poly16x8_t __ret; \
42613 poly16x8_t __s0 = __p0; \
42614 uint8x16_t __s1 = __p1; \
42615 __ret = (poly16x8_t) __builtin_neon_vluti2q_laneq_p16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \
42616 __ret; \
42617})
42618#else
42619#define vluti2q_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42620 poly16x8_t __ret; \
42621 poly16x8_t __s0 = __p0; \
42622 uint8x16_t __s1 = __p1; \
42623 poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42624 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42625 __ret = (poly16x8_t) __builtin_neon_vluti2q_laneq_p16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \
42626 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42627 __ret; \
42628})
42629#endif
42630
42631#ifdef __LITTLE_ENDIAN__
42632#define vluti2q_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42633 uint16x8_t __ret; \
42634 uint16x8_t __s0 = __p0; \
42635 uint8x16_t __s1 = __p1; \
42636 __ret = (uint16x8_t) __builtin_neon_vluti2q_laneq_u16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \
42637 __ret; \
42638})
42639#else
42640#define vluti2q_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42641 uint16x8_t __ret; \
42642 uint16x8_t __s0 = __p0; \
42643 uint8x16_t __s1 = __p1; \
42644 uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42645 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42646 __ret = (uint16x8_t) __builtin_neon_vluti2q_laneq_u16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \
42647 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42648 __ret; \
42649})
42650#endif
42651
42652#ifdef __LITTLE_ENDIAN__
42653#define vluti2q_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42654 float16x8_t __ret; \
42655 float16x8_t __s0 = __p0; \
42656 uint8x16_t __s1 = __p1; \
42657 __ret = (float16x8_t) __builtin_neon_vluti2q_laneq_f16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 40); \
42658 __ret; \
42659})
42660#else
42661#define vluti2q_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42662 float16x8_t __ret; \
42663 float16x8_t __s0 = __p0; \
42664 uint8x16_t __s1 = __p1; \
42665 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42666 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42667 __ret = (float16x8_t) __builtin_neon_vluti2q_laneq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 40); \
42668 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42669 __ret; \
42670})
42671#endif
42672
42673#ifdef __LITTLE_ENDIAN__
42674#define vluti2q_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42675 int16x8_t __ret; \
42676 int16x8_t __s0 = __p0; \
42677 uint8x16_t __s1 = __p1; \
42678 __ret = (int16x8_t) __builtin_neon_vluti2q_laneq_s16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \
42679 __ret; \
42680})
42681#else
42682#define vluti2q_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42683 int16x8_t __ret; \
42684 int16x8_t __s0 = __p0; \
42685 uint8x16_t __s1 = __p1; \
42686 int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42687 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42688 __ret = (int16x8_t) __builtin_neon_vluti2q_laneq_s16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \
42689 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42690 __ret; \
42691})
42692#endif
42693
42694#ifdef __LITTLE_ENDIAN__
42695#define vluti2_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42696 uint16x8_t __ret; \
42697 uint16x4_t __s0 = __p0; \
42698 uint8x16_t __s1 = __p1; \
42699 __ret = (uint16x8_t) __builtin_neon_vluti2_laneq_u16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 49); \
42700 __ret; \
42701})
42702#else
42703#define vluti2_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42704 uint16x8_t __ret; \
42705 uint16x4_t __s0 = __p0; \
42706 uint8x16_t __s1 = __p1; \
42707 uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42708 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42709 __ret = (uint16x8_t) __builtin_neon_vluti2_laneq_u16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 49); \
42710 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42711 __ret; \
42712})
42713#endif
42714
42715#ifdef __LITTLE_ENDIAN__
42716#define vluti2_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42717 float16x8_t __ret; \
42718 float16x4_t __s0 = __p0; \
42719 uint8x16_t __s1 = __p1; \
42720 __ret = (float16x8_t) __builtin_neon_vluti2_laneq_f16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 40); \
42721 __ret; \
42722})
42723#else
42724#define vluti2_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42725 float16x8_t __ret; \
42726 float16x4_t __s0 = __p0; \
42727 uint8x16_t __s1 = __p1; \
42728 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42729 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42730 __ret = (float16x8_t) __builtin_neon_vluti2_laneq_f16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 40); \
42731 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42732 __ret; \
42733})
42734#endif
42735
42736#ifdef __LITTLE_ENDIAN__
42737#define vluti2_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42738 int16x8_t __ret; \
42739 int16x4_t __s0 = __p0; \
42740 uint8x16_t __s1 = __p1; \
42741 __ret = (int16x8_t) __builtin_neon_vluti2_laneq_s16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 33); \
42742 __ret; \
42743})
42744#else
42745#define vluti2_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42746 int16x8_t __ret; \
42747 int16x4_t __s0 = __p0; \
42748 uint8x16_t __s1 = __p1; \
42749 int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42750 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42751 __ret = (int16x8_t) __builtin_neon_vluti2_laneq_s16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 33); \
42752 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42753 __ret; \
42754})
42755#endif
42756
42757#ifdef __LITTLE_ENDIAN__
42758#define vluti4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42759 poly8x16_t __ret; \
42760 poly8x16_t __s0 = __p0; \
42761 uint8x8_t __s1 = __p1; \
42762 __ret = (poly8x16_t) __builtin_neon_vluti4q_lane_p8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 36); \
42763 __ret; \
42764})
42765#else
42766#define vluti4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42767 poly8x16_t __ret; \
42768 poly8x16_t __s0 = __p0; \
42769 uint8x8_t __s1 = __p1; \
42770 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42771 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42772 __ret = (poly8x16_t) __builtin_neon_vluti4q_lane_p8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 36); \
42773 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42774 __ret; \
42775})
42776#endif
42777
42778#ifdef __LITTLE_ENDIAN__
42779#define vluti4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42780 uint8x16_t __ret; \
42781 uint8x16_t __s0 = __p0; \
42782 uint8x8_t __s1 = __p1; \
42783 __ret = (uint8x16_t) __builtin_neon_vluti4q_lane_u8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 48); \
42784 __ret; \
42785})
42786#else
42787#define vluti4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42788 uint8x16_t __ret; \
42789 uint8x16_t __s0 = __p0; \
42790 uint8x8_t __s1 = __p1; \
42791 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42792 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42793 __ret = (uint8x16_t) __builtin_neon_vluti4q_lane_u8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 48); \
42794 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42795 __ret; \
42796})
42797#endif
42798
42799#ifdef __LITTLE_ENDIAN__
42800#define vluti4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42801 int8x16_t __ret; \
42802 int8x16_t __s0 = __p0; \
42803 uint8x8_t __s1 = __p1; \
42804 __ret = (int8x16_t) __builtin_neon_vluti4q_lane_s8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 32); \
42805 __ret; \
42806})
42807#else
42808#define vluti4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42809 int8x16_t __ret; \
42810 int8x16_t __s0 = __p0; \
42811 uint8x8_t __s1 = __p1; \
42812 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42813 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42814 __ret = (int8x16_t) __builtin_neon_vluti4q_lane_s8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 32); \
42815 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42816 __ret; \
42817})
42818#endif
42819
42820#ifdef __LITTLE_ENDIAN__
42821#define vluti4q_lane_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42822 poly16x8_t __ret; \
42823 poly16x8x2_t __s0 = __p0; \
42824 uint8x8_t __s1 = __p1; \
42825 __ret = (poly16x8_t) __builtin_neon_vluti4q_lane_p16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 37); \
42826 __ret; \
42827})
42828#else
42829#define vluti4q_lane_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42830 poly16x8_t __ret; \
42831 poly16x8x2_t __s0 = __p0; \
42832 uint8x8_t __s1 = __p1; \
42833 poly16x8x2_t __rev0; \
42834 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42835 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42836 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42837 __ret = (poly16x8_t) __builtin_neon_vluti4q_lane_p16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 37); \
42838 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42839 __ret; \
42840})
42841#endif
42842
42843#ifdef __LITTLE_ENDIAN__
42844#define vluti4q_lane_u16_x2(__p0, __p1, __p2) __extension__ ({ \
42845 uint16x8_t __ret; \
42846 uint16x8x2_t __s0 = __p0; \
42847 uint8x8_t __s1 = __p1; \
42848 __ret = (uint16x8_t) __builtin_neon_vluti4q_lane_u16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 49); \
42849 __ret; \
42850})
42851#else
42852#define vluti4q_lane_u16_x2(__p0, __p1, __p2) __extension__ ({ \
42853 uint16x8_t __ret; \
42854 uint16x8x2_t __s0 = __p0; \
42855 uint8x8_t __s1 = __p1; \
42856 uint16x8x2_t __rev0; \
42857 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42858 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42859 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42860 __ret = (uint16x8_t) __builtin_neon_vluti4q_lane_u16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 49); \
42861 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42862 __ret; \
42863})
42864#endif
42865
42866#ifdef __LITTLE_ENDIAN__
42867#define vluti4q_lane_f16_x2(__p0, __p1, __p2) __extension__ ({ \
42868 float16x8_t __ret; \
42869 float16x8x2_t __s0 = __p0; \
42870 uint8x8_t __s1 = __p1; \
42871 __ret = (float16x8_t) __builtin_neon_vluti4q_lane_f16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 40); \
42872 __ret; \
42873})
42874#else
42875#define vluti4q_lane_f16_x2(__p0, __p1, __p2) __extension__ ({ \
42876 float16x8_t __ret; \
42877 float16x8x2_t __s0 = __p0; \
42878 uint8x8_t __s1 = __p1; \
42879 float16x8x2_t __rev0; \
42880 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42881 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42882 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42883 __ret = (float16x8_t) __builtin_neon_vluti4q_lane_f16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 40); \
42884 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42885 __ret; \
42886})
42887#endif
42888
42889#ifdef __LITTLE_ENDIAN__
42890#define vluti4q_lane_s16_x2(__p0, __p1, __p2) __extension__ ({ \
42891 int16x8_t __ret; \
42892 int16x8x2_t __s0 = __p0; \
42893 uint8x8_t __s1 = __p1; \
42894 __ret = (int16x8_t) __builtin_neon_vluti4q_lane_s16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 33); \
42895 __ret; \
42896})
42897#else
42898#define vluti4q_lane_s16_x2(__p0, __p1, __p2) __extension__ ({ \
42899 int16x8_t __ret; \
42900 int16x8x2_t __s0 = __p0; \
42901 uint8x8_t __s1 = __p1; \
42902 int16x8x2_t __rev0; \
42903 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42904 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42905 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42906 __ret = (int16x8_t) __builtin_neon_vluti4q_lane_s16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 33); \
42907 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42908 __ret; \
42909})
42910#endif
42911
42912#ifdef __LITTLE_ENDIAN__
42913#define vluti4q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42914 poly8x16_t __ret; \
42915 poly8x16_t __s0 = __p0; \
42916 uint8x16_t __s1 = __p1; \
42917 __ret = (poly8x16_t) __builtin_neon_vluti4q_laneq_p8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \
42918 __ret; \
42919})
42920#else
42921#define vluti4q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42922 poly8x16_t __ret; \
42923 poly8x16_t __s0 = __p0; \
42924 uint8x16_t __s1 = __p1; \
42925 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42926 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42927 __ret = (poly8x16_t) __builtin_neon_vluti4q_laneq_p8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \
42928 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42929 __ret; \
42930})
42931#endif
42932
42933#ifdef __LITTLE_ENDIAN__
42934#define vluti4q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42935 uint8x16_t __ret; \
42936 uint8x16_t __s0 = __p0; \
42937 uint8x16_t __s1 = __p1; \
42938 __ret = (uint8x16_t) __builtin_neon_vluti4q_laneq_u8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \
42939 __ret; \
42940})
42941#else
42942#define vluti4q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42943 uint8x16_t __ret; \
42944 uint8x16_t __s0 = __p0; \
42945 uint8x16_t __s1 = __p1; \
42946 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42947 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42948 __ret = (uint8x16_t) __builtin_neon_vluti4q_laneq_u8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \
42949 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42950 __ret; \
42951})
42952#endif
42953
42954#ifdef __LITTLE_ENDIAN__
42955#define vluti4q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42956 int8x16_t __ret; \
42957 int8x16_t __s0 = __p0; \
42958 uint8x16_t __s1 = __p1; \
42959 __ret = (int8x16_t) __builtin_neon_vluti4q_laneq_s8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \
42960 __ret; \
42961})
42962#else
42963#define vluti4q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42964 int8x16_t __ret; \
42965 int8x16_t __s0 = __p0; \
42966 uint8x16_t __s1 = __p1; \
42967 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42968 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42969 __ret = (int8x16_t) __builtin_neon_vluti4q_laneq_s8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \
42970 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42971 __ret; \
42972})
42973#endif
42974
42975#ifdef __LITTLE_ENDIAN__
42976#define vluti4q_laneq_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42977 poly16x8_t __ret; \
42978 poly16x8x2_t __s0 = __p0; \
42979 uint8x16_t __s1 = __p1; \
42980 __ret = (poly16x8_t) __builtin_neon_vluti4q_laneq_p16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 37); \
42981 __ret; \
42982})
42983#else
42984#define vluti4q_laneq_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42985 poly16x8_t __ret; \
42986 poly16x8x2_t __s0 = __p0; \
42987 uint8x16_t __s1 = __p1; \
42988 poly16x8x2_t __rev0; \
42989 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42990 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42991 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42992 __ret = (poly16x8_t) __builtin_neon_vluti4q_laneq_p16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 37); \
42993 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42994 __ret; \
42995})
42996#endif
42997
42998#ifdef __LITTLE_ENDIAN__
42999#define vluti4q_laneq_u16_x2(__p0, __p1, __p2) __extension__ ({ \
43000 uint16x8_t __ret; \
43001 uint16x8x2_t __s0 = __p0; \
43002 uint8x16_t __s1 = __p1; \
43003 __ret = (uint16x8_t) __builtin_neon_vluti4q_laneq_u16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 49); \
43004 __ret; \
43005})
43006#else
43007#define vluti4q_laneq_u16_x2(__p0, __p1, __p2) __extension__ ({ \
43008 uint16x8_t __ret; \
43009 uint16x8x2_t __s0 = __p0; \
43010 uint8x16_t __s1 = __p1; \
43011 uint16x8x2_t __rev0; \
43012 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43013 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43014 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43015 __ret = (uint16x8_t) __builtin_neon_vluti4q_laneq_u16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 49); \
43016 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43017 __ret; \
43018})
43019#endif
43020
43021#ifdef __LITTLE_ENDIAN__
43022#define vluti4q_laneq_f16_x2(__p0, __p1, __p2) __extension__ ({ \
43023 float16x8_t __ret; \
43024 float16x8x2_t __s0 = __p0; \
43025 uint8x16_t __s1 = __p1; \
43026 __ret = (float16x8_t) __builtin_neon_vluti4q_laneq_f16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 40); \
43027 __ret; \
43028})
43029#else
43030#define vluti4q_laneq_f16_x2(__p0, __p1, __p2) __extension__ ({ \
43031 float16x8_t __ret; \
43032 float16x8x2_t __s0 = __p0; \
43033 uint8x16_t __s1 = __p1; \
43034 float16x8x2_t __rev0; \
43035 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43036 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43037 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43038 __ret = (float16x8_t) __builtin_neon_vluti4q_laneq_f16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 40); \
43039 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43040 __ret; \
43041})
43042#endif
43043
43044#ifdef __LITTLE_ENDIAN__
43045#define vluti4q_laneq_s16_x2(__p0, __p1, __p2) __extension__ ({ \
43046 int16x8_t __ret; \
43047 int16x8x2_t __s0 = __p0; \
43048 uint8x16_t __s1 = __p1; \
43049 __ret = (int16x8_t) __builtin_neon_vluti4q_laneq_s16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 33); \
43050 __ret; \
43051})
43052#else
43053#define vluti4q_laneq_s16_x2(__p0, __p1, __p2) __extension__ ({ \
43054 int16x8_t __ret; \
43055 int16x8x2_t __s0 = __p0; \
43056 uint8x16_t __s1 = __p1; \
43057 int16x8x2_t __rev0; \
43058 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43059 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43060 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43061 __ret = (int16x8_t) __builtin_neon_vluti4q_laneq_s16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 33); \
43062 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43063 __ret; \
43064})
43065#endif
43066
43067#ifdef __LITTLE_ENDIAN__
43068#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43069 bfloat16x8_t __ret; \
43070 bfloat16x8_t __s0 = __p0; \
43071 uint8x8_t __s1 = __p1; \
43072 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_lane_bf16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 43); \
43073 __ret; \
43074})
43075#else
43076#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43077 bfloat16x8_t __ret; \
43078 bfloat16x8_t __s0 = __p0; \
43079 uint8x8_t __s1 = __p1; \
43080 bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
43081 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
43082 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_lane_bf16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 43); \
43083 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43084 __ret; \
43085})
43086#endif
43087
43088#ifdef __LITTLE_ENDIAN__
43089#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43090 bfloat16x8_t __ret; \
43091 bfloat16x4_t __s0 = __p0; \
43092 uint8x8_t __s1 = __p1; \
43093 __ret = (bfloat16x8_t) __builtin_neon_vluti2_lane_bf16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 43); \
43094 __ret; \
43095})
43096#else
43097#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43098 bfloat16x8_t __ret; \
43099 bfloat16x4_t __s0 = __p0; \
43100 uint8x8_t __s1 = __p1; \
43101 bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
43102 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
43103 __ret = (bfloat16x8_t) __builtin_neon_vluti2_lane_bf16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 43); \
43104 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43105 __ret; \
43106})
43107#endif
43108
43109#ifdef __LITTLE_ENDIAN__
43110#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43111 bfloat16x8_t __ret; \
43112 bfloat16x8_t __s0 = __p0; \
43113 uint8x16_t __s1 = __p1; \
43114 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_laneq_bf16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 43); \
43115 __ret; \
43116})
43117#else
43118#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43119 bfloat16x8_t __ret; \
43120 bfloat16x8_t __s0 = __p0; \
43121 uint8x16_t __s1 = __p1; \
43122 bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
43123 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43124 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_laneq_bf16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 43); \
43125 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43126 __ret; \
43127})
43128#endif
43129
43130#ifdef __LITTLE_ENDIAN__
43131#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43132 bfloat16x8_t __ret; \
43133 bfloat16x4_t __s0 = __p0; \
43134 uint8x16_t __s1 = __p1; \
43135 __ret = (bfloat16x8_t) __builtin_neon_vluti2_laneq_bf16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 43); \
43136 __ret; \
43137})
43138#else
43139#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43140 bfloat16x8_t __ret; \
43141 bfloat16x4_t __s0 = __p0; \
43142 uint8x16_t __s1 = __p1; \
43143 bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
43144 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43145 __ret = (bfloat16x8_t) __builtin_neon_vluti2_laneq_bf16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 43); \
43146 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43147 __ret; \
43148})
43149#endif
43150
43151#ifdef __LITTLE_ENDIAN__
43152#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43153 bfloat16x8_t __ret; \
43154 bfloat16x8x2_t __s0 = __p0; \
43155 uint8x8_t __s1 = __p1; \
43156 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_lane_bf16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 43); \
43157 __ret; \
43158})
43159#else
43160#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43161 bfloat16x8_t __ret; \
43162 bfloat16x8x2_t __s0 = __p0; \
43163 uint8x8_t __s1 = __p1; \
43164 bfloat16x8x2_t __rev0; \
43165 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43166 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43167 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
43168 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_lane_bf16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 43); \
43169 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43170 __ret; \
43171})
43172#endif
43173
43174#ifdef __LITTLE_ENDIAN__
43175#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43176 bfloat16x8_t __ret; \
43177 bfloat16x8x2_t __s0 = __p0; \
43178 uint8x16_t __s1 = __p1; \
43179 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_laneq_bf16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 43); \
43180 __ret; \
43181})
43182#else
43183#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43184 bfloat16x8_t __ret; \
43185 bfloat16x8x2_t __s0 = __p0; \
43186 uint8x16_t __s1 = __p1; \
43187 bfloat16x8x2_t __rev0; \
43188 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43189 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43190 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43191 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_laneq_bf16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 43); \
43192 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43193 __ret; \
43194})
43195#endif
43196
43197#ifdef __LITTLE_ENDIAN__
43198__ai __attribute__((target("neon,faminmax"))) float64x2_t vamaxq_f64(float64x2_t __p0, float64x2_t __p1) {
43199 float64x2_t __ret;
43200 __ret = (float64x2_t) __builtin_neon_vamaxq_f64((int8x16_t)__p0, (int8x16_t)__p1, 42);
41256 return __ret;43201 return __ret;
41257}43202}
43203#else
43204__ai __attribute__((target("neon,faminmax"))) float64x2_t vamaxq_f64(float64x2_t __p0, float64x2_t __p1) {
43205 float64x2_t __ret;
43206 float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43207 float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43208 __ret = (float64x2_t) __builtin_neon_vamaxq_f64((int8x16_t)__rev0, (int8x16_t)__rev1, 42);
43209 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
43210 return __ret;
43211}
43212#endif
43213
41258#ifdef __LITTLE_ENDIAN__43214#ifdef __LITTLE_ENDIAN__
41259__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __a64_vcvtq_low_bf16_f32(float32x4_t __p0) {43215__ai __attribute__((target("neon,faminmax"))) float32x4_t vamaxq_f32(float32x4_t __p0, float32x4_t __p1) {
41260 bfloat16x8_t __ret;43216 float32x4_t __ret;
41261 __ret = (bfloat16x8_t) __builtin_neon___a64_vcvtq_low_bf16_f32((int8x16_t)__p0, 43);43217 __ret = (float32x4_t) __builtin_neon_vamaxq_f32((int8x16_t)__p0, (int8x16_t)__p1, 41);
41262 return __ret;43218 return __ret;
41263}43219}
41264#else43220#else
41265__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __a64_vcvtq_low_bf16_f32(float32x4_t __p0) {43221__ai __attribute__((target("neon,faminmax"))) float32x4_t vamaxq_f32(float32x4_t __p0, float32x4_t __p1) {
41266 bfloat16x8_t __ret;43222 float32x4_t __ret;
41267 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);43223 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41268 __ret = (bfloat16x8_t) __builtin_neon___a64_vcvtq_low_bf16_f32((int8x16_t)__rev0, 43);43224 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43225 __ret = (float32x4_t) __builtin_neon_vamaxq_f32((int8x16_t)__rev0, (int8x16_t)__rev1, 41);
43226 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43227 return __ret;
43228}
43229#endif
43230
43231#ifdef __LITTLE_ENDIAN__
43232__ai __attribute__((target("neon,faminmax"))) float16x8_t vamaxq_f16(float16x8_t __p0, float16x8_t __p1) {
43233 float16x8_t __ret;
43234 __ret = (float16x8_t) __builtin_neon_vamaxq_f16((int8x16_t)__p0, (int8x16_t)__p1, 40);
43235 return __ret;
43236}
43237#else
43238__ai __attribute__((target("neon,faminmax"))) float16x8_t vamaxq_f16(float16x8_t __p0, float16x8_t __p1) {
43239 float16x8_t __ret;
43240 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
43241 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
43242 __ret = (float16x8_t) __builtin_neon_vamaxq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, 40);
41269 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);43243 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41270 return __ret;43244 return __ret;
41271}43245}
41272__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap___a64_vcvtq_low_bf16_f32(float32x4_t __p0) {43246#endif
41273 bfloat16x8_t __ret;43247
41274 __ret = (bfloat16x8_t) __builtin_neon___a64_vcvtq_low_bf16_f32((int8x16_t)__p0, 43);43248#ifdef __LITTLE_ENDIAN__
43249__ai __attribute__((target("neon,faminmax"))) float32x2_t vamax_f32(float32x2_t __p0, float32x2_t __p1) {
43250 float32x2_t __ret;
43251 __ret = (float32x2_t) __builtin_neon_vamax_f32((int8x8_t)__p0, (int8x8_t)__p1, 9);
43252 return __ret;
43253}
43254#else
43255__ai __attribute__((target("neon,faminmax"))) float32x2_t vamax_f32(float32x2_t __p0, float32x2_t __p1) {
43256 float32x2_t __ret;
43257 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43258 float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43259 __ret = (float32x2_t) __builtin_neon_vamax_f32((int8x8_t)__rev0, (int8x8_t)__rev1, 9);
43260 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
43261 return __ret;
43262}
43263#endif
43264
43265#ifdef __LITTLE_ENDIAN__
43266__ai __attribute__((target("neon,faminmax"))) float16x4_t vamax_f16(float16x4_t __p0, float16x4_t __p1) {
43267 float16x4_t __ret;
43268 __ret = (float16x4_t) __builtin_neon_vamax_f16((int8x8_t)__p0, (int8x8_t)__p1, 8);
43269 return __ret;
43270}
43271#else
43272__ai __attribute__((target("neon,faminmax"))) float16x4_t vamax_f16(float16x4_t __p0, float16x4_t __p1) {
43273 float16x4_t __ret;
43274 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
43275 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43276 __ret = (float16x4_t) __builtin_neon_vamax_f16((int8x8_t)__rev0, (int8x8_t)__rev1, 8);
43277 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43278 return __ret;
43279}
43280#endif
43281
43282#ifdef __LITTLE_ENDIAN__
43283__ai __attribute__((target("neon,faminmax"))) float64x2_t vaminq_f64(float64x2_t __p0, float64x2_t __p1) {
43284 float64x2_t __ret;
43285 __ret = (float64x2_t) __builtin_neon_vaminq_f64((int8x16_t)__p0, (int8x16_t)__p1, 42);
43286 return __ret;
43287}
43288#else
43289__ai __attribute__((target("neon,faminmax"))) float64x2_t vaminq_f64(float64x2_t __p0, float64x2_t __p1) {
43290 float64x2_t __ret;
43291 float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43292 float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43293 __ret = (float64x2_t) __builtin_neon_vaminq_f64((int8x16_t)__rev0, (int8x16_t)__rev1, 42);
43294 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
43295 return __ret;
43296}
43297#endif
43298
43299#ifdef __LITTLE_ENDIAN__
43300__ai __attribute__((target("neon,faminmax"))) float32x4_t vaminq_f32(float32x4_t __p0, float32x4_t __p1) {
43301 float32x4_t __ret;
43302 __ret = (float32x4_t) __builtin_neon_vaminq_f32((int8x16_t)__p0, (int8x16_t)__p1, 41);
43303 return __ret;
43304}
43305#else
43306__ai __attribute__((target("neon,faminmax"))) float32x4_t vaminq_f32(float32x4_t __p0, float32x4_t __p1) {
43307 float32x4_t __ret;
43308 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
43309 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43310 __ret = (float32x4_t) __builtin_neon_vaminq_f32((int8x16_t)__rev0, (int8x16_t)__rev1, 41);
43311 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43312 return __ret;
43313}
43314#endif
43315
43316#ifdef __LITTLE_ENDIAN__
43317__ai __attribute__((target("neon,faminmax"))) float16x8_t vaminq_f16(float16x8_t __p0, float16x8_t __p1) {
43318 float16x8_t __ret;
43319 __ret = (float16x8_t) __builtin_neon_vaminq_f16((int8x16_t)__p0, (int8x16_t)__p1, 40);
43320 return __ret;
43321}
43322#else
43323__ai __attribute__((target("neon,faminmax"))) float16x8_t vaminq_f16(float16x8_t __p0, float16x8_t __p1) {
43324 float16x8_t __ret;
43325 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
43326 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
43327 __ret = (float16x8_t) __builtin_neon_vaminq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, 40);
43328 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
43329 return __ret;
43330}
43331#endif
43332
43333#ifdef __LITTLE_ENDIAN__
43334__ai __attribute__((target("neon,faminmax"))) float32x2_t vamin_f32(float32x2_t __p0, float32x2_t __p1) {
43335 float32x2_t __ret;
43336 __ret = (float32x2_t) __builtin_neon_vamin_f32((int8x8_t)__p0, (int8x8_t)__p1, 9);
43337 return __ret;
43338}
43339#else
43340__ai __attribute__((target("neon,faminmax"))) float32x2_t vamin_f32(float32x2_t __p0, float32x2_t __p1) {
43341 float32x2_t __ret;
43342 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43343 float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43344 __ret = (float32x2_t) __builtin_neon_vamin_f32((int8x8_t)__rev0, (int8x8_t)__rev1, 9);
43345 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41275 return __ret;43346 return __ret;
41276}43347}
41277#endif43348#endif
4127843349
43350#ifdef __LITTLE_ENDIAN__
43351__ai __attribute__((target("neon,faminmax"))) float16x4_t vamin_f16(float16x4_t __p0, float16x4_t __p1) {
43352 float16x4_t __ret;
43353 __ret = (float16x4_t) __builtin_neon_vamin_f16((int8x8_t)__p0, (int8x8_t)__p1, 8);
43354 return __ret;
43355}
43356#else
43357__ai __attribute__((target("neon,faminmax"))) float16x4_t vamin_f16(float16x4_t __p0, float16x4_t __p1) {
43358 float16x4_t __ret;
43359 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
43360 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43361 __ret = (float16x4_t) __builtin_neon_vamin_f16((int8x8_t)__rev0, (int8x8_t)__rev1, 8);
43362 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43363 return __ret;
43364}
43365#endif
43366
43367#endif
43368#if defined(__aarch64__) || defined(__arm64ec__)
43369__ai __attribute__((target("aes,neon"))) poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) {
43370 poly128_t __ret;
43371 __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1);
43372 return __ret;
43373}
41279#ifdef __LITTLE_ENDIAN__43374#ifdef __LITTLE_ENDIAN__
41280#define vcopyq_lane_bf16(__p0_230, __p1_230, __p2_230, __p3_230) __extension__ ({ \43375#define vcopyq_lane_bf16(__p0_230, __p1_230, __p2_230, __p3_230) __extension__ ({ \
41281 bfloat16x8_t __ret_230; \43376 bfloat16x8_t __ret_230; \
...@@ -41363,14 +43458,14 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap___a64_vcvtq_low_...@@ -41363,14 +43458,14 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap___a64_vcvtq_low_
41363#ifdef __LITTLE_ENDIAN__43458#ifdef __LITTLE_ENDIAN__
41364__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) {43459__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) {
41365 bfloat16x4_t __ret;43460 bfloat16x4_t __ret;
41366 __ret = vget_low_bf16(__a64_vcvtq_low_bf16_f32(__p0));43461 __ret = (bfloat16x4_t) __builtin_neon_vcvt_bf16_f32((int8x16_t)__p0, 11);
41367 return __ret;43462 return __ret;
41368}43463}
41369#else43464#else
41370__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) {43465__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) {
41371 bfloat16x4_t __ret;43466 bfloat16x4_t __ret;
41372 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);43467 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41373 __ret = __noswap_vget_low_bf16(__noswap___a64_vcvtq_low_bf16_f32(__rev0));43468 __ret = (bfloat16x4_t) __builtin_neon_vcvt_bf16_f32((int8x16_t)__rev0, 11);
41374 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);43469 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41375 return __ret;43470 return __ret;
41376}43471}
...@@ -41396,14 +43491,14 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloa...@@ -41396,14 +43491,14 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloa
41396#ifdef __LITTLE_ENDIAN__43491#ifdef __LITTLE_ENDIAN__
41397__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) {43492__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) {
41398 bfloat16x8_t __ret;43493 bfloat16x8_t __ret;
41399 __ret = __a64_vcvtq_low_bf16_f32(__p0);43494 __ret = (bfloat16x8_t) __builtin_neon_vcvtq_low_bf16_f32((int8x16_t)__p0, 43);
41400 return __ret;43495 return __ret;
41401}43496}
41402#else43497#else
41403__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) {43498__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) {
41404 bfloat16x8_t __ret;43499 bfloat16x8_t __ret;
41405 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);43500 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41406 __ret = __noswap___a64_vcvtq_low_bf16_f32(__rev0);43501 __ret = (bfloat16x8_t) __builtin_neon_vcvtq_low_bf16_f32((int8x16_t)__rev0, 43);
41407 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);43502 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41408 return __ret;43503 return __ret;
41409}43504}
...@@ -41705,9 +43800,7 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s16(int...@@ -41705,9 +43800,7 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s16(int
41705 uint32x4_t __s0_238 = __p0_238; \43800 uint32x4_t __s0_238 = __p0_238; \
41706 uint8x16_t __s1_238 = __p1_238; \43801 uint8x16_t __s1_238 = __p1_238; \
41707 uint8x16_t __s2_238 = __p2_238; \43802 uint8x16_t __s2_238 = __p2_238; \
41708uint8x16_t __reint_238 = __s2_238; \43803 __ret_238 = vdotq_u32(__s0_238, __s1_238, __builtin_bit_cast(uint8x16_t, splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_238), __p3_238))); \
41709uint32x4_t __reint1_238 = splatq_laneq_u32(*(uint32x4_t *) &__reint_238, __p3_238); \
41710 __ret_238 = vdotq_u32(__s0_238, __s1_238, *(uint8x16_t *) &__reint1_238); \
41711 __ret_238; \43804 __ret_238; \
41712})43805})
41713#else43806#else
...@@ -41719,9 +43812,7 @@ uint32x4_t __reint1_238 = splatq_laneq_u32(*(uint32x4_t *) &__reint_238, __p3_23...@@ -41719,9 +43812,7 @@ uint32x4_t __reint1_238 = splatq_laneq_u32(*(uint32x4_t *) &__reint_238, __p3_23
41719 uint32x4_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, 3, 2, 1, 0); \43812 uint32x4_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, 3, 2, 1, 0); \
41720 uint8x16_t __rev1_239; __rev1_239 = __builtin_shufflevector(__s1_239, __s1_239, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \43813 uint8x16_t __rev1_239; __rev1_239 = __builtin_shufflevector(__s1_239, __s1_239, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41721 uint8x16_t __rev2_239; __rev2_239 = __builtin_shufflevector(__s2_239, __s2_239, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \43814 uint8x16_t __rev2_239; __rev2_239 = __builtin_shufflevector(__s2_239, __s2_239, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41722uint8x16_t __reint_239 = __rev2_239; \43815 __ret_239 = __noswap_vdotq_u32(__rev0_239, __rev1_239, __builtin_bit_cast(uint8x16_t, __noswap_splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_239), __p3_239))); \
41723uint32x4_t __reint1_239 = __noswap_splatq_laneq_u32(*(uint32x4_t *) &__reint_239, __p3_239); \
41724 __ret_239 = __noswap_vdotq_u32(__rev0_239, __rev1_239, *(uint8x16_t *) &__reint1_239); \
41725 __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, 3, 2, 1, 0); \43816 __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, 3, 2, 1, 0); \
41726 __ret_239; \43817 __ret_239; \
41727})43818})
...@@ -41733,9 +43824,7 @@ uint32x4_t __reint1_239 = __noswap_splatq_laneq_u32(*(uint32x4_t *) &__reint_239...@@ -41733,9 +43824,7 @@ uint32x4_t __reint1_239 = __noswap_splatq_laneq_u32(*(uint32x4_t *) &__reint_239
41733 int32x4_t __s0_240 = __p0_240; \43824 int32x4_t __s0_240 = __p0_240; \
41734 int8x16_t __s1_240 = __p1_240; \43825 int8x16_t __s1_240 = __p1_240; \
41735 int8x16_t __s2_240 = __p2_240; \43826 int8x16_t __s2_240 = __p2_240; \
41736int8x16_t __reint_240 = __s2_240; \43827 __ret_240 = vdotq_s32(__s0_240, __s1_240, __builtin_bit_cast(int8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_240), __p3_240))); \
41737int32x4_t __reint1_240 = splatq_laneq_s32(*(int32x4_t *) &__reint_240, __p3_240); \
41738 __ret_240 = vdotq_s32(__s0_240, __s1_240, *(int8x16_t *) &__reint1_240); \
41739 __ret_240; \43828 __ret_240; \
41740})43829})
41741#else43830#else
...@@ -41747,9 +43836,7 @@ int32x4_t __reint1_240 = splatq_laneq_s32(*(int32x4_t *) &__reint_240, __p3_240)...@@ -41747,9 +43836,7 @@ int32x4_t __reint1_240 = splatq_laneq_s32(*(int32x4_t *) &__reint_240, __p3_240)
41747 int32x4_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, 3, 2, 1, 0); \43836 int32x4_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, 3, 2, 1, 0); \
41748 int8x16_t __rev1_241; __rev1_241 = __builtin_shufflevector(__s1_241, __s1_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \43837 int8x16_t __rev1_241; __rev1_241 = __builtin_shufflevector(__s1_241, __s1_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41749 int8x16_t __rev2_241; __rev2_241 = __builtin_shufflevector(__s2_241, __s2_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \43838 int8x16_t __rev2_241; __rev2_241 = __builtin_shufflevector(__s2_241, __s2_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41750int8x16_t __reint_241 = __rev2_241; \43839 __ret_241 = __noswap_vdotq_s32(__rev0_241, __rev1_241, __builtin_bit_cast(int8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_241), __p3_241))); \
41751int32x4_t __reint1_241 = __noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_241, __p3_241); \
41752 __ret_241 = __noswap_vdotq_s32(__rev0_241, __rev1_241, *(int8x16_t *) &__reint1_241); \
41753 __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, 3, 2, 1, 0); \43840 __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, 3, 2, 1, 0); \
41754 __ret_241; \43841 __ret_241; \
41755})43842})
...@@ -41761,9 +43848,7 @@ int32x4_t __reint1_241 = __noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_241,...@@ -41761,9 +43848,7 @@ int32x4_t __reint1_241 = __noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_241,
41761 uint32x2_t __s0_242 = __p0_242; \43848 uint32x2_t __s0_242 = __p0_242; \
41762 uint8x8_t __s1_242 = __p1_242; \43849 uint8x8_t __s1_242 = __p1_242; \
41763 uint8x16_t __s2_242 = __p2_242; \43850 uint8x16_t __s2_242 = __p2_242; \
41764uint8x16_t __reint_242 = __s2_242; \43851 __ret_242 = vdot_u32(__s0_242, __s1_242, __builtin_bit_cast(uint8x8_t, splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_242), __p3_242))); \
41765uint32x2_t __reint1_242 = splat_laneq_u32(*(uint32x4_t *) &__reint_242, __p3_242); \
41766 __ret_242 = vdot_u32(__s0_242, __s1_242, *(uint8x8_t *) &__reint1_242); \
41767 __ret_242; \43852 __ret_242; \
41768})43853})
41769#else43854#else
...@@ -41775,9 +43860,7 @@ uint32x2_t __reint1_242 = splat_laneq_u32(*(uint32x4_t *) &__reint_242, __p3_242...@@ -41775,9 +43860,7 @@ uint32x2_t __reint1_242 = splat_laneq_u32(*(uint32x4_t *) &__reint_242, __p3_242
41775 uint32x2_t __rev0_243; __rev0_243 = __builtin_shufflevector(__s0_243, __s0_243, 1, 0); \43860 uint32x2_t __rev0_243; __rev0_243 = __builtin_shufflevector(__s0_243, __s0_243, 1, 0); \
41776 uint8x8_t __rev1_243; __rev1_243 = __builtin_shufflevector(__s1_243, __s1_243, 7, 6, 5, 4, 3, 2, 1, 0); \43861 uint8x8_t __rev1_243; __rev1_243 = __builtin_shufflevector(__s1_243, __s1_243, 7, 6, 5, 4, 3, 2, 1, 0); \
41777 uint8x16_t __rev2_243; __rev2_243 = __builtin_shufflevector(__s2_243, __s2_243, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \43862 uint8x16_t __rev2_243; __rev2_243 = __builtin_shufflevector(__s2_243, __s2_243, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41778uint8x16_t __reint_243 = __rev2_243; \43863 __ret_243 = __noswap_vdot_u32(__rev0_243, __rev1_243, __builtin_bit_cast(uint8x8_t, __noswap_splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_243), __p3_243))); \
41779uint32x2_t __reint1_243 = __noswap_splat_laneq_u32(*(uint32x4_t *) &__reint_243, __p3_243); \
41780 __ret_243 = __noswap_vdot_u32(__rev0_243, __rev1_243, *(uint8x8_t *) &__reint1_243); \
41781 __ret_243 = __builtin_shufflevector(__ret_243, __ret_243, 1, 0); \43864 __ret_243 = __builtin_shufflevector(__ret_243, __ret_243, 1, 0); \
41782 __ret_243; \43865 __ret_243; \
41783})43866})
...@@ -41789,9 +43872,7 @@ uint32x2_t __reint1_243 = __noswap_splat_laneq_u32(*(uint32x4_t *) &__reint_243,...@@ -41789,9 +43872,7 @@ uint32x2_t __reint1_243 = __noswap_splat_laneq_u32(*(uint32x4_t *) &__reint_243,
41789 int32x2_t __s0_244 = __p0_244; \43872 int32x2_t __s0_244 = __p0_244; \
41790 int8x8_t __s1_244 = __p1_244; \43873 int8x8_t __s1_244 = __p1_244; \
41791 int8x16_t __s2_244 = __p2_244; \43874 int8x16_t __s2_244 = __p2_244; \
41792int8x16_t __reint_244 = __s2_244; \43875 __ret_244 = vdot_s32(__s0_244, __s1_244, __builtin_bit_cast(int8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_244), __p3_244))); \
41793int32x2_t __reint1_244 = splat_laneq_s32(*(int32x4_t *) &__reint_244, __p3_244); \
41794 __ret_244 = vdot_s32(__s0_244, __s1_244, *(int8x8_t *) &__reint1_244); \
41795 __ret_244; \43876 __ret_244; \
41796})43877})
41797#else43878#else
...@@ -41803,9 +43884,7 @@ int32x2_t __reint1_244 = splat_laneq_s32(*(int32x4_t *) &__reint_244, __p3_244);...@@ -41803,9 +43884,7 @@ int32x2_t __reint1_244 = splat_laneq_s32(*(int32x4_t *) &__reint_244, __p3_244);
41803 int32x2_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, 1, 0); \43884 int32x2_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, 1, 0); \
41804 int8x8_t __rev1_245; __rev1_245 = __builtin_shufflevector(__s1_245, __s1_245, 7, 6, 5, 4, 3, 2, 1, 0); \43885 int8x8_t __rev1_245; __rev1_245 = __builtin_shufflevector(__s1_245, __s1_245, 7, 6, 5, 4, 3, 2, 1, 0); \
41805 int8x16_t __rev2_245; __rev2_245 = __builtin_shufflevector(__s2_245, __s2_245, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \43886 int8x16_t __rev2_245; __rev2_245 = __builtin_shufflevector(__s2_245, __s2_245, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41806int8x16_t __reint_245 = __rev2_245; \43887 __ret_245 = __noswap_vdot_s32(__rev0_245, __rev1_245, __builtin_bit_cast(int8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_245), __p3_245))); \
41807int32x2_t __reint1_245 = __noswap_splat_laneq_s32(*(int32x4_t *) &__reint_245, __p3_245); \
41808 __ret_245 = __noswap_vdot_s32(__rev0_245, __rev1_245, *(int8x8_t *) &__reint1_245); \
41809 __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, 1, 0); \43888 __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, 1, 0); \
41810 __ret_245; \43889 __ret_245; \
41811})43890})
...@@ -43018,8 +45097,7 @@ __ai __attribute__((target("fullfp16,neon"))) float16x4_t vsqrt_f16(float16x4_t...@@ -43018,8 +45097,7 @@ __ai __attribute__((target("fullfp16,neon"))) float16x4_t vsqrt_f16(float16x4_t
43018 int32x4_t __s0_270 = __p0_270; \45097 int32x4_t __s0_270 = __p0_270; \
43019 int8x16_t __s1_270 = __p1_270; \45098 int8x16_t __s1_270 = __p1_270; \
43020 uint8x16_t __s2_270 = __p2_270; \45099 uint8x16_t __s2_270 = __p2_270; \
43021uint8x16_t __reint_270 = __s2_270; \45100 __ret_270 = vusdotq_s32(__s0_270, (uint8x16_t)(splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_270), __p3_270)), __s1_270); \
43022 __ret_270 = vusdotq_s32(__s0_270, (uint8x16_t)(splatq_laneq_s32(*(int32x4_t *) &__reint_270, __p3_270)), __s1_270); \
43023 __ret_270; \45101 __ret_270; \
43024})45102})
43025#else45103#else
...@@ -43031,8 +45109,7 @@ uint8x16_t __reint_270 = __s2_270; \...@@ -43031,8 +45109,7 @@ uint8x16_t __reint_270 = __s2_270; \
43031 int32x4_t __rev0_271; __rev0_271 = __builtin_shufflevector(__s0_271, __s0_271, 3, 2, 1, 0); \45109 int32x4_t __rev0_271; __rev0_271 = __builtin_shufflevector(__s0_271, __s0_271, 3, 2, 1, 0); \
43032 int8x16_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \45110 int8x16_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43033 uint8x16_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \45111 uint8x16_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43034uint8x16_t __reint_271 = __rev2_271; \45112 __ret_271 = __noswap_vusdotq_s32(__rev0_271, (uint8x16_t)(__noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_271), __p3_271)), __rev1_271); \
43035 __ret_271 = __noswap_vusdotq_s32(__rev0_271, (uint8x16_t)(__noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_271, __p3_271)), __rev1_271); \
43036 __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, 3, 2, 1, 0); \45113 __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, 3, 2, 1, 0); \
43037 __ret_271; \45114 __ret_271; \
43038})45115})
...@@ -43044,8 +45121,7 @@ uint8x16_t __reint_271 = __rev2_271; \...@@ -43044,8 +45121,7 @@ uint8x16_t __reint_271 = __rev2_271; \
43044 int32x2_t __s0_272 = __p0_272; \45121 int32x2_t __s0_272 = __p0_272; \
43045 int8x8_t __s1_272 = __p1_272; \45122 int8x8_t __s1_272 = __p1_272; \
43046 uint8x16_t __s2_272 = __p2_272; \45123 uint8x16_t __s2_272 = __p2_272; \
43047uint8x16_t __reint_272 = __s2_272; \45124 __ret_272 = vusdot_s32(__s0_272, (uint8x8_t)(splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_272), __p3_272)), __s1_272); \
43048 __ret_272 = vusdot_s32(__s0_272, (uint8x8_t)(splat_laneq_s32(*(int32x4_t *) &__reint_272, __p3_272)), __s1_272); \
43049 __ret_272; \45125 __ret_272; \
43050})45126})
43051#else45127#else
...@@ -43057,8 +45133,7 @@ uint8x16_t __reint_272 = __s2_272; \...@@ -43057,8 +45133,7 @@ uint8x16_t __reint_272 = __s2_272; \
43057 int32x2_t __rev0_273; __rev0_273 = __builtin_shufflevector(__s0_273, __s0_273, 1, 0); \45133 int32x2_t __rev0_273; __rev0_273 = __builtin_shufflevector(__s0_273, __s0_273, 1, 0); \
43058 int8x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, 7, 6, 5, 4, 3, 2, 1, 0); \45134 int8x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, 7, 6, 5, 4, 3, 2, 1, 0); \
43059 uint8x16_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \45135 uint8x16_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43060uint8x16_t __reint_273 = __rev2_273; \45136 __ret_273 = __noswap_vusdot_s32(__rev0_273, (uint8x8_t)(__noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_273), __p3_273)), __rev1_273); \
43061 __ret_273 = __noswap_vusdot_s32(__rev0_273, (uint8x8_t)(__noswap_splat_laneq_s32(*(int32x4_t *) &__reint_273, __p3_273)), __rev1_273); \
43062 __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, 1, 0); \45137 __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, 1, 0); \
43063 __ret_273; \45138 __ret_273; \
43064})45139})
...@@ -43070,8 +45145,7 @@ uint8x16_t __reint_273 = __rev2_273; \...@@ -43070,8 +45145,7 @@ uint8x16_t __reint_273 = __rev2_273; \
43070 int32x4_t __s0_274 = __p0_274; \45145 int32x4_t __s0_274 = __p0_274; \
43071 uint8x16_t __s1_274 = __p1_274; \45146 uint8x16_t __s1_274 = __p1_274; \
43072 int8x16_t __s2_274 = __p2_274; \45147 int8x16_t __s2_274 = __p2_274; \
43073int8x16_t __reint_274 = __s2_274; \45148 __ret_274 = vusdotq_s32(__s0_274, __s1_274, (int8x16_t)(splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_274), __p3_274))); \
43074 __ret_274 = vusdotq_s32(__s0_274, __s1_274, (int8x16_t)(splatq_laneq_s32(*(int32x4_t *) &__reint_274, __p3_274))); \
43075 __ret_274; \45149 __ret_274; \
43076})45150})
43077#else45151#else
...@@ -43083,8 +45157,7 @@ int8x16_t __reint_274 = __s2_274; \...@@ -43083,8 +45157,7 @@ int8x16_t __reint_274 = __s2_274; \
43083 int32x4_t __rev0_275; __rev0_275 = __builtin_shufflevector(__s0_275, __s0_275, 3, 2, 1, 0); \45157 int32x4_t __rev0_275; __rev0_275 = __builtin_shufflevector(__s0_275, __s0_275, 3, 2, 1, 0); \
43084 uint8x16_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \45158 uint8x16_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43085 int8x16_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \45159 int8x16_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43086int8x16_t __reint_275 = __rev2_275; \45160 __ret_275 = __noswap_vusdotq_s32(__rev0_275, __rev1_275, (int8x16_t)(__noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_275), __p3_275))); \
43087 __ret_275 = __noswap_vusdotq_s32(__rev0_275, __rev1_275, (int8x16_t)(__noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_275, __p3_275))); \
43088 __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, 3, 2, 1, 0); \45161 __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, 3, 2, 1, 0); \
43089 __ret_275; \45162 __ret_275; \
43090})45163})
...@@ -43096,8 +45169,7 @@ int8x16_t __reint_275 = __rev2_275; \...@@ -43096,8 +45169,7 @@ int8x16_t __reint_275 = __rev2_275; \
43096 int32x2_t __s0_276 = __p0_276; \45169 int32x2_t __s0_276 = __p0_276; \
43097 uint8x8_t __s1_276 = __p1_276; \45170 uint8x8_t __s1_276 = __p1_276; \
43098 int8x16_t __s2_276 = __p2_276; \45171 int8x16_t __s2_276 = __p2_276; \
43099int8x16_t __reint_276 = __s2_276; \45172 __ret_276 = vusdot_s32(__s0_276, __s1_276, (int8x8_t)(splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_276), __p3_276))); \
43100 __ret_276 = vusdot_s32(__s0_276, __s1_276, (int8x8_t)(splat_laneq_s32(*(int32x4_t *) &__reint_276, __p3_276))); \
43101 __ret_276; \45173 __ret_276; \
43102})45174})
43103#else45175#else
...@@ -43109,8 +45181,7 @@ int8x16_t __reint_276 = __s2_276; \...@@ -43109,8 +45181,7 @@ int8x16_t __reint_276 = __s2_276; \
43109 int32x2_t __rev0_277; __rev0_277 = __builtin_shufflevector(__s0_277, __s0_277, 1, 0); \45181 int32x2_t __rev0_277; __rev0_277 = __builtin_shufflevector(__s0_277, __s0_277, 1, 0); \
43110 uint8x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, 7, 6, 5, 4, 3, 2, 1, 0); \45182 uint8x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, 7, 6, 5, 4, 3, 2, 1, 0); \
43111 int8x16_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \45183 int8x16_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43112int8x16_t __reint_277 = __rev2_277; \45184 __ret_277 = __noswap_vusdot_s32(__rev0_277, __rev1_277, (int8x8_t)(__noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_277), __p3_277))); \
43113 __ret_277 = __noswap_vusdot_s32(__rev0_277, __rev1_277, (int8x8_t)(__noswap_splat_laneq_s32(*(int32x4_t *) &__reint_277, __p3_277))); \
43114 __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, 1, 0); \45185 __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, 1, 0); \
43115 __ret_277; \45186 __ret_277; \
43116})45187})
...@@ -57773,6 +59844,11 @@ __ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_s64(int64x1_t __p...@@ -57773,6 +59844,11 @@ __ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_s64(int64x1_t __p
57773 __ret = (poly8x8_t)(__p0);59844 __ret = (poly8x8_t)(__p0);
57774 return __ret;59845 return __ret;
57775}59846}
59847__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_mf8(mfloat8x8_t __p0) {
59848 poly8x8_t __ret;
59849 __ret = (poly8x8_t)(__p0);
59850 return __ret;
59851}
57776__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) {59852__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) {
57777 poly8x8_t __ret;59853 poly8x8_t __ret;
57778 __ret = (poly8x8_t)(__p0);59854 __ret = (poly8x8_t)(__p0);
...@@ -57838,6 +59914,11 @@ __ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_s64(int64x1_t _...@@ -57838,6 +59914,11 @@ __ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_s64(int64x1_t _
57838 __ret = (poly64x1_t)(__p0);59914 __ret = (poly64x1_t)(__p0);
57839 return __ret;59915 return __ret;
57840}59916}
59917__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_mf8(mfloat8x8_t __p0) {
59918 poly64x1_t __ret;
59919 __ret = (poly64x1_t)(__p0);
59920 return __ret;
59921}
57841__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) {59922__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) {
57842 poly64x1_t __ret;59923 poly64x1_t __ret;
57843 __ret = (poly64x1_t)(__p0);59924 __ret = (poly64x1_t)(__p0);
...@@ -57903,6 +59984,11 @@ __ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_s64(int64x1_t _...@@ -57903,6 +59984,11 @@ __ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_s64(int64x1_t _
57903 __ret = (poly16x4_t)(__p0);59984 __ret = (poly16x4_t)(__p0);
57904 return __ret;59985 return __ret;
57905}59986}
59987__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_mf8(mfloat8x8_t __p0) {
59988 poly16x4_t __ret;
59989 __ret = (poly16x4_t)(__p0);
59990 return __ret;
59991}
57906__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) {59992__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) {
57907 poly16x4_t __ret;59993 poly16x4_t __ret;
57908 __ret = (poly16x4_t)(__p0);59994 __ret = (poly16x4_t)(__p0);
...@@ -57973,6 +60059,11 @@ __ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_s64(int64x2_t _...@@ -57973,6 +60059,11 @@ __ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_s64(int64x2_t _
57973 __ret = (poly8x16_t)(__p0);60059 __ret = (poly8x16_t)(__p0);
57974 return __ret;60060 return __ret;
57975}60061}
60062__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_mf8(mfloat8x16_t __p0) {
60063 poly8x16_t __ret;
60064 __ret = (poly8x16_t)(__p0);
60065 return __ret;
60066}
57976__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) {60067__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) {
57977 poly8x16_t __ret;60068 poly8x16_t __ret;
57978 __ret = (poly8x16_t)(__p0);60069 __ret = (poly8x16_t)(__p0);
...@@ -58043,6 +60134,11 @@ __ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_s64(int64x2_t...@@ -58043,6 +60134,11 @@ __ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_s64(int64x2_t
58043 __ret = (poly128_t)(__p0);60134 __ret = (poly128_t)(__p0);
58044 return __ret;60135 return __ret;
58045}60136}
60137__ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_mf8(mfloat8x16_t __p0) {
60138 poly128_t __ret;
60139 __ret = (poly128_t)(__p0);
60140 return __ret;
60141}
58046__ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_s16(int16x8_t __p0) {60142__ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_s16(int16x8_t __p0) {
58047 poly128_t __ret;60143 poly128_t __ret;
58048 __ret = (poly128_t)(__p0);60144 __ret = (poly128_t)(__p0);
...@@ -58113,6 +60209,11 @@ __ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_s64(int64x2_t...@@ -58113,6 +60209,11 @@ __ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_s64(int64x2_t
58113 __ret = (poly64x2_t)(__p0);60209 __ret = (poly64x2_t)(__p0);
58114 return __ret;60210 return __ret;
58115}60211}
60212__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_mf8(mfloat8x16_t __p0) {
60213 poly64x2_t __ret;
60214 __ret = (poly64x2_t)(__p0);
60215 return __ret;
60216}
58116__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) {60217__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) {
58117 poly64x2_t __ret;60218 poly64x2_t __ret;
58118 __ret = (poly64x2_t)(__p0);60219 __ret = (poly64x2_t)(__p0);
...@@ -58183,6 +60284,11 @@ __ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_s64(int64x2_t...@@ -58183,6 +60284,11 @@ __ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_s64(int64x2_t
58183 __ret = (poly16x8_t)(__p0);60284 __ret = (poly16x8_t)(__p0);
58184 return __ret;60285 return __ret;
58185}60286}
60287__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_mf8(mfloat8x16_t __p0) {
60288 poly16x8_t __ret;
60289 __ret = (poly16x8_t)(__p0);
60290 return __ret;
60291}
58186__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) {60292__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) {
58187 poly16x8_t __ret;60293 poly16x8_t __ret;
58188 __ret = (poly16x8_t)(__p0);60294 __ret = (poly16x8_t)(__p0);
...@@ -58253,6 +60359,11 @@ __ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_s64(int64x2_t _...@@ -58253,6 +60359,11 @@ __ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_s64(int64x2_t _
58253 __ret = (uint8x16_t)(__p0);60359 __ret = (uint8x16_t)(__p0);
58254 return __ret;60360 return __ret;
58255}60361}
60362__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_mf8(mfloat8x16_t __p0) {
60363 uint8x16_t __ret;
60364 __ret = (uint8x16_t)(__p0);
60365 return __ret;
60366}
58256__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) {60367__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) {
58257 uint8x16_t __ret;60368 uint8x16_t __ret;
58258 __ret = (uint8x16_t)(__p0);60369 __ret = (uint8x16_t)(__p0);
...@@ -58323,6 +60434,11 @@ __ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_s64(int64x2_t...@@ -58323,6 +60434,11 @@ __ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_s64(int64x2_t
58323 __ret = (uint32x4_t)(__p0);60434 __ret = (uint32x4_t)(__p0);
58324 return __ret;60435 return __ret;
58325}60436}
60437__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_mf8(mfloat8x16_t __p0) {
60438 uint32x4_t __ret;
60439 __ret = (uint32x4_t)(__p0);
60440 return __ret;
60441}
58326__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) {60442__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) {
58327 uint32x4_t __ret;60443 uint32x4_t __ret;
58328 __ret = (uint32x4_t)(__p0);60444 __ret = (uint32x4_t)(__p0);
...@@ -58393,6 +60509,11 @@ __ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_s64(int64x2_t...@@ -58393,6 +60509,11 @@ __ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_s64(int64x2_t
58393 __ret = (uint64x2_t)(__p0);60509 __ret = (uint64x2_t)(__p0);
58394 return __ret;60510 return __ret;
58395}60511}
60512__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_mf8(mfloat8x16_t __p0) {
60513 uint64x2_t __ret;
60514 __ret = (uint64x2_t)(__p0);
60515 return __ret;
60516}
58396__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) {60517__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) {
58397 uint64x2_t __ret;60518 uint64x2_t __ret;
58398 __ret = (uint64x2_t)(__p0);60519 __ret = (uint64x2_t)(__p0);
...@@ -58463,6 +60584,11 @@ __ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_s64(int64x2_t...@@ -58463,6 +60584,11 @@ __ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_s64(int64x2_t
58463 __ret = (uint16x8_t)(__p0);60584 __ret = (uint16x8_t)(__p0);
58464 return __ret;60585 return __ret;
58465}60586}
60587__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_mf8(mfloat8x16_t __p0) {
60588 uint16x8_t __ret;
60589 __ret = (uint16x8_t)(__p0);
60590 return __ret;
60591}
58466__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) {60592__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) {
58467 uint16x8_t __ret;60593 uint16x8_t __ret;
58468 __ret = (uint16x8_t)(__p0);60594 __ret = (uint16x8_t)(__p0);
...@@ -58533,6 +60659,11 @@ __ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_s64(int64x2_t __...@@ -58533,6 +60659,11 @@ __ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_s64(int64x2_t __
58533 __ret = (int8x16_t)(__p0);60659 __ret = (int8x16_t)(__p0);
58534 return __ret;60660 return __ret;
58535}60661}
60662__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_mf8(mfloat8x16_t __p0) {
60663 int8x16_t __ret;
60664 __ret = (int8x16_t)(__p0);
60665 return __ret;
60666}
58536__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) {60667__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) {
58537 int8x16_t __ret;60668 int8x16_t __ret;
58538 __ret = (int8x16_t)(__p0);60669 __ret = (int8x16_t)(__p0);
...@@ -58603,6 +60734,11 @@ __ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_s64(int64x2_t...@@ -58603,6 +60734,11 @@ __ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_s64(int64x2_t
58603 __ret = (float64x2_t)(__p0);60734 __ret = (float64x2_t)(__p0);
58604 return __ret;60735 return __ret;
58605}60736}
60737__ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_mf8(mfloat8x16_t __p0) {
60738 float64x2_t __ret;
60739 __ret = (float64x2_t)(__p0);
60740 return __ret;
60741}
58606__ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) {60742__ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) {
58607 float64x2_t __ret;60743 float64x2_t __ret;
58608 __ret = (float64x2_t)(__p0);60744 __ret = (float64x2_t)(__p0);
...@@ -58673,6 +60809,11 @@ __ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_s64(int64x2_t...@@ -58673,6 +60809,11 @@ __ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_s64(int64x2_t
58673 __ret = (float32x4_t)(__p0);60809 __ret = (float32x4_t)(__p0);
58674 return __ret;60810 return __ret;
58675}60811}
60812__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_mf8(mfloat8x16_t __p0) {
60813 float32x4_t __ret;
60814 __ret = (float32x4_t)(__p0);
60815 return __ret;
60816}
58676__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) {60817__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) {
58677 float32x4_t __ret;60818 float32x4_t __ret;
58678 __ret = (float32x4_t)(__p0);60819 __ret = (float32x4_t)(__p0);
...@@ -58743,6 +60884,11 @@ __ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_s64(int64x2_t...@@ -58743,6 +60884,11 @@ __ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_s64(int64x2_t
58743 __ret = (float16x8_t)(__p0);60884 __ret = (float16x8_t)(__p0);
58744 return __ret;60885 return __ret;
58745}60886}
60887__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_mf8(mfloat8x16_t __p0) {
60888 float16x8_t __ret;
60889 __ret = (float16x8_t)(__p0);
60890 return __ret;
60891}
58746__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) {60892__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) {
58747 float16x8_t __ret;60893 float16x8_t __ret;
58748 __ret = (float16x8_t)(__p0);60894 __ret = (float16x8_t)(__p0);
...@@ -58813,6 +60959,11 @@ __ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_s64(int64x2_t _...@@ -58813,6 +60959,11 @@ __ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_s64(int64x2_t _
58813 __ret = (int32x4_t)(__p0);60959 __ret = (int32x4_t)(__p0);
58814 return __ret;60960 return __ret;
58815}60961}
60962__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_mf8(mfloat8x16_t __p0) {
60963 int32x4_t __ret;
60964 __ret = (int32x4_t)(__p0);
60965 return __ret;
60966}
58816__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) {60967__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) {
58817 int32x4_t __ret;60968 int32x4_t __ret;
58818 __ret = (int32x4_t)(__p0);60969 __ret = (int32x4_t)(__p0);
...@@ -58883,11 +61034,91 @@ __ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_s32(int32x4_t _...@@ -58883,11 +61034,91 @@ __ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_s32(int32x4_t _
58883 __ret = (int64x2_t)(__p0);61034 __ret = (int64x2_t)(__p0);
58884 return __ret;61035 return __ret;
58885}61036}
61037__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_mf8(mfloat8x16_t __p0) {
61038 int64x2_t __ret;
61039 __ret = (int64x2_t)(__p0);
61040 return __ret;
61041}
58886__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) {61042__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) {
58887 int64x2_t __ret;61043 int64x2_t __ret;
58888 __ret = (int64x2_t)(__p0);61044 __ret = (int64x2_t)(__p0);
58889 return __ret;61045 return __ret;
58890}61046}
61047__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p8(poly8x16_t __p0) {
61048 mfloat8x16_t __ret;
61049 __ret = (mfloat8x16_t)(__p0);
61050 return __ret;
61051}
61052__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p128(poly128_t __p0) {
61053 mfloat8x16_t __ret;
61054 __ret = (mfloat8x16_t)(__p0);
61055 return __ret;
61056}
61057__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p64(poly64x2_t __p0) {
61058 mfloat8x16_t __ret;
61059 __ret = (mfloat8x16_t)(__p0);
61060 return __ret;
61061}
61062__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p16(poly16x8_t __p0) {
61063 mfloat8x16_t __ret;
61064 __ret = (mfloat8x16_t)(__p0);
61065 return __ret;
61066}
61067__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u8(uint8x16_t __p0) {
61068 mfloat8x16_t __ret;
61069 __ret = (mfloat8x16_t)(__p0);
61070 return __ret;
61071}
61072__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u32(uint32x4_t __p0) {
61073 mfloat8x16_t __ret;
61074 __ret = (mfloat8x16_t)(__p0);
61075 return __ret;
61076}
61077__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u64(uint64x2_t __p0) {
61078 mfloat8x16_t __ret;
61079 __ret = (mfloat8x16_t)(__p0);
61080 return __ret;
61081}
61082__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u16(uint16x8_t __p0) {
61083 mfloat8x16_t __ret;
61084 __ret = (mfloat8x16_t)(__p0);
61085 return __ret;
61086}
61087__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s8(int8x16_t __p0) {
61088 mfloat8x16_t __ret;
61089 __ret = (mfloat8x16_t)(__p0);
61090 return __ret;
61091}
61092__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_f64(float64x2_t __p0) {
61093 mfloat8x16_t __ret;
61094 __ret = (mfloat8x16_t)(__p0);
61095 return __ret;
61096}
61097__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_f32(float32x4_t __p0) {
61098 mfloat8x16_t __ret;
61099 __ret = (mfloat8x16_t)(__p0);
61100 return __ret;
61101}
61102__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_f16(float16x8_t __p0) {
61103 mfloat8x16_t __ret;
61104 __ret = (mfloat8x16_t)(__p0);
61105 return __ret;
61106}
61107__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s32(int32x4_t __p0) {
61108 mfloat8x16_t __ret;
61109 __ret = (mfloat8x16_t)(__p0);
61110 return __ret;
61111}
61112__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s64(int64x2_t __p0) {
61113 mfloat8x16_t __ret;
61114 __ret = (mfloat8x16_t)(__p0);
61115 return __ret;
61116}
61117__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s16(int16x8_t __p0) {
61118 mfloat8x16_t __ret;
61119 __ret = (mfloat8x16_t)(__p0);
61120 return __ret;
61121}
58891__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) {61122__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) {
58892 int16x8_t __ret;61123 int16x8_t __ret;
58893 __ret = (int16x8_t)(__p0);61124 __ret = (int16x8_t)(__p0);
...@@ -58958,6 +61189,11 @@ __ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_s64(int64x2_t _...@@ -58958,6 +61189,11 @@ __ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_s64(int64x2_t _
58958 __ret = (int16x8_t)(__p0);61189 __ret = (int16x8_t)(__p0);
58959 return __ret;61190 return __ret;
58960}61191}
61192__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_mf8(mfloat8x16_t __p0) {
61193 int16x8_t __ret;
61194 __ret = (int16x8_t)(__p0);
61195 return __ret;
61196}
58961__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) {61197__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) {
58962 uint8x8_t __ret;61198 uint8x8_t __ret;
58963 __ret = (uint8x8_t)(__p0);61199 __ret = (uint8x8_t)(__p0);
...@@ -59018,6 +61254,11 @@ __ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_s64(int64x1_t __p...@@ -59018,6 +61254,11 @@ __ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_s64(int64x1_t __p
59018 __ret = (uint8x8_t)(__p0);61254 __ret = (uint8x8_t)(__p0);
59019 return __ret;61255 return __ret;
59020}61256}
61257__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_mf8(mfloat8x8_t __p0) {
61258 uint8x8_t __ret;
61259 __ret = (uint8x8_t)(__p0);
61260 return __ret;
61261}
59021__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) {61262__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) {
59022 uint8x8_t __ret;61263 uint8x8_t __ret;
59023 __ret = (uint8x8_t)(__p0);61264 __ret = (uint8x8_t)(__p0);
...@@ -59083,6 +61324,11 @@ __ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_s64(int64x1_t _...@@ -59083,6 +61324,11 @@ __ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_s64(int64x1_t _
59083 __ret = (uint32x2_t)(__p0);61324 __ret = (uint32x2_t)(__p0);
59084 return __ret;61325 return __ret;
59085}61326}
61327__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_mf8(mfloat8x8_t __p0) {
61328 uint32x2_t __ret;
61329 __ret = (uint32x2_t)(__p0);
61330 return __ret;
61331}
59086__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) {61332__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) {
59087 uint32x2_t __ret;61333 uint32x2_t __ret;
59088 __ret = (uint32x2_t)(__p0);61334 __ret = (uint32x2_t)(__p0);
...@@ -59148,6 +61394,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_s64(int64x1_t _...@@ -59148,6 +61394,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_s64(int64x1_t _
59148 __ret = (uint64x1_t)(__p0);61394 __ret = (uint64x1_t)(__p0);
59149 return __ret;61395 return __ret;
59150}61396}
61397__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_mf8(mfloat8x8_t __p0) {
61398 uint64x1_t __ret;
61399 __ret = (uint64x1_t)(__p0);
61400 return __ret;
61401}
59151__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) {61402__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) {
59152 uint64x1_t __ret;61403 uint64x1_t __ret;
59153 __ret = (uint64x1_t)(__p0);61404 __ret = (uint64x1_t)(__p0);
...@@ -59213,6 +61464,11 @@ __ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_s64(int64x1_t _...@@ -59213,6 +61464,11 @@ __ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_s64(int64x1_t _
59213 __ret = (uint16x4_t)(__p0);61464 __ret = (uint16x4_t)(__p0);
59214 return __ret;61465 return __ret;
59215}61466}
61467__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_mf8(mfloat8x8_t __p0) {
61468 uint16x4_t __ret;
61469 __ret = (uint16x4_t)(__p0);
61470 return __ret;
61471}
59216__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) {61472__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) {
59217 uint16x4_t __ret;61473 uint16x4_t __ret;
59218 __ret = (uint16x4_t)(__p0);61474 __ret = (uint16x4_t)(__p0);
...@@ -59278,6 +61534,11 @@ __ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_s64(int64x1_t __p0...@@ -59278,6 +61534,11 @@ __ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_s64(int64x1_t __p0
59278 __ret = (int8x8_t)(__p0);61534 __ret = (int8x8_t)(__p0);
59279 return __ret;61535 return __ret;
59280}61536}
61537__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_mf8(mfloat8x8_t __p0) {
61538 int8x8_t __ret;
61539 __ret = (int8x8_t)(__p0);
61540 return __ret;
61541}
59281__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_s16(int16x4_t __p0) {61542__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_s16(int16x4_t __p0) {
59282 int8x8_t __ret;61543 int8x8_t __ret;
59283 __ret = (int8x8_t)(__p0);61544 __ret = (int8x8_t)(__p0);
...@@ -59343,6 +61604,11 @@ __ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_s64(int64x1_t...@@ -59343,6 +61604,11 @@ __ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_s64(int64x1_t
59343 __ret = (float64x1_t)(__p0);61604 __ret = (float64x1_t)(__p0);
59344 return __ret;61605 return __ret;
59345}61606}
61607__ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_mf8(mfloat8x8_t __p0) {
61608 float64x1_t __ret;
61609 __ret = (float64x1_t)(__p0);
61610 return __ret;
61611}
59346__ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_s16(int16x4_t __p0) {61612__ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_s16(int16x4_t __p0) {
59347 float64x1_t __ret;61613 float64x1_t __ret;
59348 __ret = (float64x1_t)(__p0);61614 __ret = (float64x1_t)(__p0);
...@@ -59408,6 +61674,11 @@ __ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_s64(int64x1_t...@@ -59408,6 +61674,11 @@ __ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_s64(int64x1_t
59408 __ret = (float32x2_t)(__p0);61674 __ret = (float32x2_t)(__p0);
59409 return __ret;61675 return __ret;
59410}61676}
61677__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_mf8(mfloat8x8_t __p0) {
61678 float32x2_t __ret;
61679 __ret = (float32x2_t)(__p0);
61680 return __ret;
61681}
59411__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_s16(int16x4_t __p0) {61682__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_s16(int16x4_t __p0) {
59412 float32x2_t __ret;61683 float32x2_t __ret;
59413 __ret = (float32x2_t)(__p0);61684 __ret = (float32x2_t)(__p0);
...@@ -59473,6 +61744,11 @@ __ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_s64(int64x1_t...@@ -59473,6 +61744,11 @@ __ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_s64(int64x1_t
59473 __ret = (float16x4_t)(__p0);61744 __ret = (float16x4_t)(__p0);
59474 return __ret;61745 return __ret;
59475}61746}
61747__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_mf8(mfloat8x8_t __p0) {
61748 float16x4_t __ret;
61749 __ret = (float16x4_t)(__p0);
61750 return __ret;
61751}
59476__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_s16(int16x4_t __p0) {61752__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_s16(int16x4_t __p0) {
59477 float16x4_t __ret;61753 float16x4_t __ret;
59478 __ret = (float16x4_t)(__p0);61754 __ret = (float16x4_t)(__p0);
...@@ -59538,6 +61814,11 @@ __ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_s64(int64x1_t __...@@ -59538,6 +61814,11 @@ __ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_s64(int64x1_t __
59538 __ret = (int32x2_t)(__p0);61814 __ret = (int32x2_t)(__p0);
59539 return __ret;61815 return __ret;
59540}61816}
61817__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_mf8(mfloat8x8_t __p0) {
61818 int32x2_t __ret;
61819 __ret = (int32x2_t)(__p0);
61820 return __ret;
61821}
59541__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_s16(int16x4_t __p0) {61822__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_s16(int16x4_t __p0) {
59542 int32x2_t __ret;61823 int32x2_t __ret;
59543 __ret = (int32x2_t)(__p0);61824 __ret = (int32x2_t)(__p0);
...@@ -59603,11 +61884,86 @@ __ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_s32(int32x2_t __...@@ -59603,11 +61884,86 @@ __ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_s32(int32x2_t __
59603 __ret = (int64x1_t)(__p0);61884 __ret = (int64x1_t)(__p0);
59604 return __ret;61885 return __ret;
59605}61886}
61887__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_mf8(mfloat8x8_t __p0) {
61888 int64x1_t __ret;
61889 __ret = (int64x1_t)(__p0);
61890 return __ret;
61891}
59606__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_s16(int16x4_t __p0) {61892__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_s16(int16x4_t __p0) {
59607 int64x1_t __ret;61893 int64x1_t __ret;
59608 __ret = (int64x1_t)(__p0);61894 __ret = (int64x1_t)(__p0);
59609 return __ret;61895 return __ret;
59610}61896}
61897__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_p8(poly8x8_t __p0) {
61898 mfloat8x8_t __ret;
61899 __ret = (mfloat8x8_t)(__p0);
61900 return __ret;
61901}
61902__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_p64(poly64x1_t __p0) {
61903 mfloat8x8_t __ret;
61904 __ret = (mfloat8x8_t)(__p0);
61905 return __ret;
61906}
61907__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_p16(poly16x4_t __p0) {
61908 mfloat8x8_t __ret;
61909 __ret = (mfloat8x8_t)(__p0);
61910 return __ret;
61911}
61912__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u8(uint8x8_t __p0) {
61913 mfloat8x8_t __ret;
61914 __ret = (mfloat8x8_t)(__p0);
61915 return __ret;
61916}
61917__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u32(uint32x2_t __p0) {
61918 mfloat8x8_t __ret;
61919 __ret = (mfloat8x8_t)(__p0);
61920 return __ret;
61921}
61922__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u64(uint64x1_t __p0) {
61923 mfloat8x8_t __ret;
61924 __ret = (mfloat8x8_t)(__p0);
61925 return __ret;
61926}
61927__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u16(uint16x4_t __p0) {
61928 mfloat8x8_t __ret;
61929 __ret = (mfloat8x8_t)(__p0);
61930 return __ret;
61931}
61932__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s8(int8x8_t __p0) {
61933 mfloat8x8_t __ret;
61934 __ret = (mfloat8x8_t)(__p0);
61935 return __ret;
61936}
61937__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_f64(float64x1_t __p0) {
61938 mfloat8x8_t __ret;
61939 __ret = (mfloat8x8_t)(__p0);
61940 return __ret;
61941}
61942__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_f32(float32x2_t __p0) {
61943 mfloat8x8_t __ret;
61944 __ret = (mfloat8x8_t)(__p0);
61945 return __ret;
61946}
61947__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_f16(float16x4_t __p0) {
61948 mfloat8x8_t __ret;
61949 __ret = (mfloat8x8_t)(__p0);
61950 return __ret;
61951}
61952__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s32(int32x2_t __p0) {
61953 mfloat8x8_t __ret;
61954 __ret = (mfloat8x8_t)(__p0);
61955 return __ret;
61956}
61957__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s64(int64x1_t __p0) {
61958 mfloat8x8_t __ret;
61959 __ret = (mfloat8x8_t)(__p0);
61960 return __ret;
61961}
61962__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s16(int16x4_t __p0) {
61963 mfloat8x8_t __ret;
61964 __ret = (mfloat8x8_t)(__p0);
61965 return __ret;
61966}
59611__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) {61967__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) {
59612 int16x4_t __ret;61968 int16x4_t __ret;
59613 __ret = (int16x4_t)(__p0);61969 __ret = (int16x4_t)(__p0);
...@@ -59673,6 +62029,11 @@ __ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_s64(int64x1_t __...@@ -59673,6 +62029,11 @@ __ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_s64(int64x1_t __
59673 __ret = (int16x4_t)(__p0);62029 __ret = (int16x4_t)(__p0);
59674 return __ret;62030 return __ret;
59675}62031}
62032__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_mf8(mfloat8x8_t __p0) {
62033 int16x4_t __ret;
62034 __ret = (int16x4_t)(__p0);
62035 return __ret;
62036}
59676__ai __attribute__((target("neon"))) uint64_t vrshld_u64(uint64_t __p0, int64_t __p1) {62037__ai __attribute__((target("neon"))) uint64_t vrshld_u64(uint64_t __p0, int64_t __p1) {
59677 uint64_t __ret;62038 uint64_t __ret;
59678 __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1);62039 __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1);
...@@ -65616,106 +67977,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_f64(float64x2_t __...@@ -65616,106 +67977,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_f64(float64x2_t __
65616 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);67977 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
65617 return __ret;67978 return __ret;
65618}67979}
65619__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65620 float64x2_t __ret;
65621 __ret = (float64x2_t) __builtin_neon_vcmlaq_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65622 return __ret;
65623}
65624#endif
65625
65626__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65627 float64x1_t __ret;
65628 __ret = (float64x1_t) __builtin_neon_vcmla_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65629 return __ret;
65630}
65631#define vcmla_lane_f64(__p0_792, __p1_792, __p2_792, __p3_792) __extension__ ({ \
65632 float64x1_t __ret_792; \
65633 float64x1_t __s0_792 = __p0_792; \
65634 float64x1_t __s1_792 = __p1_792; \
65635 float64x1_t __s2_792 = __p2_792; \
65636float64x1_t __reint_792 = __s2_792; \
65637uint64x2_t __reint1_792 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_792, __p3_792), vgetq_lane_u64(*(uint64x2_t *) &__reint_792, __p3_792)}; \
65638 __ret_792 = vcmla_f64(__s0_792, __s1_792, *(float64x1_t *) &__reint1_792); \
65639 __ret_792; \
65640})
65641#ifdef __LITTLE_ENDIAN__
65642#define vcmlaq_lane_f64(__p0_793, __p1_793, __p2_793, __p3_793) __extension__ ({ \
65643 float64x2_t __ret_793; \
65644 float64x2_t __s0_793 = __p0_793; \
65645 float64x2_t __s1_793 = __p1_793; \
65646 float64x1_t __s2_793 = __p2_793; \
65647float64x1_t __reint_793 = __s2_793; \
65648uint64x2_t __reint1_793 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_793, __p3_793), vgetq_lane_u64(*(uint64x2_t *) &__reint_793, __p3_793)}; \
65649 __ret_793 = vcmlaq_f64(__s0_793, __s1_793, *(float64x2_t *) &__reint1_793); \
65650 __ret_793; \
65651})
65652#else
65653#define vcmlaq_lane_f64(__p0_794, __p1_794, __p2_794, __p3_794) __extension__ ({ \
65654 float64x2_t __ret_794; \
65655 float64x2_t __s0_794 = __p0_794; \
65656 float64x2_t __s1_794 = __p1_794; \
65657 float64x1_t __s2_794 = __p2_794; \
65658 float64x2_t __rev0_794; __rev0_794 = __builtin_shufflevector(__s0_794, __s0_794, 1, 0); \
65659 float64x2_t __rev1_794; __rev1_794 = __builtin_shufflevector(__s1_794, __s1_794, 1, 0); \
65660float64x1_t __reint_794 = __s2_794; \
65661uint64x2_t __reint1_794 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_794, __p3_794), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_794, __p3_794)}; \
65662 __ret_794 = __noswap_vcmlaq_f64(__rev0_794, __rev1_794, *(float64x2_t *) &__reint1_794); \
65663 __ret_794 = __builtin_shufflevector(__ret_794, __ret_794, 1, 0); \
65664 __ret_794; \
65665})
65666#endif
65667
65668#ifdef __LITTLE_ENDIAN__
65669#define vcmla_laneq_f64(__p0_795, __p1_795, __p2_795, __p3_795) __extension__ ({ \
65670 float64x1_t __ret_795; \
65671 float64x1_t __s0_795 = __p0_795; \
65672 float64x1_t __s1_795 = __p1_795; \
65673 float64x2_t __s2_795 = __p2_795; \
65674float64x2_t __reint_795 = __s2_795; \
65675uint64x2_t __reint1_795 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_795, __p3_795), vgetq_lane_u64(*(uint64x2_t *) &__reint_795, __p3_795)}; \
65676 __ret_795 = vcmla_f64(__s0_795, __s1_795, *(float64x1_t *) &__reint1_795); \
65677 __ret_795; \
65678})
65679#else
65680#define vcmla_laneq_f64(__p0_796, __p1_796, __p2_796, __p3_796) __extension__ ({ \
65681 float64x1_t __ret_796; \
65682 float64x1_t __s0_796 = __p0_796; \
65683 float64x1_t __s1_796 = __p1_796; \
65684 float64x2_t __s2_796 = __p2_796; \
65685 float64x2_t __rev2_796; __rev2_796 = __builtin_shufflevector(__s2_796, __s2_796, 1, 0); \
65686float64x2_t __reint_796 = __rev2_796; \
65687uint64x2_t __reint1_796 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_796, __p3_796), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_796, __p3_796)}; \
65688 __ret_796 = vcmla_f64(__s0_796, __s1_796, *(float64x1_t *) &__reint1_796); \
65689 __ret_796; \
65690})
65691#endif
65692
65693#ifdef __LITTLE_ENDIAN__
65694#define vcmlaq_laneq_f64(__p0_797, __p1_797, __p2_797, __p3_797) __extension__ ({ \
65695 float64x2_t __ret_797; \
65696 float64x2_t __s0_797 = __p0_797; \
65697 float64x2_t __s1_797 = __p1_797; \
65698 float64x2_t __s2_797 = __p2_797; \
65699float64x2_t __reint_797 = __s2_797; \
65700uint64x2_t __reint1_797 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_797, __p3_797), vgetq_lane_u64(*(uint64x2_t *) &__reint_797, __p3_797)}; \
65701 __ret_797 = vcmlaq_f64(__s0_797, __s1_797, *(float64x2_t *) &__reint1_797); \
65702 __ret_797; \
65703})
65704#else
65705#define vcmlaq_laneq_f64(__p0_798, __p1_798, __p2_798, __p3_798) __extension__ ({ \
65706 float64x2_t __ret_798; \
65707 float64x2_t __s0_798 = __p0_798; \
65708 float64x2_t __s1_798 = __p1_798; \
65709 float64x2_t __s2_798 = __p2_798; \
65710 float64x2_t __rev0_798; __rev0_798 = __builtin_shufflevector(__s0_798, __s0_798, 1, 0); \
65711 float64x2_t __rev1_798; __rev1_798 = __builtin_shufflevector(__s1_798, __s1_798, 1, 0); \
65712 float64x2_t __rev2_798; __rev2_798 = __builtin_shufflevector(__s2_798, __s2_798, 1, 0); \
65713float64x2_t __reint_798 = __rev2_798; \
65714uint64x2_t __reint1_798 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_798, __p3_798), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_798, __p3_798)}; \
65715 __ret_798 = __noswap_vcmlaq_f64(__rev0_798, __rev1_798, *(float64x2_t *) &__reint1_798); \
65716 __ret_798 = __builtin_shufflevector(__ret_798, __ret_798, 1, 0); \
65717 __ret_798; \
65718})
65719#endif67980#endif
6572067981
65721#ifdef __LITTLE_ENDIAN__67982#ifdef __LITTLE_ENDIAN__
...@@ -65734,106 +67995,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot180_f64(float64...@@ -65734,106 +67995,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot180_f64(float64
65734 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);67995 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
65735 return __ret;67996 return __ret;
65736}67997}
65737__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_rot180_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65738 float64x2_t __ret;
65739 __ret = (float64x2_t) __builtin_neon_vcmlaq_rot180_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65740 return __ret;
65741}
65742#endif
65743
65744__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_rot180_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65745 float64x1_t __ret;
65746 __ret = (float64x1_t) __builtin_neon_vcmla_rot180_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65747 return __ret;
65748}
65749#define vcmla_rot180_lane_f64(__p0_799, __p1_799, __p2_799, __p3_799) __extension__ ({ \
65750 float64x1_t __ret_799; \
65751 float64x1_t __s0_799 = __p0_799; \
65752 float64x1_t __s1_799 = __p1_799; \
65753 float64x1_t __s2_799 = __p2_799; \
65754float64x1_t __reint_799 = __s2_799; \
65755uint64x2_t __reint1_799 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_799, __p3_799), vgetq_lane_u64(*(uint64x2_t *) &__reint_799, __p3_799)}; \
65756 __ret_799 = vcmla_rot180_f64(__s0_799, __s1_799, *(float64x1_t *) &__reint1_799); \
65757 __ret_799; \
65758})
65759#ifdef __LITTLE_ENDIAN__
65760#define vcmlaq_rot180_lane_f64(__p0_800, __p1_800, __p2_800, __p3_800) __extension__ ({ \
65761 float64x2_t __ret_800; \
65762 float64x2_t __s0_800 = __p0_800; \
65763 float64x2_t __s1_800 = __p1_800; \
65764 float64x1_t __s2_800 = __p2_800; \
65765float64x1_t __reint_800 = __s2_800; \
65766uint64x2_t __reint1_800 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_800, __p3_800), vgetq_lane_u64(*(uint64x2_t *) &__reint_800, __p3_800)}; \
65767 __ret_800 = vcmlaq_rot180_f64(__s0_800, __s1_800, *(float64x2_t *) &__reint1_800); \
65768 __ret_800; \
65769})
65770#else
65771#define vcmlaq_rot180_lane_f64(__p0_801, __p1_801, __p2_801, __p3_801) __extension__ ({ \
65772 float64x2_t __ret_801; \
65773 float64x2_t __s0_801 = __p0_801; \
65774 float64x2_t __s1_801 = __p1_801; \
65775 float64x1_t __s2_801 = __p2_801; \
65776 float64x2_t __rev0_801; __rev0_801 = __builtin_shufflevector(__s0_801, __s0_801, 1, 0); \
65777 float64x2_t __rev1_801; __rev1_801 = __builtin_shufflevector(__s1_801, __s1_801, 1, 0); \
65778float64x1_t __reint_801 = __s2_801; \
65779uint64x2_t __reint1_801 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_801, __p3_801), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_801, __p3_801)}; \
65780 __ret_801 = __noswap_vcmlaq_rot180_f64(__rev0_801, __rev1_801, *(float64x2_t *) &__reint1_801); \
65781 __ret_801 = __builtin_shufflevector(__ret_801, __ret_801, 1, 0); \
65782 __ret_801; \
65783})
65784#endif
65785
65786#ifdef __LITTLE_ENDIAN__
65787#define vcmla_rot180_laneq_f64(__p0_802, __p1_802, __p2_802, __p3_802) __extension__ ({ \
65788 float64x1_t __ret_802; \
65789 float64x1_t __s0_802 = __p0_802; \
65790 float64x1_t __s1_802 = __p1_802; \
65791 float64x2_t __s2_802 = __p2_802; \
65792float64x2_t __reint_802 = __s2_802; \
65793uint64x2_t __reint1_802 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_802, __p3_802), vgetq_lane_u64(*(uint64x2_t *) &__reint_802, __p3_802)}; \
65794 __ret_802 = vcmla_rot180_f64(__s0_802, __s1_802, *(float64x1_t *) &__reint1_802); \
65795 __ret_802; \
65796})
65797#else
65798#define vcmla_rot180_laneq_f64(__p0_803, __p1_803, __p2_803, __p3_803) __extension__ ({ \
65799 float64x1_t __ret_803; \
65800 float64x1_t __s0_803 = __p0_803; \
65801 float64x1_t __s1_803 = __p1_803; \
65802 float64x2_t __s2_803 = __p2_803; \
65803 float64x2_t __rev2_803; __rev2_803 = __builtin_shufflevector(__s2_803, __s2_803, 1, 0); \
65804float64x2_t __reint_803 = __rev2_803; \
65805uint64x2_t __reint1_803 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_803, __p3_803), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_803, __p3_803)}; \
65806 __ret_803 = vcmla_rot180_f64(__s0_803, __s1_803, *(float64x1_t *) &__reint1_803); \
65807 __ret_803; \
65808})
65809#endif
65810
65811#ifdef __LITTLE_ENDIAN__
65812#define vcmlaq_rot180_laneq_f64(__p0_804, __p1_804, __p2_804, __p3_804) __extension__ ({ \
65813 float64x2_t __ret_804; \
65814 float64x2_t __s0_804 = __p0_804; \
65815 float64x2_t __s1_804 = __p1_804; \
65816 float64x2_t __s2_804 = __p2_804; \
65817float64x2_t __reint_804 = __s2_804; \
65818uint64x2_t __reint1_804 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_804, __p3_804), vgetq_lane_u64(*(uint64x2_t *) &__reint_804, __p3_804)}; \
65819 __ret_804 = vcmlaq_rot180_f64(__s0_804, __s1_804, *(float64x2_t *) &__reint1_804); \
65820 __ret_804; \
65821})
65822#else
65823#define vcmlaq_rot180_laneq_f64(__p0_805, __p1_805, __p2_805, __p3_805) __extension__ ({ \
65824 float64x2_t __ret_805; \
65825 float64x2_t __s0_805 = __p0_805; \
65826 float64x2_t __s1_805 = __p1_805; \
65827 float64x2_t __s2_805 = __p2_805; \
65828 float64x2_t __rev0_805; __rev0_805 = __builtin_shufflevector(__s0_805, __s0_805, 1, 0); \
65829 float64x2_t __rev1_805; __rev1_805 = __builtin_shufflevector(__s1_805, __s1_805, 1, 0); \
65830 float64x2_t __rev2_805; __rev2_805 = __builtin_shufflevector(__s2_805, __s2_805, 1, 0); \
65831float64x2_t __reint_805 = __rev2_805; \
65832uint64x2_t __reint1_805 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_805, __p3_805), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_805, __p3_805)}; \
65833 __ret_805 = __noswap_vcmlaq_rot180_f64(__rev0_805, __rev1_805, *(float64x2_t *) &__reint1_805); \
65834 __ret_805 = __builtin_shufflevector(__ret_805, __ret_805, 1, 0); \
65835 __ret_805; \
65836})
65837#endif67998#endif
6583867999
65839#ifdef __LITTLE_ENDIAN__68000#ifdef __LITTLE_ENDIAN__
...@@ -65852,106 +68013,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot270_f64(float64...@@ -65852,106 +68013,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot270_f64(float64
65852 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);68013 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
65853 return __ret;68014 return __ret;
65854}68015}
65855__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_rot270_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65856 float64x2_t __ret;
65857 __ret = (float64x2_t) __builtin_neon_vcmlaq_rot270_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65858 return __ret;
65859}
65860#endif
65861
65862__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_rot270_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65863 float64x1_t __ret;
65864 __ret = (float64x1_t) __builtin_neon_vcmla_rot270_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65865 return __ret;
65866}
65867#define vcmla_rot270_lane_f64(__p0_806, __p1_806, __p2_806, __p3_806) __extension__ ({ \
65868 float64x1_t __ret_806; \
65869 float64x1_t __s0_806 = __p0_806; \
65870 float64x1_t __s1_806 = __p1_806; \
65871 float64x1_t __s2_806 = __p2_806; \
65872float64x1_t __reint_806 = __s2_806; \
65873uint64x2_t __reint1_806 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_806, __p3_806), vgetq_lane_u64(*(uint64x2_t *) &__reint_806, __p3_806)}; \
65874 __ret_806 = vcmla_rot270_f64(__s0_806, __s1_806, *(float64x1_t *) &__reint1_806); \
65875 __ret_806; \
65876})
65877#ifdef __LITTLE_ENDIAN__
65878#define vcmlaq_rot270_lane_f64(__p0_807, __p1_807, __p2_807, __p3_807) __extension__ ({ \
65879 float64x2_t __ret_807; \
65880 float64x2_t __s0_807 = __p0_807; \
65881 float64x2_t __s1_807 = __p1_807; \
65882 float64x1_t __s2_807 = __p2_807; \
65883float64x1_t __reint_807 = __s2_807; \
65884uint64x2_t __reint1_807 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_807, __p3_807), vgetq_lane_u64(*(uint64x2_t *) &__reint_807, __p3_807)}; \
65885 __ret_807 = vcmlaq_rot270_f64(__s0_807, __s1_807, *(float64x2_t *) &__reint1_807); \
65886 __ret_807; \
65887})
65888#else
65889#define vcmlaq_rot270_lane_f64(__p0_808, __p1_808, __p2_808, __p3_808) __extension__ ({ \
65890 float64x2_t __ret_808; \
65891 float64x2_t __s0_808 = __p0_808; \
65892 float64x2_t __s1_808 = __p1_808; \
65893 float64x1_t __s2_808 = __p2_808; \
65894 float64x2_t __rev0_808; __rev0_808 = __builtin_shufflevector(__s0_808, __s0_808, 1, 0); \
65895 float64x2_t __rev1_808; __rev1_808 = __builtin_shufflevector(__s1_808, __s1_808, 1, 0); \
65896float64x1_t __reint_808 = __s2_808; \
65897uint64x2_t __reint1_808 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_808, __p3_808), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_808, __p3_808)}; \
65898 __ret_808 = __noswap_vcmlaq_rot270_f64(__rev0_808, __rev1_808, *(float64x2_t *) &__reint1_808); \
65899 __ret_808 = __builtin_shufflevector(__ret_808, __ret_808, 1, 0); \
65900 __ret_808; \
65901})
65902#endif
65903
65904#ifdef __LITTLE_ENDIAN__
65905#define vcmla_rot270_laneq_f64(__p0_809, __p1_809, __p2_809, __p3_809) __extension__ ({ \
65906 float64x1_t __ret_809; \
65907 float64x1_t __s0_809 = __p0_809; \
65908 float64x1_t __s1_809 = __p1_809; \
65909 float64x2_t __s2_809 = __p2_809; \
65910float64x2_t __reint_809 = __s2_809; \
65911uint64x2_t __reint1_809 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_809, __p3_809), vgetq_lane_u64(*(uint64x2_t *) &__reint_809, __p3_809)}; \
65912 __ret_809 = vcmla_rot270_f64(__s0_809, __s1_809, *(float64x1_t *) &__reint1_809); \
65913 __ret_809; \
65914})
65915#else
65916#define vcmla_rot270_laneq_f64(__p0_810, __p1_810, __p2_810, __p3_810) __extension__ ({ \
65917 float64x1_t __ret_810; \
65918 float64x1_t __s0_810 = __p0_810; \
65919 float64x1_t __s1_810 = __p1_810; \
65920 float64x2_t __s2_810 = __p2_810; \
65921 float64x2_t __rev2_810; __rev2_810 = __builtin_shufflevector(__s2_810, __s2_810, 1, 0); \
65922float64x2_t __reint_810 = __rev2_810; \
65923uint64x2_t __reint1_810 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_810, __p3_810), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_810, __p3_810)}; \
65924 __ret_810 = vcmla_rot270_f64(__s0_810, __s1_810, *(float64x1_t *) &__reint1_810); \
65925 __ret_810; \
65926})
65927#endif
65928
65929#ifdef __LITTLE_ENDIAN__
65930#define vcmlaq_rot270_laneq_f64(__p0_811, __p1_811, __p2_811, __p3_811) __extension__ ({ \
65931 float64x2_t __ret_811; \
65932 float64x2_t __s0_811 = __p0_811; \
65933 float64x2_t __s1_811 = __p1_811; \
65934 float64x2_t __s2_811 = __p2_811; \
65935float64x2_t __reint_811 = __s2_811; \
65936uint64x2_t __reint1_811 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_811, __p3_811), vgetq_lane_u64(*(uint64x2_t *) &__reint_811, __p3_811)}; \
65937 __ret_811 = vcmlaq_rot270_f64(__s0_811, __s1_811, *(float64x2_t *) &__reint1_811); \
65938 __ret_811; \
65939})
65940#else
65941#define vcmlaq_rot270_laneq_f64(__p0_812, __p1_812, __p2_812, __p3_812) __extension__ ({ \
65942 float64x2_t __ret_812; \
65943 float64x2_t __s0_812 = __p0_812; \
65944 float64x2_t __s1_812 = __p1_812; \
65945 float64x2_t __s2_812 = __p2_812; \
65946 float64x2_t __rev0_812; __rev0_812 = __builtin_shufflevector(__s0_812, __s0_812, 1, 0); \
65947 float64x2_t __rev1_812; __rev1_812 = __builtin_shufflevector(__s1_812, __s1_812, 1, 0); \
65948 float64x2_t __rev2_812; __rev2_812 = __builtin_shufflevector(__s2_812, __s2_812, 1, 0); \
65949float64x2_t __reint_812 = __rev2_812; \
65950uint64x2_t __reint1_812 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_812, __p3_812), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_812, __p3_812)}; \
65951 __ret_812 = __noswap_vcmlaq_rot270_f64(__rev0_812, __rev1_812, *(float64x2_t *) &__reint1_812); \
65952 __ret_812 = __builtin_shufflevector(__ret_812, __ret_812, 1, 0); \
65953 __ret_812; \
65954})
65955#endif68016#endif
6595668017
65957#ifdef __LITTLE_ENDIAN__68018#ifdef __LITTLE_ENDIAN__
...@@ -65970,106 +68031,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot90_f64(float64x...@@ -65970,106 +68031,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot90_f64(float64x
65970 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);68031 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
65971 return __ret;68032 return __ret;
65972}68033}
65973__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_rot90_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65974 float64x2_t __ret;
65975 __ret = (float64x2_t) __builtin_neon_vcmlaq_rot90_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65976 return __ret;
65977}
65978#endif
65979
65980__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_rot90_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65981 float64x1_t __ret;
65982 __ret = (float64x1_t) __builtin_neon_vcmla_rot90_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65983 return __ret;
65984}
65985#define vcmla_rot90_lane_f64(__p0_813, __p1_813, __p2_813, __p3_813) __extension__ ({ \
65986 float64x1_t __ret_813; \
65987 float64x1_t __s0_813 = __p0_813; \
65988 float64x1_t __s1_813 = __p1_813; \
65989 float64x1_t __s2_813 = __p2_813; \
65990float64x1_t __reint_813 = __s2_813; \
65991uint64x2_t __reint1_813 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_813, __p3_813), vgetq_lane_u64(*(uint64x2_t *) &__reint_813, __p3_813)}; \
65992 __ret_813 = vcmla_rot90_f64(__s0_813, __s1_813, *(float64x1_t *) &__reint1_813); \
65993 __ret_813; \
65994})
65995#ifdef __LITTLE_ENDIAN__
65996#define vcmlaq_rot90_lane_f64(__p0_814, __p1_814, __p2_814, __p3_814) __extension__ ({ \
65997 float64x2_t __ret_814; \
65998 float64x2_t __s0_814 = __p0_814; \
65999 float64x2_t __s1_814 = __p1_814; \
66000 float64x1_t __s2_814 = __p2_814; \
66001float64x1_t __reint_814 = __s2_814; \
66002uint64x2_t __reint1_814 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_814, __p3_814), vgetq_lane_u64(*(uint64x2_t *) &__reint_814, __p3_814)}; \
66003 __ret_814 = vcmlaq_rot90_f64(__s0_814, __s1_814, *(float64x2_t *) &__reint1_814); \
66004 __ret_814; \
66005})
66006#else
66007#define vcmlaq_rot90_lane_f64(__p0_815, __p1_815, __p2_815, __p3_815) __extension__ ({ \
66008 float64x2_t __ret_815; \
66009 float64x2_t __s0_815 = __p0_815; \
66010 float64x2_t __s1_815 = __p1_815; \
66011 float64x1_t __s2_815 = __p2_815; \
66012 float64x2_t __rev0_815; __rev0_815 = __builtin_shufflevector(__s0_815, __s0_815, 1, 0); \
66013 float64x2_t __rev1_815; __rev1_815 = __builtin_shufflevector(__s1_815, __s1_815, 1, 0); \
66014float64x1_t __reint_815 = __s2_815; \
66015uint64x2_t __reint1_815 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_815, __p3_815), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_815, __p3_815)}; \
66016 __ret_815 = __noswap_vcmlaq_rot90_f64(__rev0_815, __rev1_815, *(float64x2_t *) &__reint1_815); \
66017 __ret_815 = __builtin_shufflevector(__ret_815, __ret_815, 1, 0); \
66018 __ret_815; \
66019})
66020#endif
66021
66022#ifdef __LITTLE_ENDIAN__
66023#define vcmla_rot90_laneq_f64(__p0_816, __p1_816, __p2_816, __p3_816) __extension__ ({ \
66024 float64x1_t __ret_816; \
66025 float64x1_t __s0_816 = __p0_816; \
66026 float64x1_t __s1_816 = __p1_816; \
66027 float64x2_t __s2_816 = __p2_816; \
66028float64x2_t __reint_816 = __s2_816; \
66029uint64x2_t __reint1_816 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_816, __p3_816), vgetq_lane_u64(*(uint64x2_t *) &__reint_816, __p3_816)}; \
66030 __ret_816 = vcmla_rot90_f64(__s0_816, __s1_816, *(float64x1_t *) &__reint1_816); \
66031 __ret_816; \
66032})
66033#else
66034#define vcmla_rot90_laneq_f64(__p0_817, __p1_817, __p2_817, __p3_817) __extension__ ({ \
66035 float64x1_t __ret_817; \
66036 float64x1_t __s0_817 = __p0_817; \
66037 float64x1_t __s1_817 = __p1_817; \
66038 float64x2_t __s2_817 = __p2_817; \
66039 float64x2_t __rev2_817; __rev2_817 = __builtin_shufflevector(__s2_817, __s2_817, 1, 0); \
66040float64x2_t __reint_817 = __rev2_817; \
66041uint64x2_t __reint1_817 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_817, __p3_817), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_817, __p3_817)}; \
66042 __ret_817 = vcmla_rot90_f64(__s0_817, __s1_817, *(float64x1_t *) &__reint1_817); \
66043 __ret_817; \
66044})
66045#endif
66046
66047#ifdef __LITTLE_ENDIAN__
66048#define vcmlaq_rot90_laneq_f64(__p0_818, __p1_818, __p2_818, __p3_818) __extension__ ({ \
66049 float64x2_t __ret_818; \
66050 float64x2_t __s0_818 = __p0_818; \
66051 float64x2_t __s1_818 = __p1_818; \
66052 float64x2_t __s2_818 = __p2_818; \
66053float64x2_t __reint_818 = __s2_818; \
66054uint64x2_t __reint1_818 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_818, __p3_818), vgetq_lane_u64(*(uint64x2_t *) &__reint_818, __p3_818)}; \
66055 __ret_818 = vcmlaq_rot90_f64(__s0_818, __s1_818, *(float64x2_t *) &__reint1_818); \
66056 __ret_818; \
66057})
66058#else
66059#define vcmlaq_rot90_laneq_f64(__p0_819, __p1_819, __p2_819, __p3_819) __extension__ ({ \
66060 float64x2_t __ret_819; \
66061 float64x2_t __s0_819 = __p0_819; \
66062 float64x2_t __s1_819 = __p1_819; \
66063 float64x2_t __s2_819 = __p2_819; \
66064 float64x2_t __rev0_819; __rev0_819 = __builtin_shufflevector(__s0_819, __s0_819, 1, 0); \
66065 float64x2_t __rev1_819; __rev1_819 = __builtin_shufflevector(__s1_819, __s1_819, 1, 0); \
66066 float64x2_t __rev2_819; __rev2_819 = __builtin_shufflevector(__s2_819, __s2_819, 1, 0); \
66067float64x2_t __reint_819 = __rev2_819; \
66068uint64x2_t __reint1_819 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_819, __p3_819), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_819, __p3_819)}; \
66069 __ret_819 = __noswap_vcmlaq_rot90_f64(__rev0_819, __rev1_819, *(float64x2_t *) &__reint1_819); \
66070 __ret_819 = __builtin_shufflevector(__ret_819, __ret_819, 1, 0); \
66071 __ret_819; \
66072})
66073#endif68034#endif
6607468035
66075#ifdef __LITTLE_ENDIAN__68036#ifdef __LITTLE_ENDIAN__
...@@ -66286,237 +68247,215 @@ __ai __attribute__((target("v8.5a,neon"))) float64x1_t vrnd64z_f64(float64x1_t _...@@ -66286,237 +68247,215 @@ __ai __attribute__((target("v8.5a,neon"))) float64x1_t vrnd64z_f64(float64x1_t _
66286}68247}
66287#endif68248#endif
66288#ifdef __LITTLE_ENDIAN__68249#ifdef __LITTLE_ENDIAN__
66289#define vbfdotq_lane_f32(__p0_820, __p1_820, __p2_820, __p3_820) __extension__ ({ \68250#define vbfdotq_lane_f32(__p0_792, __p1_792, __p2_792, __p3_792) __extension__ ({ \
66290 float32x4_t __ret_820; \68251 float32x4_t __ret_792; \
66291 float32x4_t __s0_820 = __p0_820; \68252 float32x4_t __s0_792 = __p0_792; \
66292 bfloat16x8_t __s1_820 = __p1_820; \68253 bfloat16x8_t __s1_792 = __p1_792; \
66293 bfloat16x4_t __s2_820 = __p2_820; \68254 bfloat16x4_t __s2_792 = __p2_792; \
66294bfloat16x4_t __reint_820 = __s2_820; \68255 __ret_792 = vbfdotq_f32(__s0_792, __s1_792, __builtin_bit_cast(bfloat16x8_t, splatq_lane_f32(__builtin_bit_cast(float32x2_t, __s2_792), __p3_792))); \
66295float32x4_t __reint1_820 = splatq_lane_f32(*(float32x2_t *) &__reint_820, __p3_820); \68256 __ret_792; \
66296 __ret_820 = vbfdotq_f32(__s0_820, __s1_820, *(bfloat16x8_t *) &__reint1_820); \
66297 __ret_820; \
66298})68257})
66299#else68258#else
66300#define vbfdotq_lane_f32(__p0_821, __p1_821, __p2_821, __p3_821) __extension__ ({ \68259#define vbfdotq_lane_f32(__p0_793, __p1_793, __p2_793, __p3_793) __extension__ ({ \
66301 float32x4_t __ret_821; \68260 float32x4_t __ret_793; \
66302 float32x4_t __s0_821 = __p0_821; \68261 float32x4_t __s0_793 = __p0_793; \
66303 bfloat16x8_t __s1_821 = __p1_821; \68262 bfloat16x8_t __s1_793 = __p1_793; \
66304 bfloat16x4_t __s2_821 = __p2_821; \68263 bfloat16x4_t __s2_793 = __p2_793; \
66305 float32x4_t __rev0_821; __rev0_821 = __builtin_shufflevector(__s0_821, __s0_821, 3, 2, 1, 0); \68264 float32x4_t __rev0_793; __rev0_793 = __builtin_shufflevector(__s0_793, __s0_793, 3, 2, 1, 0); \
66306 bfloat16x8_t __rev1_821; __rev1_821 = __builtin_shufflevector(__s1_821, __s1_821, 7, 6, 5, 4, 3, 2, 1, 0); \68265 bfloat16x8_t __rev1_793; __rev1_793 = __builtin_shufflevector(__s1_793, __s1_793, 7, 6, 5, 4, 3, 2, 1, 0); \
66307 bfloat16x4_t __rev2_821; __rev2_821 = __builtin_shufflevector(__s2_821, __s2_821, 3, 2, 1, 0); \68266 bfloat16x4_t __rev2_793; __rev2_793 = __builtin_shufflevector(__s2_793, __s2_793, 3, 2, 1, 0); \
66308bfloat16x4_t __reint_821 = __rev2_821; \68267 __ret_793 = __noswap_vbfdotq_f32(__rev0_793, __rev1_793, __builtin_bit_cast(bfloat16x8_t, __noswap_splatq_lane_f32(__builtin_bit_cast(float32x2_t, __rev2_793), __p3_793))); \
66309float32x4_t __reint1_821 = __noswap_splatq_lane_f32(*(float32x2_t *) &__reint_821, __p3_821); \68268 __ret_793 = __builtin_shufflevector(__ret_793, __ret_793, 3, 2, 1, 0); \
66310 __ret_821 = __noswap_vbfdotq_f32(__rev0_821, __rev1_821, *(bfloat16x8_t *) &__reint1_821); \68269 __ret_793; \
66311 __ret_821 = __builtin_shufflevector(__ret_821, __ret_821, 3, 2, 1, 0); \
66312 __ret_821; \
66313})68270})
66314#endif68271#endif
6631568272
66316#ifdef __LITTLE_ENDIAN__68273#ifdef __LITTLE_ENDIAN__
66317#define vbfdot_lane_f32(__p0_822, __p1_822, __p2_822, __p3_822) __extension__ ({ \68274#define vbfdot_lane_f32(__p0_794, __p1_794, __p2_794, __p3_794) __extension__ ({ \
66318 float32x2_t __ret_822; \68275 float32x2_t __ret_794; \
66319 float32x2_t __s0_822 = __p0_822; \68276 float32x2_t __s0_794 = __p0_794; \
66320 bfloat16x4_t __s1_822 = __p1_822; \68277 bfloat16x4_t __s1_794 = __p1_794; \
66321 bfloat16x4_t __s2_822 = __p2_822; \68278 bfloat16x4_t __s2_794 = __p2_794; \
66322bfloat16x4_t __reint_822 = __s2_822; \68279 __ret_794 = vbfdot_f32(__s0_794, __s1_794, __builtin_bit_cast(bfloat16x4_t, splat_lane_f32(__builtin_bit_cast(float32x2_t, __s2_794), __p3_794))); \
66323float32x2_t __reint1_822 = splat_lane_f32(*(float32x2_t *) &__reint_822, __p3_822); \68280 __ret_794; \
66324 __ret_822 = vbfdot_f32(__s0_822, __s1_822, *(bfloat16x4_t *) &__reint1_822); \
66325 __ret_822; \
66326})68281})
66327#else68282#else
66328#define vbfdot_lane_f32(__p0_823, __p1_823, __p2_823, __p3_823) __extension__ ({ \68283#define vbfdot_lane_f32(__p0_795, __p1_795, __p2_795, __p3_795) __extension__ ({ \
66329 float32x2_t __ret_823; \68284 float32x2_t __ret_795; \
66330 float32x2_t __s0_823 = __p0_823; \68285 float32x2_t __s0_795 = __p0_795; \
66331 bfloat16x4_t __s1_823 = __p1_823; \68286 bfloat16x4_t __s1_795 = __p1_795; \
66332 bfloat16x4_t __s2_823 = __p2_823; \68287 bfloat16x4_t __s2_795 = __p2_795; \
66333 float32x2_t __rev0_823; __rev0_823 = __builtin_shufflevector(__s0_823, __s0_823, 1, 0); \68288 float32x2_t __rev0_795; __rev0_795 = __builtin_shufflevector(__s0_795, __s0_795, 1, 0); \
66334 bfloat16x4_t __rev1_823; __rev1_823 = __builtin_shufflevector(__s1_823, __s1_823, 3, 2, 1, 0); \68289 bfloat16x4_t __rev1_795; __rev1_795 = __builtin_shufflevector(__s1_795, __s1_795, 3, 2, 1, 0); \
66335 bfloat16x4_t __rev2_823; __rev2_823 = __builtin_shufflevector(__s2_823, __s2_823, 3, 2, 1, 0); \68290 bfloat16x4_t __rev2_795; __rev2_795 = __builtin_shufflevector(__s2_795, __s2_795, 3, 2, 1, 0); \
66336bfloat16x4_t __reint_823 = __rev2_823; \68291 __ret_795 = __noswap_vbfdot_f32(__rev0_795, __rev1_795, __builtin_bit_cast(bfloat16x4_t, __noswap_splat_lane_f32(__builtin_bit_cast(float32x2_t, __rev2_795), __p3_795))); \
66337float32x2_t __reint1_823 = __noswap_splat_lane_f32(*(float32x2_t *) &__reint_823, __p3_823); \68292 __ret_795 = __builtin_shufflevector(__ret_795, __ret_795, 1, 0); \
66338 __ret_823 = __noswap_vbfdot_f32(__rev0_823, __rev1_823, *(bfloat16x4_t *) &__reint1_823); \68293 __ret_795; \
66339 __ret_823 = __builtin_shufflevector(__ret_823, __ret_823, 1, 0); \
66340 __ret_823; \
66341})68294})
66342#endif68295#endif
6634368296
66344#ifdef __LITTLE_ENDIAN__68297#ifdef __LITTLE_ENDIAN__
66345#define vbfdotq_laneq_f32(__p0_824, __p1_824, __p2_824, __p3_824) __extension__ ({ \68298#define vbfdotq_laneq_f32(__p0_796, __p1_796, __p2_796, __p3_796) __extension__ ({ \
66346 float32x4_t __ret_824; \68299 float32x4_t __ret_796; \
66347 float32x4_t __s0_824 = __p0_824; \68300 float32x4_t __s0_796 = __p0_796; \
66348 bfloat16x8_t __s1_824 = __p1_824; \68301 bfloat16x8_t __s1_796 = __p1_796; \
66349 bfloat16x8_t __s2_824 = __p2_824; \68302 bfloat16x8_t __s2_796 = __p2_796; \
66350bfloat16x8_t __reint_824 = __s2_824; \68303 __ret_796 = vbfdotq_f32(__s0_796, __s1_796, __builtin_bit_cast(bfloat16x8_t, splatq_laneq_f32(__builtin_bit_cast(float32x4_t, __s2_796), __p3_796))); \
66351float32x4_t __reint1_824 = splatq_laneq_f32(*(float32x4_t *) &__reint_824, __p3_824); \68304 __ret_796; \
66352 __ret_824 = vbfdotq_f32(__s0_824, __s1_824, *(bfloat16x8_t *) &__reint1_824); \
66353 __ret_824; \
66354})68305})
66355#else68306#else
66356#define vbfdotq_laneq_f32(__p0_825, __p1_825, __p2_825, __p3_825) __extension__ ({ \68307#define vbfdotq_laneq_f32(__p0_797, __p1_797, __p2_797, __p3_797) __extension__ ({ \
66357 float32x4_t __ret_825; \68308 float32x4_t __ret_797; \
66358 float32x4_t __s0_825 = __p0_825; \68309 float32x4_t __s0_797 = __p0_797; \
66359 bfloat16x8_t __s1_825 = __p1_825; \68310 bfloat16x8_t __s1_797 = __p1_797; \
66360 bfloat16x8_t __s2_825 = __p2_825; \68311 bfloat16x8_t __s2_797 = __p2_797; \
66361 float32x4_t __rev0_825; __rev0_825 = __builtin_shufflevector(__s0_825, __s0_825, 3, 2, 1, 0); \68312 float32x4_t __rev0_797; __rev0_797 = __builtin_shufflevector(__s0_797, __s0_797, 3, 2, 1, 0); \
66362 bfloat16x8_t __rev1_825; __rev1_825 = __builtin_shufflevector(__s1_825, __s1_825, 7, 6, 5, 4, 3, 2, 1, 0); \68313 bfloat16x8_t __rev1_797; __rev1_797 = __builtin_shufflevector(__s1_797, __s1_797, 7, 6, 5, 4, 3, 2, 1, 0); \
66363 bfloat16x8_t __rev2_825; __rev2_825 = __builtin_shufflevector(__s2_825, __s2_825, 7, 6, 5, 4, 3, 2, 1, 0); \68314 bfloat16x8_t __rev2_797; __rev2_797 = __builtin_shufflevector(__s2_797, __s2_797, 7, 6, 5, 4, 3, 2, 1, 0); \
66364bfloat16x8_t __reint_825 = __rev2_825; \68315 __ret_797 = __noswap_vbfdotq_f32(__rev0_797, __rev1_797, __builtin_bit_cast(bfloat16x8_t, __noswap_splatq_laneq_f32(__builtin_bit_cast(float32x4_t, __rev2_797), __p3_797))); \
66365float32x4_t __reint1_825 = __noswap_splatq_laneq_f32(*(float32x4_t *) &__reint_825, __p3_825); \68316 __ret_797 = __builtin_shufflevector(__ret_797, __ret_797, 3, 2, 1, 0); \
66366 __ret_825 = __noswap_vbfdotq_f32(__rev0_825, __rev1_825, *(bfloat16x8_t *) &__reint1_825); \68317 __ret_797; \
66367 __ret_825 = __builtin_shufflevector(__ret_825, __ret_825, 3, 2, 1, 0); \
66368 __ret_825; \
66369})68318})
66370#endif68319#endif
6637168320
66372#ifdef __LITTLE_ENDIAN__68321#ifdef __LITTLE_ENDIAN__
66373#define vbfdot_laneq_f32(__p0_826, __p1_826, __p2_826, __p3_826) __extension__ ({ \68322#define vbfdot_laneq_f32(__p0_798, __p1_798, __p2_798, __p3_798) __extension__ ({ \
66374 float32x2_t __ret_826; \68323 float32x2_t __ret_798; \
66375 float32x2_t __s0_826 = __p0_826; \68324 float32x2_t __s0_798 = __p0_798; \
66376 bfloat16x4_t __s1_826 = __p1_826; \68325 bfloat16x4_t __s1_798 = __p1_798; \
66377 bfloat16x8_t __s2_826 = __p2_826; \68326 bfloat16x8_t __s2_798 = __p2_798; \
66378bfloat16x8_t __reint_826 = __s2_826; \68327 __ret_798 = vbfdot_f32(__s0_798, __s1_798, __builtin_bit_cast(bfloat16x4_t, splat_laneq_f32(__builtin_bit_cast(float32x4_t, __s2_798), __p3_798))); \
66379float32x2_t __reint1_826 = splat_laneq_f32(*(float32x4_t *) &__reint_826, __p3_826); \68328 __ret_798; \
66380 __ret_826 = vbfdot_f32(__s0_826, __s1_826, *(bfloat16x4_t *) &__reint1_826); \
66381 __ret_826; \
66382})68329})
66383#else68330#else
66384#define vbfdot_laneq_f32(__p0_827, __p1_827, __p2_827, __p3_827) __extension__ ({ \68331#define vbfdot_laneq_f32(__p0_799, __p1_799, __p2_799, __p3_799) __extension__ ({ \
66385 float32x2_t __ret_827; \68332 float32x2_t __ret_799; \
66386 float32x2_t __s0_827 = __p0_827; \68333 float32x2_t __s0_799 = __p0_799; \
66387 bfloat16x4_t __s1_827 = __p1_827; \68334 bfloat16x4_t __s1_799 = __p1_799; \
66388 bfloat16x8_t __s2_827 = __p2_827; \68335 bfloat16x8_t __s2_799 = __p2_799; \
66389 float32x2_t __rev0_827; __rev0_827 = __builtin_shufflevector(__s0_827, __s0_827, 1, 0); \68336 float32x2_t __rev0_799; __rev0_799 = __builtin_shufflevector(__s0_799, __s0_799, 1, 0); \
66390 bfloat16x4_t __rev1_827; __rev1_827 = __builtin_shufflevector(__s1_827, __s1_827, 3, 2, 1, 0); \68337 bfloat16x4_t __rev1_799; __rev1_799 = __builtin_shufflevector(__s1_799, __s1_799, 3, 2, 1, 0); \
66391 bfloat16x8_t __rev2_827; __rev2_827 = __builtin_shufflevector(__s2_827, __s2_827, 7, 6, 5, 4, 3, 2, 1, 0); \68338 bfloat16x8_t __rev2_799; __rev2_799 = __builtin_shufflevector(__s2_799, __s2_799, 7, 6, 5, 4, 3, 2, 1, 0); \
66392bfloat16x8_t __reint_827 = __rev2_827; \68339 __ret_799 = __noswap_vbfdot_f32(__rev0_799, __rev1_799, __builtin_bit_cast(bfloat16x4_t, __noswap_splat_laneq_f32(__builtin_bit_cast(float32x4_t, __rev2_799), __p3_799))); \
66393float32x2_t __reint1_827 = __noswap_splat_laneq_f32(*(float32x4_t *) &__reint_827, __p3_827); \68340 __ret_799 = __builtin_shufflevector(__ret_799, __ret_799, 1, 0); \
66394 __ret_827 = __noswap_vbfdot_f32(__rev0_827, __rev1_827, *(bfloat16x4_t *) &__reint1_827); \68341 __ret_799; \
66395 __ret_827 = __builtin_shufflevector(__ret_827, __ret_827, 1, 0); \
66396 __ret_827; \
66397})68342})
66398#endif68343#endif
6639968344
66400#ifdef __LITTLE_ENDIAN__68345#ifdef __LITTLE_ENDIAN__
66401#define vbfmlalbq_lane_f32(__p0_828, __p1_828, __p2_828, __p3_828) __extension__ ({ \68346#define vbfmlalbq_lane_f32(__p0_800, __p1_800, __p2_800, __p3_800) __extension__ ({ \
66402 float32x4_t __ret_828; \68347 float32x4_t __ret_800; \
66403 float32x4_t __s0_828 = __p0_828; \68348 float32x4_t __s0_800 = __p0_800; \
66404 bfloat16x8_t __s1_828 = __p1_828; \68349 bfloat16x8_t __s1_800 = __p1_800; \
66405 bfloat16x4_t __s2_828 = __p2_828; \68350 bfloat16x4_t __s2_800 = __p2_800; \
66406 __ret_828 = vbfmlalbq_f32(__s0_828, __s1_828, (bfloat16x8_t) {vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828)}); \68351 __ret_800 = vbfmlalbq_f32(__s0_800, __s1_800, (bfloat16x8_t) {vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800)}); \
66407 __ret_828; \68352 __ret_800; \
66408})68353})
66409#else68354#else
66410#define vbfmlalbq_lane_f32(__p0_829, __p1_829, __p2_829, __p3_829) __extension__ ({ \68355#define vbfmlalbq_lane_f32(__p0_801, __p1_801, __p2_801, __p3_801) __extension__ ({ \
66411 float32x4_t __ret_829; \68356 float32x4_t __ret_801; \
66412 float32x4_t __s0_829 = __p0_829; \68357 float32x4_t __s0_801 = __p0_801; \
66413 bfloat16x8_t __s1_829 = __p1_829; \68358 bfloat16x8_t __s1_801 = __p1_801; \
66414 bfloat16x4_t __s2_829 = __p2_829; \68359 bfloat16x4_t __s2_801 = __p2_801; \
66415 float32x4_t __rev0_829; __rev0_829 = __builtin_shufflevector(__s0_829, __s0_829, 3, 2, 1, 0); \68360 float32x4_t __rev0_801; __rev0_801 = __builtin_shufflevector(__s0_801, __s0_801, 3, 2, 1, 0); \
66416 bfloat16x8_t __rev1_829; __rev1_829 = __builtin_shufflevector(__s1_829, __s1_829, 7, 6, 5, 4, 3, 2, 1, 0); \68361 bfloat16x8_t __rev1_801; __rev1_801 = __builtin_shufflevector(__s1_801, __s1_801, 7, 6, 5, 4, 3, 2, 1, 0); \
66417 bfloat16x4_t __rev2_829; __rev2_829 = __builtin_shufflevector(__s2_829, __s2_829, 3, 2, 1, 0); \68362 bfloat16x4_t __rev2_801; __rev2_801 = __builtin_shufflevector(__s2_801, __s2_801, 3, 2, 1, 0); \
66418 __ret_829 = __noswap_vbfmlalbq_f32(__rev0_829, __rev1_829, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829)}); \68363 __ret_801 = __noswap_vbfmlalbq_f32(__rev0_801, __rev1_801, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801)}); \
66419 __ret_829 = __builtin_shufflevector(__ret_829, __ret_829, 3, 2, 1, 0); \68364 __ret_801 = __builtin_shufflevector(__ret_801, __ret_801, 3, 2, 1, 0); \
66420 __ret_829; \68365 __ret_801; \
66421})68366})
66422#endif68367#endif
6642368368
66424#ifdef __LITTLE_ENDIAN__68369#ifdef __LITTLE_ENDIAN__
66425#define vbfmlalbq_laneq_f32(__p0_830, __p1_830, __p2_830, __p3_830) __extension__ ({ \68370#define vbfmlalbq_laneq_f32(__p0_802, __p1_802, __p2_802, __p3_802) __extension__ ({ \
66426 float32x4_t __ret_830; \68371 float32x4_t __ret_802; \
66427 float32x4_t __s0_830 = __p0_830; \68372 float32x4_t __s0_802 = __p0_802; \
66428 bfloat16x8_t __s1_830 = __p1_830; \68373 bfloat16x8_t __s1_802 = __p1_802; \
66429 bfloat16x8_t __s2_830 = __p2_830; \68374 bfloat16x8_t __s2_802 = __p2_802; \
66430 __ret_830 = vbfmlalbq_f32(__s0_830, __s1_830, (bfloat16x8_t) {vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830)}); \68375 __ret_802 = vbfmlalbq_f32(__s0_802, __s1_802, (bfloat16x8_t) {vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802)}); \
66431 __ret_830; \68376 __ret_802; \
66432})68377})
66433#else68378#else
66434#define vbfmlalbq_laneq_f32(__p0_831, __p1_831, __p2_831, __p3_831) __extension__ ({ \68379#define vbfmlalbq_laneq_f32(__p0_803, __p1_803, __p2_803, __p3_803) __extension__ ({ \
66435 float32x4_t __ret_831; \68380 float32x4_t __ret_803; \
66436 float32x4_t __s0_831 = __p0_831; \68381 float32x4_t __s0_803 = __p0_803; \
66437 bfloat16x8_t __s1_831 = __p1_831; \68382 bfloat16x8_t __s1_803 = __p1_803; \
66438 bfloat16x8_t __s2_831 = __p2_831; \68383 bfloat16x8_t __s2_803 = __p2_803; \
66439 float32x4_t __rev0_831; __rev0_831 = __builtin_shufflevector(__s0_831, __s0_831, 3, 2, 1, 0); \68384 float32x4_t __rev0_803; __rev0_803 = __builtin_shufflevector(__s0_803, __s0_803, 3, 2, 1, 0); \
66440 bfloat16x8_t __rev1_831; __rev1_831 = __builtin_shufflevector(__s1_831, __s1_831, 7, 6, 5, 4, 3, 2, 1, 0); \68385 bfloat16x8_t __rev1_803; __rev1_803 = __builtin_shufflevector(__s1_803, __s1_803, 7, 6, 5, 4, 3, 2, 1, 0); \
66441 bfloat16x8_t __rev2_831; __rev2_831 = __builtin_shufflevector(__s2_831, __s2_831, 7, 6, 5, 4, 3, 2, 1, 0); \68386 bfloat16x8_t __rev2_803; __rev2_803 = __builtin_shufflevector(__s2_803, __s2_803, 7, 6, 5, 4, 3, 2, 1, 0); \
66442 __ret_831 = __noswap_vbfmlalbq_f32(__rev0_831, __rev1_831, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831)}); \68387 __ret_803 = __noswap_vbfmlalbq_f32(__rev0_803, __rev1_803, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803)}); \
66443 __ret_831 = __builtin_shufflevector(__ret_831, __ret_831, 3, 2, 1, 0); \68388 __ret_803 = __builtin_shufflevector(__ret_803, __ret_803, 3, 2, 1, 0); \
66444 __ret_831; \68389 __ret_803; \
66445})68390})
66446#endif68391#endif
6644768392
66448#ifdef __LITTLE_ENDIAN__68393#ifdef __LITTLE_ENDIAN__
66449#define vbfmlaltq_lane_f32(__p0_832, __p1_832, __p2_832, __p3_832) __extension__ ({ \68394#define vbfmlaltq_lane_f32(__p0_804, __p1_804, __p2_804, __p3_804) __extension__ ({ \
66450 float32x4_t __ret_832; \68395 float32x4_t __ret_804; \
66451 float32x4_t __s0_832 = __p0_832; \68396 float32x4_t __s0_804 = __p0_804; \
66452 bfloat16x8_t __s1_832 = __p1_832; \68397 bfloat16x8_t __s1_804 = __p1_804; \
66453 bfloat16x4_t __s2_832 = __p2_832; \68398 bfloat16x4_t __s2_804 = __p2_804; \
66454 __ret_832 = vbfmlaltq_f32(__s0_832, __s1_832, (bfloat16x8_t) {vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832)}); \68399 __ret_804 = vbfmlaltq_f32(__s0_804, __s1_804, (bfloat16x8_t) {vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804)}); \
66455 __ret_832; \68400 __ret_804; \
66456})68401})
66457#else68402#else
66458#define vbfmlaltq_lane_f32(__p0_833, __p1_833, __p2_833, __p3_833) __extension__ ({ \68403#define vbfmlaltq_lane_f32(__p0_805, __p1_805, __p2_805, __p3_805) __extension__ ({ \
66459 float32x4_t __ret_833; \68404 float32x4_t __ret_805; \
66460 float32x4_t __s0_833 = __p0_833; \68405 float32x4_t __s0_805 = __p0_805; \
66461 bfloat16x8_t __s1_833 = __p1_833; \68406 bfloat16x8_t __s1_805 = __p1_805; \
66462 bfloat16x4_t __s2_833 = __p2_833; \68407 bfloat16x4_t __s2_805 = __p2_805; \
66463 float32x4_t __rev0_833; __rev0_833 = __builtin_shufflevector(__s0_833, __s0_833, 3, 2, 1, 0); \68408 float32x4_t __rev0_805; __rev0_805 = __builtin_shufflevector(__s0_805, __s0_805, 3, 2, 1, 0); \
66464 bfloat16x8_t __rev1_833; __rev1_833 = __builtin_shufflevector(__s1_833, __s1_833, 7, 6, 5, 4, 3, 2, 1, 0); \68409 bfloat16x8_t __rev1_805; __rev1_805 = __builtin_shufflevector(__s1_805, __s1_805, 7, 6, 5, 4, 3, 2, 1, 0); \
66465 bfloat16x4_t __rev2_833; __rev2_833 = __builtin_shufflevector(__s2_833, __s2_833, 3, 2, 1, 0); \68410 bfloat16x4_t __rev2_805; __rev2_805 = __builtin_shufflevector(__s2_805, __s2_805, 3, 2, 1, 0); \
66466 __ret_833 = __noswap_vbfmlaltq_f32(__rev0_833, __rev1_833, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833)}); \68411 __ret_805 = __noswap_vbfmlaltq_f32(__rev0_805, __rev1_805, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805)}); \
66467 __ret_833 = __builtin_shufflevector(__ret_833, __ret_833, 3, 2, 1, 0); \68412 __ret_805 = __builtin_shufflevector(__ret_805, __ret_805, 3, 2, 1, 0); \
66468 __ret_833; \68413 __ret_805; \
66469})68414})
66470#endif68415#endif
6647168416
66472#ifdef __LITTLE_ENDIAN__68417#ifdef __LITTLE_ENDIAN__
66473#define vbfmlaltq_laneq_f32(__p0_834, __p1_834, __p2_834, __p3_834) __extension__ ({ \68418#define vbfmlaltq_laneq_f32(__p0_806, __p1_806, __p2_806, __p3_806) __extension__ ({ \
66474 float32x4_t __ret_834; \68419 float32x4_t __ret_806; \
66475 float32x4_t __s0_834 = __p0_834; \68420 float32x4_t __s0_806 = __p0_806; \
66476 bfloat16x8_t __s1_834 = __p1_834; \68421 bfloat16x8_t __s1_806 = __p1_806; \
66477 bfloat16x8_t __s2_834 = __p2_834; \68422 bfloat16x8_t __s2_806 = __p2_806; \
66478 __ret_834 = vbfmlaltq_f32(__s0_834, __s1_834, (bfloat16x8_t) {vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834)}); \68423 __ret_806 = vbfmlaltq_f32(__s0_806, __s1_806, (bfloat16x8_t) {vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806)}); \
66479 __ret_834; \68424 __ret_806; \
66480})68425})
66481#else68426#else
66482#define vbfmlaltq_laneq_f32(__p0_835, __p1_835, __p2_835, __p3_835) __extension__ ({ \68427#define vbfmlaltq_laneq_f32(__p0_807, __p1_807, __p2_807, __p3_807) __extension__ ({ \
66483 float32x4_t __ret_835; \68428 float32x4_t __ret_807; \
66484 float32x4_t __s0_835 = __p0_835; \68429 float32x4_t __s0_807 = __p0_807; \
66485 bfloat16x8_t __s1_835 = __p1_835; \68430 bfloat16x8_t __s1_807 = __p1_807; \
66486 bfloat16x8_t __s2_835 = __p2_835; \68431 bfloat16x8_t __s2_807 = __p2_807; \
66487 float32x4_t __rev0_835; __rev0_835 = __builtin_shufflevector(__s0_835, __s0_835, 3, 2, 1, 0); \68432 float32x4_t __rev0_807; __rev0_807 = __builtin_shufflevector(__s0_807, __s0_807, 3, 2, 1, 0); \
66488 bfloat16x8_t __rev1_835; __rev1_835 = __builtin_shufflevector(__s1_835, __s1_835, 7, 6, 5, 4, 3, 2, 1, 0); \68433 bfloat16x8_t __rev1_807; __rev1_807 = __builtin_shufflevector(__s1_807, __s1_807, 7, 6, 5, 4, 3, 2, 1, 0); \
66489 bfloat16x8_t __rev2_835; __rev2_835 = __builtin_shufflevector(__s2_835, __s2_835, 7, 6, 5, 4, 3, 2, 1, 0); \68434 bfloat16x8_t __rev2_807; __rev2_807 = __builtin_shufflevector(__s2_807, __s2_807, 7, 6, 5, 4, 3, 2, 1, 0); \
66490 __ret_835 = __noswap_vbfmlaltq_f32(__rev0_835, __rev1_835, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835)}); \68435 __ret_807 = __noswap_vbfmlaltq_f32(__rev0_807, __rev1_807, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807)}); \
66491 __ret_835 = __builtin_shufflevector(__ret_835, __ret_835, 3, 2, 1, 0); \68436 __ret_807 = __builtin_shufflevector(__ret_807, __ret_807, 3, 2, 1, 0); \
66492 __ret_835; \68437 __ret_807; \
66493})68438})
66494#endif68439#endif
6649568440
66496#ifdef __LITTLE_ENDIAN__68441#ifdef __LITTLE_ENDIAN__
66497__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_836) {68442__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_808) {
66498 float32x4_t __ret_836;68443 float32x4_t __ret_808;
66499bfloat16x4_t __reint_836 = __p0_836;68444 __ret_808 = __builtin_bit_cast(float32x4_t, vshll_n_u16(__builtin_bit_cast(uint16x4_t, __p0_808), 16));
66500int32x4_t __reint1_836 = vshll_n_s16(*(int16x4_t *) &__reint_836, 16);68445 return __ret_808;
66501 __ret_836 = *(float32x4_t *) &__reint1_836;
66502 return __ret_836;
66503}68446}
66504#else68447#else
66505__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_837) {68448__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_809) {
66506 float32x4_t __ret_837;68449 float32x4_t __ret_809;
66507 bfloat16x4_t __rev0_837; __rev0_837 = __builtin_shufflevector(__p0_837, __p0_837, 3, 2, 1, 0);68450 bfloat16x4_t __rev0_809; __rev0_809 = __builtin_shufflevector(__p0_809, __p0_809, 3, 2, 1, 0);
66508bfloat16x4_t __reint_837 = __rev0_837;68451 __ret_809 = __builtin_bit_cast(float32x4_t, __noswap_vshll_n_u16(__builtin_bit_cast(uint16x4_t, __rev0_809), 16));
66509int32x4_t __reint1_837 = __noswap_vshll_n_s16(*(int16x4_t *) &__reint_837, 16);68452 __ret_809 = __builtin_shufflevector(__ret_809, __ret_809, 3, 2, 1, 0);
66510 __ret_837 = *(float32x4_t *) &__reint1_837;68453 return __ret_809;
66511 __ret_837 = __builtin_shufflevector(__ret_837, __ret_837, 3, 2, 1, 0);
66512 return __ret_837;
66513}68454}
66514__ai __attribute__((target("bf16,neon"))) float32x4_t __noswap_vcvt_f32_bf16(bfloat16x4_t __p0_838) {68455__ai __attribute__((target("bf16,neon"))) float32x4_t __noswap_vcvt_f32_bf16(bfloat16x4_t __p0_810) {
66515 float32x4_t __ret_838;68456 float32x4_t __ret_810;
66516bfloat16x4_t __reint_838 = __p0_838;68457 __ret_810 = __builtin_bit_cast(float32x4_t, __noswap_vshll_n_u16(__builtin_bit_cast(uint16x4_t, __p0_810), 16));
66517int32x4_t __reint1_838 = __noswap_vshll_n_s16(*(int16x4_t *) &__reint_838, 16);68458 return __ret_810;
66518 __ret_838 = *(float32x4_t *) &__reint1_838;
66519 return __ret_838;
66520}68459}
66521#endif68460#endif
6652268461
...@@ -66553,260 +68492,236 @@ __ai __attribute__((target("bf16,neon"))) float32x4_t vcvtq_low_f32_bf16(bfloat1...@@ -66553,260 +68492,236 @@ __ai __attribute__((target("bf16,neon"))) float32x4_t vcvtq_low_f32_bf16(bfloat1
66553#endif68492#endif
6655468493
66555#ifdef __LITTLE_ENDIAN__68494#ifdef __LITTLE_ENDIAN__
66556#define vdotq_lane_u32(__p0_839, __p1_839, __p2_839, __p3_839) __extension__ ({ \68495#define vdotq_lane_u32(__p0_811, __p1_811, __p2_811, __p3_811) __extension__ ({ \
66557 uint32x4_t __ret_839; \68496 uint32x4_t __ret_811; \
66558 uint32x4_t __s0_839 = __p0_839; \68497 uint32x4_t __s0_811 = __p0_811; \
66559 uint8x16_t __s1_839 = __p1_839; \68498 uint8x16_t __s1_811 = __p1_811; \
66560 uint8x8_t __s2_839 = __p2_839; \68499 uint8x8_t __s2_811 = __p2_811; \
66561uint8x8_t __reint_839 = __s2_839; \68500 __ret_811 = vdotq_u32(__s0_811, __s1_811, __builtin_bit_cast(uint8x16_t, splatq_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_811), __p3_811))); \
66562uint32x4_t __reint1_839 = splatq_lane_u32(*(uint32x2_t *) &__reint_839, __p3_839); \68501 __ret_811; \
66563 __ret_839 = vdotq_u32(__s0_839, __s1_839, *(uint8x16_t *) &__reint1_839); \
66564 __ret_839; \
66565})68502})
66566#else68503#else
66567#define vdotq_lane_u32(__p0_840, __p1_840, __p2_840, __p3_840) __extension__ ({ \68504#define vdotq_lane_u32(__p0_812, __p1_812, __p2_812, __p3_812) __extension__ ({ \
66568 uint32x4_t __ret_840; \68505 uint32x4_t __ret_812; \
66569 uint32x4_t __s0_840 = __p0_840; \68506 uint32x4_t __s0_812 = __p0_812; \
66570 uint8x16_t __s1_840 = __p1_840; \68507 uint8x16_t __s1_812 = __p1_812; \
66571 uint8x8_t __s2_840 = __p2_840; \68508 uint8x8_t __s2_812 = __p2_812; \
66572 uint32x4_t __rev0_840; __rev0_840 = __builtin_shufflevector(__s0_840, __s0_840, 3, 2, 1, 0); \68509 uint32x4_t __rev0_812; __rev0_812 = __builtin_shufflevector(__s0_812, __s0_812, 3, 2, 1, 0); \
66573 uint8x16_t __rev1_840; __rev1_840 = __builtin_shufflevector(__s1_840, __s1_840, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \68510 uint8x16_t __rev1_812; __rev1_812 = __builtin_shufflevector(__s1_812, __s1_812, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66574 uint8x8_t __rev2_840; __rev2_840 = __builtin_shufflevector(__s2_840, __s2_840, 7, 6, 5, 4, 3, 2, 1, 0); \68511 uint8x8_t __rev2_812; __rev2_812 = __builtin_shufflevector(__s2_812, __s2_812, 7, 6, 5, 4, 3, 2, 1, 0); \
66575uint8x8_t __reint_840 = __rev2_840; \68512 __ret_812 = __noswap_vdotq_u32(__rev0_812, __rev1_812, __builtin_bit_cast(uint8x16_t, __noswap_splatq_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_812), __p3_812))); \
66576uint32x4_t __reint1_840 = __noswap_splatq_lane_u32(*(uint32x2_t *) &__reint_840, __p3_840); \68513 __ret_812 = __builtin_shufflevector(__ret_812, __ret_812, 3, 2, 1, 0); \
66577 __ret_840 = __noswap_vdotq_u32(__rev0_840, __rev1_840, *(uint8x16_t *) &__reint1_840); \68514 __ret_812; \
66578 __ret_840 = __builtin_shufflevector(__ret_840, __ret_840, 3, 2, 1, 0); \
66579 __ret_840; \
66580})68515})
66581#endif68516#endif
6658268517
66583#ifdef __LITTLE_ENDIAN__68518#ifdef __LITTLE_ENDIAN__
66584#define vdotq_lane_s32(__p0_841, __p1_841, __p2_841, __p3_841) __extension__ ({ \68519#define vdotq_lane_s32(__p0_813, __p1_813, __p2_813, __p3_813) __extension__ ({ \
66585 int32x4_t __ret_841; \68520 int32x4_t __ret_813; \
66586 int32x4_t __s0_841 = __p0_841; \68521 int32x4_t __s0_813 = __p0_813; \
66587 int8x16_t __s1_841 = __p1_841; \68522 int8x16_t __s1_813 = __p1_813; \
66588 int8x8_t __s2_841 = __p2_841; \68523 int8x8_t __s2_813 = __p2_813; \
66589int8x8_t __reint_841 = __s2_841; \68524 __ret_813 = vdotq_s32(__s0_813, __s1_813, __builtin_bit_cast(int8x16_t, splatq_lane_s32(__builtin_bit_cast(int32x2_t, __s2_813), __p3_813))); \
66590int32x4_t __reint1_841 = splatq_lane_s32(*(int32x2_t *) &__reint_841, __p3_841); \68525 __ret_813; \
66591 __ret_841 = vdotq_s32(__s0_841, __s1_841, *(int8x16_t *) &__reint1_841); \
66592 __ret_841; \
66593})68526})
66594#else68527#else
66595#define vdotq_lane_s32(__p0_842, __p1_842, __p2_842, __p3_842) __extension__ ({ \68528#define vdotq_lane_s32(__p0_814, __p1_814, __p2_814, __p3_814) __extension__ ({ \
66596 int32x4_t __ret_842; \68529 int32x4_t __ret_814; \
66597 int32x4_t __s0_842 = __p0_842; \68530 int32x4_t __s0_814 = __p0_814; \
66598 int8x16_t __s1_842 = __p1_842; \68531 int8x16_t __s1_814 = __p1_814; \
66599 int8x8_t __s2_842 = __p2_842; \68532 int8x8_t __s2_814 = __p2_814; \
66600 int32x4_t __rev0_842; __rev0_842 = __builtin_shufflevector(__s0_842, __s0_842, 3, 2, 1, 0); \68533 int32x4_t __rev0_814; __rev0_814 = __builtin_shufflevector(__s0_814, __s0_814, 3, 2, 1, 0); \
66601 int8x16_t __rev1_842; __rev1_842 = __builtin_shufflevector(__s1_842, __s1_842, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \68534 int8x16_t __rev1_814; __rev1_814 = __builtin_shufflevector(__s1_814, __s1_814, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66602 int8x8_t __rev2_842; __rev2_842 = __builtin_shufflevector(__s2_842, __s2_842, 7, 6, 5, 4, 3, 2, 1, 0); \68535 int8x8_t __rev2_814; __rev2_814 = __builtin_shufflevector(__s2_814, __s2_814, 7, 6, 5, 4, 3, 2, 1, 0); \
66603int8x8_t __reint_842 = __rev2_842; \68536 __ret_814 = __noswap_vdotq_s32(__rev0_814, __rev1_814, __builtin_bit_cast(int8x16_t, __noswap_splatq_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_814), __p3_814))); \
66604int32x4_t __reint1_842 = __noswap_splatq_lane_s32(*(int32x2_t *) &__reint_842, __p3_842); \68537 __ret_814 = __builtin_shufflevector(__ret_814, __ret_814, 3, 2, 1, 0); \
66605 __ret_842 = __noswap_vdotq_s32(__rev0_842, __rev1_842, *(int8x16_t *) &__reint1_842); \68538 __ret_814; \
66606 __ret_842 = __builtin_shufflevector(__ret_842, __ret_842, 3, 2, 1, 0); \
66607 __ret_842; \
66608})68539})
66609#endif68540#endif
6661068541
66611#ifdef __LITTLE_ENDIAN__68542#ifdef __LITTLE_ENDIAN__
66612#define vdot_lane_u32(__p0_843, __p1_843, __p2_843, __p3_843) __extension__ ({ \68543#define vdot_lane_u32(__p0_815, __p1_815, __p2_815, __p3_815) __extension__ ({ \
66613 uint32x2_t __ret_843; \68544 uint32x2_t __ret_815; \
66614 uint32x2_t __s0_843 = __p0_843; \68545 uint32x2_t __s0_815 = __p0_815; \
66615 uint8x8_t __s1_843 = __p1_843; \68546 uint8x8_t __s1_815 = __p1_815; \
66616 uint8x8_t __s2_843 = __p2_843; \68547 uint8x8_t __s2_815 = __p2_815; \
66617uint8x8_t __reint_843 = __s2_843; \68548 __ret_815 = vdot_u32(__s0_815, __s1_815, __builtin_bit_cast(uint8x8_t, splat_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_815), __p3_815))); \
66618uint32x2_t __reint1_843 = splat_lane_u32(*(uint32x2_t *) &__reint_843, __p3_843); \68549 __ret_815; \
66619 __ret_843 = vdot_u32(__s0_843, __s1_843, *(uint8x8_t *) &__reint1_843); \
66620 __ret_843; \
66621})68550})
66622#else68551#else
66623#define vdot_lane_u32(__p0_844, __p1_844, __p2_844, __p3_844) __extension__ ({ \68552#define vdot_lane_u32(__p0_816, __p1_816, __p2_816, __p3_816) __extension__ ({ \
66624 uint32x2_t __ret_844; \68553 uint32x2_t __ret_816; \
66625 uint32x2_t __s0_844 = __p0_844; \68554 uint32x2_t __s0_816 = __p0_816; \
66626 uint8x8_t __s1_844 = __p1_844; \68555 uint8x8_t __s1_816 = __p1_816; \
66627 uint8x8_t __s2_844 = __p2_844; \68556 uint8x8_t __s2_816 = __p2_816; \
66628 uint32x2_t __rev0_844; __rev0_844 = __builtin_shufflevector(__s0_844, __s0_844, 1, 0); \68557 uint32x2_t __rev0_816; __rev0_816 = __builtin_shufflevector(__s0_816, __s0_816, 1, 0); \
66629 uint8x8_t __rev1_844; __rev1_844 = __builtin_shufflevector(__s1_844, __s1_844, 7, 6, 5, 4, 3, 2, 1, 0); \68558 uint8x8_t __rev1_816; __rev1_816 = __builtin_shufflevector(__s1_816, __s1_816, 7, 6, 5, 4, 3, 2, 1, 0); \
66630 uint8x8_t __rev2_844; __rev2_844 = __builtin_shufflevector(__s2_844, __s2_844, 7, 6, 5, 4, 3, 2, 1, 0); \68559 uint8x8_t __rev2_816; __rev2_816 = __builtin_shufflevector(__s2_816, __s2_816, 7, 6, 5, 4, 3, 2, 1, 0); \
66631uint8x8_t __reint_844 = __rev2_844; \68560 __ret_816 = __noswap_vdot_u32(__rev0_816, __rev1_816, __builtin_bit_cast(uint8x8_t, __noswap_splat_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_816), __p3_816))); \
66632uint32x2_t __reint1_844 = __noswap_splat_lane_u32(*(uint32x2_t *) &__reint_844, __p3_844); \68561 __ret_816 = __builtin_shufflevector(__ret_816, __ret_816, 1, 0); \
66633 __ret_844 = __noswap_vdot_u32(__rev0_844, __rev1_844, *(uint8x8_t *) &__reint1_844); \68562 __ret_816; \
66634 __ret_844 = __builtin_shufflevector(__ret_844, __ret_844, 1, 0); \
66635 __ret_844; \
66636})68563})
66637#endif68564#endif
6663868565
66639#ifdef __LITTLE_ENDIAN__68566#ifdef __LITTLE_ENDIAN__
66640#define vdot_lane_s32(__p0_845, __p1_845, __p2_845, __p3_845) __extension__ ({ \68567#define vdot_lane_s32(__p0_817, __p1_817, __p2_817, __p3_817) __extension__ ({ \
66641 int32x2_t __ret_845; \68568 int32x2_t __ret_817; \
66642 int32x2_t __s0_845 = __p0_845; \68569 int32x2_t __s0_817 = __p0_817; \
66643 int8x8_t __s1_845 = __p1_845; \68570 int8x8_t __s1_817 = __p1_817; \
66644 int8x8_t __s2_845 = __p2_845; \68571 int8x8_t __s2_817 = __p2_817; \
66645int8x8_t __reint_845 = __s2_845; \68572 __ret_817 = vdot_s32(__s0_817, __s1_817, __builtin_bit_cast(int8x8_t, splat_lane_s32(__builtin_bit_cast(int32x2_t, __s2_817), __p3_817))); \
66646int32x2_t __reint1_845 = splat_lane_s32(*(int32x2_t *) &__reint_845, __p3_845); \68573 __ret_817; \
66647 __ret_845 = vdot_s32(__s0_845, __s1_845, *(int8x8_t *) &__reint1_845); \
66648 __ret_845; \
66649})68574})
66650#else68575#else
66651#define vdot_lane_s32(__p0_846, __p1_846, __p2_846, __p3_846) __extension__ ({ \68576#define vdot_lane_s32(__p0_818, __p1_818, __p2_818, __p3_818) __extension__ ({ \
66652 int32x2_t __ret_846; \68577 int32x2_t __ret_818; \
66653 int32x2_t __s0_846 = __p0_846; \68578 int32x2_t __s0_818 = __p0_818; \
66654 int8x8_t __s1_846 = __p1_846; \68579 int8x8_t __s1_818 = __p1_818; \
66655 int8x8_t __s2_846 = __p2_846; \68580 int8x8_t __s2_818 = __p2_818; \
66656 int32x2_t __rev0_846; __rev0_846 = __builtin_shufflevector(__s0_846, __s0_846, 1, 0); \68581 int32x2_t __rev0_818; __rev0_818 = __builtin_shufflevector(__s0_818, __s0_818, 1, 0); \
66657 int8x8_t __rev1_846; __rev1_846 = __builtin_shufflevector(__s1_846, __s1_846, 7, 6, 5, 4, 3, 2, 1, 0); \68582 int8x8_t __rev1_818; __rev1_818 = __builtin_shufflevector(__s1_818, __s1_818, 7, 6, 5, 4, 3, 2, 1, 0); \
66658 int8x8_t __rev2_846; __rev2_846 = __builtin_shufflevector(__s2_846, __s2_846, 7, 6, 5, 4, 3, 2, 1, 0); \68583 int8x8_t __rev2_818; __rev2_818 = __builtin_shufflevector(__s2_818, __s2_818, 7, 6, 5, 4, 3, 2, 1, 0); \
66659int8x8_t __reint_846 = __rev2_846; \68584 __ret_818 = __noswap_vdot_s32(__rev0_818, __rev1_818, __builtin_bit_cast(int8x8_t, __noswap_splat_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_818), __p3_818))); \
66660int32x2_t __reint1_846 = __noswap_splat_lane_s32(*(int32x2_t *) &__reint_846, __p3_846); \68585 __ret_818 = __builtin_shufflevector(__ret_818, __ret_818, 1, 0); \
66661 __ret_846 = __noswap_vdot_s32(__rev0_846, __rev1_846, *(int8x8_t *) &__reint1_846); \68586 __ret_818; \
66662 __ret_846 = __builtin_shufflevector(__ret_846, __ret_846, 1, 0); \
66663 __ret_846; \
66664})68587})
66665#endif68588#endif
6666668589
66667#ifdef __LITTLE_ENDIAN__68590#ifdef __LITTLE_ENDIAN__
66668#define vmulq_lane_f16(__p0_847, __p1_847, __p2_847) __extension__ ({ \68591#define vmulq_lane_f16(__p0_819, __p1_819, __p2_819) __extension__ ({ \
66669 float16x8_t __ret_847; \68592 float16x8_t __ret_819; \
66670 float16x8_t __s0_847 = __p0_847; \68593 float16x8_t __s0_819 = __p0_819; \
66671 float16x4_t __s1_847 = __p1_847; \68594 float16x4_t __s1_819 = __p1_819; \
66672 __ret_847 = __s0_847 * splatq_lane_f16(__s1_847, __p2_847); \68595 __ret_819 = __s0_819 * splatq_lane_f16(__s1_819, __p2_819); \
66673 __ret_847; \68596 __ret_819; \
66674})68597})
66675#else68598#else
66676#define vmulq_lane_f16(__p0_848, __p1_848, __p2_848) __extension__ ({ \68599#define vmulq_lane_f16(__p0_820, __p1_820, __p2_820) __extension__ ({ \
66677 float16x8_t __ret_848; \68600 float16x8_t __ret_820; \
66678 float16x8_t __s0_848 = __p0_848; \68601 float16x8_t __s0_820 = __p0_820; \
66679 float16x4_t __s1_848 = __p1_848; \68602 float16x4_t __s1_820 = __p1_820; \
66680 float16x8_t __rev0_848; __rev0_848 = __builtin_shufflevector(__s0_848, __s0_848, 7, 6, 5, 4, 3, 2, 1, 0); \68603 float16x8_t __rev0_820; __rev0_820 = __builtin_shufflevector(__s0_820, __s0_820, 7, 6, 5, 4, 3, 2, 1, 0); \
66681 float16x4_t __rev1_848; __rev1_848 = __builtin_shufflevector(__s1_848, __s1_848, 3, 2, 1, 0); \68604 float16x4_t __rev1_820; __rev1_820 = __builtin_shufflevector(__s1_820, __s1_820, 3, 2, 1, 0); \
66682 __ret_848 = __rev0_848 * __noswap_splatq_lane_f16(__rev1_848, __p2_848); \68605 __ret_820 = __rev0_820 * __noswap_splatq_lane_f16(__rev1_820, __p2_820); \
66683 __ret_848 = __builtin_shufflevector(__ret_848, __ret_848, 7, 6, 5, 4, 3, 2, 1, 0); \68606 __ret_820 = __builtin_shufflevector(__ret_820, __ret_820, 7, 6, 5, 4, 3, 2, 1, 0); \
66684 __ret_848; \68607 __ret_820; \
66685})68608})
66686#endif68609#endif
6668768610
66688#ifdef __LITTLE_ENDIAN__68611#ifdef __LITTLE_ENDIAN__
66689#define vmul_lane_f16(__p0_849, __p1_849, __p2_849) __extension__ ({ \68612#define vmul_lane_f16(__p0_821, __p1_821, __p2_821) __extension__ ({ \
66690 float16x4_t __ret_849; \68613 float16x4_t __ret_821; \
66691 float16x4_t __s0_849 = __p0_849; \68614 float16x4_t __s0_821 = __p0_821; \
66692 float16x4_t __s1_849 = __p1_849; \68615 float16x4_t __s1_821 = __p1_821; \
66693 __ret_849 = __s0_849 * splat_lane_f16(__s1_849, __p2_849); \68616 __ret_821 = __s0_821 * splat_lane_f16(__s1_821, __p2_821); \
66694 __ret_849; \68617 __ret_821; \
66695})68618})
66696#else68619#else
66697#define vmul_lane_f16(__p0_850, __p1_850, __p2_850) __extension__ ({ \68620#define vmul_lane_f16(__p0_822, __p1_822, __p2_822) __extension__ ({ \
66698 float16x4_t __ret_850; \68621 float16x4_t __ret_822; \
66699 float16x4_t __s0_850 = __p0_850; \68622 float16x4_t __s0_822 = __p0_822; \
66700 float16x4_t __s1_850 = __p1_850; \68623 float16x4_t __s1_822 = __p1_822; \
66701 float16x4_t __rev0_850; __rev0_850 = __builtin_shufflevector(__s0_850, __s0_850, 3, 2, 1, 0); \68624 float16x4_t __rev0_822; __rev0_822 = __builtin_shufflevector(__s0_822, __s0_822, 3, 2, 1, 0); \
66702 float16x4_t __rev1_850; __rev1_850 = __builtin_shufflevector(__s1_850, __s1_850, 3, 2, 1, 0); \68625 float16x4_t __rev1_822; __rev1_822 = __builtin_shufflevector(__s1_822, __s1_822, 3, 2, 1, 0); \
66703 __ret_850 = __rev0_850 * __noswap_splat_lane_f16(__rev1_850, __p2_850); \68626 __ret_822 = __rev0_822 * __noswap_splat_lane_f16(__rev1_822, __p2_822); \
66704 __ret_850 = __builtin_shufflevector(__ret_850, __ret_850, 3, 2, 1, 0); \68627 __ret_822 = __builtin_shufflevector(__ret_822, __ret_822, 3, 2, 1, 0); \
66705 __ret_850; \68628 __ret_822; \
66706})68629})
66707#endif68630#endif
6670868631
66709#ifdef __LITTLE_ENDIAN__68632#ifdef __LITTLE_ENDIAN__
66710#define vsudotq_lane_s32(__p0_851, __p1_851, __p2_851, __p3_851) __extension__ ({ \68633#define vsudotq_lane_s32(__p0_823, __p1_823, __p2_823, __p3_823) __extension__ ({ \
66711 int32x4_t __ret_851; \68634 int32x4_t __ret_823; \
66712 int32x4_t __s0_851 = __p0_851; \68635 int32x4_t __s0_823 = __p0_823; \
66713 int8x16_t __s1_851 = __p1_851; \68636 int8x16_t __s1_823 = __p1_823; \
66714 uint8x8_t __s2_851 = __p2_851; \68637 uint8x8_t __s2_823 = __p2_823; \
66715uint8x8_t __reint_851 = __s2_851; \68638 __ret_823 = vusdotq_s32(__s0_823, (uint8x16_t)(splatq_lane_s32(__builtin_bit_cast(int32x2_t, __s2_823), __p3_823)), __s1_823); \
66716 __ret_851 = vusdotq_s32(__s0_851, (uint8x16_t)(splatq_lane_s32(*(int32x2_t *) &__reint_851, __p3_851)), __s1_851); \68639 __ret_823; \
66717 __ret_851; \
66718})68640})
66719#else68641#else
66720#define vsudotq_lane_s32(__p0_852, __p1_852, __p2_852, __p3_852) __extension__ ({ \68642#define vsudotq_lane_s32(__p0_824, __p1_824, __p2_824, __p3_824) __extension__ ({ \
66721 int32x4_t __ret_852; \68643 int32x4_t __ret_824; \
66722 int32x4_t __s0_852 = __p0_852; \68644 int32x4_t __s0_824 = __p0_824; \
66723 int8x16_t __s1_852 = __p1_852; \68645 int8x16_t __s1_824 = __p1_824; \
66724 uint8x8_t __s2_852 = __p2_852; \68646 uint8x8_t __s2_824 = __p2_824; \
66725 int32x4_t __rev0_852; __rev0_852 = __builtin_shufflevector(__s0_852, __s0_852, 3, 2, 1, 0); \68647 int32x4_t __rev0_824; __rev0_824 = __builtin_shufflevector(__s0_824, __s0_824, 3, 2, 1, 0); \
66726 int8x16_t __rev1_852; __rev1_852 = __builtin_shufflevector(__s1_852, __s1_852, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \68648 int8x16_t __rev1_824; __rev1_824 = __builtin_shufflevector(__s1_824, __s1_824, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66727 uint8x8_t __rev2_852; __rev2_852 = __builtin_shufflevector(__s2_852, __s2_852, 7, 6, 5, 4, 3, 2, 1, 0); \68649 uint8x8_t __rev2_824; __rev2_824 = __builtin_shufflevector(__s2_824, __s2_824, 7, 6, 5, 4, 3, 2, 1, 0); \
66728uint8x8_t __reint_852 = __rev2_852; \68650 __ret_824 = __noswap_vusdotq_s32(__rev0_824, (uint8x16_t)(__noswap_splatq_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_824), __p3_824)), __rev1_824); \
66729 __ret_852 = __noswap_vusdotq_s32(__rev0_852, (uint8x16_t)(__noswap_splatq_lane_s32(*(int32x2_t *) &__reint_852, __p3_852)), __rev1_852); \68651 __ret_824 = __builtin_shufflevector(__ret_824, __ret_824, 3, 2, 1, 0); \
66730 __ret_852 = __builtin_shufflevector(__ret_852, __ret_852, 3, 2, 1, 0); \68652 __ret_824; \
66731 __ret_852; \
66732})68653})
66733#endif68654#endif
6673468655
66735#ifdef __LITTLE_ENDIAN__68656#ifdef __LITTLE_ENDIAN__
66736#define vsudot_lane_s32(__p0_853, __p1_853, __p2_853, __p3_853) __extension__ ({ \68657#define vsudot_lane_s32(__p0_825, __p1_825, __p2_825, __p3_825) __extension__ ({ \
66737 int32x2_t __ret_853; \68658 int32x2_t __ret_825; \
66738 int32x2_t __s0_853 = __p0_853; \68659 int32x2_t __s0_825 = __p0_825; \
66739 int8x8_t __s1_853 = __p1_853; \68660 int8x8_t __s1_825 = __p1_825; \
66740 uint8x8_t __s2_853 = __p2_853; \68661 uint8x8_t __s2_825 = __p2_825; \
66741uint8x8_t __reint_853 = __s2_853; \68662 __ret_825 = vusdot_s32(__s0_825, (uint8x8_t)(splat_lane_s32(__builtin_bit_cast(int32x2_t, __s2_825), __p3_825)), __s1_825); \
66742 __ret_853 = vusdot_s32(__s0_853, (uint8x8_t)(splat_lane_s32(*(int32x2_t *) &__reint_853, __p3_853)), __s1_853); \68663 __ret_825; \
66743 __ret_853; \
66744})68664})
66745#else68665#else
66746#define vsudot_lane_s32(__p0_854, __p1_854, __p2_854, __p3_854) __extension__ ({ \68666#define vsudot_lane_s32(__p0_826, __p1_826, __p2_826, __p3_826) __extension__ ({ \
66747 int32x2_t __ret_854; \68667 int32x2_t __ret_826; \
66748 int32x2_t __s0_854 = __p0_854; \68668 int32x2_t __s0_826 = __p0_826; \
66749 int8x8_t __s1_854 = __p1_854; \68669 int8x8_t __s1_826 = __p1_826; \
66750 uint8x8_t __s2_854 = __p2_854; \68670 uint8x8_t __s2_826 = __p2_826; \
66751 int32x2_t __rev0_854; __rev0_854 = __builtin_shufflevector(__s0_854, __s0_854, 1, 0); \68671 int32x2_t __rev0_826; __rev0_826 = __builtin_shufflevector(__s0_826, __s0_826, 1, 0); \
66752 int8x8_t __rev1_854; __rev1_854 = __builtin_shufflevector(__s1_854, __s1_854, 7, 6, 5, 4, 3, 2, 1, 0); \68672 int8x8_t __rev1_826; __rev1_826 = __builtin_shufflevector(__s1_826, __s1_826, 7, 6, 5, 4, 3, 2, 1, 0); \
66753 uint8x8_t __rev2_854; __rev2_854 = __builtin_shufflevector(__s2_854, __s2_854, 7, 6, 5, 4, 3, 2, 1, 0); \68673 uint8x8_t __rev2_826; __rev2_826 = __builtin_shufflevector(__s2_826, __s2_826, 7, 6, 5, 4, 3, 2, 1, 0); \
66754uint8x8_t __reint_854 = __rev2_854; \68674 __ret_826 = __noswap_vusdot_s32(__rev0_826, (uint8x8_t)(__noswap_splat_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_826), __p3_826)), __rev1_826); \
66755 __ret_854 = __noswap_vusdot_s32(__rev0_854, (uint8x8_t)(__noswap_splat_lane_s32(*(int32x2_t *) &__reint_854, __p3_854)), __rev1_854); \68675 __ret_826 = __builtin_shufflevector(__ret_826, __ret_826, 1, 0); \
66756 __ret_854 = __builtin_shufflevector(__ret_854, __ret_854, 1, 0); \68676 __ret_826; \
66757 __ret_854; \
66758})68677})
66759#endif68678#endif
6676068679
66761#ifdef __LITTLE_ENDIAN__68680#ifdef __LITTLE_ENDIAN__
66762#define vusdotq_lane_s32(__p0_855, __p1_855, __p2_855, __p3_855) __extension__ ({ \68681#define vusdotq_lane_s32(__p0_827, __p1_827, __p2_827, __p3_827) __extension__ ({ \
66763 int32x4_t __ret_855; \68682 int32x4_t __ret_827; \
66764 int32x4_t __s0_855 = __p0_855; \68683 int32x4_t __s0_827 = __p0_827; \
66765 uint8x16_t __s1_855 = __p1_855; \68684 uint8x16_t __s1_827 = __p1_827; \
66766 int8x8_t __s2_855 = __p2_855; \68685 int8x8_t __s2_827 = __p2_827; \
66767int8x8_t __reint_855 = __s2_855; \68686 __ret_827 = vusdotq_s32(__s0_827, __s1_827, (int8x16_t)(splatq_lane_s32(__builtin_bit_cast(int32x2_t, __s2_827), __p3_827))); \
66768 __ret_855 = vusdotq_s32(__s0_855, __s1_855, (int8x16_t)(splatq_lane_s32(*(int32x2_t *) &__reint_855, __p3_855))); \68687 __ret_827; \
66769 __ret_855; \
66770})68688})
66771#else68689#else
66772#define vusdotq_lane_s32(__p0_856, __p1_856, __p2_856, __p3_856) __extension__ ({ \68690#define vusdotq_lane_s32(__p0_828, __p1_828, __p2_828, __p3_828) __extension__ ({ \
66773 int32x4_t __ret_856; \68691 int32x4_t __ret_828; \
66774 int32x4_t __s0_856 = __p0_856; \68692 int32x4_t __s0_828 = __p0_828; \
66775 uint8x16_t __s1_856 = __p1_856; \68693 uint8x16_t __s1_828 = __p1_828; \
66776 int8x8_t __s2_856 = __p2_856; \68694 int8x8_t __s2_828 = __p2_828; \
66777 int32x4_t __rev0_856; __rev0_856 = __builtin_shufflevector(__s0_856, __s0_856, 3, 2, 1, 0); \68695 int32x4_t __rev0_828; __rev0_828 = __builtin_shufflevector(__s0_828, __s0_828, 3, 2, 1, 0); \
66778 uint8x16_t __rev1_856; __rev1_856 = __builtin_shufflevector(__s1_856, __s1_856, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \68696 uint8x16_t __rev1_828; __rev1_828 = __builtin_shufflevector(__s1_828, __s1_828, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66779 int8x8_t __rev2_856; __rev2_856 = __builtin_shufflevector(__s2_856, __s2_856, 7, 6, 5, 4, 3, 2, 1, 0); \68697 int8x8_t __rev2_828; __rev2_828 = __builtin_shufflevector(__s2_828, __s2_828, 7, 6, 5, 4, 3, 2, 1, 0); \
66780int8x8_t __reint_856 = __rev2_856; \68698 __ret_828 = __noswap_vusdotq_s32(__rev0_828, __rev1_828, (int8x16_t)(__noswap_splatq_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_828), __p3_828))); \
66781 __ret_856 = __noswap_vusdotq_s32(__rev0_856, __rev1_856, (int8x16_t)(__noswap_splatq_lane_s32(*(int32x2_t *) &__reint_856, __p3_856))); \68699 __ret_828 = __builtin_shufflevector(__ret_828, __ret_828, 3, 2, 1, 0); \
66782 __ret_856 = __builtin_shufflevector(__ret_856, __ret_856, 3, 2, 1, 0); \68700 __ret_828; \
66783 __ret_856; \
66784})68701})
66785#endif68702#endif
6678668703
66787#ifdef __LITTLE_ENDIAN__68704#ifdef __LITTLE_ENDIAN__
66788#define vusdot_lane_s32(__p0_857, __p1_857, __p2_857, __p3_857) __extension__ ({ \68705#define vusdot_lane_s32(__p0_829, __p1_829, __p2_829, __p3_829) __extension__ ({ \
66789 int32x2_t __ret_857; \68706 int32x2_t __ret_829; \
66790 int32x2_t __s0_857 = __p0_857; \68707 int32x2_t __s0_829 = __p0_829; \
66791 uint8x8_t __s1_857 = __p1_857; \68708 uint8x8_t __s1_829 = __p1_829; \
66792 int8x8_t __s2_857 = __p2_857; \68709 int8x8_t __s2_829 = __p2_829; \
66793int8x8_t __reint_857 = __s2_857; \68710 __ret_829 = vusdot_s32(__s0_829, __s1_829, (int8x8_t)(splat_lane_s32(__builtin_bit_cast(int32x2_t, __s2_829), __p3_829))); \
66794 __ret_857 = vusdot_s32(__s0_857, __s1_857, (int8x8_t)(splat_lane_s32(*(int32x2_t *) &__reint_857, __p3_857))); \68711 __ret_829; \
66795 __ret_857; \
66796})68712})
66797#else68713#else
66798#define vusdot_lane_s32(__p0_858, __p1_858, __p2_858, __p3_858) __extension__ ({ \68714#define vusdot_lane_s32(__p0_830, __p1_830, __p2_830, __p3_830) __extension__ ({ \
66799 int32x2_t __ret_858; \68715 int32x2_t __ret_830; \
66800 int32x2_t __s0_858 = __p0_858; \68716 int32x2_t __s0_830 = __p0_830; \
66801 uint8x8_t __s1_858 = __p1_858; \68717 uint8x8_t __s1_830 = __p1_830; \
66802 int8x8_t __s2_858 = __p2_858; \68718 int8x8_t __s2_830 = __p2_830; \
66803 int32x2_t __rev0_858; __rev0_858 = __builtin_shufflevector(__s0_858, __s0_858, 1, 0); \68719 int32x2_t __rev0_830; __rev0_830 = __builtin_shufflevector(__s0_830, __s0_830, 1, 0); \
66804 uint8x8_t __rev1_858; __rev1_858 = __builtin_shufflevector(__s1_858, __s1_858, 7, 6, 5, 4, 3, 2, 1, 0); \68720 uint8x8_t __rev1_830; __rev1_830 = __builtin_shufflevector(__s1_830, __s1_830, 7, 6, 5, 4, 3, 2, 1, 0); \
66805 int8x8_t __rev2_858; __rev2_858 = __builtin_shufflevector(__s2_858, __s2_858, 7, 6, 5, 4, 3, 2, 1, 0); \68721 int8x8_t __rev2_830; __rev2_830 = __builtin_shufflevector(__s2_830, __s2_830, 7, 6, 5, 4, 3, 2, 1, 0); \
66806int8x8_t __reint_858 = __rev2_858; \68722 __ret_830 = __noswap_vusdot_s32(__rev0_830, __rev1_830, (int8x8_t)(__noswap_splat_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_830), __p3_830))); \
66807 __ret_858 = __noswap_vusdot_s32(__rev0_858, __rev1_858, (int8x8_t)(__noswap_splat_lane_s32(*(int32x2_t *) &__reint_858, __p3_858))); \68723 __ret_830 = __builtin_shufflevector(__ret_830, __ret_830, 1, 0); \
66808 __ret_858 = __builtin_shufflevector(__ret_858, __ret_858, 1, 0); \68724 __ret_830; \
66809 __ret_858; \
66810})68725})
66811#endif68726#endif
6681268727
...@@ -67363,60 +69278,48 @@ __ai __attribute__((target("neon"))) int32x4_t vaddw_s16(int32x4_t __p0, int16x4...@@ -67363,60 +69278,48 @@ __ai __attribute__((target("neon"))) int32x4_t vaddw_s16(int32x4_t __p0, int16x4
67363#endif69278#endif
6736469279
67365#ifdef __LITTLE_ENDIAN__69280#ifdef __LITTLE_ENDIAN__
67366#define vget_lane_f16(__p0_859, __p1_859) __extension__ ({ \69281#define vget_lane_f16(__p0_831, __p1_831) __extension__ ({ \
67367 float16_t __ret_859; \69282 float16_t __ret_831; \
67368 float16x4_t __s0_859 = __p0_859; \69283 float16x4_t __s0_831 = __p0_831; \
67369float16x4_t __reint_859 = __s0_859; \69284 __ret_831 = __builtin_bit_cast(float16_t, vget_lane_s16(__builtin_bit_cast(int16x4_t, __s0_831), __p1_831)); \
67370int16_t __reint1_859 = vget_lane_s16(*(int16x4_t *) &__reint_859, __p1_859); \69285 __ret_831; \
67371 __ret_859 = *(float16_t *) &__reint1_859; \
67372 __ret_859; \
67373})69286})
67374#else69287#else
67375#define vget_lane_f16(__p0_860, __p1_860) __extension__ ({ \69288#define vget_lane_f16(__p0_832, __p1_832) __extension__ ({ \
67376 float16_t __ret_860; \69289 float16_t __ret_832; \
67377 float16x4_t __s0_860 = __p0_860; \69290 float16x4_t __s0_832 = __p0_832; \
67378 float16x4_t __rev0_860; __rev0_860 = __builtin_shufflevector(__s0_860, __s0_860, 3, 2, 1, 0); \69291 float16x4_t __rev0_832; __rev0_832 = __builtin_shufflevector(__s0_832, __s0_832, 3, 2, 1, 0); \
67379float16x4_t __reint_860 = __rev0_860; \69292 __ret_832 = __builtin_bit_cast(float16_t, __noswap_vget_lane_s16(__builtin_bit_cast(int16x4_t, __rev0_832), __p1_832)); \
67380int16_t __reint1_860 = __noswap_vget_lane_s16(*(int16x4_t *) &__reint_860, __p1_860); \69293 __ret_832; \
67381 __ret_860 = *(float16_t *) &__reint1_860; \
67382 __ret_860; \
67383})69294})
67384#define __noswap_vget_lane_f16(__p0_861, __p1_861) __extension__ ({ \69295#define __noswap_vget_lane_f16(__p0_833, __p1_833) __extension__ ({ \
67385 float16_t __ret_861; \69296 float16_t __ret_833; \
67386 float16x4_t __s0_861 = __p0_861; \69297 float16x4_t __s0_833 = __p0_833; \
67387float16x4_t __reint_861 = __s0_861; \69298 __ret_833 = __builtin_bit_cast(float16_t, __noswap_vget_lane_s16(__builtin_bit_cast(int16x4_t, __s0_833), __p1_833)); \
67388int16_t __reint1_861 = __noswap_vget_lane_s16(*(int16x4_t *) &__reint_861, __p1_861); \69299 __ret_833; \
67389 __ret_861 = *(float16_t *) &__reint1_861; \
67390 __ret_861; \
67391})69300})
67392#endif69301#endif
6739369302
67394#ifdef __LITTLE_ENDIAN__69303#ifdef __LITTLE_ENDIAN__
67395#define vgetq_lane_f16(__p0_862, __p1_862) __extension__ ({ \69304#define vgetq_lane_f16(__p0_834, __p1_834) __extension__ ({ \
67396 float16_t __ret_862; \69305 float16_t __ret_834; \
67397 float16x8_t __s0_862 = __p0_862; \69306 float16x8_t __s0_834 = __p0_834; \
67398float16x8_t __reint_862 = __s0_862; \69307 __ret_834 = __builtin_bit_cast(float16_t, vgetq_lane_s16(__builtin_bit_cast(int16x8_t, __s0_834), __p1_834)); \
67399int16_t __reint1_862 = vgetq_lane_s16(*(int16x8_t *) &__reint_862, __p1_862); \69308 __ret_834; \
67400 __ret_862 = *(float16_t *) &__reint1_862; \
67401 __ret_862; \
67402})69309})
67403#else69310#else
67404#define vgetq_lane_f16(__p0_863, __p1_863) __extension__ ({ \69311#define vgetq_lane_f16(__p0_835, __p1_835) __extension__ ({ \
67405 float16_t __ret_863; \69312 float16_t __ret_835; \
67406 float16x8_t __s0_863 = __p0_863; \69313 float16x8_t __s0_835 = __p0_835; \
67407 float16x8_t __rev0_863; __rev0_863 = __builtin_shufflevector(__s0_863, __s0_863, 7, 6, 5, 4, 3, 2, 1, 0); \69314 float16x8_t __rev0_835; __rev0_835 = __builtin_shufflevector(__s0_835, __s0_835, 7, 6, 5, 4, 3, 2, 1, 0); \
67408float16x8_t __reint_863 = __rev0_863; \69315 __ret_835 = __builtin_bit_cast(float16_t, __noswap_vgetq_lane_s16(__builtin_bit_cast(int16x8_t, __rev0_835), __p1_835)); \
67409int16_t __reint1_863 = __noswap_vgetq_lane_s16(*(int16x8_t *) &__reint_863, __p1_863); \69316 __ret_835; \
67410 __ret_863 = *(float16_t *) &__reint1_863; \
67411 __ret_863; \
67412})69317})
67413#define __noswap_vgetq_lane_f16(__p0_864, __p1_864) __extension__ ({ \69318#define __noswap_vgetq_lane_f16(__p0_836, __p1_836) __extension__ ({ \
67414 float16_t __ret_864; \69319 float16_t __ret_836; \
67415 float16x8_t __s0_864 = __p0_864; \69320 float16x8_t __s0_836 = __p0_836; \
67416float16x8_t __reint_864 = __s0_864; \69321 __ret_836 = __builtin_bit_cast(float16_t, __noswap_vgetq_lane_s16(__builtin_bit_cast(int16x8_t, __s0_836), __p1_836)); \
67417int16_t __reint1_864 = __noswap_vgetq_lane_s16(*(int16x8_t *) &__reint_864, __p1_864); \69322 __ret_836; \
67418 __ret_864 = *(float16_t *) &__reint1_864; \
67419 __ret_864; \
67420})69323})
67421#endif69324#endif
6742269325
...@@ -67559,98 +69462,98 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlal_s16(int32x4_t __p0...@@ -67559,98 +69462,98 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlal_s16(int32x4_t __p0
67559#endif69462#endif
6756069463
67561#ifdef __LITTLE_ENDIAN__69464#ifdef __LITTLE_ENDIAN__
67562#define vmlal_lane_u32(__p0_865, __p1_865, __p2_865, __p3_865) __extension__ ({ \69465#define vmlal_lane_u32(__p0_837, __p1_837, __p2_837, __p3_837) __extension__ ({ \
67563 uint64x2_t __ret_865; \69466 uint64x2_t __ret_837; \
67564 uint64x2_t __s0_865 = __p0_865; \69467 uint64x2_t __s0_837 = __p0_837; \
67565 uint32x2_t __s1_865 = __p1_865; \69468 uint32x2_t __s1_837 = __p1_837; \
67566 uint32x2_t __s2_865 = __p2_865; \69469 uint32x2_t __s2_837 = __p2_837; \
67567 __ret_865 = __s0_865 + vmull_u32(__s1_865, splat_lane_u32(__s2_865, __p3_865)); \69470 __ret_837 = __s0_837 + vmull_u32(__s1_837, splat_lane_u32(__s2_837, __p3_837)); \
67568 __ret_865; \69471 __ret_837; \
67569})69472})
67570#else69473#else
67571#define vmlal_lane_u32(__p0_866, __p1_866, __p2_866, __p3_866) __extension__ ({ \69474#define vmlal_lane_u32(__p0_838, __p1_838, __p2_838, __p3_838) __extension__ ({ \
67572 uint64x2_t __ret_866; \69475 uint64x2_t __ret_838; \
67573 uint64x2_t __s0_866 = __p0_866; \69476 uint64x2_t __s0_838 = __p0_838; \
67574 uint32x2_t __s1_866 = __p1_866; \69477 uint32x2_t __s1_838 = __p1_838; \
67575 uint32x2_t __s2_866 = __p2_866; \69478 uint32x2_t __s2_838 = __p2_838; \
67576 uint64x2_t __rev0_866; __rev0_866 = __builtin_shufflevector(__s0_866, __s0_866, 1, 0); \69479 uint64x2_t __rev0_838; __rev0_838 = __builtin_shufflevector(__s0_838, __s0_838, 1, 0); \
67577 uint32x2_t __rev1_866; __rev1_866 = __builtin_shufflevector(__s1_866, __s1_866, 1, 0); \69480 uint32x2_t __rev1_838; __rev1_838 = __builtin_shufflevector(__s1_838, __s1_838, 1, 0); \
67578 uint32x2_t __rev2_866; __rev2_866 = __builtin_shufflevector(__s2_866, __s2_866, 1, 0); \69481 uint32x2_t __rev2_838; __rev2_838 = __builtin_shufflevector(__s2_838, __s2_838, 1, 0); \
67579 __ret_866 = __rev0_866 + __noswap_vmull_u32(__rev1_866, __noswap_splat_lane_u32(__rev2_866, __p3_866)); \69482 __ret_838 = __rev0_838 + __noswap_vmull_u32(__rev1_838, __noswap_splat_lane_u32(__rev2_838, __p3_838)); \
67580 __ret_866 = __builtin_shufflevector(__ret_866, __ret_866, 1, 0); \69483 __ret_838 = __builtin_shufflevector(__ret_838, __ret_838, 1, 0); \
67581 __ret_866; \69484 __ret_838; \
67582})69485})
67583#endif69486#endif
6758469487
67585#ifdef __LITTLE_ENDIAN__69488#ifdef __LITTLE_ENDIAN__
67586#define vmlal_lane_u16(__p0_867, __p1_867, __p2_867, __p3_867) __extension__ ({ \69489#define vmlal_lane_u16(__p0_839, __p1_839, __p2_839, __p3_839) __extension__ ({ \
67587 uint32x4_t __ret_867; \69490 uint32x4_t __ret_839; \
67588 uint32x4_t __s0_867 = __p0_867; \69491 uint32x4_t __s0_839 = __p0_839; \
67589 uint16x4_t __s1_867 = __p1_867; \69492 uint16x4_t __s1_839 = __p1_839; \
67590 uint16x4_t __s2_867 = __p2_867; \69493 uint16x4_t __s2_839 = __p2_839; \
67591 __ret_867 = __s0_867 + vmull_u16(__s1_867, splat_lane_u16(__s2_867, __p3_867)); \69494 __ret_839 = __s0_839 + vmull_u16(__s1_839, splat_lane_u16(__s2_839, __p3_839)); \
67592 __ret_867; \69495 __ret_839; \
67593})69496})
67594#else69497#else
67595#define vmlal_lane_u16(__p0_868, __p1_868, __p2_868, __p3_868) __extension__ ({ \69498#define vmlal_lane_u16(__p0_840, __p1_840, __p2_840, __p3_840) __extension__ ({ \
67596 uint32x4_t __ret_868; \69499 uint32x4_t __ret_840; \
67597 uint32x4_t __s0_868 = __p0_868; \69500 uint32x4_t __s0_840 = __p0_840; \
67598 uint16x4_t __s1_868 = __p1_868; \69501 uint16x4_t __s1_840 = __p1_840; \
67599 uint16x4_t __s2_868 = __p2_868; \69502 uint16x4_t __s2_840 = __p2_840; \
67600 uint32x4_t __rev0_868; __rev0_868 = __builtin_shufflevector(__s0_868, __s0_868, 3, 2, 1, 0); \69503 uint32x4_t __rev0_840; __rev0_840 = __builtin_shufflevector(__s0_840, __s0_840, 3, 2, 1, 0); \
67601 uint16x4_t __rev1_868; __rev1_868 = __builtin_shufflevector(__s1_868, __s1_868, 3, 2, 1, 0); \69504 uint16x4_t __rev1_840; __rev1_840 = __builtin_shufflevector(__s1_840, __s1_840, 3, 2, 1, 0); \
67602 uint16x4_t __rev2_868; __rev2_868 = __builtin_shufflevector(__s2_868, __s2_868, 3, 2, 1, 0); \69505 uint16x4_t __rev2_840; __rev2_840 = __builtin_shufflevector(__s2_840, __s2_840, 3, 2, 1, 0); \
67603 __ret_868 = __rev0_868 + __noswap_vmull_u16(__rev1_868, __noswap_splat_lane_u16(__rev2_868, __p3_868)); \69506 __ret_840 = __rev0_840 + __noswap_vmull_u16(__rev1_840, __noswap_splat_lane_u16(__rev2_840, __p3_840)); \
67604 __ret_868 = __builtin_shufflevector(__ret_868, __ret_868, 3, 2, 1, 0); \69507 __ret_840 = __builtin_shufflevector(__ret_840, __ret_840, 3, 2, 1, 0); \
67605 __ret_868; \69508 __ret_840; \
67606})69509})
67607#endif69510#endif
6760869511
67609#ifdef __LITTLE_ENDIAN__69512#ifdef __LITTLE_ENDIAN__
67610#define vmlal_lane_s32(__p0_869, __p1_869, __p2_869, __p3_869) __extension__ ({ \69513#define vmlal_lane_s32(__p0_841, __p1_841, __p2_841, __p3_841) __extension__ ({ \
67611 int64x2_t __ret_869; \69514 int64x2_t __ret_841; \
67612 int64x2_t __s0_869 = __p0_869; \69515 int64x2_t __s0_841 = __p0_841; \
67613 int32x2_t __s1_869 = __p1_869; \69516 int32x2_t __s1_841 = __p1_841; \
67614 int32x2_t __s2_869 = __p2_869; \69517 int32x2_t __s2_841 = __p2_841; \
67615 __ret_869 = __s0_869 + vmull_s32(__s1_869, splat_lane_s32(__s2_869, __p3_869)); \69518 __ret_841 = __s0_841 + vmull_s32(__s1_841, splat_lane_s32(__s2_841, __p3_841)); \
67616 __ret_869; \69519 __ret_841; \
67617})69520})
67618#else69521#else
67619#define vmlal_lane_s32(__p0_870, __p1_870, __p2_870, __p3_870) __extension__ ({ \69522#define vmlal_lane_s32(__p0_842, __p1_842, __p2_842, __p3_842) __extension__ ({ \
67620 int64x2_t __ret_870; \69523 int64x2_t __ret_842; \
67621 int64x2_t __s0_870 = __p0_870; \69524 int64x2_t __s0_842 = __p0_842; \
67622 int32x2_t __s1_870 = __p1_870; \69525 int32x2_t __s1_842 = __p1_842; \
67623 int32x2_t __s2_870 = __p2_870; \69526 int32x2_t __s2_842 = __p2_842; \
67624 int64x2_t __rev0_870; __rev0_870 = __builtin_shufflevector(__s0_870, __s0_870, 1, 0); \69527 int64x2_t __rev0_842; __rev0_842 = __builtin_shufflevector(__s0_842, __s0_842, 1, 0); \
67625 int32x2_t __rev1_870; __rev1_870 = __builtin_shufflevector(__s1_870, __s1_870, 1, 0); \69528 int32x2_t __rev1_842; __rev1_842 = __builtin_shufflevector(__s1_842, __s1_842, 1, 0); \
67626 int32x2_t __rev2_870; __rev2_870 = __builtin_shufflevector(__s2_870, __s2_870, 1, 0); \69529 int32x2_t __rev2_842; __rev2_842 = __builtin_shufflevector(__s2_842, __s2_842, 1, 0); \
67627 __ret_870 = __rev0_870 + __noswap_vmull_s32(__rev1_870, __noswap_splat_lane_s32(__rev2_870, __p3_870)); \69530 __ret_842 = __rev0_842 + __noswap_vmull_s32(__rev1_842, __noswap_splat_lane_s32(__rev2_842, __p3_842)); \
67628 __ret_870 = __builtin_shufflevector(__ret_870, __ret_870, 1, 0); \69531 __ret_842 = __builtin_shufflevector(__ret_842, __ret_842, 1, 0); \
67629 __ret_870; \69532 __ret_842; \
67630})69533})
67631#endif69534#endif
6763269535
67633#ifdef __LITTLE_ENDIAN__69536#ifdef __LITTLE_ENDIAN__
67634#define vmlal_lane_s16(__p0_871, __p1_871, __p2_871, __p3_871) __extension__ ({ \69537#define vmlal_lane_s16(__p0_843, __p1_843, __p2_843, __p3_843) __extension__ ({ \
67635 int32x4_t __ret_871; \69538 int32x4_t __ret_843; \
67636 int32x4_t __s0_871 = __p0_871; \69539 int32x4_t __s0_843 = __p0_843; \
67637 int16x4_t __s1_871 = __p1_871; \69540 int16x4_t __s1_843 = __p1_843; \
67638 int16x4_t __s2_871 = __p2_871; \69541 int16x4_t __s2_843 = __p2_843; \
67639 __ret_871 = __s0_871 + vmull_s16(__s1_871, splat_lane_s16(__s2_871, __p3_871)); \69542 __ret_843 = __s0_843 + vmull_s16(__s1_843, splat_lane_s16(__s2_843, __p3_843)); \
67640 __ret_871; \69543 __ret_843; \
67641})69544})
67642#else69545#else
67643#define vmlal_lane_s16(__p0_872, __p1_872, __p2_872, __p3_872) __extension__ ({ \69546#define vmlal_lane_s16(__p0_844, __p1_844, __p2_844, __p3_844) __extension__ ({ \
67644 int32x4_t __ret_872; \69547 int32x4_t __ret_844; \
67645 int32x4_t __s0_872 = __p0_872; \69548 int32x4_t __s0_844 = __p0_844; \
67646 int16x4_t __s1_872 = __p1_872; \69549 int16x4_t __s1_844 = __p1_844; \
67647 int16x4_t __s2_872 = __p2_872; \69550 int16x4_t __s2_844 = __p2_844; \
67648 int32x4_t __rev0_872; __rev0_872 = __builtin_shufflevector(__s0_872, __s0_872, 3, 2, 1, 0); \69551 int32x4_t __rev0_844; __rev0_844 = __builtin_shufflevector(__s0_844, __s0_844, 3, 2, 1, 0); \
67649 int16x4_t __rev1_872; __rev1_872 = __builtin_shufflevector(__s1_872, __s1_872, 3, 2, 1, 0); \69552 int16x4_t __rev1_844; __rev1_844 = __builtin_shufflevector(__s1_844, __s1_844, 3, 2, 1, 0); \
67650 int16x4_t __rev2_872; __rev2_872 = __builtin_shufflevector(__s2_872, __s2_872, 3, 2, 1, 0); \69553 int16x4_t __rev2_844; __rev2_844 = __builtin_shufflevector(__s2_844, __s2_844, 3, 2, 1, 0); \
67651 __ret_872 = __rev0_872 + __noswap_vmull_s16(__rev1_872, __noswap_splat_lane_s16(__rev2_872, __p3_872)); \69554 __ret_844 = __rev0_844 + __noswap_vmull_s16(__rev1_844, __noswap_splat_lane_s16(__rev2_844, __p3_844)); \
67652 __ret_872 = __builtin_shufflevector(__ret_872, __ret_872, 3, 2, 1, 0); \69555 __ret_844 = __builtin_shufflevector(__ret_844, __ret_844, 3, 2, 1, 0); \
67653 __ret_872; \69556 __ret_844; \
67654})69557})
67655#endif69558#endif
6765669559
...@@ -67881,98 +69784,98 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlsl_s16(int32x4_t __p0...@@ -67881,98 +69784,98 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlsl_s16(int32x4_t __p0
67881#endif69784#endif
6788269785
67883#ifdef __LITTLE_ENDIAN__69786#ifdef __LITTLE_ENDIAN__
67884#define vmlsl_lane_u32(__p0_873, __p1_873, __p2_873, __p3_873) __extension__ ({ \69787#define vmlsl_lane_u32(__p0_845, __p1_845, __p2_845, __p3_845) __extension__ ({ \
67885 uint64x2_t __ret_873; \69788 uint64x2_t __ret_845; \
67886 uint64x2_t __s0_873 = __p0_873; \69789 uint64x2_t __s0_845 = __p0_845; \
67887 uint32x2_t __s1_873 = __p1_873; \69790 uint32x2_t __s1_845 = __p1_845; \
67888 uint32x2_t __s2_873 = __p2_873; \69791 uint32x2_t __s2_845 = __p2_845; \
67889 __ret_873 = __s0_873 - vmull_u32(__s1_873, splat_lane_u32(__s2_873, __p3_873)); \69792 __ret_845 = __s0_845 - vmull_u32(__s1_845, splat_lane_u32(__s2_845, __p3_845)); \
67890 __ret_873; \69793 __ret_845; \
67891})69794})
67892#else69795#else
67893#define vmlsl_lane_u32(__p0_874, __p1_874, __p2_874, __p3_874) __extension__ ({ \69796#define vmlsl_lane_u32(__p0_846, __p1_846, __p2_846, __p3_846) __extension__ ({ \
67894 uint64x2_t __ret_874; \69797 uint64x2_t __ret_846; \
67895 uint64x2_t __s0_874 = __p0_874; \69798 uint64x2_t __s0_846 = __p0_846; \
67896 uint32x2_t __s1_874 = __p1_874; \69799 uint32x2_t __s1_846 = __p1_846; \
67897 uint32x2_t __s2_874 = __p2_874; \69800 uint32x2_t __s2_846 = __p2_846; \
67898 uint64x2_t __rev0_874; __rev0_874 = __builtin_shufflevector(__s0_874, __s0_874, 1, 0); \69801 uint64x2_t __rev0_846; __rev0_846 = __builtin_shufflevector(__s0_846, __s0_846, 1, 0); \
67899 uint32x2_t __rev1_874; __rev1_874 = __builtin_shufflevector(__s1_874, __s1_874, 1, 0); \69802 uint32x2_t __rev1_846; __rev1_846 = __builtin_shufflevector(__s1_846, __s1_846, 1, 0); \
67900 uint32x2_t __rev2_874; __rev2_874 = __builtin_shufflevector(__s2_874, __s2_874, 1, 0); \69803 uint32x2_t __rev2_846; __rev2_846 = __builtin_shufflevector(__s2_846, __s2_846, 1, 0); \
67901 __ret_874 = __rev0_874 - __noswap_vmull_u32(__rev1_874, __noswap_splat_lane_u32(__rev2_874, __p3_874)); \69804 __ret_846 = __rev0_846 - __noswap_vmull_u32(__rev1_846, __noswap_splat_lane_u32(__rev2_846, __p3_846)); \
67902 __ret_874 = __builtin_shufflevector(__ret_874, __ret_874, 1, 0); \69805 __ret_846 = __builtin_shufflevector(__ret_846, __ret_846, 1, 0); \
67903 __ret_874; \69806 __ret_846; \
67904})69807})
67905#endif69808#endif
6790669809
67907#ifdef __LITTLE_ENDIAN__69810#ifdef __LITTLE_ENDIAN__
67908#define vmlsl_lane_u16(__p0_875, __p1_875, __p2_875, __p3_875) __extension__ ({ \69811#define vmlsl_lane_u16(__p0_847, __p1_847, __p2_847, __p3_847) __extension__ ({ \
67909 uint32x4_t __ret_875; \69812 uint32x4_t __ret_847; \
67910 uint32x4_t __s0_875 = __p0_875; \69813 uint32x4_t __s0_847 = __p0_847; \
67911 uint16x4_t __s1_875 = __p1_875; \69814 uint16x4_t __s1_847 = __p1_847; \
67912 uint16x4_t __s2_875 = __p2_875; \69815 uint16x4_t __s2_847 = __p2_847; \
67913 __ret_875 = __s0_875 - vmull_u16(__s1_875, splat_lane_u16(__s2_875, __p3_875)); \69816 __ret_847 = __s0_847 - vmull_u16(__s1_847, splat_lane_u16(__s2_847, __p3_847)); \
67914 __ret_875; \69817 __ret_847; \
67915})69818})
67916#else69819#else
67917#define vmlsl_lane_u16(__p0_876, __p1_876, __p2_876, __p3_876) __extension__ ({ \69820#define vmlsl_lane_u16(__p0_848, __p1_848, __p2_848, __p3_848) __extension__ ({ \
67918 uint32x4_t __ret_876; \69821 uint32x4_t __ret_848; \
67919 uint32x4_t __s0_876 = __p0_876; \69822 uint32x4_t __s0_848 = __p0_848; \
67920 uint16x4_t __s1_876 = __p1_876; \69823 uint16x4_t __s1_848 = __p1_848; \
67921 uint16x4_t __s2_876 = __p2_876; \69824 uint16x4_t __s2_848 = __p2_848; \
67922 uint32x4_t __rev0_876; __rev0_876 = __builtin_shufflevector(__s0_876, __s0_876, 3, 2, 1, 0); \69825 uint32x4_t __rev0_848; __rev0_848 = __builtin_shufflevector(__s0_848, __s0_848, 3, 2, 1, 0); \
67923 uint16x4_t __rev1_876; __rev1_876 = __builtin_shufflevector(__s1_876, __s1_876, 3, 2, 1, 0); \69826 uint16x4_t __rev1_848; __rev1_848 = __builtin_shufflevector(__s1_848, __s1_848, 3, 2, 1, 0); \
67924 uint16x4_t __rev2_876; __rev2_876 = __builtin_shufflevector(__s2_876, __s2_876, 3, 2, 1, 0); \69827 uint16x4_t __rev2_848; __rev2_848 = __builtin_shufflevector(__s2_848, __s2_848, 3, 2, 1, 0); \
67925 __ret_876 = __rev0_876 - __noswap_vmull_u16(__rev1_876, __noswap_splat_lane_u16(__rev2_876, __p3_876)); \69828 __ret_848 = __rev0_848 - __noswap_vmull_u16(__rev1_848, __noswap_splat_lane_u16(__rev2_848, __p3_848)); \
67926 __ret_876 = __builtin_shufflevector(__ret_876, __ret_876, 3, 2, 1, 0); \69829 __ret_848 = __builtin_shufflevector(__ret_848, __ret_848, 3, 2, 1, 0); \
67927 __ret_876; \69830 __ret_848; \
67928})69831})
67929#endif69832#endif
6793069833
67931#ifdef __LITTLE_ENDIAN__69834#ifdef __LITTLE_ENDIAN__
67932#define vmlsl_lane_s32(__p0_877, __p1_877, __p2_877, __p3_877) __extension__ ({ \69835#define vmlsl_lane_s32(__p0_849, __p1_849, __p2_849, __p3_849) __extension__ ({ \
67933 int64x2_t __ret_877; \69836 int64x2_t __ret_849; \
67934 int64x2_t __s0_877 = __p0_877; \69837 int64x2_t __s0_849 = __p0_849; \
67935 int32x2_t __s1_877 = __p1_877; \69838 int32x2_t __s1_849 = __p1_849; \
67936 int32x2_t __s2_877 = __p2_877; \69839 int32x2_t __s2_849 = __p2_849; \
67937 __ret_877 = __s0_877 - vmull_s32(__s1_877, splat_lane_s32(__s2_877, __p3_877)); \69840 __ret_849 = __s0_849 - vmull_s32(__s1_849, splat_lane_s32(__s2_849, __p3_849)); \
67938 __ret_877; \69841 __ret_849; \
67939})69842})
67940#else69843#else
67941#define vmlsl_lane_s32(__p0_878, __p1_878, __p2_878, __p3_878) __extension__ ({ \69844#define vmlsl_lane_s32(__p0_850, __p1_850, __p2_850, __p3_850) __extension__ ({ \
67942 int64x2_t __ret_878; \69845 int64x2_t __ret_850; \
67943 int64x2_t __s0_878 = __p0_878; \69846 int64x2_t __s0_850 = __p0_850; \
67944 int32x2_t __s1_878 = __p1_878; \69847 int32x2_t __s1_850 = __p1_850; \
67945 int32x2_t __s2_878 = __p2_878; \69848 int32x2_t __s2_850 = __p2_850; \
67946 int64x2_t __rev0_878; __rev0_878 = __builtin_shufflevector(__s0_878, __s0_878, 1, 0); \69849 int64x2_t __rev0_850; __rev0_850 = __builtin_shufflevector(__s0_850, __s0_850, 1, 0); \
67947 int32x2_t __rev1_878; __rev1_878 = __builtin_shufflevector(__s1_878, __s1_878, 1, 0); \69850 int32x2_t __rev1_850; __rev1_850 = __builtin_shufflevector(__s1_850, __s1_850, 1, 0); \
67948 int32x2_t __rev2_878; __rev2_878 = __builtin_shufflevector(__s2_878, __s2_878, 1, 0); \69851 int32x2_t __rev2_850; __rev2_850 = __builtin_shufflevector(__s2_850, __s2_850, 1, 0); \
67949 __ret_878 = __rev0_878 - __noswap_vmull_s32(__rev1_878, __noswap_splat_lane_s32(__rev2_878, __p3_878)); \69852 __ret_850 = __rev0_850 - __noswap_vmull_s32(__rev1_850, __noswap_splat_lane_s32(__rev2_850, __p3_850)); \
67950 __ret_878 = __builtin_shufflevector(__ret_878, __ret_878, 1, 0); \69853 __ret_850 = __builtin_shufflevector(__ret_850, __ret_850, 1, 0); \
67951 __ret_878; \69854 __ret_850; \
67952})69855})
67953#endif69856#endif
6795469857
67955#ifdef __LITTLE_ENDIAN__69858#ifdef __LITTLE_ENDIAN__
67956#define vmlsl_lane_s16(__p0_879, __p1_879, __p2_879, __p3_879) __extension__ ({ \69859#define vmlsl_lane_s16(__p0_851, __p1_851, __p2_851, __p3_851) __extension__ ({ \
67957 int32x4_t __ret_879; \69860 int32x4_t __ret_851; \
67958 int32x4_t __s0_879 = __p0_879; \69861 int32x4_t __s0_851 = __p0_851; \
67959 int16x4_t __s1_879 = __p1_879; \69862 int16x4_t __s1_851 = __p1_851; \
67960 int16x4_t __s2_879 = __p2_879; \69863 int16x4_t __s2_851 = __p2_851; \
67961 __ret_879 = __s0_879 - vmull_s16(__s1_879, splat_lane_s16(__s2_879, __p3_879)); \69864 __ret_851 = __s0_851 - vmull_s16(__s1_851, splat_lane_s16(__s2_851, __p3_851)); \
67962 __ret_879; \69865 __ret_851; \
67963})69866})
67964#else69867#else
67965#define vmlsl_lane_s16(__p0_880, __p1_880, __p2_880, __p3_880) __extension__ ({ \69868#define vmlsl_lane_s16(__p0_852, __p1_852, __p2_852, __p3_852) __extension__ ({ \
67966 int32x4_t __ret_880; \69869 int32x4_t __ret_852; \
67967 int32x4_t __s0_880 = __p0_880; \69870 int32x4_t __s0_852 = __p0_852; \
67968 int16x4_t __s1_880 = __p1_880; \69871 int16x4_t __s1_852 = __p1_852; \
67969 int16x4_t __s2_880 = __p2_880; \69872 int16x4_t __s2_852 = __p2_852; \
67970 int32x4_t __rev0_880; __rev0_880 = __builtin_shufflevector(__s0_880, __s0_880, 3, 2, 1, 0); \69873 int32x4_t __rev0_852; __rev0_852 = __builtin_shufflevector(__s0_852, __s0_852, 3, 2, 1, 0); \
67971 int16x4_t __rev1_880; __rev1_880 = __builtin_shufflevector(__s1_880, __s1_880, 3, 2, 1, 0); \69874 int16x4_t __rev1_852; __rev1_852 = __builtin_shufflevector(__s1_852, __s1_852, 3, 2, 1, 0); \
67972 int16x4_t __rev2_880; __rev2_880 = __builtin_shufflevector(__s2_880, __s2_880, 3, 2, 1, 0); \69875 int16x4_t __rev2_852; __rev2_852 = __builtin_shufflevector(__s2_852, __s2_852, 3, 2, 1, 0); \
67973 __ret_880 = __rev0_880 - __noswap_vmull_s16(__rev1_880, __noswap_splat_lane_s16(__rev2_880, __p3_880)); \69876 __ret_852 = __rev0_852 - __noswap_vmull_s16(__rev1_852, __noswap_splat_lane_s16(__rev2_852, __p3_852)); \
67974 __ret_880 = __builtin_shufflevector(__ret_880, __ret_880, 3, 2, 1, 0); \69877 __ret_852 = __builtin_shufflevector(__ret_852, __ret_852, 3, 2, 1, 0); \
67975 __ret_880; \69878 __ret_852; \
67976})69879})
67977#endif69880#endif
6797869881
...@@ -68065,54 +69968,42 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlsl_n_s16(int32x4_t __...@@ -68065,54 +69968,42 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlsl_n_s16(int32x4_t __
68065#endif69968#endif
6806669969
68067#ifdef __LITTLE_ENDIAN__69970#ifdef __LITTLE_ENDIAN__
68068#define vset_lane_f16(__p0_881, __p1_881, __p2_881) __extension__ ({ \69971#define vset_lane_f16(__p0_853, __p1_853, __p2_853) __extension__ ({ \
68069 float16x4_t __ret_881; \69972 float16x4_t __ret_853; \
68070 float16_t __s0_881 = __p0_881; \69973 float16_t __s0_853 = __p0_853; \
68071 float16x4_t __s1_881 = __p1_881; \69974 float16x4_t __s1_853 = __p1_853; \
68072float16_t __reint_881 = __s0_881; \69975 __ret_853 = __builtin_bit_cast(float16x4_t, vset_lane_s16(__builtin_bit_cast(int16_t, __s0_853), __builtin_bit_cast(int16x4_t, __s1_853), __p2_853)); \
68073float16x4_t __reint1_881 = __s1_881; \69976 __ret_853; \
68074int16x4_t __reint2_881 = vset_lane_s16(*(int16_t *) &__reint_881, *(int16x4_t *) &__reint1_881, __p2_881); \
68075 __ret_881 = *(float16x4_t *) &__reint2_881; \
68076 __ret_881; \
68077})69977})
68078#else69978#else
68079#define vset_lane_f16(__p0_882, __p1_882, __p2_882) __extension__ ({ \69979#define vset_lane_f16(__p0_854, __p1_854, __p2_854) __extension__ ({ \
68080 float16x4_t __ret_882; \69980 float16x4_t __ret_854; \
68081 float16_t __s0_882 = __p0_882; \69981 float16_t __s0_854 = __p0_854; \
68082 float16x4_t __s1_882 = __p1_882; \69982 float16x4_t __s1_854 = __p1_854; \
68083 float16x4_t __rev1_882; __rev1_882 = __builtin_shufflevector(__s1_882, __s1_882, 3, 2, 1, 0); \69983 float16x4_t __rev1_854; __rev1_854 = __builtin_shufflevector(__s1_854, __s1_854, 3, 2, 1, 0); \
68084float16_t __reint_882 = __s0_882; \69984 __ret_854 = __builtin_bit_cast(float16x4_t, __noswap_vset_lane_s16(__builtin_bit_cast(int16_t, __s0_854), __builtin_bit_cast(int16x4_t, __rev1_854), __p2_854)); \
68085float16x4_t __reint1_882 = __rev1_882; \69985 __ret_854 = __builtin_shufflevector(__ret_854, __ret_854, 3, 2, 1, 0); \
68086int16x4_t __reint2_882 = __noswap_vset_lane_s16(*(int16_t *) &__reint_882, *(int16x4_t *) &__reint1_882, __p2_882); \69986 __ret_854; \
68087 __ret_882 = *(float16x4_t *) &__reint2_882; \
68088 __ret_882 = __builtin_shufflevector(__ret_882, __ret_882, 3, 2, 1, 0); \
68089 __ret_882; \
68090})69987})
68091#endif69988#endif
6809269989
68093#ifdef __LITTLE_ENDIAN__69990#ifdef __LITTLE_ENDIAN__
68094#define vsetq_lane_f16(__p0_883, __p1_883, __p2_883) __extension__ ({ \69991#define vsetq_lane_f16(__p0_855, __p1_855, __p2_855) __extension__ ({ \
68095 float16x8_t __ret_883; \69992 float16x8_t __ret_855; \
68096 float16_t __s0_883 = __p0_883; \69993 float16_t __s0_855 = __p0_855; \
68097 float16x8_t __s1_883 = __p1_883; \69994 float16x8_t __s1_855 = __p1_855; \
68098float16_t __reint_883 = __s0_883; \69995 __ret_855 = __builtin_bit_cast(float16x8_t, vsetq_lane_s16(__builtin_bit_cast(int16_t, __s0_855), __builtin_bit_cast(int16x8_t, __s1_855), __p2_855)); \
68099float16x8_t __reint1_883 = __s1_883; \69996 __ret_855; \
68100int16x8_t __reint2_883 = vsetq_lane_s16(*(int16_t *) &__reint_883, *(int16x8_t *) &__reint1_883, __p2_883); \
68101 __ret_883 = *(float16x8_t *) &__reint2_883; \
68102 __ret_883; \
68103})69997})
68104#else69998#else
68105#define vsetq_lane_f16(__p0_884, __p1_884, __p2_884) __extension__ ({ \69999#define vsetq_lane_f16(__p0_856, __p1_856, __p2_856) __extension__ ({ \
68106 float16x8_t __ret_884; \70000 float16x8_t __ret_856; \
68107 float16_t __s0_884 = __p0_884; \70001 float16_t __s0_856 = __p0_856; \
68108 float16x8_t __s1_884 = __p1_884; \70002 float16x8_t __s1_856 = __p1_856; \
68109 float16x8_t __rev1_884; __rev1_884 = __builtin_shufflevector(__s1_884, __s1_884, 7, 6, 5, 4, 3, 2, 1, 0); \70003 float16x8_t __rev1_856; __rev1_856 = __builtin_shufflevector(__s1_856, __s1_856, 7, 6, 5, 4, 3, 2, 1, 0); \
68110float16_t __reint_884 = __s0_884; \70004 __ret_856 = __builtin_bit_cast(float16x8_t, __noswap_vsetq_lane_s16(__builtin_bit_cast(int16_t, __s0_856), __builtin_bit_cast(int16x8_t, __rev1_856), __p2_856)); \
68111float16x8_t __reint1_884 = __rev1_884; \70005 __ret_856 = __builtin_shufflevector(__ret_856, __ret_856, 7, 6, 5, 4, 3, 2, 1, 0); \
68112int16x8_t __reint2_884 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_884, *(int16x8_t *) &__reint1_884, __p2_884); \70006 __ret_856; \
68113 __ret_884 = *(float16x8_t *) &__reint2_884; \
68114 __ret_884 = __builtin_shufflevector(__ret_884, __ret_884, 7, 6, 5, 4, 3, 2, 1, 0); \
68115 __ret_884; \
68116})70007})
68117#endif70008#endif
6811870009
...@@ -68134,424 +70025,424 @@ __ai __attribute__((target("aes,neon"))) poly128_t vmull_high_p64(poly64x2_t __p...@@ -68134,424 +70025,424 @@ __ai __attribute__((target("aes,neon"))) poly128_t vmull_high_p64(poly64x2_t __p
68134#endif70025#endif
6813570026
68136#ifdef __LITTLE_ENDIAN__70027#ifdef __LITTLE_ENDIAN__
68137#define vfmlalq_lane_high_f16(__p0_885, __p1_885, __p2_885, __p3_885) __extension__ ({ \70028#define vfmlalq_lane_high_f16(__p0_857, __p1_857, __p2_857, __p3_857) __extension__ ({ \
68138 float32x4_t __ret_885; \70029 float32x4_t __ret_857; \
68139 float32x4_t __s0_885 = __p0_885; \70030 float32x4_t __s0_857 = __p0_857; \
68140 float16x8_t __s1_885 = __p1_885; \70031 float16x8_t __s1_857 = __p1_857; \
68141 float16x4_t __s2_885 = __p2_885; \70032 float16x4_t __s2_857 = __p2_857; \
68142 __ret_885 = vfmlalq_high_f16(__s0_885, __s1_885, (float16x8_t) {vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885)}); \70033 __ret_857 = vfmlalq_high_f16(__s0_857, __s1_857, (float16x8_t) {vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857)}); \
68143 __ret_885; \70034 __ret_857; \
68144})70035})
68145#else70036#else
68146#define vfmlalq_lane_high_f16(__p0_886, __p1_886, __p2_886, __p3_886) __extension__ ({ \70037#define vfmlalq_lane_high_f16(__p0_858, __p1_858, __p2_858, __p3_858) __extension__ ({ \
68147 float32x4_t __ret_886; \70038 float32x4_t __ret_858; \
68148 float32x4_t __s0_886 = __p0_886; \70039 float32x4_t __s0_858 = __p0_858; \
68149 float16x8_t __s1_886 = __p1_886; \70040 float16x8_t __s1_858 = __p1_858; \
68150 float16x4_t __s2_886 = __p2_886; \70041 float16x4_t __s2_858 = __p2_858; \
68151 float32x4_t __rev0_886; __rev0_886 = __builtin_shufflevector(__s0_886, __s0_886, 3, 2, 1, 0); \70042 float32x4_t __rev0_858; __rev0_858 = __builtin_shufflevector(__s0_858, __s0_858, 3, 2, 1, 0); \
68152 float16x8_t __rev1_886; __rev1_886 = __builtin_shufflevector(__s1_886, __s1_886, 7, 6, 5, 4, 3, 2, 1, 0); \70043 float16x8_t __rev1_858; __rev1_858 = __builtin_shufflevector(__s1_858, __s1_858, 7, 6, 5, 4, 3, 2, 1, 0); \
68153 float16x4_t __rev2_886; __rev2_886 = __builtin_shufflevector(__s2_886, __s2_886, 3, 2, 1, 0); \70044 float16x4_t __rev2_858; __rev2_858 = __builtin_shufflevector(__s2_858, __s2_858, 3, 2, 1, 0); \
68154 __ret_886 = __noswap_vfmlalq_high_f16(__rev0_886, __rev1_886, (float16x8_t) {__noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886)}); \70045 __ret_858 = __noswap_vfmlalq_high_f16(__rev0_858, __rev1_858, (float16x8_t) {__noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858)}); \
68155 __ret_886 = __builtin_shufflevector(__ret_886, __ret_886, 3, 2, 1, 0); \70046 __ret_858 = __builtin_shufflevector(__ret_858, __ret_858, 3, 2, 1, 0); \
68156 __ret_886; \70047 __ret_858; \
68157})70048})
68158#endif70049#endif
6815970050
68160#ifdef __LITTLE_ENDIAN__70051#ifdef __LITTLE_ENDIAN__
68161#define vfmlal_lane_high_f16(__p0_887, __p1_887, __p2_887, __p3_887) __extension__ ({ \70052#define vfmlal_lane_high_f16(__p0_859, __p1_859, __p2_859, __p3_859) __extension__ ({ \
68162 float32x2_t __ret_887; \70053 float32x2_t __ret_859; \
68163 float32x2_t __s0_887 = __p0_887; \70054 float32x2_t __s0_859 = __p0_859; \
68164 float16x4_t __s1_887 = __p1_887; \70055 float16x4_t __s1_859 = __p1_859; \
68165 float16x4_t __s2_887 = __p2_887; \70056 float16x4_t __s2_859 = __p2_859; \
68166 __ret_887 = vfmlal_high_f16(__s0_887, __s1_887, (float16x4_t) {vget_lane_f16(__s2_887, __p3_887), vget_lane_f16(__s2_887, __p3_887), vget_lane_f16(__s2_887, __p3_887), vget_lane_f16(__s2_887, __p3_887)}); \70057 __ret_859 = vfmlal_high_f16(__s0_859, __s1_859, (float16x4_t) {vget_lane_f16(__s2_859, __p3_859), vget_lane_f16(__s2_859, __p3_859), vget_lane_f16(__s2_859, __p3_859), vget_lane_f16(__s2_859, __p3_859)}); \
68167 __ret_887; \70058 __ret_859; \
68168})70059})
68169#else70060#else
68170#define vfmlal_lane_high_f16(__p0_888, __p1_888, __p2_888, __p3_888) __extension__ ({ \70061#define vfmlal_lane_high_f16(__p0_860, __p1_860, __p2_860, __p3_860) __extension__ ({ \
68171 float32x2_t __ret_888; \70062 float32x2_t __ret_860; \
68172 float32x2_t __s0_888 = __p0_888; \70063 float32x2_t __s0_860 = __p0_860; \
68173 float16x4_t __s1_888 = __p1_888; \70064 float16x4_t __s1_860 = __p1_860; \
68174 float16x4_t __s2_888 = __p2_888; \70065 float16x4_t __s2_860 = __p2_860; \
68175 float32x2_t __rev0_888; __rev0_888 = __builtin_shufflevector(__s0_888, __s0_888, 1, 0); \70066 float32x2_t __rev0_860; __rev0_860 = __builtin_shufflevector(__s0_860, __s0_860, 1, 0); \
68176 float16x4_t __rev1_888; __rev1_888 = __builtin_shufflevector(__s1_888, __s1_888, 3, 2, 1, 0); \70067 float16x4_t __rev1_860; __rev1_860 = __builtin_shufflevector(__s1_860, __s1_860, 3, 2, 1, 0); \
68177 float16x4_t __rev2_888; __rev2_888 = __builtin_shufflevector(__s2_888, __s2_888, 3, 2, 1, 0); \70068 float16x4_t __rev2_860; __rev2_860 = __builtin_shufflevector(__s2_860, __s2_860, 3, 2, 1, 0); \
68178 __ret_888 = __noswap_vfmlal_high_f16(__rev0_888, __rev1_888, (float16x4_t) {__noswap_vget_lane_f16(__rev2_888, __p3_888), __noswap_vget_lane_f16(__rev2_888, __p3_888), __noswap_vget_lane_f16(__rev2_888, __p3_888), __noswap_vget_lane_f16(__rev2_888, __p3_888)}); \70069 __ret_860 = __noswap_vfmlal_high_f16(__rev0_860, __rev1_860, (float16x4_t) {__noswap_vget_lane_f16(__rev2_860, __p3_860), __noswap_vget_lane_f16(__rev2_860, __p3_860), __noswap_vget_lane_f16(__rev2_860, __p3_860), __noswap_vget_lane_f16(__rev2_860, __p3_860)}); \
68179 __ret_888 = __builtin_shufflevector(__ret_888, __ret_888, 1, 0); \70070 __ret_860 = __builtin_shufflevector(__ret_860, __ret_860, 1, 0); \
68180 __ret_888; \70071 __ret_860; \
68181})70072})
68182#endif70073#endif
6818370074
68184#ifdef __LITTLE_ENDIAN__70075#ifdef __LITTLE_ENDIAN__
68185#define vfmlalq_lane_low_f16(__p0_889, __p1_889, __p2_889, __p3_889) __extension__ ({ \70076#define vfmlalq_lane_low_f16(__p0_861, __p1_861, __p2_861, __p3_861) __extension__ ({ \
68186 float32x4_t __ret_889; \70077 float32x4_t __ret_861; \
68187 float32x4_t __s0_889 = __p0_889; \70078 float32x4_t __s0_861 = __p0_861; \
68188 float16x8_t __s1_889 = __p1_889; \70079 float16x8_t __s1_861 = __p1_861; \
68189 float16x4_t __s2_889 = __p2_889; \70080 float16x4_t __s2_861 = __p2_861; \
68190 __ret_889 = vfmlalq_low_f16(__s0_889, __s1_889, (float16x8_t) {vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889)}); \70081 __ret_861 = vfmlalq_low_f16(__s0_861, __s1_861, (float16x8_t) {vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861)}); \
68191 __ret_889; \70082 __ret_861; \
68192})70083})
68193#else70084#else
68194#define vfmlalq_lane_low_f16(__p0_890, __p1_890, __p2_890, __p3_890) __extension__ ({ \70085#define vfmlalq_lane_low_f16(__p0_862, __p1_862, __p2_862, __p3_862) __extension__ ({ \
68195 float32x4_t __ret_890; \70086 float32x4_t __ret_862; \
68196 float32x4_t __s0_890 = __p0_890; \70087 float32x4_t __s0_862 = __p0_862; \
68197 float16x8_t __s1_890 = __p1_890; \70088 float16x8_t __s1_862 = __p1_862; \
68198 float16x4_t __s2_890 = __p2_890; \70089 float16x4_t __s2_862 = __p2_862; \
68199 float32x4_t __rev0_890; __rev0_890 = __builtin_shufflevector(__s0_890, __s0_890, 3, 2, 1, 0); \70090 float32x4_t __rev0_862; __rev0_862 = __builtin_shufflevector(__s0_862, __s0_862, 3, 2, 1, 0); \
68200 float16x8_t __rev1_890; __rev1_890 = __builtin_shufflevector(__s1_890, __s1_890, 7, 6, 5, 4, 3, 2, 1, 0); \70091 float16x8_t __rev1_862; __rev1_862 = __builtin_shufflevector(__s1_862, __s1_862, 7, 6, 5, 4, 3, 2, 1, 0); \
68201 float16x4_t __rev2_890; __rev2_890 = __builtin_shufflevector(__s2_890, __s2_890, 3, 2, 1, 0); \70092 float16x4_t __rev2_862; __rev2_862 = __builtin_shufflevector(__s2_862, __s2_862, 3, 2, 1, 0); \
68202 __ret_890 = __noswap_vfmlalq_low_f16(__rev0_890, __rev1_890, (float16x8_t) {__noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890)}); \70093 __ret_862 = __noswap_vfmlalq_low_f16(__rev0_862, __rev1_862, (float16x8_t) {__noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862)}); \
68203 __ret_890 = __builtin_shufflevector(__ret_890, __ret_890, 3, 2, 1, 0); \70094 __ret_862 = __builtin_shufflevector(__ret_862, __ret_862, 3, 2, 1, 0); \
68204 __ret_890; \70095 __ret_862; \
68205})70096})
68206#endif70097#endif
6820770098
68208#ifdef __LITTLE_ENDIAN__70099#ifdef __LITTLE_ENDIAN__
68209#define vfmlal_lane_low_f16(__p0_891, __p1_891, __p2_891, __p3_891) __extension__ ({ \70100#define vfmlal_lane_low_f16(__p0_863, __p1_863, __p2_863, __p3_863) __extension__ ({ \
68210 float32x2_t __ret_891; \70101 float32x2_t __ret_863; \
68211 float32x2_t __s0_891 = __p0_891; \70102 float32x2_t __s0_863 = __p0_863; \
68212 float16x4_t __s1_891 = __p1_891; \70103 float16x4_t __s1_863 = __p1_863; \
68213 float16x4_t __s2_891 = __p2_891; \70104 float16x4_t __s2_863 = __p2_863; \
68214 __ret_891 = vfmlal_low_f16(__s0_891, __s1_891, (float16x4_t) {vget_lane_f16(__s2_891, __p3_891), vget_lane_f16(__s2_891, __p3_891), vget_lane_f16(__s2_891, __p3_891), vget_lane_f16(__s2_891, __p3_891)}); \70105 __ret_863 = vfmlal_low_f16(__s0_863, __s1_863, (float16x4_t) {vget_lane_f16(__s2_863, __p3_863), vget_lane_f16(__s2_863, __p3_863), vget_lane_f16(__s2_863, __p3_863), vget_lane_f16(__s2_863, __p3_863)}); \
68215 __ret_891; \70106 __ret_863; \
68216})70107})
68217#else70108#else
68218#define vfmlal_lane_low_f16(__p0_892, __p1_892, __p2_892, __p3_892) __extension__ ({ \70109#define vfmlal_lane_low_f16(__p0_864, __p1_864, __p2_864, __p3_864) __extension__ ({ \
68219 float32x2_t __ret_892; \70110 float32x2_t __ret_864; \
68220 float32x2_t __s0_892 = __p0_892; \70111 float32x2_t __s0_864 = __p0_864; \
68221 float16x4_t __s1_892 = __p1_892; \70112 float16x4_t __s1_864 = __p1_864; \
68222 float16x4_t __s2_892 = __p2_892; \70113 float16x4_t __s2_864 = __p2_864; \
68223 float32x2_t __rev0_892; __rev0_892 = __builtin_shufflevector(__s0_892, __s0_892, 1, 0); \70114 float32x2_t __rev0_864; __rev0_864 = __builtin_shufflevector(__s0_864, __s0_864, 1, 0); \
68224 float16x4_t __rev1_892; __rev1_892 = __builtin_shufflevector(__s1_892, __s1_892, 3, 2, 1, 0); \70115 float16x4_t __rev1_864; __rev1_864 = __builtin_shufflevector(__s1_864, __s1_864, 3, 2, 1, 0); \
68225 float16x4_t __rev2_892; __rev2_892 = __builtin_shufflevector(__s2_892, __s2_892, 3, 2, 1, 0); \70116 float16x4_t __rev2_864; __rev2_864 = __builtin_shufflevector(__s2_864, __s2_864, 3, 2, 1, 0); \
68226 __ret_892 = __noswap_vfmlal_low_f16(__rev0_892, __rev1_892, (float16x4_t) {__noswap_vget_lane_f16(__rev2_892, __p3_892), __noswap_vget_lane_f16(__rev2_892, __p3_892), __noswap_vget_lane_f16(__rev2_892, __p3_892), __noswap_vget_lane_f16(__rev2_892, __p3_892)}); \70117 __ret_864 = __noswap_vfmlal_low_f16(__rev0_864, __rev1_864, (float16x4_t) {__noswap_vget_lane_f16(__rev2_864, __p3_864), __noswap_vget_lane_f16(__rev2_864, __p3_864), __noswap_vget_lane_f16(__rev2_864, __p3_864), __noswap_vget_lane_f16(__rev2_864, __p3_864)}); \
68227 __ret_892 = __builtin_shufflevector(__ret_892, __ret_892, 1, 0); \70118 __ret_864 = __builtin_shufflevector(__ret_864, __ret_864, 1, 0); \
68228 __ret_892; \70119 __ret_864; \
68229})70120})
68230#endif70121#endif
6823170122
68232#ifdef __LITTLE_ENDIAN__70123#ifdef __LITTLE_ENDIAN__
68233#define vfmlalq_laneq_high_f16(__p0_893, __p1_893, __p2_893, __p3_893) __extension__ ({ \70124#define vfmlalq_laneq_high_f16(__p0_865, __p1_865, __p2_865, __p3_865) __extension__ ({ \
68234 float32x4_t __ret_893; \70125 float32x4_t __ret_865; \
68235 float32x4_t __s0_893 = __p0_893; \70126 float32x4_t __s0_865 = __p0_865; \
68236 float16x8_t __s1_893 = __p1_893; \70127 float16x8_t __s1_865 = __p1_865; \
68237 float16x8_t __s2_893 = __p2_893; \70128 float16x8_t __s2_865 = __p2_865; \
68238 __ret_893 = vfmlalq_high_f16(__s0_893, __s1_893, (float16x8_t) {vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893)}); \70129 __ret_865 = vfmlalq_high_f16(__s0_865, __s1_865, (float16x8_t) {vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865)}); \
68239 __ret_893; \70130 __ret_865; \
68240})70131})
68241#else70132#else
68242#define vfmlalq_laneq_high_f16(__p0_894, __p1_894, __p2_894, __p3_894) __extension__ ({ \70133#define vfmlalq_laneq_high_f16(__p0_866, __p1_866, __p2_866, __p3_866) __extension__ ({ \
68243 float32x4_t __ret_894; \70134 float32x4_t __ret_866; \
68244 float32x4_t __s0_894 = __p0_894; \70135 float32x4_t __s0_866 = __p0_866; \
68245 float16x8_t __s1_894 = __p1_894; \70136 float16x8_t __s1_866 = __p1_866; \
68246 float16x8_t __s2_894 = __p2_894; \70137 float16x8_t __s2_866 = __p2_866; \
68247 float32x4_t __rev0_894; __rev0_894 = __builtin_shufflevector(__s0_894, __s0_894, 3, 2, 1, 0); \70138 float32x4_t __rev0_866; __rev0_866 = __builtin_shufflevector(__s0_866, __s0_866, 3, 2, 1, 0); \
68248 float16x8_t __rev1_894; __rev1_894 = __builtin_shufflevector(__s1_894, __s1_894, 7, 6, 5, 4, 3, 2, 1, 0); \70139 float16x8_t __rev1_866; __rev1_866 = __builtin_shufflevector(__s1_866, __s1_866, 7, 6, 5, 4, 3, 2, 1, 0); \
68249 float16x8_t __rev2_894; __rev2_894 = __builtin_shufflevector(__s2_894, __s2_894, 7, 6, 5, 4, 3, 2, 1, 0); \70140 float16x8_t __rev2_866; __rev2_866 = __builtin_shufflevector(__s2_866, __s2_866, 7, 6, 5, 4, 3, 2, 1, 0); \
68250 __ret_894 = __noswap_vfmlalq_high_f16(__rev0_894, __rev1_894, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894)}); \70141 __ret_866 = __noswap_vfmlalq_high_f16(__rev0_866, __rev1_866, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866)}); \
68251 __ret_894 = __builtin_shufflevector(__ret_894, __ret_894, 3, 2, 1, 0); \70142 __ret_866 = __builtin_shufflevector(__ret_866, __ret_866, 3, 2, 1, 0); \
68252 __ret_894; \70143 __ret_866; \
68253})70144})
68254#endif70145#endif
6825570146
68256#ifdef __LITTLE_ENDIAN__70147#ifdef __LITTLE_ENDIAN__
68257#define vfmlal_laneq_high_f16(__p0_895, __p1_895, __p2_895, __p3_895) __extension__ ({ \70148#define vfmlal_laneq_high_f16(__p0_867, __p1_867, __p2_867, __p3_867) __extension__ ({ \
68258 float32x2_t __ret_895; \70149 float32x2_t __ret_867; \
68259 float32x2_t __s0_895 = __p0_895; \70150 float32x2_t __s0_867 = __p0_867; \
68260 float16x4_t __s1_895 = __p1_895; \70151 float16x4_t __s1_867 = __p1_867; \
68261 float16x8_t __s2_895 = __p2_895; \70152 float16x8_t __s2_867 = __p2_867; \
68262 __ret_895 = vfmlal_high_f16(__s0_895, __s1_895, (float16x4_t) {vgetq_lane_f16(__s2_895, __p3_895), vgetq_lane_f16(__s2_895, __p3_895), vgetq_lane_f16(__s2_895, __p3_895), vgetq_lane_f16(__s2_895, __p3_895)}); \70153 __ret_867 = vfmlal_high_f16(__s0_867, __s1_867, (float16x4_t) {vgetq_lane_f16(__s2_867, __p3_867), vgetq_lane_f16(__s2_867, __p3_867), vgetq_lane_f16(__s2_867, __p3_867), vgetq_lane_f16(__s2_867, __p3_867)}); \
68263 __ret_895; \70154 __ret_867; \
68264})70155})
68265#else70156#else
68266#define vfmlal_laneq_high_f16(__p0_896, __p1_896, __p2_896, __p3_896) __extension__ ({ \70157#define vfmlal_laneq_high_f16(__p0_868, __p1_868, __p2_868, __p3_868) __extension__ ({ \
68267 float32x2_t __ret_896; \70158 float32x2_t __ret_868; \
68268 float32x2_t __s0_896 = __p0_896; \70159 float32x2_t __s0_868 = __p0_868; \
68269 float16x4_t __s1_896 = __p1_896; \70160 float16x4_t __s1_868 = __p1_868; \
68270 float16x8_t __s2_896 = __p2_896; \70161 float16x8_t __s2_868 = __p2_868; \
68271 float32x2_t __rev0_896; __rev0_896 = __builtin_shufflevector(__s0_896, __s0_896, 1, 0); \70162 float32x2_t __rev0_868; __rev0_868 = __builtin_shufflevector(__s0_868, __s0_868, 1, 0); \
68272 float16x4_t __rev1_896; __rev1_896 = __builtin_shufflevector(__s1_896, __s1_896, 3, 2, 1, 0); \70163 float16x4_t __rev1_868; __rev1_868 = __builtin_shufflevector(__s1_868, __s1_868, 3, 2, 1, 0); \
68273 float16x8_t __rev2_896; __rev2_896 = __builtin_shufflevector(__s2_896, __s2_896, 7, 6, 5, 4, 3, 2, 1, 0); \70164 float16x8_t __rev2_868; __rev2_868 = __builtin_shufflevector(__s2_868, __s2_868, 7, 6, 5, 4, 3, 2, 1, 0); \
68274 __ret_896 = __noswap_vfmlal_high_f16(__rev0_896, __rev1_896, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_896, __p3_896), __noswap_vgetq_lane_f16(__rev2_896, __p3_896), __noswap_vgetq_lane_f16(__rev2_896, __p3_896), __noswap_vgetq_lane_f16(__rev2_896, __p3_896)}); \70165 __ret_868 = __noswap_vfmlal_high_f16(__rev0_868, __rev1_868, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_868, __p3_868), __noswap_vgetq_lane_f16(__rev2_868, __p3_868), __noswap_vgetq_lane_f16(__rev2_868, __p3_868), __noswap_vgetq_lane_f16(__rev2_868, __p3_868)}); \
68275 __ret_896 = __builtin_shufflevector(__ret_896, __ret_896, 1, 0); \70166 __ret_868 = __builtin_shufflevector(__ret_868, __ret_868, 1, 0); \
68276 __ret_896; \70167 __ret_868; \
68277})70168})
68278#endif70169#endif
6827970170
68280#ifdef __LITTLE_ENDIAN__70171#ifdef __LITTLE_ENDIAN__
68281#define vfmlalq_laneq_low_f16(__p0_897, __p1_897, __p2_897, __p3_897) __extension__ ({ \70172#define vfmlalq_laneq_low_f16(__p0_869, __p1_869, __p2_869, __p3_869) __extension__ ({ \
68282 float32x4_t __ret_897; \70173 float32x4_t __ret_869; \
68283 float32x4_t __s0_897 = __p0_897; \70174 float32x4_t __s0_869 = __p0_869; \
68284 float16x8_t __s1_897 = __p1_897; \70175 float16x8_t __s1_869 = __p1_869; \
68285 float16x8_t __s2_897 = __p2_897; \70176 float16x8_t __s2_869 = __p2_869; \
68286 __ret_897 = vfmlalq_low_f16(__s0_897, __s1_897, (float16x8_t) {vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897)}); \70177 __ret_869 = vfmlalq_low_f16(__s0_869, __s1_869, (float16x8_t) {vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869)}); \
68287 __ret_897; \70178 __ret_869; \
68288})70179})
68289#else70180#else
68290#define vfmlalq_laneq_low_f16(__p0_898, __p1_898, __p2_898, __p3_898) __extension__ ({ \70181#define vfmlalq_laneq_low_f16(__p0_870, __p1_870, __p2_870, __p3_870) __extension__ ({ \
68291 float32x4_t __ret_898; \70182 float32x4_t __ret_870; \
68292 float32x4_t __s0_898 = __p0_898; \70183 float32x4_t __s0_870 = __p0_870; \
68293 float16x8_t __s1_898 = __p1_898; \70184 float16x8_t __s1_870 = __p1_870; \
68294 float16x8_t __s2_898 = __p2_898; \70185 float16x8_t __s2_870 = __p2_870; \
68295 float32x4_t __rev0_898; __rev0_898 = __builtin_shufflevector(__s0_898, __s0_898, 3, 2, 1, 0); \70186 float32x4_t __rev0_870; __rev0_870 = __builtin_shufflevector(__s0_870, __s0_870, 3, 2, 1, 0); \
68296 float16x8_t __rev1_898; __rev1_898 = __builtin_shufflevector(__s1_898, __s1_898, 7, 6, 5, 4, 3, 2, 1, 0); \70187 float16x8_t __rev1_870; __rev1_870 = __builtin_shufflevector(__s1_870, __s1_870, 7, 6, 5, 4, 3, 2, 1, 0); \
68297 float16x8_t __rev2_898; __rev2_898 = __builtin_shufflevector(__s2_898, __s2_898, 7, 6, 5, 4, 3, 2, 1, 0); \70188 float16x8_t __rev2_870; __rev2_870 = __builtin_shufflevector(__s2_870, __s2_870, 7, 6, 5, 4, 3, 2, 1, 0); \
68298 __ret_898 = __noswap_vfmlalq_low_f16(__rev0_898, __rev1_898, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898)}); \70189 __ret_870 = __noswap_vfmlalq_low_f16(__rev0_870, __rev1_870, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870)}); \
68299 __ret_898 = __builtin_shufflevector(__ret_898, __ret_898, 3, 2, 1, 0); \70190 __ret_870 = __builtin_shufflevector(__ret_870, __ret_870, 3, 2, 1, 0); \
68300 __ret_898; \70191 __ret_870; \
68301})70192})
68302#endif70193#endif
6830370194
68304#ifdef __LITTLE_ENDIAN__70195#ifdef __LITTLE_ENDIAN__
68305#define vfmlal_laneq_low_f16(__p0_899, __p1_899, __p2_899, __p3_899) __extension__ ({ \70196#define vfmlal_laneq_low_f16(__p0_871, __p1_871, __p2_871, __p3_871) __extension__ ({ \
68306 float32x2_t __ret_899; \70197 float32x2_t __ret_871; \
68307 float32x2_t __s0_899 = __p0_899; \70198 float32x2_t __s0_871 = __p0_871; \
68308 float16x4_t __s1_899 = __p1_899; \70199 float16x4_t __s1_871 = __p1_871; \
68309 float16x8_t __s2_899 = __p2_899; \70200 float16x8_t __s2_871 = __p2_871; \
68310 __ret_899 = vfmlal_low_f16(__s0_899, __s1_899, (float16x4_t) {vgetq_lane_f16(__s2_899, __p3_899), vgetq_lane_f16(__s2_899, __p3_899), vgetq_lane_f16(__s2_899, __p3_899), vgetq_lane_f16(__s2_899, __p3_899)}); \70201 __ret_871 = vfmlal_low_f16(__s0_871, __s1_871, (float16x4_t) {vgetq_lane_f16(__s2_871, __p3_871), vgetq_lane_f16(__s2_871, __p3_871), vgetq_lane_f16(__s2_871, __p3_871), vgetq_lane_f16(__s2_871, __p3_871)}); \
68311 __ret_899; \70202 __ret_871; \
68312})70203})
68313#else70204#else
68314#define vfmlal_laneq_low_f16(__p0_900, __p1_900, __p2_900, __p3_900) __extension__ ({ \70205#define vfmlal_laneq_low_f16(__p0_872, __p1_872, __p2_872, __p3_872) __extension__ ({ \
68315 float32x2_t __ret_900; \70206 float32x2_t __ret_872; \
68316 float32x2_t __s0_900 = __p0_900; \70207 float32x2_t __s0_872 = __p0_872; \
68317 float16x4_t __s1_900 = __p1_900; \70208 float16x4_t __s1_872 = __p1_872; \
68318 float16x8_t __s2_900 = __p2_900; \70209 float16x8_t __s2_872 = __p2_872; \
68319 float32x2_t __rev0_900; __rev0_900 = __builtin_shufflevector(__s0_900, __s0_900, 1, 0); \70210 float32x2_t __rev0_872; __rev0_872 = __builtin_shufflevector(__s0_872, __s0_872, 1, 0); \
68320 float16x4_t __rev1_900; __rev1_900 = __builtin_shufflevector(__s1_900, __s1_900, 3, 2, 1, 0); \70211 float16x4_t __rev1_872; __rev1_872 = __builtin_shufflevector(__s1_872, __s1_872, 3, 2, 1, 0); \
68321 float16x8_t __rev2_900; __rev2_900 = __builtin_shufflevector(__s2_900, __s2_900, 7, 6, 5, 4, 3, 2, 1, 0); \70212 float16x8_t __rev2_872; __rev2_872 = __builtin_shufflevector(__s2_872, __s2_872, 7, 6, 5, 4, 3, 2, 1, 0); \
68322 __ret_900 = __noswap_vfmlal_low_f16(__rev0_900, __rev1_900, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_900, __p3_900), __noswap_vgetq_lane_f16(__rev2_900, __p3_900), __noswap_vgetq_lane_f16(__rev2_900, __p3_900), __noswap_vgetq_lane_f16(__rev2_900, __p3_900)}); \70213 __ret_872 = __noswap_vfmlal_low_f16(__rev0_872, __rev1_872, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_872, __p3_872), __noswap_vgetq_lane_f16(__rev2_872, __p3_872), __noswap_vgetq_lane_f16(__rev2_872, __p3_872), __noswap_vgetq_lane_f16(__rev2_872, __p3_872)}); \
68323 __ret_900 = __builtin_shufflevector(__ret_900, __ret_900, 1, 0); \70214 __ret_872 = __builtin_shufflevector(__ret_872, __ret_872, 1, 0); \
68324 __ret_900; \70215 __ret_872; \
68325})70216})
68326#endif70217#endif
6832770218
68328#ifdef __LITTLE_ENDIAN__70219#ifdef __LITTLE_ENDIAN__
68329#define vfmlslq_lane_high_f16(__p0_901, __p1_901, __p2_901, __p3_901) __extension__ ({ \70220#define vfmlslq_lane_high_f16(__p0_873, __p1_873, __p2_873, __p3_873) __extension__ ({ \
68330 float32x4_t __ret_901; \70221 float32x4_t __ret_873; \
68331 float32x4_t __s0_901 = __p0_901; \70222 float32x4_t __s0_873 = __p0_873; \
68332 float16x8_t __s1_901 = __p1_901; \70223 float16x8_t __s1_873 = __p1_873; \
68333 float16x4_t __s2_901 = __p2_901; \70224 float16x4_t __s2_873 = __p2_873; \
68334 __ret_901 = vfmlslq_high_f16(__s0_901, __s1_901, (float16x8_t) {vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901)}); \70225 __ret_873 = vfmlslq_high_f16(__s0_873, __s1_873, (float16x8_t) {vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873)}); \
68335 __ret_901; \70226 __ret_873; \
68336})70227})
68337#else70228#else
68338#define vfmlslq_lane_high_f16(__p0_902, __p1_902, __p2_902, __p3_902) __extension__ ({ \70229#define vfmlslq_lane_high_f16(__p0_874, __p1_874, __p2_874, __p3_874) __extension__ ({ \
68339 float32x4_t __ret_902; \70230 float32x4_t __ret_874; \
68340 float32x4_t __s0_902 = __p0_902; \70231 float32x4_t __s0_874 = __p0_874; \
68341 float16x8_t __s1_902 = __p1_902; \70232 float16x8_t __s1_874 = __p1_874; \
68342 float16x4_t __s2_902 = __p2_902; \70233 float16x4_t __s2_874 = __p2_874; \
68343 float32x4_t __rev0_902; __rev0_902 = __builtin_shufflevector(__s0_902, __s0_902, 3, 2, 1, 0); \70234 float32x4_t __rev0_874; __rev0_874 = __builtin_shufflevector(__s0_874, __s0_874, 3, 2, 1, 0); \
68344 float16x8_t __rev1_902; __rev1_902 = __builtin_shufflevector(__s1_902, __s1_902, 7, 6, 5, 4, 3, 2, 1, 0); \70235 float16x8_t __rev1_874; __rev1_874 = __builtin_shufflevector(__s1_874, __s1_874, 7, 6, 5, 4, 3, 2, 1, 0); \
68345 float16x4_t __rev2_902; __rev2_902 = __builtin_shufflevector(__s2_902, __s2_902, 3, 2, 1, 0); \70236 float16x4_t __rev2_874; __rev2_874 = __builtin_shufflevector(__s2_874, __s2_874, 3, 2, 1, 0); \
68346 __ret_902 = __noswap_vfmlslq_high_f16(__rev0_902, __rev1_902, (float16x8_t) {__noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902)}); \70237 __ret_874 = __noswap_vfmlslq_high_f16(__rev0_874, __rev1_874, (float16x8_t) {__noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874)}); \
68347 __ret_902 = __builtin_shufflevector(__ret_902, __ret_902, 3, 2, 1, 0); \70238 __ret_874 = __builtin_shufflevector(__ret_874, __ret_874, 3, 2, 1, 0); \
68348 __ret_902; \70239 __ret_874; \
68349})70240})
68350#endif70241#endif
6835170242
68352#ifdef __LITTLE_ENDIAN__70243#ifdef __LITTLE_ENDIAN__
68353#define vfmlsl_lane_high_f16(__p0_903, __p1_903, __p2_903, __p3_903) __extension__ ({ \70244#define vfmlsl_lane_high_f16(__p0_875, __p1_875, __p2_875, __p3_875) __extension__ ({ \
68354 float32x2_t __ret_903; \70245 float32x2_t __ret_875; \
68355 float32x2_t __s0_903 = __p0_903; \70246 float32x2_t __s0_875 = __p0_875; \
68356 float16x4_t __s1_903 = __p1_903; \70247 float16x4_t __s1_875 = __p1_875; \
68357 float16x4_t __s2_903 = __p2_903; \70248 float16x4_t __s2_875 = __p2_875; \
68358 __ret_903 = vfmlsl_high_f16(__s0_903, __s1_903, (float16x4_t) {vget_lane_f16(__s2_903, __p3_903), vget_lane_f16(__s2_903, __p3_903), vget_lane_f16(__s2_903, __p3_903), vget_lane_f16(__s2_903, __p3_903)}); \70249 __ret_875 = vfmlsl_high_f16(__s0_875, __s1_875, (float16x4_t) {vget_lane_f16(__s2_875, __p3_875), vget_lane_f16(__s2_875, __p3_875), vget_lane_f16(__s2_875, __p3_875), vget_lane_f16(__s2_875, __p3_875)}); \
68359 __ret_903; \70250 __ret_875; \
68360})70251})
68361#else70252#else
68362#define vfmlsl_lane_high_f16(__p0_904, __p1_904, __p2_904, __p3_904) __extension__ ({ \70253#define vfmlsl_lane_high_f16(__p0_876, __p1_876, __p2_876, __p3_876) __extension__ ({ \
68363 float32x2_t __ret_904; \70254 float32x2_t __ret_876; \
68364 float32x2_t __s0_904 = __p0_904; \70255 float32x2_t __s0_876 = __p0_876; \
68365 float16x4_t __s1_904 = __p1_904; \70256 float16x4_t __s1_876 = __p1_876; \
68366 float16x4_t __s2_904 = __p2_904; \70257 float16x4_t __s2_876 = __p2_876; \
68367 float32x2_t __rev0_904; __rev0_904 = __builtin_shufflevector(__s0_904, __s0_904, 1, 0); \70258 float32x2_t __rev0_876; __rev0_876 = __builtin_shufflevector(__s0_876, __s0_876, 1, 0); \
68368 float16x4_t __rev1_904; __rev1_904 = __builtin_shufflevector(__s1_904, __s1_904, 3, 2, 1, 0); \70259 float16x4_t __rev1_876; __rev1_876 = __builtin_shufflevector(__s1_876, __s1_876, 3, 2, 1, 0); \
68369 float16x4_t __rev2_904; __rev2_904 = __builtin_shufflevector(__s2_904, __s2_904, 3, 2, 1, 0); \70260 float16x4_t __rev2_876; __rev2_876 = __builtin_shufflevector(__s2_876, __s2_876, 3, 2, 1, 0); \
68370 __ret_904 = __noswap_vfmlsl_high_f16(__rev0_904, __rev1_904, (float16x4_t) {__noswap_vget_lane_f16(__rev2_904, __p3_904), __noswap_vget_lane_f16(__rev2_904, __p3_904), __noswap_vget_lane_f16(__rev2_904, __p3_904), __noswap_vget_lane_f16(__rev2_904, __p3_904)}); \70261 __ret_876 = __noswap_vfmlsl_high_f16(__rev0_876, __rev1_876, (float16x4_t) {__noswap_vget_lane_f16(__rev2_876, __p3_876), __noswap_vget_lane_f16(__rev2_876, __p3_876), __noswap_vget_lane_f16(__rev2_876, __p3_876), __noswap_vget_lane_f16(__rev2_876, __p3_876)}); \
68371 __ret_904 = __builtin_shufflevector(__ret_904, __ret_904, 1, 0); \70262 __ret_876 = __builtin_shufflevector(__ret_876, __ret_876, 1, 0); \
68372 __ret_904; \70263 __ret_876; \
68373})70264})
68374#endif70265#endif
6837570266
68376#ifdef __LITTLE_ENDIAN__70267#ifdef __LITTLE_ENDIAN__
68377#define vfmlslq_lane_low_f16(__p0_905, __p1_905, __p2_905, __p3_905) __extension__ ({ \70268#define vfmlslq_lane_low_f16(__p0_877, __p1_877, __p2_877, __p3_877) __extension__ ({ \
68378 float32x4_t __ret_905; \70269 float32x4_t __ret_877; \
68379 float32x4_t __s0_905 = __p0_905; \70270 float32x4_t __s0_877 = __p0_877; \
68380 float16x8_t __s1_905 = __p1_905; \70271 float16x8_t __s1_877 = __p1_877; \
68381 float16x4_t __s2_905 = __p2_905; \70272 float16x4_t __s2_877 = __p2_877; \
68382 __ret_905 = vfmlslq_low_f16(__s0_905, __s1_905, (float16x8_t) {vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905)}); \70273 __ret_877 = vfmlslq_low_f16(__s0_877, __s1_877, (float16x8_t) {vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877)}); \
68383 __ret_905; \70274 __ret_877; \
68384})70275})
68385#else70276#else
68386#define vfmlslq_lane_low_f16(__p0_906, __p1_906, __p2_906, __p3_906) __extension__ ({ \70277#define vfmlslq_lane_low_f16(__p0_878, __p1_878, __p2_878, __p3_878) __extension__ ({ \
68387 float32x4_t __ret_906; \70278 float32x4_t __ret_878; \
68388 float32x4_t __s0_906 = __p0_906; \70279 float32x4_t __s0_878 = __p0_878; \
68389 float16x8_t __s1_906 = __p1_906; \70280 float16x8_t __s1_878 = __p1_878; \
68390 float16x4_t __s2_906 = __p2_906; \70281 float16x4_t __s2_878 = __p2_878; \
68391 float32x4_t __rev0_906; __rev0_906 = __builtin_shufflevector(__s0_906, __s0_906, 3, 2, 1, 0); \70282 float32x4_t __rev0_878; __rev0_878 = __builtin_shufflevector(__s0_878, __s0_878, 3, 2, 1, 0); \
68392 float16x8_t __rev1_906; __rev1_906 = __builtin_shufflevector(__s1_906, __s1_906, 7, 6, 5, 4, 3, 2, 1, 0); \70283 float16x8_t __rev1_878; __rev1_878 = __builtin_shufflevector(__s1_878, __s1_878, 7, 6, 5, 4, 3, 2, 1, 0); \
68393 float16x4_t __rev2_906; __rev2_906 = __builtin_shufflevector(__s2_906, __s2_906, 3, 2, 1, 0); \70284 float16x4_t __rev2_878; __rev2_878 = __builtin_shufflevector(__s2_878, __s2_878, 3, 2, 1, 0); \
68394 __ret_906 = __noswap_vfmlslq_low_f16(__rev0_906, __rev1_906, (float16x8_t) {__noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906)}); \70285 __ret_878 = __noswap_vfmlslq_low_f16(__rev0_878, __rev1_878, (float16x8_t) {__noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878)}); \
68395 __ret_906 = __builtin_shufflevector(__ret_906, __ret_906, 3, 2, 1, 0); \70286 __ret_878 = __builtin_shufflevector(__ret_878, __ret_878, 3, 2, 1, 0); \
68396 __ret_906; \70287 __ret_878; \
68397})70288})
68398#endif70289#endif
6839970290
68400#ifdef __LITTLE_ENDIAN__70291#ifdef __LITTLE_ENDIAN__
68401#define vfmlsl_lane_low_f16(__p0_907, __p1_907, __p2_907, __p3_907) __extension__ ({ \70292#define vfmlsl_lane_low_f16(__p0_879, __p1_879, __p2_879, __p3_879) __extension__ ({ \
68402 float32x2_t __ret_907; \70293 float32x2_t __ret_879; \
68403 float32x2_t __s0_907 = __p0_907; \70294 float32x2_t __s0_879 = __p0_879; \
68404 float16x4_t __s1_907 = __p1_907; \70295 float16x4_t __s1_879 = __p1_879; \
68405 float16x4_t __s2_907 = __p2_907; \70296 float16x4_t __s2_879 = __p2_879; \
68406 __ret_907 = vfmlsl_low_f16(__s0_907, __s1_907, (float16x4_t) {vget_lane_f16(__s2_907, __p3_907), vget_lane_f16(__s2_907, __p3_907), vget_lane_f16(__s2_907, __p3_907), vget_lane_f16(__s2_907, __p3_907)}); \70297 __ret_879 = vfmlsl_low_f16(__s0_879, __s1_879, (float16x4_t) {vget_lane_f16(__s2_879, __p3_879), vget_lane_f16(__s2_879, __p3_879), vget_lane_f16(__s2_879, __p3_879), vget_lane_f16(__s2_879, __p3_879)}); \
68407 __ret_907; \70298 __ret_879; \
68408})70299})
68409#else70300#else
68410#define vfmlsl_lane_low_f16(__p0_908, __p1_908, __p2_908, __p3_908) __extension__ ({ \70301#define vfmlsl_lane_low_f16(__p0_880, __p1_880, __p2_880, __p3_880) __extension__ ({ \
68411 float32x2_t __ret_908; \70302 float32x2_t __ret_880; \
68412 float32x2_t __s0_908 = __p0_908; \70303 float32x2_t __s0_880 = __p0_880; \
68413 float16x4_t __s1_908 = __p1_908; \70304 float16x4_t __s1_880 = __p1_880; \
68414 float16x4_t __s2_908 = __p2_908; \70305 float16x4_t __s2_880 = __p2_880; \
68415 float32x2_t __rev0_908; __rev0_908 = __builtin_shufflevector(__s0_908, __s0_908, 1, 0); \70306 float32x2_t __rev0_880; __rev0_880 = __builtin_shufflevector(__s0_880, __s0_880, 1, 0); \
68416 float16x4_t __rev1_908; __rev1_908 = __builtin_shufflevector(__s1_908, __s1_908, 3, 2, 1, 0); \70307 float16x4_t __rev1_880; __rev1_880 = __builtin_shufflevector(__s1_880, __s1_880, 3, 2, 1, 0); \
68417 float16x4_t __rev2_908; __rev2_908 = __builtin_shufflevector(__s2_908, __s2_908, 3, 2, 1, 0); \70308 float16x4_t __rev2_880; __rev2_880 = __builtin_shufflevector(__s2_880, __s2_880, 3, 2, 1, 0); \
68418 __ret_908 = __noswap_vfmlsl_low_f16(__rev0_908, __rev1_908, (float16x4_t) {__noswap_vget_lane_f16(__rev2_908, __p3_908), __noswap_vget_lane_f16(__rev2_908, __p3_908), __noswap_vget_lane_f16(__rev2_908, __p3_908), __noswap_vget_lane_f16(__rev2_908, __p3_908)}); \70309 __ret_880 = __noswap_vfmlsl_low_f16(__rev0_880, __rev1_880, (float16x4_t) {__noswap_vget_lane_f16(__rev2_880, __p3_880), __noswap_vget_lane_f16(__rev2_880, __p3_880), __noswap_vget_lane_f16(__rev2_880, __p3_880), __noswap_vget_lane_f16(__rev2_880, __p3_880)}); \
68419 __ret_908 = __builtin_shufflevector(__ret_908, __ret_908, 1, 0); \70310 __ret_880 = __builtin_shufflevector(__ret_880, __ret_880, 1, 0); \
68420 __ret_908; \70311 __ret_880; \
68421})70312})
68422#endif70313#endif
6842370314
68424#ifdef __LITTLE_ENDIAN__70315#ifdef __LITTLE_ENDIAN__
68425#define vfmlslq_laneq_high_f16(__p0_909, __p1_909, __p2_909, __p3_909) __extension__ ({ \70316#define vfmlslq_laneq_high_f16(__p0_881, __p1_881, __p2_881, __p3_881) __extension__ ({ \
68426 float32x4_t __ret_909; \70317 float32x4_t __ret_881; \
68427 float32x4_t __s0_909 = __p0_909; \70318 float32x4_t __s0_881 = __p0_881; \
68428 float16x8_t __s1_909 = __p1_909; \70319 float16x8_t __s1_881 = __p1_881; \
68429 float16x8_t __s2_909 = __p2_909; \70320 float16x8_t __s2_881 = __p2_881; \
68430 __ret_909 = vfmlslq_high_f16(__s0_909, __s1_909, (float16x8_t) {vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909)}); \70321 __ret_881 = vfmlslq_high_f16(__s0_881, __s1_881, (float16x8_t) {vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881)}); \
68431 __ret_909; \70322 __ret_881; \
68432})70323})
68433#else70324#else
68434#define vfmlslq_laneq_high_f16(__p0_910, __p1_910, __p2_910, __p3_910) __extension__ ({ \70325#define vfmlslq_laneq_high_f16(__p0_882, __p1_882, __p2_882, __p3_882) __extension__ ({ \
68435 float32x4_t __ret_910; \70326 float32x4_t __ret_882; \
68436 float32x4_t __s0_910 = __p0_910; \70327 float32x4_t __s0_882 = __p0_882; \
68437 float16x8_t __s1_910 = __p1_910; \70328 float16x8_t __s1_882 = __p1_882; \
68438 float16x8_t __s2_910 = __p2_910; \70329 float16x8_t __s2_882 = __p2_882; \
68439 float32x4_t __rev0_910; __rev0_910 = __builtin_shufflevector(__s0_910, __s0_910, 3, 2, 1, 0); \70330 float32x4_t __rev0_882; __rev0_882 = __builtin_shufflevector(__s0_882, __s0_882, 3, 2, 1, 0); \
68440 float16x8_t __rev1_910; __rev1_910 = __builtin_shufflevector(__s1_910, __s1_910, 7, 6, 5, 4, 3, 2, 1, 0); \70331 float16x8_t __rev1_882; __rev1_882 = __builtin_shufflevector(__s1_882, __s1_882, 7, 6, 5, 4, 3, 2, 1, 0); \
68441 float16x8_t __rev2_910; __rev2_910 = __builtin_shufflevector(__s2_910, __s2_910, 7, 6, 5, 4, 3, 2, 1, 0); \70332 float16x8_t __rev2_882; __rev2_882 = __builtin_shufflevector(__s2_882, __s2_882, 7, 6, 5, 4, 3, 2, 1, 0); \
68442 __ret_910 = __noswap_vfmlslq_high_f16(__rev0_910, __rev1_910, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910)}); \70333 __ret_882 = __noswap_vfmlslq_high_f16(__rev0_882, __rev1_882, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882)}); \
68443 __ret_910 = __builtin_shufflevector(__ret_910, __ret_910, 3, 2, 1, 0); \70334 __ret_882 = __builtin_shufflevector(__ret_882, __ret_882, 3, 2, 1, 0); \
68444 __ret_910; \70335 __ret_882; \
68445})70336})
68446#endif70337#endif
6844770338
68448#ifdef __LITTLE_ENDIAN__70339#ifdef __LITTLE_ENDIAN__
68449#define vfmlsl_laneq_high_f16(__p0_911, __p1_911, __p2_911, __p3_911) __extension__ ({ \70340#define vfmlsl_laneq_high_f16(__p0_883, __p1_883, __p2_883, __p3_883) __extension__ ({ \
68450 float32x2_t __ret_911; \70341 float32x2_t __ret_883; \
68451 float32x2_t __s0_911 = __p0_911; \70342 float32x2_t __s0_883 = __p0_883; \
68452 float16x4_t __s1_911 = __p1_911; \70343 float16x4_t __s1_883 = __p1_883; \
68453 float16x8_t __s2_911 = __p2_911; \70344 float16x8_t __s2_883 = __p2_883; \
68454 __ret_911 = vfmlsl_high_f16(__s0_911, __s1_911, (float16x4_t) {vgetq_lane_f16(__s2_911, __p3_911), vgetq_lane_f16(__s2_911, __p3_911), vgetq_lane_f16(__s2_911, __p3_911), vgetq_lane_f16(__s2_911, __p3_911)}); \70345 __ret_883 = vfmlsl_high_f16(__s0_883, __s1_883, (float16x4_t) {vgetq_lane_f16(__s2_883, __p3_883), vgetq_lane_f16(__s2_883, __p3_883), vgetq_lane_f16(__s2_883, __p3_883), vgetq_lane_f16(__s2_883, __p3_883)}); \
68455 __ret_911; \70346 __ret_883; \
68456})70347})
68457#else70348#else
68458#define vfmlsl_laneq_high_f16(__p0_912, __p1_912, __p2_912, __p3_912) __extension__ ({ \70349#define vfmlsl_laneq_high_f16(__p0_884, __p1_884, __p2_884, __p3_884) __extension__ ({ \
68459 float32x2_t __ret_912; \70350 float32x2_t __ret_884; \
68460 float32x2_t __s0_912 = __p0_912; \70351 float32x2_t __s0_884 = __p0_884; \
68461 float16x4_t __s1_912 = __p1_912; \70352 float16x4_t __s1_884 = __p1_884; \
68462 float16x8_t __s2_912 = __p2_912; \70353 float16x8_t __s2_884 = __p2_884; \
68463 float32x2_t __rev0_912; __rev0_912 = __builtin_shufflevector(__s0_912, __s0_912, 1, 0); \70354 float32x2_t __rev0_884; __rev0_884 = __builtin_shufflevector(__s0_884, __s0_884, 1, 0); \
68464 float16x4_t __rev1_912; __rev1_912 = __builtin_shufflevector(__s1_912, __s1_912, 3, 2, 1, 0); \70355 float16x4_t __rev1_884; __rev1_884 = __builtin_shufflevector(__s1_884, __s1_884, 3, 2, 1, 0); \
68465 float16x8_t __rev2_912; __rev2_912 = __builtin_shufflevector(__s2_912, __s2_912, 7, 6, 5, 4, 3, 2, 1, 0); \70356 float16x8_t __rev2_884; __rev2_884 = __builtin_shufflevector(__s2_884, __s2_884, 7, 6, 5, 4, 3, 2, 1, 0); \
68466 __ret_912 = __noswap_vfmlsl_high_f16(__rev0_912, __rev1_912, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_912, __p3_912), __noswap_vgetq_lane_f16(__rev2_912, __p3_912), __noswap_vgetq_lane_f16(__rev2_912, __p3_912), __noswap_vgetq_lane_f16(__rev2_912, __p3_912)}); \70357 __ret_884 = __noswap_vfmlsl_high_f16(__rev0_884, __rev1_884, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_884, __p3_884), __noswap_vgetq_lane_f16(__rev2_884, __p3_884), __noswap_vgetq_lane_f16(__rev2_884, __p3_884), __noswap_vgetq_lane_f16(__rev2_884, __p3_884)}); \
68467 __ret_912 = __builtin_shufflevector(__ret_912, __ret_912, 1, 0); \70358 __ret_884 = __builtin_shufflevector(__ret_884, __ret_884, 1, 0); \
68468 __ret_912; \70359 __ret_884; \
68469})70360})
68470#endif70361#endif
6847170362
68472#ifdef __LITTLE_ENDIAN__70363#ifdef __LITTLE_ENDIAN__
68473#define vfmlslq_laneq_low_f16(__p0_913, __p1_913, __p2_913, __p3_913) __extension__ ({ \70364#define vfmlslq_laneq_low_f16(__p0_885, __p1_885, __p2_885, __p3_885) __extension__ ({ \
68474 float32x4_t __ret_913; \70365 float32x4_t __ret_885; \
68475 float32x4_t __s0_913 = __p0_913; \70366 float32x4_t __s0_885 = __p0_885; \
68476 float16x8_t __s1_913 = __p1_913; \70367 float16x8_t __s1_885 = __p1_885; \
68477 float16x8_t __s2_913 = __p2_913; \70368 float16x8_t __s2_885 = __p2_885; \
68478 __ret_913 = vfmlslq_low_f16(__s0_913, __s1_913, (float16x8_t) {vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913)}); \70369 __ret_885 = vfmlslq_low_f16(__s0_885, __s1_885, (float16x8_t) {vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885)}); \
68479 __ret_913; \70370 __ret_885; \
68480})70371})
68481#else70372#else
68482#define vfmlslq_laneq_low_f16(__p0_914, __p1_914, __p2_914, __p3_914) __extension__ ({ \70373#define vfmlslq_laneq_low_f16(__p0_886, __p1_886, __p2_886, __p3_886) __extension__ ({ \
68483 float32x4_t __ret_914; \70374 float32x4_t __ret_886; \
68484 float32x4_t __s0_914 = __p0_914; \70375 float32x4_t __s0_886 = __p0_886; \
68485 float16x8_t __s1_914 = __p1_914; \70376 float16x8_t __s1_886 = __p1_886; \
68486 float16x8_t __s2_914 = __p2_914; \70377 float16x8_t __s2_886 = __p2_886; \
68487 float32x4_t __rev0_914; __rev0_914 = __builtin_shufflevector(__s0_914, __s0_914, 3, 2, 1, 0); \70378 float32x4_t __rev0_886; __rev0_886 = __builtin_shufflevector(__s0_886, __s0_886, 3, 2, 1, 0); \
68488 float16x8_t __rev1_914; __rev1_914 = __builtin_shufflevector(__s1_914, __s1_914, 7, 6, 5, 4, 3, 2, 1, 0); \70379 float16x8_t __rev1_886; __rev1_886 = __builtin_shufflevector(__s1_886, __s1_886, 7, 6, 5, 4, 3, 2, 1, 0); \
68489 float16x8_t __rev2_914; __rev2_914 = __builtin_shufflevector(__s2_914, __s2_914, 7, 6, 5, 4, 3, 2, 1, 0); \70380 float16x8_t __rev2_886; __rev2_886 = __builtin_shufflevector(__s2_886, __s2_886, 7, 6, 5, 4, 3, 2, 1, 0); \
68490 __ret_914 = __noswap_vfmlslq_low_f16(__rev0_914, __rev1_914, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914)}); \70381 __ret_886 = __noswap_vfmlslq_low_f16(__rev0_886, __rev1_886, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886)}); \
68491 __ret_914 = __builtin_shufflevector(__ret_914, __ret_914, 3, 2, 1, 0); \70382 __ret_886 = __builtin_shufflevector(__ret_886, __ret_886, 3, 2, 1, 0); \
68492 __ret_914; \70383 __ret_886; \
68493})70384})
68494#endif70385#endif
6849570386
68496#ifdef __LITTLE_ENDIAN__70387#ifdef __LITTLE_ENDIAN__
68497#define vfmlsl_laneq_low_f16(__p0_915, __p1_915, __p2_915, __p3_915) __extension__ ({ \70388#define vfmlsl_laneq_low_f16(__p0_887, __p1_887, __p2_887, __p3_887) __extension__ ({ \
68498 float32x2_t __ret_915; \70389 float32x2_t __ret_887; \
68499 float32x2_t __s0_915 = __p0_915; \70390 float32x2_t __s0_887 = __p0_887; \
68500 float16x4_t __s1_915 = __p1_915; \70391 float16x4_t __s1_887 = __p1_887; \
68501 float16x8_t __s2_915 = __p2_915; \70392 float16x8_t __s2_887 = __p2_887; \
68502 __ret_915 = vfmlsl_low_f16(__s0_915, __s1_915, (float16x4_t) {vgetq_lane_f16(__s2_915, __p3_915), vgetq_lane_f16(__s2_915, __p3_915), vgetq_lane_f16(__s2_915, __p3_915), vgetq_lane_f16(__s2_915, __p3_915)}); \70393 __ret_887 = vfmlsl_low_f16(__s0_887, __s1_887, (float16x4_t) {vgetq_lane_f16(__s2_887, __p3_887), vgetq_lane_f16(__s2_887, __p3_887), vgetq_lane_f16(__s2_887, __p3_887), vgetq_lane_f16(__s2_887, __p3_887)}); \
68503 __ret_915; \70394 __ret_887; \
68504})70395})
68505#else70396#else
68506#define vfmlsl_laneq_low_f16(__p0_916, __p1_916, __p2_916, __p3_916) __extension__ ({ \70397#define vfmlsl_laneq_low_f16(__p0_888, __p1_888, __p2_888, __p3_888) __extension__ ({ \
68507 float32x2_t __ret_916; \70398 float32x2_t __ret_888; \
68508 float32x2_t __s0_916 = __p0_916; \70399 float32x2_t __s0_888 = __p0_888; \
68509 float16x4_t __s1_916 = __p1_916; \70400 float16x4_t __s1_888 = __p1_888; \
68510 float16x8_t __s2_916 = __p2_916; \70401 float16x8_t __s2_888 = __p2_888; \
68511 float32x2_t __rev0_916; __rev0_916 = __builtin_shufflevector(__s0_916, __s0_916, 1, 0); \70402 float32x2_t __rev0_888; __rev0_888 = __builtin_shufflevector(__s0_888, __s0_888, 1, 0); \
68512 float16x4_t __rev1_916; __rev1_916 = __builtin_shufflevector(__s1_916, __s1_916, 3, 2, 1, 0); \70403 float16x4_t __rev1_888; __rev1_888 = __builtin_shufflevector(__s1_888, __s1_888, 3, 2, 1, 0); \
68513 float16x8_t __rev2_916; __rev2_916 = __builtin_shufflevector(__s2_916, __s2_916, 7, 6, 5, 4, 3, 2, 1, 0); \70404 float16x8_t __rev2_888; __rev2_888 = __builtin_shufflevector(__s2_888, __s2_888, 7, 6, 5, 4, 3, 2, 1, 0); \
68514 __ret_916 = __noswap_vfmlsl_low_f16(__rev0_916, __rev1_916, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_916, __p3_916), __noswap_vgetq_lane_f16(__rev2_916, __p3_916), __noswap_vgetq_lane_f16(__rev2_916, __p3_916), __noswap_vgetq_lane_f16(__rev2_916, __p3_916)}); \70405 __ret_888 = __noswap_vfmlsl_low_f16(__rev0_888, __rev1_888, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_888, __p3_888), __noswap_vgetq_lane_f16(__rev2_888, __p3_888), __noswap_vgetq_lane_f16(__rev2_888, __p3_888), __noswap_vgetq_lane_f16(__rev2_888, __p3_888)}); \
68515 __ret_916 = __builtin_shufflevector(__ret_916, __ret_916, 1, 0); \70406 __ret_888 = __builtin_shufflevector(__ret_888, __ret_888, 1, 0); \
68516 __ret_916; \70407 __ret_888; \
68517})70408})
68518#endif70409#endif
6851970410
68520#ifdef __LITTLE_ENDIAN__70411#ifdef __LITTLE_ENDIAN__
68521#define vmulh_lane_f16(__p0_917, __p1_917, __p2_917) __extension__ ({ \70412#define vmulh_lane_f16(__p0_889, __p1_889, __p2_889) __extension__ ({ \
68522 float16_t __ret_917; \70413 float16_t __ret_889; \
68523 float16_t __s0_917 = __p0_917; \70414 float16_t __s0_889 = __p0_889; \
68524 float16x4_t __s1_917 = __p1_917; \70415 float16x4_t __s1_889 = __p1_889; \
68525 __ret_917 = __s0_917 * vget_lane_f16(__s1_917, __p2_917); \70416 __ret_889 = __s0_889 * vget_lane_f16(__s1_889, __p2_889); \
68526 __ret_917; \70417 __ret_889; \
68527})70418})
68528#else70419#else
68529#define vmulh_lane_f16(__p0_918, __p1_918, __p2_918) __extension__ ({ \70420#define vmulh_lane_f16(__p0_890, __p1_890, __p2_890) __extension__ ({ \
68530 float16_t __ret_918; \70421 float16_t __ret_890; \
68531 float16_t __s0_918 = __p0_918; \70422 float16_t __s0_890 = __p0_890; \
68532 float16x4_t __s1_918 = __p1_918; \70423 float16x4_t __s1_890 = __p1_890; \
68533 float16x4_t __rev1_918; __rev1_918 = __builtin_shufflevector(__s1_918, __s1_918, 3, 2, 1, 0); \70424 float16x4_t __rev1_890; __rev1_890 = __builtin_shufflevector(__s1_890, __s1_890, 3, 2, 1, 0); \
68534 __ret_918 = __s0_918 * __noswap_vget_lane_f16(__rev1_918, __p2_918); \70425 __ret_890 = __s0_890 * __noswap_vget_lane_f16(__rev1_890, __p2_890); \
68535 __ret_918; \70426 __ret_890; \
68536})70427})
68537#endif70428#endif
6853870429
68539#ifdef __LITTLE_ENDIAN__70430#ifdef __LITTLE_ENDIAN__
68540#define vmulh_laneq_f16(__p0_919, __p1_919, __p2_919) __extension__ ({ \70431#define vmulh_laneq_f16(__p0_891, __p1_891, __p2_891) __extension__ ({ \
68541 float16_t __ret_919; \70432 float16_t __ret_891; \
68542 float16_t __s0_919 = __p0_919; \70433 float16_t __s0_891 = __p0_891; \
68543 float16x8_t __s1_919 = __p1_919; \70434 float16x8_t __s1_891 = __p1_891; \
68544 __ret_919 = __s0_919 * vgetq_lane_f16(__s1_919, __p2_919); \70435 __ret_891 = __s0_891 * vgetq_lane_f16(__s1_891, __p2_891); \
68545 __ret_919; \70436 __ret_891; \
68546})70437})
68547#else70438#else
68548#define vmulh_laneq_f16(__p0_920, __p1_920, __p2_920) __extension__ ({ \70439#define vmulh_laneq_f16(__p0_892, __p1_892, __p2_892) __extension__ ({ \
68549 float16_t __ret_920; \70440 float16_t __ret_892; \
68550 float16_t __s0_920 = __p0_920; \70441 float16_t __s0_892 = __p0_892; \
68551 float16x8_t __s1_920 = __p1_920; \70442 float16x8_t __s1_892 = __p1_892; \
68552 float16x8_t __rev1_920; __rev1_920 = __builtin_shufflevector(__s1_920, __s1_920, 7, 6, 5, 4, 3, 2, 1, 0); \70443 float16x8_t __rev1_892; __rev1_892 = __builtin_shufflevector(__s1_892, __s1_892, 7, 6, 5, 4, 3, 2, 1, 0); \
68553 __ret_920 = __s0_920 * __noswap_vgetq_lane_f16(__rev1_920, __p2_920); \70444 __ret_892 = __s0_892 * __noswap_vgetq_lane_f16(__rev1_892, __p2_892); \
68554 __ret_920; \70445 __ret_892; \
68555})70446})
68556#endif70447#endif
6855770448
...@@ -68862,136 +70753,136 @@ __ai __attribute__((target("neon"))) int32x4_t vaddw_high_s16(int32x4_t __p0, in...@@ -68862,136 +70753,136 @@ __ai __attribute__((target("neon"))) int32x4_t vaddw_high_s16(int32x4_t __p0, in
68862#endif70753#endif
6886370754
68864#ifdef __LITTLE_ENDIAN__70755#ifdef __LITTLE_ENDIAN__
68865#define vcopyq_lane_p64(__p0_921, __p1_921, __p2_921, __p3_921) __extension__ ({ \70756#define vcopyq_lane_p64(__p0_893, __p1_893, __p2_893, __p3_893) __extension__ ({ \
68866 poly64x2_t __ret_921; \70757 poly64x2_t __ret_893; \
68867 poly64x2_t __s0_921 = __p0_921; \70758 poly64x2_t __s0_893 = __p0_893; \
68868 poly64x1_t __s2_921 = __p2_921; \70759 poly64x1_t __s2_893 = __p2_893; \
68869 __ret_921 = vsetq_lane_p64(vget_lane_p64(__s2_921, __p3_921), __s0_921, __p1_921); \70760 __ret_893 = vsetq_lane_p64(vget_lane_p64(__s2_893, __p3_893), __s0_893, __p1_893); \
68870 __ret_921; \70761 __ret_893; \
68871})70762})
68872#else70763#else
68873#define vcopyq_lane_p64(__p0_922, __p1_922, __p2_922, __p3_922) __extension__ ({ \70764#define vcopyq_lane_p64(__p0_894, __p1_894, __p2_894, __p3_894) __extension__ ({ \
68874 poly64x2_t __ret_922; \70765 poly64x2_t __ret_894; \
68875 poly64x2_t __s0_922 = __p0_922; \70766 poly64x2_t __s0_894 = __p0_894; \
68876 poly64x1_t __s2_922 = __p2_922; \70767 poly64x1_t __s2_894 = __p2_894; \
68877 poly64x2_t __rev0_922; __rev0_922 = __builtin_shufflevector(__s0_922, __s0_922, 1, 0); \70768 poly64x2_t __rev0_894; __rev0_894 = __builtin_shufflevector(__s0_894, __s0_894, 1, 0); \
68878 __ret_922 = __noswap_vsetq_lane_p64(vget_lane_p64(__s2_922, __p3_922), __rev0_922, __p1_922); \70769 __ret_894 = __noswap_vsetq_lane_p64(vget_lane_p64(__s2_894, __p3_894), __rev0_894, __p1_894); \
68879 __ret_922 = __builtin_shufflevector(__ret_922, __ret_922, 1, 0); \70770 __ret_894 = __builtin_shufflevector(__ret_894, __ret_894, 1, 0); \
68880 __ret_922; \70771 __ret_894; \
68881})70772})
68882#endif70773#endif
6888370774
68884#ifdef __LITTLE_ENDIAN__70775#ifdef __LITTLE_ENDIAN__
68885#define vcopyq_lane_f64(__p0_923, __p1_923, __p2_923, __p3_923) __extension__ ({ \70776#define vcopyq_lane_f64(__p0_895, __p1_895, __p2_895, __p3_895) __extension__ ({ \
68886 float64x2_t __ret_923; \70777 float64x2_t __ret_895; \
68887 float64x2_t __s0_923 = __p0_923; \70778 float64x2_t __s0_895 = __p0_895; \
68888 float64x1_t __s2_923 = __p2_923; \70779 float64x1_t __s2_895 = __p2_895; \
68889 __ret_923 = vsetq_lane_f64(vget_lane_f64(__s2_923, __p3_923), __s0_923, __p1_923); \70780 __ret_895 = vsetq_lane_f64(vget_lane_f64(__s2_895, __p3_895), __s0_895, __p1_895); \
68890 __ret_923; \70781 __ret_895; \
68891})70782})
68892#else70783#else
68893#define vcopyq_lane_f64(__p0_924, __p1_924, __p2_924, __p3_924) __extension__ ({ \70784#define vcopyq_lane_f64(__p0_896, __p1_896, __p2_896, __p3_896) __extension__ ({ \
68894 float64x2_t __ret_924; \70785 float64x2_t __ret_896; \
68895 float64x2_t __s0_924 = __p0_924; \70786 float64x2_t __s0_896 = __p0_896; \
68896 float64x1_t __s2_924 = __p2_924; \70787 float64x1_t __s2_896 = __p2_896; \
68897 float64x2_t __rev0_924; __rev0_924 = __builtin_shufflevector(__s0_924, __s0_924, 1, 0); \70788 float64x2_t __rev0_896; __rev0_896 = __builtin_shufflevector(__s0_896, __s0_896, 1, 0); \
68898 __ret_924 = __noswap_vsetq_lane_f64(vget_lane_f64(__s2_924, __p3_924), __rev0_924, __p1_924); \70789 __ret_896 = __noswap_vsetq_lane_f64(vget_lane_f64(__s2_896, __p3_896), __rev0_896, __p1_896); \
68899 __ret_924 = __builtin_shufflevector(__ret_924, __ret_924, 1, 0); \70790 __ret_896 = __builtin_shufflevector(__ret_896, __ret_896, 1, 0); \
68900 __ret_924; \70791 __ret_896; \
68901})70792})
68902#endif70793#endif
6890370794
68904#define vcopy_lane_p64(__p0_925, __p1_925, __p2_925, __p3_925) __extension__ ({ \70795#define vcopy_lane_p64(__p0_897, __p1_897, __p2_897, __p3_897) __extension__ ({ \
68905 poly64x1_t __ret_925; \70796 poly64x1_t __ret_897; \
68906 poly64x1_t __s0_925 = __p0_925; \70797 poly64x1_t __s0_897 = __p0_897; \
68907 poly64x1_t __s2_925 = __p2_925; \70798 poly64x1_t __s2_897 = __p2_897; \
68908 __ret_925 = vset_lane_p64(vget_lane_p64(__s2_925, __p3_925), __s0_925, __p1_925); \70799 __ret_897 = vset_lane_p64(vget_lane_p64(__s2_897, __p3_897), __s0_897, __p1_897); \
68909 __ret_925; \70800 __ret_897; \
68910})70801})
68911#define vcopy_lane_f64(__p0_926, __p1_926, __p2_926, __p3_926) __extension__ ({ \70802#define vcopy_lane_f64(__p0_898, __p1_898, __p2_898, __p3_898) __extension__ ({ \
68912 float64x1_t __ret_926; \70803 float64x1_t __ret_898; \
68913 float64x1_t __s0_926 = __p0_926; \70804 float64x1_t __s0_898 = __p0_898; \
68914 float64x1_t __s2_926 = __p2_926; \70805 float64x1_t __s2_898 = __p2_898; \
68915 __ret_926 = vset_lane_f64(vget_lane_f64(__s2_926, __p3_926), __s0_926, __p1_926); \70806 __ret_898 = vset_lane_f64(vget_lane_f64(__s2_898, __p3_898), __s0_898, __p1_898); \
68916 __ret_926; \70807 __ret_898; \
68917})70808})
68918#ifdef __LITTLE_ENDIAN__70809#ifdef __LITTLE_ENDIAN__
68919#define vcopyq_laneq_p64(__p0_927, __p1_927, __p2_927, __p3_927) __extension__ ({ \70810#define vcopyq_laneq_p64(__p0_899, __p1_899, __p2_899, __p3_899) __extension__ ({ \
68920 poly64x2_t __ret_927; \70811 poly64x2_t __ret_899; \
68921 poly64x2_t __s0_927 = __p0_927; \70812 poly64x2_t __s0_899 = __p0_899; \
68922 poly64x2_t __s2_927 = __p2_927; \70813 poly64x2_t __s2_899 = __p2_899; \
68923 __ret_927 = vsetq_lane_p64(vgetq_lane_p64(__s2_927, __p3_927), __s0_927, __p1_927); \70814 __ret_899 = vsetq_lane_p64(vgetq_lane_p64(__s2_899, __p3_899), __s0_899, __p1_899); \
68924 __ret_927; \70815 __ret_899; \
68925})70816})
68926#else70817#else
68927#define vcopyq_laneq_p64(__p0_928, __p1_928, __p2_928, __p3_928) __extension__ ({ \70818#define vcopyq_laneq_p64(__p0_900, __p1_900, __p2_900, __p3_900) __extension__ ({ \
68928 poly64x2_t __ret_928; \70819 poly64x2_t __ret_900; \
68929 poly64x2_t __s0_928 = __p0_928; \70820 poly64x2_t __s0_900 = __p0_900; \
68930 poly64x2_t __s2_928 = __p2_928; \70821 poly64x2_t __s2_900 = __p2_900; \
68931 poly64x2_t __rev0_928; __rev0_928 = __builtin_shufflevector(__s0_928, __s0_928, 1, 0); \70822 poly64x2_t __rev0_900; __rev0_900 = __builtin_shufflevector(__s0_900, __s0_900, 1, 0); \
68932 poly64x2_t __rev2_928; __rev2_928 = __builtin_shufflevector(__s2_928, __s2_928, 1, 0); \70823 poly64x2_t __rev2_900; __rev2_900 = __builtin_shufflevector(__s2_900, __s2_900, 1, 0); \
68933 __ret_928 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_928, __p3_928), __rev0_928, __p1_928); \70824 __ret_900 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_900, __p3_900), __rev0_900, __p1_900); \
68934 __ret_928 = __builtin_shufflevector(__ret_928, __ret_928, 1, 0); \70825 __ret_900 = __builtin_shufflevector(__ret_900, __ret_900, 1, 0); \
68935 __ret_928; \70826 __ret_900; \
68936})70827})
68937#endif70828#endif
6893870829
68939#ifdef __LITTLE_ENDIAN__70830#ifdef __LITTLE_ENDIAN__
68940#define vcopyq_laneq_f64(__p0_929, __p1_929, __p2_929, __p3_929) __extension__ ({ \70831#define vcopyq_laneq_f64(__p0_901, __p1_901, __p2_901, __p3_901) __extension__ ({ \
68941 float64x2_t __ret_929; \70832 float64x2_t __ret_901; \
68942 float64x2_t __s0_929 = __p0_929; \70833 float64x2_t __s0_901 = __p0_901; \
68943 float64x2_t __s2_929 = __p2_929; \70834 float64x2_t __s2_901 = __p2_901; \
68944 __ret_929 = vsetq_lane_f64(vgetq_lane_f64(__s2_929, __p3_929), __s0_929, __p1_929); \70835 __ret_901 = vsetq_lane_f64(vgetq_lane_f64(__s2_901, __p3_901), __s0_901, __p1_901); \
68945 __ret_929; \70836 __ret_901; \
68946})70837})
68947#else70838#else
68948#define vcopyq_laneq_f64(__p0_930, __p1_930, __p2_930, __p3_930) __extension__ ({ \70839#define vcopyq_laneq_f64(__p0_902, __p1_902, __p2_902, __p3_902) __extension__ ({ \
68949 float64x2_t __ret_930; \70840 float64x2_t __ret_902; \
68950 float64x2_t __s0_930 = __p0_930; \70841 float64x2_t __s0_902 = __p0_902; \
68951 float64x2_t __s2_930 = __p2_930; \70842 float64x2_t __s2_902 = __p2_902; \
68952 float64x2_t __rev0_930; __rev0_930 = __builtin_shufflevector(__s0_930, __s0_930, 1, 0); \70843 float64x2_t __rev0_902; __rev0_902 = __builtin_shufflevector(__s0_902, __s0_902, 1, 0); \
68953 float64x2_t __rev2_930; __rev2_930 = __builtin_shufflevector(__s2_930, __s2_930, 1, 0); \70844 float64x2_t __rev2_902; __rev2_902 = __builtin_shufflevector(__s2_902, __s2_902, 1, 0); \
68954 __ret_930 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_930, __p3_930), __rev0_930, __p1_930); \70845 __ret_902 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_902, __p3_902), __rev0_902, __p1_902); \
68955 __ret_930 = __builtin_shufflevector(__ret_930, __ret_930, 1, 0); \70846 __ret_902 = __builtin_shufflevector(__ret_902, __ret_902, 1, 0); \
68956 __ret_930; \70847 __ret_902; \
68957})70848})
68958#endif70849#endif
6895970850
68960#ifdef __LITTLE_ENDIAN__70851#ifdef __LITTLE_ENDIAN__
68961#define vcopy_laneq_p64(__p0_931, __p1_931, __p2_931, __p3_931) __extension__ ({ \70852#define vcopy_laneq_p64(__p0_903, __p1_903, __p2_903, __p3_903) __extension__ ({ \
68962 poly64x1_t __ret_931; \70853 poly64x1_t __ret_903; \
68963 poly64x1_t __s0_931 = __p0_931; \70854 poly64x1_t __s0_903 = __p0_903; \
68964 poly64x2_t __s2_931 = __p2_931; \70855 poly64x2_t __s2_903 = __p2_903; \
68965 __ret_931 = vset_lane_p64(vgetq_lane_p64(__s2_931, __p3_931), __s0_931, __p1_931); \70856 __ret_903 = vset_lane_p64(vgetq_lane_p64(__s2_903, __p3_903), __s0_903, __p1_903); \
68966 __ret_931; \70857 __ret_903; \
68967})70858})
68968#else70859#else
68969#define vcopy_laneq_p64(__p0_932, __p1_932, __p2_932, __p3_932) __extension__ ({ \70860#define vcopy_laneq_p64(__p0_904, __p1_904, __p2_904, __p3_904) __extension__ ({ \
68970 poly64x1_t __ret_932; \70861 poly64x1_t __ret_904; \
68971 poly64x1_t __s0_932 = __p0_932; \70862 poly64x1_t __s0_904 = __p0_904; \
68972 poly64x2_t __s2_932 = __p2_932; \70863 poly64x2_t __s2_904 = __p2_904; \
68973 poly64x2_t __rev2_932; __rev2_932 = __builtin_shufflevector(__s2_932, __s2_932, 1, 0); \70864 poly64x2_t __rev2_904; __rev2_904 = __builtin_shufflevector(__s2_904, __s2_904, 1, 0); \
68974 __ret_932 = vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_932, __p3_932), __s0_932, __p1_932); \70865 __ret_904 = vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_904, __p3_904), __s0_904, __p1_904); \
68975 __ret_932; \70866 __ret_904; \
68976})70867})
68977#endif70868#endif
6897870869
68979#ifdef __LITTLE_ENDIAN__70870#ifdef __LITTLE_ENDIAN__
68980#define vcopy_laneq_f64(__p0_933, __p1_933, __p2_933, __p3_933) __extension__ ({ \70871#define vcopy_laneq_f64(__p0_905, __p1_905, __p2_905, __p3_905) __extension__ ({ \
68981 float64x1_t __ret_933; \70872 float64x1_t __ret_905; \
68982 float64x1_t __s0_933 = __p0_933; \70873 float64x1_t __s0_905 = __p0_905; \
68983 float64x2_t __s2_933 = __p2_933; \70874 float64x2_t __s2_905 = __p2_905; \
68984 __ret_933 = vset_lane_f64(vgetq_lane_f64(__s2_933, __p3_933), __s0_933, __p1_933); \70875 __ret_905 = vset_lane_f64(vgetq_lane_f64(__s2_905, __p3_905), __s0_905, __p1_905); \
68985 __ret_933; \70876 __ret_905; \
68986})70877})
68987#else70878#else
68988#define vcopy_laneq_f64(__p0_934, __p1_934, __p2_934, __p3_934) __extension__ ({ \70879#define vcopy_laneq_f64(__p0_906, __p1_906, __p2_906, __p3_906) __extension__ ({ \
68989 float64x1_t __ret_934; \70880 float64x1_t __ret_906; \
68990 float64x1_t __s0_934 = __p0_934; \70881 float64x1_t __s0_906 = __p0_906; \
68991 float64x2_t __s2_934 = __p2_934; \70882 float64x2_t __s2_906 = __p2_906; \
68992 float64x2_t __rev2_934; __rev2_934 = __builtin_shufflevector(__s2_934, __s2_934, 1, 0); \70883 float64x2_t __rev2_906; __rev2_906 = __builtin_shufflevector(__s2_906, __s2_906, 1, 0); \
68993 __ret_934 = vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_934, __p3_934), __s0_934, __p1_934); \70884 __ret_906 = vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_906, __p3_906), __s0_906, __p1_906); \
68994 __ret_934; \70885 __ret_906; \
68995})70886})
68996#endif70887#endif
6899770888
...@@ -69347,38 +71238,38 @@ __ai __attribute__((target("neon"))) int32x4_t vmlsl_high_n_s16(int32x4_t __p0,...@@ -69347,38 +71238,38 @@ __ai __attribute__((target("neon"))) int32x4_t vmlsl_high_n_s16(int32x4_t __p0,
69347}71238}
69348#endif71239#endif
6934971240
69350#define vmulx_lane_f64(__p0_935, __p1_935, __p2_935) __extension__ ({ \71241#define vmulx_lane_f64(__p0_907, __p1_907, __p2_907) __extension__ ({ \
69351 float64x1_t __ret_935; \71242 float64x1_t __ret_907; \
69352 float64x1_t __s0_935 = __p0_935; \71243 float64x1_t __s0_907 = __p0_907; \
69353 float64x1_t __s1_935 = __p1_935; \71244 float64x1_t __s1_907 = __p1_907; \
69354 float64_t __x_935 = vget_lane_f64(__s0_935, 0); \71245 float64_t __x_907 = vget_lane_f64(__s0_907, 0); \
69355 float64_t __y_935 = vget_lane_f64(__s1_935, __p2_935); \71246 float64_t __y_907 = vget_lane_f64(__s1_907, __p2_907); \
69356 float64_t __z_935 = vmulxd_f64(__x_935, __y_935); \71247 float64_t __z_907 = vmulxd_f64(__x_907, __y_907); \
69357 __ret_935 = vset_lane_f64(__z_935, __s0_935, __p2_935); \71248 __ret_907 = vset_lane_f64(__z_907, __s0_907, __p2_907); \
69358 __ret_935; \71249 __ret_907; \
69359})71250})
69360#ifdef __LITTLE_ENDIAN__71251#ifdef __LITTLE_ENDIAN__
69361#define vmulx_laneq_f64(__p0_936, __p1_936, __p2_936) __extension__ ({ \71252#define vmulx_laneq_f64(__p0_908, __p1_908, __p2_908) __extension__ ({ \
69362 float64x1_t __ret_936; \71253 float64x1_t __ret_908; \
69363 float64x1_t __s0_936 = __p0_936; \71254 float64x1_t __s0_908 = __p0_908; \
69364 float64x2_t __s1_936 = __p1_936; \71255 float64x2_t __s1_908 = __p1_908; \
69365 float64_t __x_936 = vget_lane_f64(__s0_936, 0); \71256 float64_t __x_908 = vget_lane_f64(__s0_908, 0); \
69366 float64_t __y_936 = vgetq_lane_f64(__s1_936, __p2_936); \71257 float64_t __y_908 = vgetq_lane_f64(__s1_908, __p2_908); \
69367 float64_t __z_936 = vmulxd_f64(__x_936, __y_936); \71258 float64_t __z_908 = vmulxd_f64(__x_908, __y_908); \
69368 __ret_936 = vset_lane_f64(__z_936, __s0_936, 0); \71259 __ret_908 = vset_lane_f64(__z_908, __s0_908, 0); \
69369 __ret_936; \71260 __ret_908; \
69370})71261})
69371#else71262#else
69372#define vmulx_laneq_f64(__p0_937, __p1_937, __p2_937) __extension__ ({ \71263#define vmulx_laneq_f64(__p0_909, __p1_909, __p2_909) __extension__ ({ \
69373 float64x1_t __ret_937; \71264 float64x1_t __ret_909; \
69374 float64x1_t __s0_937 = __p0_937; \71265 float64x1_t __s0_909 = __p0_909; \
69375 float64x2_t __s1_937 = __p1_937; \71266 float64x2_t __s1_909 = __p1_909; \
69376 float64x2_t __rev1_937; __rev1_937 = __builtin_shufflevector(__s1_937, __s1_937, 1, 0); \71267 float64x2_t __rev1_909; __rev1_909 = __builtin_shufflevector(__s1_909, __s1_909, 1, 0); \
69377 float64_t __x_937 = vget_lane_f64(__s0_937, 0); \71268 float64_t __x_909 = vget_lane_f64(__s0_909, 0); \
69378 float64_t __y_937 = __noswap_vgetq_lane_f64(__rev1_937, __p2_937); \71269 float64_t __y_909 = __noswap_vgetq_lane_f64(__rev1_909, __p2_909); \
69379 float64_t __z_937 = vmulxd_f64(__x_937, __y_937); \71270 float64_t __z_909 = vmulxd_f64(__x_909, __y_909); \
69380 __ret_937 = vset_lane_f64(__z_937, __s0_937, 0); \71271 __ret_909 = vset_lane_f64(__z_909, __s0_909, 0); \
69381 __ret_937; \71272 __ret_909; \
69382})71273})
69383#endif71274#endif
6938471275
lib/include/arm_sme.h+322-94
...@@ -35,12 +35,6 @@ __ai bool __arm_has_sme(void) __arm_streaming_compatible {...@@ -35,12 +35,6 @@ __ai bool __arm_has_sme(void) __arm_streaming_compatible {
35 return x0 & (1ULL << 63);35 return x0 & (1ULL << 63);
36}36}
3737
38__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible {
39 uint64_t x0, x1;
40 __builtin_arm_get_sme_state(&x0, &x1);
41 return x0 & 1;
42}
43
44void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;38void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;
45void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;39void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;
46void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;40void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;
...@@ -48,6 +42,8 @@ void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;...@@ -48,6 +42,8 @@ void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;
4842
49__ai __attribute__((target("sme"))) void svundef_za(void) __arm_streaming_compatible __arm_out("za") { }43__ai __attribute__((target("sme"))) void svundef_za(void) __arm_streaming_compatible __arm_out("za") { }
5044
45__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme___arm_in_streaming_mode)))
46bool __arm_in_streaming_mode(void);
51__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m)))47__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m)))
52void svaddha_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);48void svaddha_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
53__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m)))49__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m)))
...@@ -604,6 +600,94 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_...@@ -604,6 +600,94 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_
604void svwrite_ver_za8_m(uint64_t, uint32_t, svbool_t, svuint8_t);600void svwrite_ver_za8_m(uint64_t, uint32_t, svbool_t, svuint8_t);
605__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m)))601__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m)))
606void svwrite_ver_za8_m(uint64_t, uint32_t, svbool_t, svint8_t);602void svwrite_ver_za8_m(uint64_t, uint32_t, svbool_t, svint8_t);
603__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
604void svadd_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
605__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
606void svadd_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
607__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
608void svsub_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
609__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
610void svsub_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
611__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
612void svadd_za16_vg1x2(uint32_t, svfloat16x2_t);
613__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
614void svadd_za16_vg1x4(uint32_t, svfloat16x4_t);
615__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
616void svsub_za16_vg1x2(uint32_t, svfloat16x2_t);
617__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
618void svsub_za16_vg1x4(uint32_t, svfloat16x4_t);
619__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
620void svadd_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
621__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
622void svadd_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
623__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
624void svmla_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
625__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
626void svmla_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
627__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
628void svmla_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
629__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
630void svmla_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
631__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
632void svmla_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
633__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
634void svmla_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
635__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
636void svmls_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
637__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
638void svmls_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
639__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
640void svmls_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
641__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
642void svmls_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
643__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
644void svmls_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
645__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
646void svmls_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
647__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
648void svmopa_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
649__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
650void svmops_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
651__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
652void svsub_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
653__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
654void svsub_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
655__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
656void svadd_za16_vg1x2(uint32_t, svbfloat16x2_t);
657__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
658void svadd_za16_vg1x4(uint32_t, svbfloat16x4_t);
659__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
660void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
661__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
662void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
663__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
664void svmla_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
665__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
666void svmla_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
667__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
668void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
669__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
670void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
671__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
672void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
673__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
674void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
675__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
676void svmls_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
677__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
678void svmls_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
679__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
680void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
681__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
682void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
683__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
684void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
685__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
686void svmops_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
687__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
688void svsub_za16_vg1x2(uint32_t, svbfloat16x2_t);
689__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
690void svsub_za16_vg1x4(uint32_t, svbfloat16x4_t);
607__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_f16_vg1x2)))691__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_f16_vg1x2)))
608void svmla_single_za16_f16_vg1x2(uint32_t, svfloat16x2_t, svfloat16_t);692void svmla_single_za16_f16_vg1x2(uint32_t, svfloat16x2_t, svfloat16_t);
609__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_f16_vg1x4)))693__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_f16_vg1x4)))
...@@ -660,22 +744,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_f16_m))...@@ -660,22 +744,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_f16_m))
660void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);744void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
661__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_f16_m)))745__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_f16_m)))
662void svmops_za16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);746void svmops_za16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
663__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
664void svadd_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
665__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
666void svadd_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
667__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
668void svsub_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
669__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
670void svsub_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
671__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
672void svadd_za16_vg1x2(uint32_t, svfloat16x2_t);
673__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
674void svadd_za16_vg1x4(uint32_t, svfloat16x4_t);
675__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
676void svsub_za16_vg1x2(uint32_t, svfloat16x2_t);
677__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
678void svsub_za16_vg1x4(uint32_t, svfloat16x4_t);
679__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m)))747__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m)))
680void svmopa_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);748void svmopa_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
681__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m)))749__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m)))
...@@ -684,6 +752,138 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m))...@@ -684,6 +752,138 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m))
684void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);752void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
685__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m)))753__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m)))
686void svmops_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);754void svmops_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
755__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x2_fpm)))
756void svdot_single_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
757__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x4_fpm)))
758void svdot_single_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
759__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x2_fpm)))
760void svdot_lane_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
761__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x4_fpm)))
762void svdot_lane_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
763__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x2_fpm)))
764void svdot_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
765__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x4_fpm)))
766void svdot_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
767__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x1_fpm)))
768void svmla_single_za16_mf8_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
769__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x2_fpm)))
770void svmla_single_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
771__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x4_fpm)))
772void svmla_single_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
773__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x1_fpm)))
774void svmla_lane_za16_mf8_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
775__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x2_fpm)))
776void svmla_lane_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
777__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x4_fpm)))
778void svmla_lane_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
779__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x2_fpm)))
780void svmla_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
781__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x4_fpm)))
782void svmla_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
783__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_mf8_m_fpm)))
784void svmopa_za16_mf8_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
785__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdot_lane_za16_mf8_vg1x2_fpm)))
786void svvdot_lane_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
787__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x2_fpm)))
788void svdot_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
789__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x4_fpm)))
790void svdot_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
791__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x2_fpm)))
792void svdot_lane_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
793__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x4_fpm)))
794void svdot_lane_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
795__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x2_fpm)))
796void svdot_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
797__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x4_fpm)))
798void svdot_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
799__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x1_fpm)))
800void svmla_za16_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
801__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x2_fpm)))
802void svmla_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
803__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x4_fpm)))
804void svmla_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
805__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x1_fpm)))
806void svmla_lane_za16_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
807__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x2_fpm)))
808void svmla_lane_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
809__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x4_fpm)))
810void svmla_lane_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
811__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x2_fpm)))
812void svmla_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
813__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x4_fpm)))
814void svmla_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
815__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_mf8_m_fpm)))
816void svmopa_za16_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
817__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdot_lane_za16_mf8_vg1x2_fpm)))
818void svvdot_lane_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
819__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x2_fpm)))
820void svdot_single_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
821__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x4_fpm)))
822void svdot_single_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
823__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x2_fpm)))
824void svdot_lane_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
825__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x4_fpm)))
826void svdot_lane_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
827__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x2_fpm)))
828void svdot_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
829__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x4_fpm)))
830void svdot_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
831__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x1_fpm)))
832void svmla_single_za32_mf8_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
833__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x2_fpm)))
834void svmla_single_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
835__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x4_fpm)))
836void svmla_single_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
837__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x1_fpm)))
838void svmla_lane_za32_mf8_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
839__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x2_fpm)))
840void svmla_lane_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
841__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x4_fpm)))
842void svmla_lane_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
843__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x2_fpm)))
844void svmla_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
845__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x4_fpm)))
846void svmla_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
847__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_mf8_m_fpm)))
848void svmopa_za32_mf8_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
849__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdotb_lane_za32_mf8_vg1x4_fpm)))
850void svvdotb_lane_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
851__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdott_lane_za32_mf8_vg1x4_fpm)))
852void svvdott_lane_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
853__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x2_fpm)))
854void svdot_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
855__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x4_fpm)))
856void svdot_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
857__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x2_fpm)))
858void svdot_lane_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
859__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x4_fpm)))
860void svdot_lane_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
861__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x2_fpm)))
862void svdot_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
863__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x4_fpm)))
864void svdot_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
865__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x1_fpm)))
866void svmla_za32_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
867__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x2_fpm)))
868void svmla_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
869__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x4_fpm)))
870void svmla_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
871__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x1_fpm)))
872void svmla_lane_za32_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
873__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x2_fpm)))
874void svmla_lane_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
875__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x4_fpm)))
876void svmla_lane_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
877__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x2_fpm)))
878void svmla_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
879__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x4_fpm)))
880void svmla_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
881__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_mf8_m_fpm)))
882void svmopa_za32_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
883__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdotb_lane_za32_mf8_vg1x4_fpm)))
884void svvdotb_lane_za32_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
885__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdott_lane_za32_mf8_vg1x4_fpm)))
886void svvdott_lane_za32_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
687__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m)))887__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m)))
688void svaddha_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);888void svaddha_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
689__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m)))889__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m)))
...@@ -732,6 +932,106 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m...@@ -732,6 +932,106 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m
732void svusmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);932void svusmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
733__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m)))933__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m)))
734void svusmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);934void svusmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
935__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti4_zt_u8_x4)))
936svuint8x4_t svluti4_zt_u8_x4(uint64_t, svuint8x2_t);
937__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti4_zt_s8_x4)))
938svint8x4_t svluti4_zt_s8_x4(uint64_t, svuint8x2_t);
939__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u8)))
940void svwrite_lane_zt_u8(uint64_t, svuint8_t, uint64_t);
941__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u32)))
942void svwrite_lane_zt_u32(uint64_t, svuint32_t, uint64_t);
943__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u64)))
944void svwrite_lane_zt_u64(uint64_t, svuint64_t, uint64_t);
945__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u16)))
946void svwrite_lane_zt_u16(uint64_t, svuint16_t, uint64_t);
947__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_bf16)))
948void svwrite_lane_zt_bf16(uint64_t, svbfloat16_t, uint64_t);
949__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s8)))
950void svwrite_lane_zt_s8(uint64_t, svint8_t, uint64_t);
951__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f64)))
952void svwrite_lane_zt_f64(uint64_t, svfloat64_t, uint64_t);
953__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f32)))
954void svwrite_lane_zt_f32(uint64_t, svfloat32_t, uint64_t);
955__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f16)))
956void svwrite_lane_zt_f16(uint64_t, svfloat16_t, uint64_t);
957__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s32)))
958void svwrite_lane_zt_s32(uint64_t, svint32_t, uint64_t);
959__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s64)))
960void svwrite_lane_zt_s64(uint64_t, svint64_t, uint64_t);
961__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s16)))
962void svwrite_lane_zt_s16(uint64_t, svint16_t, uint64_t);
963__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u8)))
964void svwrite_zt_u8(uint64_t, svuint8_t);
965__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u32)))
966void svwrite_zt_u32(uint64_t, svuint32_t);
967__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u64)))
968void svwrite_zt_u64(uint64_t, svuint64_t);
969__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u16)))
970void svwrite_zt_u16(uint64_t, svuint16_t);
971__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_bf16)))
972void svwrite_zt_bf16(uint64_t, svbfloat16_t);
973__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s8)))
974void svwrite_zt_s8(uint64_t, svint8_t);
975__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f64)))
976void svwrite_zt_f64(uint64_t, svfloat64_t);
977__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f32)))
978void svwrite_zt_f32(uint64_t, svfloat32_t);
979__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f16)))
980void svwrite_zt_f16(uint64_t, svfloat16_t);
981__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s32)))
982void svwrite_zt_s32(uint64_t, svint32_t);
983__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s64)))
984void svwrite_zt_s64(uint64_t, svint64_t);
985__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s16)))
986void svwrite_zt_s16(uint64_t, svint16_t);
987__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u8)))
988void svwrite_lane_zt(uint64_t, svuint8_t, uint64_t);
989__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u32)))
990void svwrite_lane_zt(uint64_t, svuint32_t, uint64_t);
991__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u64)))
992void svwrite_lane_zt(uint64_t, svuint64_t, uint64_t);
993__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u16)))
994void svwrite_lane_zt(uint64_t, svuint16_t, uint64_t);
995__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_bf16)))
996void svwrite_lane_zt(uint64_t, svbfloat16_t, uint64_t);
997__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s8)))
998void svwrite_lane_zt(uint64_t, svint8_t, uint64_t);
999__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f64)))
1000void svwrite_lane_zt(uint64_t, svfloat64_t, uint64_t);
1001__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f32)))
1002void svwrite_lane_zt(uint64_t, svfloat32_t, uint64_t);
1003__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f16)))
1004void svwrite_lane_zt(uint64_t, svfloat16_t, uint64_t);
1005__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s32)))
1006void svwrite_lane_zt(uint64_t, svint32_t, uint64_t);
1007__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s64)))
1008void svwrite_lane_zt(uint64_t, svint64_t, uint64_t);
1009__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s16)))
1010void svwrite_lane_zt(uint64_t, svint16_t, uint64_t);
1011__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u8)))
1012void svwrite_zt(uint64_t, svuint8_t);
1013__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u32)))
1014void svwrite_zt(uint64_t, svuint32_t);
1015__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u64)))
1016void svwrite_zt(uint64_t, svuint64_t);
1017__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u16)))
1018void svwrite_zt(uint64_t, svuint16_t);
1019__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_bf16)))
1020void svwrite_zt(uint64_t, svbfloat16_t);
1021__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s8)))
1022void svwrite_zt(uint64_t, svint8_t);
1023__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f64)))
1024void svwrite_zt(uint64_t, svfloat64_t);
1025__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f32)))
1026void svwrite_zt(uint64_t, svfloat32_t);
1027__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f16)))
1028void svwrite_zt(uint64_t, svfloat16_t);
1029__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s32)))
1030void svwrite_zt(uint64_t, svint32_t);
1031__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s64)))
1032void svwrite_zt(uint64_t, svint64_t);
1033__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s16)))
1034void svwrite_zt(uint64_t, svint16_t);
735__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_write_single_za32_u32_vg1x2)))1035__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_write_single_za32_u32_vg1x2)))
736void svadd_write_single_za32_u32_vg1x2(uint32_t, svuint32x2_t, svuint32_t);1036void svadd_write_single_za32_u32_vg1x2(uint32_t, svuint32x2_t, svuint32_t);
737__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_write_single_za32_s32_vg1x2)))1037__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_write_single_za32_s32_vg1x2)))
...@@ -2138,78 +2438,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_za8_u8_vg1x...@@ -2138,78 +2438,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_za8_u8_vg1x
2138void svwrite_za8_vg1x4(uint32_t, svuint8x4_t);2438void svwrite_za8_vg1x4(uint32_t, svuint8x4_t);
2139__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_za8_s8_vg1x4)))2439__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_za8_s8_vg1x4)))
2140void svwrite_za8_vg1x4(uint32_t, svint8x4_t);2440void svwrite_za8_vg1x4(uint32_t, svint8x4_t);
2141__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
2142void svadd_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
2143__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
2144void svadd_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
2145__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
2146void svmla_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2147__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
2148void svmla_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2149__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
2150void svmla_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2151__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
2152void svmla_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2153__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
2154void svmla_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2155__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
2156void svmla_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2157__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
2158void svmls_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2159__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
2160void svmls_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2161__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
2162void svmls_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2163__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
2164void svmls_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2165__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
2166void svmls_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2167__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
2168void svmls_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2169__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
2170void svmopa_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2171__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
2172void svmops_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2173__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
2174void svsub_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
2175__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
2176void svsub_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
2177__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
2178void svadd_za16_vg1x2(uint32_t, svbfloat16x2_t);
2179__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
2180void svadd_za16_vg1x4(uint32_t, svbfloat16x4_t);
2181__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
2182void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2183__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
2184void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2185__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
2186void svmla_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2187__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
2188void svmla_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2189__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
2190void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2191__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
2192void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2193__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
2194void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2195__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
2196void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2197__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
2198void svmls_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2199__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
2200void svmls_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2201__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
2202void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2203__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
2204void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2205__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
2206void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2207__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
2208void svmops_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2209__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
2210void svsub_za16_vg1x2(uint32_t, svbfloat16x2_t);
2211__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
2212void svsub_za16_vg1x4(uint32_t, svbfloat16x4_t);
2213__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za64_f64_vg1x2)))2441__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za64_f64_vg1x2)))
2214void svadd_za64_f64_vg1x2(uint32_t, svfloat64x2_t);2442void svadd_za64_f64_vg1x2(uint32_t, svfloat64x2_t);
2215__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za64_f64_vg1x4)))2443__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za64_f64_vg1x4)))
lib/include/arm_sve.h+1105-84
...@@ -38,6 +38,8 @@ typedef __SVFloat16_t svfloat16_t;...@@ -38,6 +38,8 @@ typedef __SVFloat16_t svfloat16_t;
38typedef __SVBfloat16_t svbfloat16_t;38typedef __SVBfloat16_t svbfloat16_t;
39#include <arm_bf16.h>39#include <arm_bf16.h>
40#include <arm_vector_types.h>40#include <arm_vector_types.h>
41typedef __SVMfloat8_t svmfloat8_t;
42
41typedef __SVFloat32_t svfloat32_t;43typedef __SVFloat32_t svfloat32_t;
42typedef __SVFloat64_t svfloat64_t;44typedef __SVFloat64_t svfloat64_t;
43typedef __clang_svint8x2_t svint8x2_t;45typedef __clang_svint8x2_t svint8x2_t;
...@@ -80,6 +82,9 @@ typedef __clang_svboolx4_t svboolx4_t;...@@ -80,6 +82,9 @@ typedef __clang_svboolx4_t svboolx4_t;
80typedef __clang_svbfloat16x2_t svbfloat16x2_t;82typedef __clang_svbfloat16x2_t svbfloat16x2_t;
81typedef __clang_svbfloat16x3_t svbfloat16x3_t;83typedef __clang_svbfloat16x3_t svbfloat16x3_t;
82typedef __clang_svbfloat16x4_t svbfloat16x4_t;84typedef __clang_svbfloat16x4_t svbfloat16x4_t;
85typedef __clang_svmfloat8x2_t svmfloat8x2_t;
86typedef __clang_svmfloat8x3_t svmfloat8x3_t;
87typedef __clang_svmfloat8x4_t svmfloat8x4_t;
83typedef __SVCount_t svcount_t;88typedef __SVCount_t svcount_t;
8489
85enum svpattern90enum svpattern
...@@ -128,6 +133,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8))...@@ -128,6 +133,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8))
128svint8_t svreinterpret_s8_s8(svint8_t op);133svint8_t svreinterpret_s8_s8(svint8_t op);
129__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8)))134__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8)))
130svint8_t svreinterpret_s8_u8(svuint8_t op);135svint8_t svreinterpret_s8_u8(svuint8_t op);
136__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8)))
137svint8_t svreinterpret_s8_mf8(svmfloat8_t op);
131__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16)))138__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16)))
132svint8_t svreinterpret_s8_s16(svint16_t op);139svint8_t svreinterpret_s8_s16(svint16_t op);
133__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16)))140__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16)))
...@@ -152,6 +159,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8))...@@ -152,6 +159,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8))
152svuint8_t svreinterpret_u8_s8(svint8_t op);159svuint8_t svreinterpret_u8_s8(svint8_t op);
153__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8)))160__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8)))
154svuint8_t svreinterpret_u8_u8(svuint8_t op);161svuint8_t svreinterpret_u8_u8(svuint8_t op);
162__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8)))
163svuint8_t svreinterpret_u8_mf8(svmfloat8_t op);
155__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16)))164__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16)))
156svuint8_t svreinterpret_u8_s16(svint16_t op);165svuint8_t svreinterpret_u8_s16(svint16_t op);
157__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16)))166__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16)))
...@@ -172,10 +181,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32)...@@ -172,10 +181,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32)
172svuint8_t svreinterpret_u8_f32(svfloat32_t op);181svuint8_t svreinterpret_u8_f32(svfloat32_t op);
173__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64)))182__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64)))
174svuint8_t svreinterpret_u8_f64(svfloat64_t op);183svuint8_t svreinterpret_u8_f64(svfloat64_t op);
184__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8)))
185svmfloat8_t svreinterpret_mf8_s8(svint8_t op);
186__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8)))
187svmfloat8_t svreinterpret_mf8_u8(svuint8_t op);
188__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8)))
189svmfloat8_t svreinterpret_mf8_mf8(svmfloat8_t op);
190__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16)))
191svmfloat8_t svreinterpret_mf8_s16(svint16_t op);
192__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16)))
193svmfloat8_t svreinterpret_mf8_u16(svuint16_t op);
194__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32)))
195svmfloat8_t svreinterpret_mf8_s32(svint32_t op);
196__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32)))
197svmfloat8_t svreinterpret_mf8_u32(svuint32_t op);
198__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64)))
199svmfloat8_t svreinterpret_mf8_s64(svint64_t op);
200__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64)))
201svmfloat8_t svreinterpret_mf8_u64(svuint64_t op);
202__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16)))
203svmfloat8_t svreinterpret_mf8_f16(svfloat16_t op);
204__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16)))
205svmfloat8_t svreinterpret_mf8_bf16(svbfloat16_t op);
206__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32)))
207svmfloat8_t svreinterpret_mf8_f32(svfloat32_t op);
208__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64)))
209svmfloat8_t svreinterpret_mf8_f64(svfloat64_t op);
175__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8)))210__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8)))
176svint16_t svreinterpret_s16_s8(svint8_t op);211svint16_t svreinterpret_s16_s8(svint8_t op);
177__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8)))212__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8)))
178svint16_t svreinterpret_s16_u8(svuint8_t op);213svint16_t svreinterpret_s16_u8(svuint8_t op);
214__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8)))
215svint16_t svreinterpret_s16_mf8(svmfloat8_t op);
179__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16)))216__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16)))
180svint16_t svreinterpret_s16_s16(svint16_t op);217svint16_t svreinterpret_s16_s16(svint16_t op);
181__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16)))218__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16)))
...@@ -200,6 +237,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8)...@@ -200,6 +237,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8)
200svuint16_t svreinterpret_u16_s8(svint8_t op);237svuint16_t svreinterpret_u16_s8(svint8_t op);
201__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8)))238__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8)))
202svuint16_t svreinterpret_u16_u8(svuint8_t op);239svuint16_t svreinterpret_u16_u8(svuint8_t op);
240__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8)))
241svuint16_t svreinterpret_u16_mf8(svmfloat8_t op);
203__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16)))242__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16)))
204svuint16_t svreinterpret_u16_s16(svint16_t op);243svuint16_t svreinterpret_u16_s16(svint16_t op);
205__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16)))244__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16)))
...@@ -224,6 +263,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8)...@@ -224,6 +263,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8)
224svint32_t svreinterpret_s32_s8(svint8_t op);263svint32_t svreinterpret_s32_s8(svint8_t op);
225__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8)))264__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8)))
226svint32_t svreinterpret_s32_u8(svuint8_t op);265svint32_t svreinterpret_s32_u8(svuint8_t op);
266__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8)))
267svint32_t svreinterpret_s32_mf8(svmfloat8_t op);
227__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16)))268__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16)))
228svint32_t svreinterpret_s32_s16(svint16_t op);269svint32_t svreinterpret_s32_s16(svint16_t op);
229__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16)))270__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16)))
...@@ -248,6 +289,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8)...@@ -248,6 +289,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8)
248svuint32_t svreinterpret_u32_s8(svint8_t op);289svuint32_t svreinterpret_u32_s8(svint8_t op);
249__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8)))290__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8)))
250svuint32_t svreinterpret_u32_u8(svuint8_t op);291svuint32_t svreinterpret_u32_u8(svuint8_t op);
292__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8)))
293svuint32_t svreinterpret_u32_mf8(svmfloat8_t op);
251__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16)))294__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16)))
252svuint32_t svreinterpret_u32_s16(svint16_t op);295svuint32_t svreinterpret_u32_s16(svint16_t op);
253__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16)))296__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16)))
...@@ -272,6 +315,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8)...@@ -272,6 +315,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8)
272svint64_t svreinterpret_s64_s8(svint8_t op);315svint64_t svreinterpret_s64_s8(svint8_t op);
273__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8)))316__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8)))
274svint64_t svreinterpret_s64_u8(svuint8_t op);317svint64_t svreinterpret_s64_u8(svuint8_t op);
318__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8)))
319svint64_t svreinterpret_s64_mf8(svmfloat8_t op);
275__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16)))320__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16)))
276svint64_t svreinterpret_s64_s16(svint16_t op);321svint64_t svreinterpret_s64_s16(svint16_t op);
277__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16)))322__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16)))
...@@ -296,6 +341,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8)...@@ -296,6 +341,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8)
296svuint64_t svreinterpret_u64_s8(svint8_t op);341svuint64_t svreinterpret_u64_s8(svint8_t op);
297__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8)))342__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8)))
298svuint64_t svreinterpret_u64_u8(svuint8_t op);343svuint64_t svreinterpret_u64_u8(svuint8_t op);
344__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8)))
345svuint64_t svreinterpret_u64_mf8(svmfloat8_t op);
299__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16)))346__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16)))
300svuint64_t svreinterpret_u64_s16(svint16_t op);347svuint64_t svreinterpret_u64_s16(svint16_t op);
301__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16)))348__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16)))
...@@ -320,6 +367,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8)...@@ -320,6 +367,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8)
320svfloat16_t svreinterpret_f16_s8(svint8_t op);367svfloat16_t svreinterpret_f16_s8(svint8_t op);
321__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8)))368__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8)))
322svfloat16_t svreinterpret_f16_u8(svuint8_t op);369svfloat16_t svreinterpret_f16_u8(svuint8_t op);
370__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8)))
371svfloat16_t svreinterpret_f16_mf8(svmfloat8_t op);
323__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16)))372__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16)))
324svfloat16_t svreinterpret_f16_s16(svint16_t op);373svfloat16_t svreinterpret_f16_s16(svint16_t op);
325__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16)))374__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16)))
...@@ -344,6 +393,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -344,6 +393,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
344svbfloat16_t svreinterpret_bf16_s8(svint8_t op);393svbfloat16_t svreinterpret_bf16_s8(svint8_t op);
345__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8)))394__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8)))
346svbfloat16_t svreinterpret_bf16_u8(svuint8_t op);395svbfloat16_t svreinterpret_bf16_u8(svuint8_t op);
396__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8)))
397svbfloat16_t svreinterpret_bf16_mf8(svmfloat8_t op);
347__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16)))398__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16)))
348svbfloat16_t svreinterpret_bf16_s16(svint16_t op);399svbfloat16_t svreinterpret_bf16_s16(svint16_t op);
349__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16)))400__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16)))
...@@ -368,6 +419,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8)...@@ -368,6 +419,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8)
368svfloat32_t svreinterpret_f32_s8(svint8_t op);419svfloat32_t svreinterpret_f32_s8(svint8_t op);
369__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8)))420__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8)))
370svfloat32_t svreinterpret_f32_u8(svuint8_t op);421svfloat32_t svreinterpret_f32_u8(svuint8_t op);
422__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8)))
423svfloat32_t svreinterpret_f32_mf8(svmfloat8_t op);
371__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16)))424__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16)))
372svfloat32_t svreinterpret_f32_s16(svint16_t op);425svfloat32_t svreinterpret_f32_s16(svint16_t op);
373__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16)))426__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16)))
...@@ -392,6 +445,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8)...@@ -392,6 +445,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8)
392svfloat64_t svreinterpret_f64_s8(svint8_t op);445svfloat64_t svreinterpret_f64_s8(svint8_t op);
393__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8)))446__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8)))
394svfloat64_t svreinterpret_f64_u8(svuint8_t op);447svfloat64_t svreinterpret_f64_u8(svuint8_t op);
448__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8)))
449svfloat64_t svreinterpret_f64_mf8(svmfloat8_t op);
395__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16)))450__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16)))
396svfloat64_t svreinterpret_f64_s16(svint16_t op);451svfloat64_t svreinterpret_f64_s16(svint16_t op);
397__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16)))452__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16)))
...@@ -416,6 +471,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8))...@@ -416,6 +471,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8))
416svint8_t svreinterpret_s8(svint8_t op);471svint8_t svreinterpret_s8(svint8_t op);
417__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8)))472__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8)))
418svint8_t svreinterpret_s8(svuint8_t op);473svint8_t svreinterpret_s8(svuint8_t op);
474__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8)))
475svint8_t svreinterpret_s8(svmfloat8_t op);
419__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16)))476__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16)))
420svint8_t svreinterpret_s8(svint16_t op);477svint8_t svreinterpret_s8(svint16_t op);
421__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16)))478__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16)))
...@@ -440,6 +497,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8))...@@ -440,6 +497,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8))
440svuint8_t svreinterpret_u8(svint8_t op);497svuint8_t svreinterpret_u8(svint8_t op);
441__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8)))498__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8)))
442svuint8_t svreinterpret_u8(svuint8_t op);499svuint8_t svreinterpret_u8(svuint8_t op);
500__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8)))
501svuint8_t svreinterpret_u8(svmfloat8_t op);
443__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16)))502__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16)))
444svuint8_t svreinterpret_u8(svint16_t op);503svuint8_t svreinterpret_u8(svint16_t op);
445__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16)))504__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16)))
...@@ -460,10 +519,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32)...@@ -460,10 +519,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32)
460svuint8_t svreinterpret_u8(svfloat32_t op);519svuint8_t svreinterpret_u8(svfloat32_t op);
461__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64)))520__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64)))
462svuint8_t svreinterpret_u8(svfloat64_t op);521svuint8_t svreinterpret_u8(svfloat64_t op);
522__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8)))
523svmfloat8_t svreinterpret_mf8(svint8_t op);
524__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8)))
525svmfloat8_t svreinterpret_mf8(svuint8_t op);
526__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8)))
527svmfloat8_t svreinterpret_mf8(svmfloat8_t op);
528__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16)))
529svmfloat8_t svreinterpret_mf8(svint16_t op);
530__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16)))
531svmfloat8_t svreinterpret_mf8(svuint16_t op);
532__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32)))
533svmfloat8_t svreinterpret_mf8(svint32_t op);
534__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32)))
535svmfloat8_t svreinterpret_mf8(svuint32_t op);
536__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64)))
537svmfloat8_t svreinterpret_mf8(svint64_t op);
538__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64)))
539svmfloat8_t svreinterpret_mf8(svuint64_t op);
540__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16)))
541svmfloat8_t svreinterpret_mf8(svfloat16_t op);
542__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16)))
543svmfloat8_t svreinterpret_mf8(svbfloat16_t op);
544__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32)))
545svmfloat8_t svreinterpret_mf8(svfloat32_t op);
546__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64)))
547svmfloat8_t svreinterpret_mf8(svfloat64_t op);
463__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8)))548__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8)))
464svint16_t svreinterpret_s16(svint8_t op);549svint16_t svreinterpret_s16(svint8_t op);
465__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8)))550__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8)))
466svint16_t svreinterpret_s16(svuint8_t op);551svint16_t svreinterpret_s16(svuint8_t op);
552__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8)))
553svint16_t svreinterpret_s16(svmfloat8_t op);
467__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16)))554__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16)))
468svint16_t svreinterpret_s16(svint16_t op);555svint16_t svreinterpret_s16(svint16_t op);
469__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16)))556__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16)))
...@@ -488,6 +575,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8)...@@ -488,6 +575,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8)
488svuint16_t svreinterpret_u16(svint8_t op);575svuint16_t svreinterpret_u16(svint8_t op);
489__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8)))576__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8)))
490svuint16_t svreinterpret_u16(svuint8_t op);577svuint16_t svreinterpret_u16(svuint8_t op);
578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8)))
579svuint16_t svreinterpret_u16(svmfloat8_t op);
491__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16)))580__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16)))
492svuint16_t svreinterpret_u16(svint16_t op);581svuint16_t svreinterpret_u16(svint16_t op);
493__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16)))582__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16)))
...@@ -512,6 +601,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8)...@@ -512,6 +601,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8)
512svint32_t svreinterpret_s32(svint8_t op);601svint32_t svreinterpret_s32(svint8_t op);
513__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8)))602__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8)))
514svint32_t svreinterpret_s32(svuint8_t op);603svint32_t svreinterpret_s32(svuint8_t op);
604__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8)))
605svint32_t svreinterpret_s32(svmfloat8_t op);
515__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16)))606__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16)))
516svint32_t svreinterpret_s32(svint16_t op);607svint32_t svreinterpret_s32(svint16_t op);
517__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16)))608__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16)))
...@@ -536,6 +627,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8)...@@ -536,6 +627,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8)
536svuint32_t svreinterpret_u32(svint8_t op);627svuint32_t svreinterpret_u32(svint8_t op);
537__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8)))628__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8)))
538svuint32_t svreinterpret_u32(svuint8_t op);629svuint32_t svreinterpret_u32(svuint8_t op);
630__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8)))
631svuint32_t svreinterpret_u32(svmfloat8_t op);
539__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16)))632__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16)))
540svuint32_t svreinterpret_u32(svint16_t op);633svuint32_t svreinterpret_u32(svint16_t op);
541__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16)))634__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16)))
...@@ -560,6 +653,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8)...@@ -560,6 +653,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8)
560svint64_t svreinterpret_s64(svint8_t op);653svint64_t svreinterpret_s64(svint8_t op);
561__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8)))654__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8)))
562svint64_t svreinterpret_s64(svuint8_t op);655svint64_t svreinterpret_s64(svuint8_t op);
656__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8)))
657svint64_t svreinterpret_s64(svmfloat8_t op);
563__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16)))658__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16)))
564svint64_t svreinterpret_s64(svint16_t op);659svint64_t svreinterpret_s64(svint16_t op);
565__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16)))660__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16)))
...@@ -584,6 +679,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8)...@@ -584,6 +679,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8)
584svuint64_t svreinterpret_u64(svint8_t op);679svuint64_t svreinterpret_u64(svint8_t op);
585__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8)))680__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8)))
586svuint64_t svreinterpret_u64(svuint8_t op);681svuint64_t svreinterpret_u64(svuint8_t op);
682__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8)))
683svuint64_t svreinterpret_u64(svmfloat8_t op);
587__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16)))684__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16)))
588svuint64_t svreinterpret_u64(svint16_t op);685svuint64_t svreinterpret_u64(svint16_t op);
589__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16)))686__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16)))
...@@ -608,6 +705,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8)...@@ -608,6 +705,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8)
608svfloat16_t svreinterpret_f16(svint8_t op);705svfloat16_t svreinterpret_f16(svint8_t op);
609__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8)))706__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8)))
610svfloat16_t svreinterpret_f16(svuint8_t op);707svfloat16_t svreinterpret_f16(svuint8_t op);
708__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8)))
709svfloat16_t svreinterpret_f16(svmfloat8_t op);
611__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16)))710__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16)))
612svfloat16_t svreinterpret_f16(svint16_t op);711svfloat16_t svreinterpret_f16(svint16_t op);
613__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16)))712__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16)))
...@@ -632,6 +731,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -632,6 +731,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
632svbfloat16_t svreinterpret_bf16(svint8_t op);731svbfloat16_t svreinterpret_bf16(svint8_t op);
633__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8)))732__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8)))
634svbfloat16_t svreinterpret_bf16(svuint8_t op);733svbfloat16_t svreinterpret_bf16(svuint8_t op);
734__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8)))
735svbfloat16_t svreinterpret_bf16(svmfloat8_t op);
635__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16)))736__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16)))
636svbfloat16_t svreinterpret_bf16(svint16_t op);737svbfloat16_t svreinterpret_bf16(svint16_t op);
637__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16)))738__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16)))
...@@ -656,6 +757,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8)...@@ -656,6 +757,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8)
656svfloat32_t svreinterpret_f32(svint8_t op);757svfloat32_t svreinterpret_f32(svint8_t op);
657__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8)))758__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8)))
658svfloat32_t svreinterpret_f32(svuint8_t op);759svfloat32_t svreinterpret_f32(svuint8_t op);
760__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8)))
761svfloat32_t svreinterpret_f32(svmfloat8_t op);
659__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16)))762__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16)))
660svfloat32_t svreinterpret_f32(svint16_t op);763svfloat32_t svreinterpret_f32(svint16_t op);
661__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16)))764__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16)))
...@@ -680,6 +783,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8)...@@ -680,6 +783,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8)
680svfloat64_t svreinterpret_f64(svint8_t op);783svfloat64_t svreinterpret_f64(svint8_t op);
681__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8)))784__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8)))
682svfloat64_t svreinterpret_f64(svuint8_t op);785svfloat64_t svreinterpret_f64(svuint8_t op);
786__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8)))
787svfloat64_t svreinterpret_f64(svmfloat8_t op);
683__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16)))788__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16)))
684svfloat64_t svreinterpret_f64(svint16_t op);789svfloat64_t svreinterpret_f64(svint16_t op);
685__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16)))790__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16)))
...@@ -704,6 +809,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x...@@ -704,6 +809,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
704svint8x2_t svreinterpret_s8_s8_x2(svint8x2_t op);809svint8x2_t svreinterpret_s8_s8_x2(svint8x2_t op);
705__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x2)))810__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x2)))
706svint8x2_t svreinterpret_s8_u8_x2(svuint8x2_t op);811svint8x2_t svreinterpret_s8_u8_x2(svuint8x2_t op);
812__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x2)))
813svint8x2_t svreinterpret_s8_mf8_x2(svmfloat8x2_t op);
707__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x2)))814__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x2)))
708svint8x2_t svreinterpret_s8_s16_x2(svint16x2_t op);815svint8x2_t svreinterpret_s8_s16_x2(svint16x2_t op);
709__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x2)))816__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x2)))
...@@ -728,6 +835,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x...@@ -728,6 +835,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
728svuint8x2_t svreinterpret_u8_s8_x2(svint8x2_t op);835svuint8x2_t svreinterpret_u8_s8_x2(svint8x2_t op);
729__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x2)))836__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x2)))
730svuint8x2_t svreinterpret_u8_u8_x2(svuint8x2_t op);837svuint8x2_t svreinterpret_u8_u8_x2(svuint8x2_t op);
838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x2)))
839svuint8x2_t svreinterpret_u8_mf8_x2(svmfloat8x2_t op);
731__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x2)))840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x2)))
732svuint8x2_t svreinterpret_u8_s16_x2(svint16x2_t op);841svuint8x2_t svreinterpret_u8_s16_x2(svint16x2_t op);
733__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x2)))842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x2)))
...@@ -748,10 +857,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_...@@ -748,10 +857,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
748svuint8x2_t svreinterpret_u8_f32_x2(svfloat32x2_t op);857svuint8x2_t svreinterpret_u8_f32_x2(svfloat32x2_t op);
749__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x2)))858__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x2)))
750svuint8x2_t svreinterpret_u8_f64_x2(svfloat64x2_t op);859svuint8x2_t svreinterpret_u8_f64_x2(svfloat64x2_t op);
860__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x2)))
861svmfloat8x2_t svreinterpret_mf8_s8_x2(svint8x2_t op);
862__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x2)))
863svmfloat8x2_t svreinterpret_mf8_u8_x2(svuint8x2_t op);
864__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x2)))
865svmfloat8x2_t svreinterpret_mf8_mf8_x2(svmfloat8x2_t op);
866__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x2)))
867svmfloat8x2_t svreinterpret_mf8_s16_x2(svint16x2_t op);
868__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x2)))
869svmfloat8x2_t svreinterpret_mf8_u16_x2(svuint16x2_t op);
870__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x2)))
871svmfloat8x2_t svreinterpret_mf8_s32_x2(svint32x2_t op);
872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x2)))
873svmfloat8x2_t svreinterpret_mf8_u32_x2(svuint32x2_t op);
874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x2)))
875svmfloat8x2_t svreinterpret_mf8_s64_x2(svint64x2_t op);
876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x2)))
877svmfloat8x2_t svreinterpret_mf8_u64_x2(svuint64x2_t op);
878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x2)))
879svmfloat8x2_t svreinterpret_mf8_f16_x2(svfloat16x2_t op);
880__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x2)))
881svmfloat8x2_t svreinterpret_mf8_bf16_x2(svbfloat16x2_t op);
882__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x2)))
883svmfloat8x2_t svreinterpret_mf8_f32_x2(svfloat32x2_t op);
884__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x2)))
885svmfloat8x2_t svreinterpret_mf8_f64_x2(svfloat64x2_t op);
751__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x2)))886__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x2)))
752svint16x2_t svreinterpret_s16_s8_x2(svint8x2_t op);887svint16x2_t svreinterpret_s16_s8_x2(svint8x2_t op);
753__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x2)))888__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x2)))
754svint16x2_t svreinterpret_s16_u8_x2(svuint8x2_t op);889svint16x2_t svreinterpret_s16_u8_x2(svuint8x2_t op);
890__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x2)))
891svint16x2_t svreinterpret_s16_mf8_x2(svmfloat8x2_t op);
755__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x2)))892__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x2)))
756svint16x2_t svreinterpret_s16_s16_x2(svint16x2_t op);893svint16x2_t svreinterpret_s16_s16_x2(svint16x2_t op);
757__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x2)))894__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x2)))
...@@ -776,6 +913,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_...@@ -776,6 +913,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
776svuint16x2_t svreinterpret_u16_s8_x2(svint8x2_t op);913svuint16x2_t svreinterpret_u16_s8_x2(svint8x2_t op);
777__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x2)))914__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x2)))
778svuint16x2_t svreinterpret_u16_u8_x2(svuint8x2_t op);915svuint16x2_t svreinterpret_u16_u8_x2(svuint8x2_t op);
916__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x2)))
917svuint16x2_t svreinterpret_u16_mf8_x2(svmfloat8x2_t op);
779__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x2)))918__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x2)))
780svuint16x2_t svreinterpret_u16_s16_x2(svint16x2_t op);919svuint16x2_t svreinterpret_u16_s16_x2(svint16x2_t op);
781__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x2)))920__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x2)))
...@@ -800,6 +939,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_...@@ -800,6 +939,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
800svint32x2_t svreinterpret_s32_s8_x2(svint8x2_t op);939svint32x2_t svreinterpret_s32_s8_x2(svint8x2_t op);
801__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x2)))940__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x2)))
802svint32x2_t svreinterpret_s32_u8_x2(svuint8x2_t op);941svint32x2_t svreinterpret_s32_u8_x2(svuint8x2_t op);
942__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x2)))
943svint32x2_t svreinterpret_s32_mf8_x2(svmfloat8x2_t op);
803__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x2)))944__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x2)))
804svint32x2_t svreinterpret_s32_s16_x2(svint16x2_t op);945svint32x2_t svreinterpret_s32_s16_x2(svint16x2_t op);
805__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x2)))946__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x2)))
...@@ -824,6 +965,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_...@@ -824,6 +965,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
824svuint32x2_t svreinterpret_u32_s8_x2(svint8x2_t op);965svuint32x2_t svreinterpret_u32_s8_x2(svint8x2_t op);
825__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x2)))966__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x2)))
826svuint32x2_t svreinterpret_u32_u8_x2(svuint8x2_t op);967svuint32x2_t svreinterpret_u32_u8_x2(svuint8x2_t op);
968__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x2)))
969svuint32x2_t svreinterpret_u32_mf8_x2(svmfloat8x2_t op);
827__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x2)))970__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x2)))
828svuint32x2_t svreinterpret_u32_s16_x2(svint16x2_t op);971svuint32x2_t svreinterpret_u32_s16_x2(svint16x2_t op);
829__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x2)))972__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x2)))
...@@ -848,6 +991,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_...@@ -848,6 +991,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
848svint64x2_t svreinterpret_s64_s8_x2(svint8x2_t op);991svint64x2_t svreinterpret_s64_s8_x2(svint8x2_t op);
849__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x2)))992__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x2)))
850svint64x2_t svreinterpret_s64_u8_x2(svuint8x2_t op);993svint64x2_t svreinterpret_s64_u8_x2(svuint8x2_t op);
994__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x2)))
995svint64x2_t svreinterpret_s64_mf8_x2(svmfloat8x2_t op);
851__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x2)))996__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x2)))
852svint64x2_t svreinterpret_s64_s16_x2(svint16x2_t op);997svint64x2_t svreinterpret_s64_s16_x2(svint16x2_t op);
853__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x2)))998__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x2)))
...@@ -872,6 +1017,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_...@@ -872,6 +1017,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
872svuint64x2_t svreinterpret_u64_s8_x2(svint8x2_t op);1017svuint64x2_t svreinterpret_u64_s8_x2(svint8x2_t op);
873__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x2)))1018__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x2)))
874svuint64x2_t svreinterpret_u64_u8_x2(svuint8x2_t op);1019svuint64x2_t svreinterpret_u64_u8_x2(svuint8x2_t op);
1020__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x2)))
1021svuint64x2_t svreinterpret_u64_mf8_x2(svmfloat8x2_t op);
875__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x2)))1022__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x2)))
876svuint64x2_t svreinterpret_u64_s16_x2(svint16x2_t op);1023svuint64x2_t svreinterpret_u64_s16_x2(svint16x2_t op);
877__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x2)))1024__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x2)))
...@@ -896,6 +1043,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_...@@ -896,6 +1043,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
896svfloat16x2_t svreinterpret_f16_s8_x2(svint8x2_t op);1043svfloat16x2_t svreinterpret_f16_s8_x2(svint8x2_t op);
897__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x2)))1044__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x2)))
898svfloat16x2_t svreinterpret_f16_u8_x2(svuint8x2_t op);1045svfloat16x2_t svreinterpret_f16_u8_x2(svuint8x2_t op);
1046__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x2)))
1047svfloat16x2_t svreinterpret_f16_mf8_x2(svmfloat8x2_t op);
899__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x2)))1048__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x2)))
900svfloat16x2_t svreinterpret_f16_s16_x2(svint16x2_t op);1049svfloat16x2_t svreinterpret_f16_s16_x2(svint16x2_t op);
901__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x2)))1050__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x2)))
...@@ -920,6 +1069,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -920,6 +1069,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
920svbfloat16x2_t svreinterpret_bf16_s8_x2(svint8x2_t op);1069svbfloat16x2_t svreinterpret_bf16_s8_x2(svint8x2_t op);
921__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x2)))1070__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x2)))
922svbfloat16x2_t svreinterpret_bf16_u8_x2(svuint8x2_t op);1071svbfloat16x2_t svreinterpret_bf16_u8_x2(svuint8x2_t op);
1072__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x2)))
1073svbfloat16x2_t svreinterpret_bf16_mf8_x2(svmfloat8x2_t op);
923__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x2)))1074__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x2)))
924svbfloat16x2_t svreinterpret_bf16_s16_x2(svint16x2_t op);1075svbfloat16x2_t svreinterpret_bf16_s16_x2(svint16x2_t op);
925__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x2)))1076__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x2)))
...@@ -944,6 +1095,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_...@@ -944,6 +1095,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
944svfloat32x2_t svreinterpret_f32_s8_x2(svint8x2_t op);1095svfloat32x2_t svreinterpret_f32_s8_x2(svint8x2_t op);
945__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x2)))1096__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x2)))
946svfloat32x2_t svreinterpret_f32_u8_x2(svuint8x2_t op);1097svfloat32x2_t svreinterpret_f32_u8_x2(svuint8x2_t op);
1098__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x2)))
1099svfloat32x2_t svreinterpret_f32_mf8_x2(svmfloat8x2_t op);
947__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x2)))1100__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x2)))
948svfloat32x2_t svreinterpret_f32_s16_x2(svint16x2_t op);1101svfloat32x2_t svreinterpret_f32_s16_x2(svint16x2_t op);
949__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x2)))1102__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x2)))
...@@ -968,6 +1121,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_...@@ -968,6 +1121,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
968svfloat64x2_t svreinterpret_f64_s8_x2(svint8x2_t op);1121svfloat64x2_t svreinterpret_f64_s8_x2(svint8x2_t op);
969__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x2)))1122__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x2)))
970svfloat64x2_t svreinterpret_f64_u8_x2(svuint8x2_t op);1123svfloat64x2_t svreinterpret_f64_u8_x2(svuint8x2_t op);
1124__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x2)))
1125svfloat64x2_t svreinterpret_f64_mf8_x2(svmfloat8x2_t op);
971__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x2)))1126__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x2)))
972svfloat64x2_t svreinterpret_f64_s16_x2(svint16x2_t op);1127svfloat64x2_t svreinterpret_f64_s16_x2(svint16x2_t op);
973__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x2)))1128__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x2)))
...@@ -992,6 +1147,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x...@@ -992,6 +1147,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
992svint8x2_t svreinterpret_s8(svint8x2_t op);1147svint8x2_t svreinterpret_s8(svint8x2_t op);
993__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x2)))1148__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x2)))
994svint8x2_t svreinterpret_s8(svuint8x2_t op);1149svint8x2_t svreinterpret_s8(svuint8x2_t op);
1150__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x2)))
1151svint8x2_t svreinterpret_s8(svmfloat8x2_t op);
995__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x2)))1152__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x2)))
996svint8x2_t svreinterpret_s8(svint16x2_t op);1153svint8x2_t svreinterpret_s8(svint16x2_t op);
997__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x2)))1154__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x2)))
...@@ -1016,6 +1173,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x...@@ -1016,6 +1173,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
1016svuint8x2_t svreinterpret_u8(svint8x2_t op);1173svuint8x2_t svreinterpret_u8(svint8x2_t op);
1017__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x2)))1174__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x2)))
1018svuint8x2_t svreinterpret_u8(svuint8x2_t op);1175svuint8x2_t svreinterpret_u8(svuint8x2_t op);
1176__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x2)))
1177svuint8x2_t svreinterpret_u8(svmfloat8x2_t op);
1019__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x2)))1178__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x2)))
1020svuint8x2_t svreinterpret_u8(svint16x2_t op);1179svuint8x2_t svreinterpret_u8(svint16x2_t op);
1021__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x2)))1180__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x2)))
...@@ -1036,10 +1195,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_...@@ -1036,10 +1195,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
1036svuint8x2_t svreinterpret_u8(svfloat32x2_t op);1195svuint8x2_t svreinterpret_u8(svfloat32x2_t op);
1037__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x2)))1196__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x2)))
1038svuint8x2_t svreinterpret_u8(svfloat64x2_t op);1197svuint8x2_t svreinterpret_u8(svfloat64x2_t op);
1198__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x2)))
1199svmfloat8x2_t svreinterpret_mf8(svint8x2_t op);
1200__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x2)))
1201svmfloat8x2_t svreinterpret_mf8(svuint8x2_t op);
1202__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x2)))
1203svmfloat8x2_t svreinterpret_mf8(svmfloat8x2_t op);
1204__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x2)))
1205svmfloat8x2_t svreinterpret_mf8(svint16x2_t op);
1206__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x2)))
1207svmfloat8x2_t svreinterpret_mf8(svuint16x2_t op);
1208__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x2)))
1209svmfloat8x2_t svreinterpret_mf8(svint32x2_t op);
1210__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x2)))
1211svmfloat8x2_t svreinterpret_mf8(svuint32x2_t op);
1212__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x2)))
1213svmfloat8x2_t svreinterpret_mf8(svint64x2_t op);
1214__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x2)))
1215svmfloat8x2_t svreinterpret_mf8(svuint64x2_t op);
1216__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x2)))
1217svmfloat8x2_t svreinterpret_mf8(svfloat16x2_t op);
1218__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x2)))
1219svmfloat8x2_t svreinterpret_mf8(svbfloat16x2_t op);
1220__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x2)))
1221svmfloat8x2_t svreinterpret_mf8(svfloat32x2_t op);
1222__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x2)))
1223svmfloat8x2_t svreinterpret_mf8(svfloat64x2_t op);
1039__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x2)))1224__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x2)))
1040svint16x2_t svreinterpret_s16(svint8x2_t op);1225svint16x2_t svreinterpret_s16(svint8x2_t op);
1041__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x2)))1226__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x2)))
1042svint16x2_t svreinterpret_s16(svuint8x2_t op);1227svint16x2_t svreinterpret_s16(svuint8x2_t op);
1228__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x2)))
1229svint16x2_t svreinterpret_s16(svmfloat8x2_t op);
1043__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x2)))1230__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x2)))
1044svint16x2_t svreinterpret_s16(svint16x2_t op);1231svint16x2_t svreinterpret_s16(svint16x2_t op);
1045__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x2)))1232__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x2)))
...@@ -1064,6 +1251,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_...@@ -1064,6 +1251,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
1064svuint16x2_t svreinterpret_u16(svint8x2_t op);1251svuint16x2_t svreinterpret_u16(svint8x2_t op);
1065__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x2)))1252__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x2)))
1066svuint16x2_t svreinterpret_u16(svuint8x2_t op);1253svuint16x2_t svreinterpret_u16(svuint8x2_t op);
1254__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x2)))
1255svuint16x2_t svreinterpret_u16(svmfloat8x2_t op);
1067__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x2)))1256__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x2)))
1068svuint16x2_t svreinterpret_u16(svint16x2_t op);1257svuint16x2_t svreinterpret_u16(svint16x2_t op);
1069__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x2)))1258__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x2)))
...@@ -1088,6 +1277,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_...@@ -1088,6 +1277,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
1088svint32x2_t svreinterpret_s32(svint8x2_t op);1277svint32x2_t svreinterpret_s32(svint8x2_t op);
1089__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x2)))1278__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x2)))
1090svint32x2_t svreinterpret_s32(svuint8x2_t op);1279svint32x2_t svreinterpret_s32(svuint8x2_t op);
1280__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x2)))
1281svint32x2_t svreinterpret_s32(svmfloat8x2_t op);
1091__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x2)))1282__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x2)))
1092svint32x2_t svreinterpret_s32(svint16x2_t op);1283svint32x2_t svreinterpret_s32(svint16x2_t op);
1093__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x2)))1284__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x2)))
...@@ -1112,6 +1303,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_...@@ -1112,6 +1303,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
1112svuint32x2_t svreinterpret_u32(svint8x2_t op);1303svuint32x2_t svreinterpret_u32(svint8x2_t op);
1113__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x2)))1304__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x2)))
1114svuint32x2_t svreinterpret_u32(svuint8x2_t op);1305svuint32x2_t svreinterpret_u32(svuint8x2_t op);
1306__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x2)))
1307svuint32x2_t svreinterpret_u32(svmfloat8x2_t op);
1115__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x2)))1308__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x2)))
1116svuint32x2_t svreinterpret_u32(svint16x2_t op);1309svuint32x2_t svreinterpret_u32(svint16x2_t op);
1117__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x2)))1310__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x2)))
...@@ -1136,6 +1329,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_...@@ -1136,6 +1329,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
1136svint64x2_t svreinterpret_s64(svint8x2_t op);1329svint64x2_t svreinterpret_s64(svint8x2_t op);
1137__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x2)))1330__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x2)))
1138svint64x2_t svreinterpret_s64(svuint8x2_t op);1331svint64x2_t svreinterpret_s64(svuint8x2_t op);
1332__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x2)))
1333svint64x2_t svreinterpret_s64(svmfloat8x2_t op);
1139__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x2)))1334__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x2)))
1140svint64x2_t svreinterpret_s64(svint16x2_t op);1335svint64x2_t svreinterpret_s64(svint16x2_t op);
1141__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x2)))1336__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x2)))
...@@ -1160,6 +1355,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_...@@ -1160,6 +1355,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
1160svuint64x2_t svreinterpret_u64(svint8x2_t op);1355svuint64x2_t svreinterpret_u64(svint8x2_t op);
1161__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x2)))1356__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x2)))
1162svuint64x2_t svreinterpret_u64(svuint8x2_t op);1357svuint64x2_t svreinterpret_u64(svuint8x2_t op);
1358__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x2)))
1359svuint64x2_t svreinterpret_u64(svmfloat8x2_t op);
1163__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x2)))1360__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x2)))
1164svuint64x2_t svreinterpret_u64(svint16x2_t op);1361svuint64x2_t svreinterpret_u64(svint16x2_t op);
1165__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x2)))1362__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x2)))
...@@ -1184,6 +1381,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_...@@ -1184,6 +1381,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
1184svfloat16x2_t svreinterpret_f16(svint8x2_t op);1381svfloat16x2_t svreinterpret_f16(svint8x2_t op);
1185__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x2)))1382__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x2)))
1186svfloat16x2_t svreinterpret_f16(svuint8x2_t op);1383svfloat16x2_t svreinterpret_f16(svuint8x2_t op);
1384__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x2)))
1385svfloat16x2_t svreinterpret_f16(svmfloat8x2_t op);
1187__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x2)))1386__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x2)))
1188svfloat16x2_t svreinterpret_f16(svint16x2_t op);1387svfloat16x2_t svreinterpret_f16(svint16x2_t op);
1189__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x2)))1388__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x2)))
...@@ -1208,6 +1407,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -1208,6 +1407,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
1208svbfloat16x2_t svreinterpret_bf16(svint8x2_t op);1407svbfloat16x2_t svreinterpret_bf16(svint8x2_t op);
1209__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x2)))1408__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x2)))
1210svbfloat16x2_t svreinterpret_bf16(svuint8x2_t op);1409svbfloat16x2_t svreinterpret_bf16(svuint8x2_t op);
1410__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x2)))
1411svbfloat16x2_t svreinterpret_bf16(svmfloat8x2_t op);
1211__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x2)))1412__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x2)))
1212svbfloat16x2_t svreinterpret_bf16(svint16x2_t op);1413svbfloat16x2_t svreinterpret_bf16(svint16x2_t op);
1213__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x2)))1414__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x2)))
...@@ -1232,6 +1433,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_...@@ -1232,6 +1433,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
1232svfloat32x2_t svreinterpret_f32(svint8x2_t op);1433svfloat32x2_t svreinterpret_f32(svint8x2_t op);
1233__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x2)))1434__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x2)))
1234svfloat32x2_t svreinterpret_f32(svuint8x2_t op);1435svfloat32x2_t svreinterpret_f32(svuint8x2_t op);
1436__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x2)))
1437svfloat32x2_t svreinterpret_f32(svmfloat8x2_t op);
1235__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x2)))1438__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x2)))
1236svfloat32x2_t svreinterpret_f32(svint16x2_t op);1439svfloat32x2_t svreinterpret_f32(svint16x2_t op);
1237__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x2)))1440__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x2)))
...@@ -1256,6 +1459,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_...@@ -1256,6 +1459,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
1256svfloat64x2_t svreinterpret_f64(svint8x2_t op);1459svfloat64x2_t svreinterpret_f64(svint8x2_t op);
1257__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x2)))1460__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x2)))
1258svfloat64x2_t svreinterpret_f64(svuint8x2_t op);1461svfloat64x2_t svreinterpret_f64(svuint8x2_t op);
1462__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x2)))
1463svfloat64x2_t svreinterpret_f64(svmfloat8x2_t op);
1259__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x2)))1464__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x2)))
1260svfloat64x2_t svreinterpret_f64(svint16x2_t op);1465svfloat64x2_t svreinterpret_f64(svint16x2_t op);
1261__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x2)))1466__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x2)))
...@@ -1280,6 +1485,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x...@@ -1280,6 +1485,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
1280svint8x3_t svreinterpret_s8_s8_x3(svint8x3_t op);1485svint8x3_t svreinterpret_s8_s8_x3(svint8x3_t op);
1281__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x3)))1486__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x3)))
1282svint8x3_t svreinterpret_s8_u8_x3(svuint8x3_t op);1487svint8x3_t svreinterpret_s8_u8_x3(svuint8x3_t op);
1488__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x3)))
1489svint8x3_t svreinterpret_s8_mf8_x3(svmfloat8x3_t op);
1283__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x3)))1490__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x3)))
1284svint8x3_t svreinterpret_s8_s16_x3(svint16x3_t op);1491svint8x3_t svreinterpret_s8_s16_x3(svint16x3_t op);
1285__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x3)))1492__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x3)))
...@@ -1304,6 +1511,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x...@@ -1304,6 +1511,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
1304svuint8x3_t svreinterpret_u8_s8_x3(svint8x3_t op);1511svuint8x3_t svreinterpret_u8_s8_x3(svint8x3_t op);
1305__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x3)))1512__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x3)))
1306svuint8x3_t svreinterpret_u8_u8_x3(svuint8x3_t op);1513svuint8x3_t svreinterpret_u8_u8_x3(svuint8x3_t op);
1514__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x3)))
1515svuint8x3_t svreinterpret_u8_mf8_x3(svmfloat8x3_t op);
1307__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x3)))1516__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x3)))
1308svuint8x3_t svreinterpret_u8_s16_x3(svint16x3_t op);1517svuint8x3_t svreinterpret_u8_s16_x3(svint16x3_t op);
1309__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x3)))1518__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x3)))
...@@ -1324,10 +1533,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_...@@ -1324,10 +1533,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
1324svuint8x3_t svreinterpret_u8_f32_x3(svfloat32x3_t op);1533svuint8x3_t svreinterpret_u8_f32_x3(svfloat32x3_t op);
1325__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x3)))1534__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x3)))
1326svuint8x3_t svreinterpret_u8_f64_x3(svfloat64x3_t op);1535svuint8x3_t svreinterpret_u8_f64_x3(svfloat64x3_t op);
1536__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x3)))
1537svmfloat8x3_t svreinterpret_mf8_s8_x3(svint8x3_t op);
1538__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x3)))
1539svmfloat8x3_t svreinterpret_mf8_u8_x3(svuint8x3_t op);
1540__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x3)))
1541svmfloat8x3_t svreinterpret_mf8_mf8_x3(svmfloat8x3_t op);
1542__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x3)))
1543svmfloat8x3_t svreinterpret_mf8_s16_x3(svint16x3_t op);
1544__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x3)))
1545svmfloat8x3_t svreinterpret_mf8_u16_x3(svuint16x3_t op);
1546__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x3)))
1547svmfloat8x3_t svreinterpret_mf8_s32_x3(svint32x3_t op);
1548__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x3)))
1549svmfloat8x3_t svreinterpret_mf8_u32_x3(svuint32x3_t op);
1550__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x3)))
1551svmfloat8x3_t svreinterpret_mf8_s64_x3(svint64x3_t op);
1552__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x3)))
1553svmfloat8x3_t svreinterpret_mf8_u64_x3(svuint64x3_t op);
1554__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x3)))
1555svmfloat8x3_t svreinterpret_mf8_f16_x3(svfloat16x3_t op);
1556__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x3)))
1557svmfloat8x3_t svreinterpret_mf8_bf16_x3(svbfloat16x3_t op);
1558__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x3)))
1559svmfloat8x3_t svreinterpret_mf8_f32_x3(svfloat32x3_t op);
1560__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x3)))
1561svmfloat8x3_t svreinterpret_mf8_f64_x3(svfloat64x3_t op);
1327__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x3)))1562__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x3)))
1328svint16x3_t svreinterpret_s16_s8_x3(svint8x3_t op);1563svint16x3_t svreinterpret_s16_s8_x3(svint8x3_t op);
1329__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x3)))1564__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x3)))
1330svint16x3_t svreinterpret_s16_u8_x3(svuint8x3_t op);1565svint16x3_t svreinterpret_s16_u8_x3(svuint8x3_t op);
1566__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x3)))
1567svint16x3_t svreinterpret_s16_mf8_x3(svmfloat8x3_t op);
1331__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x3)))1568__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x3)))
1332svint16x3_t svreinterpret_s16_s16_x3(svint16x3_t op);1569svint16x3_t svreinterpret_s16_s16_x3(svint16x3_t op);
1333__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x3)))1570__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x3)))
...@@ -1352,6 +1589,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_...@@ -1352,6 +1589,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
1352svuint16x3_t svreinterpret_u16_s8_x3(svint8x3_t op);1589svuint16x3_t svreinterpret_u16_s8_x3(svint8x3_t op);
1353__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x3)))1590__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x3)))
1354svuint16x3_t svreinterpret_u16_u8_x3(svuint8x3_t op);1591svuint16x3_t svreinterpret_u16_u8_x3(svuint8x3_t op);
1592__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x3)))
1593svuint16x3_t svreinterpret_u16_mf8_x3(svmfloat8x3_t op);
1355__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x3)))1594__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x3)))
1356svuint16x3_t svreinterpret_u16_s16_x3(svint16x3_t op);1595svuint16x3_t svreinterpret_u16_s16_x3(svint16x3_t op);
1357__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x3)))1596__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x3)))
...@@ -1376,6 +1615,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_...@@ -1376,6 +1615,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
1376svint32x3_t svreinterpret_s32_s8_x3(svint8x3_t op);1615svint32x3_t svreinterpret_s32_s8_x3(svint8x3_t op);
1377__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x3)))1616__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x3)))
1378svint32x3_t svreinterpret_s32_u8_x3(svuint8x3_t op);1617svint32x3_t svreinterpret_s32_u8_x3(svuint8x3_t op);
1618__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x3)))
1619svint32x3_t svreinterpret_s32_mf8_x3(svmfloat8x3_t op);
1379__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x3)))1620__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x3)))
1380svint32x3_t svreinterpret_s32_s16_x3(svint16x3_t op);1621svint32x3_t svreinterpret_s32_s16_x3(svint16x3_t op);
1381__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x3)))1622__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x3)))
...@@ -1400,6 +1641,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_...@@ -1400,6 +1641,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
1400svuint32x3_t svreinterpret_u32_s8_x3(svint8x3_t op);1641svuint32x3_t svreinterpret_u32_s8_x3(svint8x3_t op);
1401__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x3)))1642__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x3)))
1402svuint32x3_t svreinterpret_u32_u8_x3(svuint8x3_t op);1643svuint32x3_t svreinterpret_u32_u8_x3(svuint8x3_t op);
1644__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x3)))
1645svuint32x3_t svreinterpret_u32_mf8_x3(svmfloat8x3_t op);
1403__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x3)))1646__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x3)))
1404svuint32x3_t svreinterpret_u32_s16_x3(svint16x3_t op);1647svuint32x3_t svreinterpret_u32_s16_x3(svint16x3_t op);
1405__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x3)))1648__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x3)))
...@@ -1424,6 +1667,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_...@@ -1424,6 +1667,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
1424svint64x3_t svreinterpret_s64_s8_x3(svint8x3_t op);1667svint64x3_t svreinterpret_s64_s8_x3(svint8x3_t op);
1425__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x3)))1668__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x3)))
1426svint64x3_t svreinterpret_s64_u8_x3(svuint8x3_t op);1669svint64x3_t svreinterpret_s64_u8_x3(svuint8x3_t op);
1670__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x3)))
1671svint64x3_t svreinterpret_s64_mf8_x3(svmfloat8x3_t op);
1427__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x3)))1672__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x3)))
1428svint64x3_t svreinterpret_s64_s16_x3(svint16x3_t op);1673svint64x3_t svreinterpret_s64_s16_x3(svint16x3_t op);
1429__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x3)))1674__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x3)))
...@@ -1448,6 +1693,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_...@@ -1448,6 +1693,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
1448svuint64x3_t svreinterpret_u64_s8_x3(svint8x3_t op);1693svuint64x3_t svreinterpret_u64_s8_x3(svint8x3_t op);
1449__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x3)))1694__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x3)))
1450svuint64x3_t svreinterpret_u64_u8_x3(svuint8x3_t op);1695svuint64x3_t svreinterpret_u64_u8_x3(svuint8x3_t op);
1696__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x3)))
1697svuint64x3_t svreinterpret_u64_mf8_x3(svmfloat8x3_t op);
1451__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x3)))1698__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x3)))
1452svuint64x3_t svreinterpret_u64_s16_x3(svint16x3_t op);1699svuint64x3_t svreinterpret_u64_s16_x3(svint16x3_t op);
1453__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x3)))1700__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x3)))
...@@ -1472,6 +1719,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_...@@ -1472,6 +1719,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
1472svfloat16x3_t svreinterpret_f16_s8_x3(svint8x3_t op);1719svfloat16x3_t svreinterpret_f16_s8_x3(svint8x3_t op);
1473__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x3)))1720__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x3)))
1474svfloat16x3_t svreinterpret_f16_u8_x3(svuint8x3_t op);1721svfloat16x3_t svreinterpret_f16_u8_x3(svuint8x3_t op);
1722__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x3)))
1723svfloat16x3_t svreinterpret_f16_mf8_x3(svmfloat8x3_t op);
1475__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x3)))1724__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x3)))
1476svfloat16x3_t svreinterpret_f16_s16_x3(svint16x3_t op);1725svfloat16x3_t svreinterpret_f16_s16_x3(svint16x3_t op);
1477__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x3)))1726__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x3)))
...@@ -1496,6 +1745,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -1496,6 +1745,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
1496svbfloat16x3_t svreinterpret_bf16_s8_x3(svint8x3_t op);1745svbfloat16x3_t svreinterpret_bf16_s8_x3(svint8x3_t op);
1497__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x3)))1746__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x3)))
1498svbfloat16x3_t svreinterpret_bf16_u8_x3(svuint8x3_t op);1747svbfloat16x3_t svreinterpret_bf16_u8_x3(svuint8x3_t op);
1748__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x3)))
1749svbfloat16x3_t svreinterpret_bf16_mf8_x3(svmfloat8x3_t op);
1499__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x3)))1750__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x3)))
1500svbfloat16x3_t svreinterpret_bf16_s16_x3(svint16x3_t op);1751svbfloat16x3_t svreinterpret_bf16_s16_x3(svint16x3_t op);
1501__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x3)))1752__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x3)))
...@@ -1520,6 +1771,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_...@@ -1520,6 +1771,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
1520svfloat32x3_t svreinterpret_f32_s8_x3(svint8x3_t op);1771svfloat32x3_t svreinterpret_f32_s8_x3(svint8x3_t op);
1521__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x3)))1772__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x3)))
1522svfloat32x3_t svreinterpret_f32_u8_x3(svuint8x3_t op);1773svfloat32x3_t svreinterpret_f32_u8_x3(svuint8x3_t op);
1774__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x3)))
1775svfloat32x3_t svreinterpret_f32_mf8_x3(svmfloat8x3_t op);
1523__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x3)))1776__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x3)))
1524svfloat32x3_t svreinterpret_f32_s16_x3(svint16x3_t op);1777svfloat32x3_t svreinterpret_f32_s16_x3(svint16x3_t op);
1525__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x3)))1778__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x3)))
...@@ -1544,6 +1797,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_...@@ -1544,6 +1797,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
1544svfloat64x3_t svreinterpret_f64_s8_x3(svint8x3_t op);1797svfloat64x3_t svreinterpret_f64_s8_x3(svint8x3_t op);
1545__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x3)))1798__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x3)))
1546svfloat64x3_t svreinterpret_f64_u8_x3(svuint8x3_t op);1799svfloat64x3_t svreinterpret_f64_u8_x3(svuint8x3_t op);
1800__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x3)))
1801svfloat64x3_t svreinterpret_f64_mf8_x3(svmfloat8x3_t op);
1547__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x3)))1802__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x3)))
1548svfloat64x3_t svreinterpret_f64_s16_x3(svint16x3_t op);1803svfloat64x3_t svreinterpret_f64_s16_x3(svint16x3_t op);
1549__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x3)))1804__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x3)))
...@@ -1568,6 +1823,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x...@@ -1568,6 +1823,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
1568svint8x3_t svreinterpret_s8(svint8x3_t op);1823svint8x3_t svreinterpret_s8(svint8x3_t op);
1569__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x3)))1824__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x3)))
1570svint8x3_t svreinterpret_s8(svuint8x3_t op);1825svint8x3_t svreinterpret_s8(svuint8x3_t op);
1826__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x3)))
1827svint8x3_t svreinterpret_s8(svmfloat8x3_t op);
1571__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x3)))1828__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x3)))
1572svint8x3_t svreinterpret_s8(svint16x3_t op);1829svint8x3_t svreinterpret_s8(svint16x3_t op);
1573__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x3)))1830__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x3)))
...@@ -1592,6 +1849,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x...@@ -1592,6 +1849,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
1592svuint8x3_t svreinterpret_u8(svint8x3_t op);1849svuint8x3_t svreinterpret_u8(svint8x3_t op);
1593__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x3)))1850__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x3)))
1594svuint8x3_t svreinterpret_u8(svuint8x3_t op);1851svuint8x3_t svreinterpret_u8(svuint8x3_t op);
1852__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x3)))
1853svuint8x3_t svreinterpret_u8(svmfloat8x3_t op);
1595__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x3)))1854__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x3)))
1596svuint8x3_t svreinterpret_u8(svint16x3_t op);1855svuint8x3_t svreinterpret_u8(svint16x3_t op);
1597__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x3)))1856__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x3)))
...@@ -1612,10 +1871,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_...@@ -1612,10 +1871,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
1612svuint8x3_t svreinterpret_u8(svfloat32x3_t op);1871svuint8x3_t svreinterpret_u8(svfloat32x3_t op);
1613__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x3)))1872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x3)))
1614svuint8x3_t svreinterpret_u8(svfloat64x3_t op);1873svuint8x3_t svreinterpret_u8(svfloat64x3_t op);
1874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x3)))
1875svmfloat8x3_t svreinterpret_mf8(svint8x3_t op);
1876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x3)))
1877svmfloat8x3_t svreinterpret_mf8(svuint8x3_t op);
1878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x3)))
1879svmfloat8x3_t svreinterpret_mf8(svmfloat8x3_t op);
1880__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x3)))
1881svmfloat8x3_t svreinterpret_mf8(svint16x3_t op);
1882__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x3)))
1883svmfloat8x3_t svreinterpret_mf8(svuint16x3_t op);
1884__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x3)))
1885svmfloat8x3_t svreinterpret_mf8(svint32x3_t op);
1886__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x3)))
1887svmfloat8x3_t svreinterpret_mf8(svuint32x3_t op);
1888__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x3)))
1889svmfloat8x3_t svreinterpret_mf8(svint64x3_t op);
1890__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x3)))
1891svmfloat8x3_t svreinterpret_mf8(svuint64x3_t op);
1892__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x3)))
1893svmfloat8x3_t svreinterpret_mf8(svfloat16x3_t op);
1894__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x3)))
1895svmfloat8x3_t svreinterpret_mf8(svbfloat16x3_t op);
1896__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x3)))
1897svmfloat8x3_t svreinterpret_mf8(svfloat32x3_t op);
1898__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x3)))
1899svmfloat8x3_t svreinterpret_mf8(svfloat64x3_t op);
1615__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x3)))1900__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x3)))
1616svint16x3_t svreinterpret_s16(svint8x3_t op);1901svint16x3_t svreinterpret_s16(svint8x3_t op);
1617__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x3)))1902__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x3)))
1618svint16x3_t svreinterpret_s16(svuint8x3_t op);1903svint16x3_t svreinterpret_s16(svuint8x3_t op);
1904__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x3)))
1905svint16x3_t svreinterpret_s16(svmfloat8x3_t op);
1619__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x3)))1906__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x3)))
1620svint16x3_t svreinterpret_s16(svint16x3_t op);1907svint16x3_t svreinterpret_s16(svint16x3_t op);
1621__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x3)))1908__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x3)))
...@@ -1640,6 +1927,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_...@@ -1640,6 +1927,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
1640svuint16x3_t svreinterpret_u16(svint8x3_t op);1927svuint16x3_t svreinterpret_u16(svint8x3_t op);
1641__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x3)))1928__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x3)))
1642svuint16x3_t svreinterpret_u16(svuint8x3_t op);1929svuint16x3_t svreinterpret_u16(svuint8x3_t op);
1930__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x3)))
1931svuint16x3_t svreinterpret_u16(svmfloat8x3_t op);
1643__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x3)))1932__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x3)))
1644svuint16x3_t svreinterpret_u16(svint16x3_t op);1933svuint16x3_t svreinterpret_u16(svint16x3_t op);
1645__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x3)))1934__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x3)))
...@@ -1664,6 +1953,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_...@@ -1664,6 +1953,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
1664svint32x3_t svreinterpret_s32(svint8x3_t op);1953svint32x3_t svreinterpret_s32(svint8x3_t op);
1665__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x3)))1954__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x3)))
1666svint32x3_t svreinterpret_s32(svuint8x3_t op);1955svint32x3_t svreinterpret_s32(svuint8x3_t op);
1956__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x3)))
1957svint32x3_t svreinterpret_s32(svmfloat8x3_t op);
1667__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x3)))1958__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x3)))
1668svint32x3_t svreinterpret_s32(svint16x3_t op);1959svint32x3_t svreinterpret_s32(svint16x3_t op);
1669__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x3)))1960__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x3)))
...@@ -1688,6 +1979,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_...@@ -1688,6 +1979,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
1688svuint32x3_t svreinterpret_u32(svint8x3_t op);1979svuint32x3_t svreinterpret_u32(svint8x3_t op);
1689__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x3)))1980__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x3)))
1690svuint32x3_t svreinterpret_u32(svuint8x3_t op);1981svuint32x3_t svreinterpret_u32(svuint8x3_t op);
1982__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x3)))
1983svuint32x3_t svreinterpret_u32(svmfloat8x3_t op);
1691__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x3)))1984__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x3)))
1692svuint32x3_t svreinterpret_u32(svint16x3_t op);1985svuint32x3_t svreinterpret_u32(svint16x3_t op);
1693__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x3)))1986__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x3)))
...@@ -1712,6 +2005,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_...@@ -1712,6 +2005,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
1712svint64x3_t svreinterpret_s64(svint8x3_t op);2005svint64x3_t svreinterpret_s64(svint8x3_t op);
1713__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x3)))2006__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x3)))
1714svint64x3_t svreinterpret_s64(svuint8x3_t op);2007svint64x3_t svreinterpret_s64(svuint8x3_t op);
2008__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x3)))
2009svint64x3_t svreinterpret_s64(svmfloat8x3_t op);
1715__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x3)))2010__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x3)))
1716svint64x3_t svreinterpret_s64(svint16x3_t op);2011svint64x3_t svreinterpret_s64(svint16x3_t op);
1717__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x3)))2012__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x3)))
...@@ -1736,6 +2031,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_...@@ -1736,6 +2031,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
1736svuint64x3_t svreinterpret_u64(svint8x3_t op);2031svuint64x3_t svreinterpret_u64(svint8x3_t op);
1737__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x3)))2032__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x3)))
1738svuint64x3_t svreinterpret_u64(svuint8x3_t op);2033svuint64x3_t svreinterpret_u64(svuint8x3_t op);
2034__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x3)))
2035svuint64x3_t svreinterpret_u64(svmfloat8x3_t op);
1739__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x3)))2036__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x3)))
1740svuint64x3_t svreinterpret_u64(svint16x3_t op);2037svuint64x3_t svreinterpret_u64(svint16x3_t op);
1741__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x3)))2038__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x3)))
...@@ -1760,6 +2057,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_...@@ -1760,6 +2057,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
1760svfloat16x3_t svreinterpret_f16(svint8x3_t op);2057svfloat16x3_t svreinterpret_f16(svint8x3_t op);
1761__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x3)))2058__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x3)))
1762svfloat16x3_t svreinterpret_f16(svuint8x3_t op);2059svfloat16x3_t svreinterpret_f16(svuint8x3_t op);
2060__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x3)))
2061svfloat16x3_t svreinterpret_f16(svmfloat8x3_t op);
1763__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x3)))2062__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x3)))
1764svfloat16x3_t svreinterpret_f16(svint16x3_t op);2063svfloat16x3_t svreinterpret_f16(svint16x3_t op);
1765__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x3)))2064__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x3)))
...@@ -1784,6 +2083,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -1784,6 +2083,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
1784svbfloat16x3_t svreinterpret_bf16(svint8x3_t op);2083svbfloat16x3_t svreinterpret_bf16(svint8x3_t op);
1785__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x3)))2084__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x3)))
1786svbfloat16x3_t svreinterpret_bf16(svuint8x3_t op);2085svbfloat16x3_t svreinterpret_bf16(svuint8x3_t op);
2086__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x3)))
2087svbfloat16x3_t svreinterpret_bf16(svmfloat8x3_t op);
1787__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x3)))2088__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x3)))
1788svbfloat16x3_t svreinterpret_bf16(svint16x3_t op);2089svbfloat16x3_t svreinterpret_bf16(svint16x3_t op);
1789__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x3)))2090__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x3)))
...@@ -1808,6 +2109,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_...@@ -1808,6 +2109,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
1808svfloat32x3_t svreinterpret_f32(svint8x3_t op);2109svfloat32x3_t svreinterpret_f32(svint8x3_t op);
1809__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x3)))2110__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x3)))
1810svfloat32x3_t svreinterpret_f32(svuint8x3_t op);2111svfloat32x3_t svreinterpret_f32(svuint8x3_t op);
2112__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x3)))
2113svfloat32x3_t svreinterpret_f32(svmfloat8x3_t op);
1811__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x3)))2114__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x3)))
1812svfloat32x3_t svreinterpret_f32(svint16x3_t op);2115svfloat32x3_t svreinterpret_f32(svint16x3_t op);
1813__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x3)))2116__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x3)))
...@@ -1832,6 +2135,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_...@@ -1832,6 +2135,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
1832svfloat64x3_t svreinterpret_f64(svint8x3_t op);2135svfloat64x3_t svreinterpret_f64(svint8x3_t op);
1833__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x3)))2136__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x3)))
1834svfloat64x3_t svreinterpret_f64(svuint8x3_t op);2137svfloat64x3_t svreinterpret_f64(svuint8x3_t op);
2138__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x3)))
2139svfloat64x3_t svreinterpret_f64(svmfloat8x3_t op);
1835__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x3)))2140__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x3)))
1836svfloat64x3_t svreinterpret_f64(svint16x3_t op);2141svfloat64x3_t svreinterpret_f64(svint16x3_t op);
1837__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x3)))2142__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x3)))
...@@ -1856,6 +2161,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x...@@ -1856,6 +2161,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
1856svint8x4_t svreinterpret_s8_s8_x4(svint8x4_t op);2161svint8x4_t svreinterpret_s8_s8_x4(svint8x4_t op);
1857__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x4)))2162__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x4)))
1858svint8x4_t svreinterpret_s8_u8_x4(svuint8x4_t op);2163svint8x4_t svreinterpret_s8_u8_x4(svuint8x4_t op);
2164__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x4)))
2165svint8x4_t svreinterpret_s8_mf8_x4(svmfloat8x4_t op);
1859__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x4)))2166__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x4)))
1860svint8x4_t svreinterpret_s8_s16_x4(svint16x4_t op);2167svint8x4_t svreinterpret_s8_s16_x4(svint16x4_t op);
1861__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x4)))2168__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x4)))
...@@ -1880,6 +2187,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x...@@ -1880,6 +2187,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
1880svuint8x4_t svreinterpret_u8_s8_x4(svint8x4_t op);2187svuint8x4_t svreinterpret_u8_s8_x4(svint8x4_t op);
1881__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x4)))2188__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x4)))
1882svuint8x4_t svreinterpret_u8_u8_x4(svuint8x4_t op);2189svuint8x4_t svreinterpret_u8_u8_x4(svuint8x4_t op);
2190__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x4)))
2191svuint8x4_t svreinterpret_u8_mf8_x4(svmfloat8x4_t op);
1883__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x4)))2192__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x4)))
1884svuint8x4_t svreinterpret_u8_s16_x4(svint16x4_t op);2193svuint8x4_t svreinterpret_u8_s16_x4(svint16x4_t op);
1885__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x4)))2194__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x4)))
...@@ -1900,10 +2209,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_...@@ -1900,10 +2209,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
1900svuint8x4_t svreinterpret_u8_f32_x4(svfloat32x4_t op);2209svuint8x4_t svreinterpret_u8_f32_x4(svfloat32x4_t op);
1901__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x4)))2210__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x4)))
1902svuint8x4_t svreinterpret_u8_f64_x4(svfloat64x4_t op);2211svuint8x4_t svreinterpret_u8_f64_x4(svfloat64x4_t op);
2212__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x4)))
2213svmfloat8x4_t svreinterpret_mf8_s8_x4(svint8x4_t op);
2214__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x4)))
2215svmfloat8x4_t svreinterpret_mf8_u8_x4(svuint8x4_t op);
2216__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x4)))
2217svmfloat8x4_t svreinterpret_mf8_mf8_x4(svmfloat8x4_t op);
2218__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x4)))
2219svmfloat8x4_t svreinterpret_mf8_s16_x4(svint16x4_t op);
2220__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x4)))
2221svmfloat8x4_t svreinterpret_mf8_u16_x4(svuint16x4_t op);
2222__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x4)))
2223svmfloat8x4_t svreinterpret_mf8_s32_x4(svint32x4_t op);
2224__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x4)))
2225svmfloat8x4_t svreinterpret_mf8_u32_x4(svuint32x4_t op);
2226__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x4)))
2227svmfloat8x4_t svreinterpret_mf8_s64_x4(svint64x4_t op);
2228__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x4)))
2229svmfloat8x4_t svreinterpret_mf8_u64_x4(svuint64x4_t op);
2230__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x4)))
2231svmfloat8x4_t svreinterpret_mf8_f16_x4(svfloat16x4_t op);
2232__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x4)))
2233svmfloat8x4_t svreinterpret_mf8_bf16_x4(svbfloat16x4_t op);
2234__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x4)))
2235svmfloat8x4_t svreinterpret_mf8_f32_x4(svfloat32x4_t op);
2236__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x4)))
2237svmfloat8x4_t svreinterpret_mf8_f64_x4(svfloat64x4_t op);
1903__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x4)))2238__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x4)))
1904svint16x4_t svreinterpret_s16_s8_x4(svint8x4_t op);2239svint16x4_t svreinterpret_s16_s8_x4(svint8x4_t op);
1905__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x4)))2240__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x4)))
1906svint16x4_t svreinterpret_s16_u8_x4(svuint8x4_t op);2241svint16x4_t svreinterpret_s16_u8_x4(svuint8x4_t op);
2242__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x4)))
2243svint16x4_t svreinterpret_s16_mf8_x4(svmfloat8x4_t op);
1907__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x4)))2244__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x4)))
1908svint16x4_t svreinterpret_s16_s16_x4(svint16x4_t op);2245svint16x4_t svreinterpret_s16_s16_x4(svint16x4_t op);
1909__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x4)))2246__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x4)))
...@@ -1928,6 +2265,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_...@@ -1928,6 +2265,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
1928svuint16x4_t svreinterpret_u16_s8_x4(svint8x4_t op);2265svuint16x4_t svreinterpret_u16_s8_x4(svint8x4_t op);
1929__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x4)))2266__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x4)))
1930svuint16x4_t svreinterpret_u16_u8_x4(svuint8x4_t op);2267svuint16x4_t svreinterpret_u16_u8_x4(svuint8x4_t op);
2268__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x4)))
2269svuint16x4_t svreinterpret_u16_mf8_x4(svmfloat8x4_t op);
1931__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x4)))2270__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x4)))
1932svuint16x4_t svreinterpret_u16_s16_x4(svint16x4_t op);2271svuint16x4_t svreinterpret_u16_s16_x4(svint16x4_t op);
1933__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x4)))2272__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x4)))
...@@ -1952,6 +2291,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_...@@ -1952,6 +2291,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
1952svint32x4_t svreinterpret_s32_s8_x4(svint8x4_t op);2291svint32x4_t svreinterpret_s32_s8_x4(svint8x4_t op);
1953__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x4)))2292__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x4)))
1954svint32x4_t svreinterpret_s32_u8_x4(svuint8x4_t op);2293svint32x4_t svreinterpret_s32_u8_x4(svuint8x4_t op);
2294__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x4)))
2295svint32x4_t svreinterpret_s32_mf8_x4(svmfloat8x4_t op);
1955__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x4)))2296__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x4)))
1956svint32x4_t svreinterpret_s32_s16_x4(svint16x4_t op);2297svint32x4_t svreinterpret_s32_s16_x4(svint16x4_t op);
1957__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x4)))2298__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x4)))
...@@ -1976,6 +2317,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_...@@ -1976,6 +2317,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
1976svuint32x4_t svreinterpret_u32_s8_x4(svint8x4_t op);2317svuint32x4_t svreinterpret_u32_s8_x4(svint8x4_t op);
1977__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x4)))2318__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x4)))
1978svuint32x4_t svreinterpret_u32_u8_x4(svuint8x4_t op);2319svuint32x4_t svreinterpret_u32_u8_x4(svuint8x4_t op);
2320__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x4)))
2321svuint32x4_t svreinterpret_u32_mf8_x4(svmfloat8x4_t op);
1979__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x4)))2322__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x4)))
1980svuint32x4_t svreinterpret_u32_s16_x4(svint16x4_t op);2323svuint32x4_t svreinterpret_u32_s16_x4(svint16x4_t op);
1981__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x4)))2324__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x4)))
...@@ -2000,6 +2343,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_...@@ -2000,6 +2343,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
2000svint64x4_t svreinterpret_s64_s8_x4(svint8x4_t op);2343svint64x4_t svreinterpret_s64_s8_x4(svint8x4_t op);
2001__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x4)))2344__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x4)))
2002svint64x4_t svreinterpret_s64_u8_x4(svuint8x4_t op);2345svint64x4_t svreinterpret_s64_u8_x4(svuint8x4_t op);
2346__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x4)))
2347svint64x4_t svreinterpret_s64_mf8_x4(svmfloat8x4_t op);
2003__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x4)))2348__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x4)))
2004svint64x4_t svreinterpret_s64_s16_x4(svint16x4_t op);2349svint64x4_t svreinterpret_s64_s16_x4(svint16x4_t op);
2005__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x4)))2350__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x4)))
...@@ -2024,6 +2369,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_...@@ -2024,6 +2369,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
2024svuint64x4_t svreinterpret_u64_s8_x4(svint8x4_t op);2369svuint64x4_t svreinterpret_u64_s8_x4(svint8x4_t op);
2025__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x4)))2370__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x4)))
2026svuint64x4_t svreinterpret_u64_u8_x4(svuint8x4_t op);2371svuint64x4_t svreinterpret_u64_u8_x4(svuint8x4_t op);
2372__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x4)))
2373svuint64x4_t svreinterpret_u64_mf8_x4(svmfloat8x4_t op);
2027__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x4)))2374__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x4)))
2028svuint64x4_t svreinterpret_u64_s16_x4(svint16x4_t op);2375svuint64x4_t svreinterpret_u64_s16_x4(svint16x4_t op);
2029__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x4)))2376__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x4)))
...@@ -2048,6 +2395,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_...@@ -2048,6 +2395,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
2048svfloat16x4_t svreinterpret_f16_s8_x4(svint8x4_t op);2395svfloat16x4_t svreinterpret_f16_s8_x4(svint8x4_t op);
2049__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x4)))2396__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x4)))
2050svfloat16x4_t svreinterpret_f16_u8_x4(svuint8x4_t op);2397svfloat16x4_t svreinterpret_f16_u8_x4(svuint8x4_t op);
2398__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x4)))
2399svfloat16x4_t svreinterpret_f16_mf8_x4(svmfloat8x4_t op);
2051__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x4)))2400__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x4)))
2052svfloat16x4_t svreinterpret_f16_s16_x4(svint16x4_t op);2401svfloat16x4_t svreinterpret_f16_s16_x4(svint16x4_t op);
2053__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x4)))2402__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x4)))
...@@ -2072,6 +2421,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -2072,6 +2421,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
2072svbfloat16x4_t svreinterpret_bf16_s8_x4(svint8x4_t op);2421svbfloat16x4_t svreinterpret_bf16_s8_x4(svint8x4_t op);
2073__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x4)))2422__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x4)))
2074svbfloat16x4_t svreinterpret_bf16_u8_x4(svuint8x4_t op);2423svbfloat16x4_t svreinterpret_bf16_u8_x4(svuint8x4_t op);
2424__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x4)))
2425svbfloat16x4_t svreinterpret_bf16_mf8_x4(svmfloat8x4_t op);
2075__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x4)))2426__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x4)))
2076svbfloat16x4_t svreinterpret_bf16_s16_x4(svint16x4_t op);2427svbfloat16x4_t svreinterpret_bf16_s16_x4(svint16x4_t op);
2077__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x4)))2428__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x4)))
...@@ -2096,6 +2447,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_...@@ -2096,6 +2447,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
2096svfloat32x4_t svreinterpret_f32_s8_x4(svint8x4_t op);2447svfloat32x4_t svreinterpret_f32_s8_x4(svint8x4_t op);
2097__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x4)))2448__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x4)))
2098svfloat32x4_t svreinterpret_f32_u8_x4(svuint8x4_t op);2449svfloat32x4_t svreinterpret_f32_u8_x4(svuint8x4_t op);
2450__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x4)))
2451svfloat32x4_t svreinterpret_f32_mf8_x4(svmfloat8x4_t op);
2099__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x4)))2452__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x4)))
2100svfloat32x4_t svreinterpret_f32_s16_x4(svint16x4_t op);2453svfloat32x4_t svreinterpret_f32_s16_x4(svint16x4_t op);
2101__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x4)))2454__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x4)))
...@@ -2120,6 +2473,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_...@@ -2120,6 +2473,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
2120svfloat64x4_t svreinterpret_f64_s8_x4(svint8x4_t op);2473svfloat64x4_t svreinterpret_f64_s8_x4(svint8x4_t op);
2121__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x4)))2474__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x4)))
2122svfloat64x4_t svreinterpret_f64_u8_x4(svuint8x4_t op);2475svfloat64x4_t svreinterpret_f64_u8_x4(svuint8x4_t op);
2476__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x4)))
2477svfloat64x4_t svreinterpret_f64_mf8_x4(svmfloat8x4_t op);
2123__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x4)))2478__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x4)))
2124svfloat64x4_t svreinterpret_f64_s16_x4(svint16x4_t op);2479svfloat64x4_t svreinterpret_f64_s16_x4(svint16x4_t op);
2125__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x4)))2480__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x4)))
...@@ -2144,6 +2499,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x...@@ -2144,6 +2499,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
2144svint8x4_t svreinterpret_s8(svint8x4_t op);2499svint8x4_t svreinterpret_s8(svint8x4_t op);
2145__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x4)))2500__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x4)))
2146svint8x4_t svreinterpret_s8(svuint8x4_t op);2501svint8x4_t svreinterpret_s8(svuint8x4_t op);
2502__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x4)))
2503svint8x4_t svreinterpret_s8(svmfloat8x4_t op);
2147__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x4)))2504__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x4)))
2148svint8x4_t svreinterpret_s8(svint16x4_t op);2505svint8x4_t svreinterpret_s8(svint16x4_t op);
2149__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x4)))2506__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x4)))
...@@ -2168,6 +2525,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x...@@ -2168,6 +2525,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
2168svuint8x4_t svreinterpret_u8(svint8x4_t op);2525svuint8x4_t svreinterpret_u8(svint8x4_t op);
2169__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x4)))2526__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x4)))
2170svuint8x4_t svreinterpret_u8(svuint8x4_t op);2527svuint8x4_t svreinterpret_u8(svuint8x4_t op);
2528__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x4)))
2529svuint8x4_t svreinterpret_u8(svmfloat8x4_t op);
2171__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x4)))2530__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x4)))
2172svuint8x4_t svreinterpret_u8(svint16x4_t op);2531svuint8x4_t svreinterpret_u8(svint16x4_t op);
2173__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x4)))2532__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x4)))
...@@ -2188,10 +2547,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_...@@ -2188,10 +2547,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
2188svuint8x4_t svreinterpret_u8(svfloat32x4_t op);2547svuint8x4_t svreinterpret_u8(svfloat32x4_t op);
2189__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x4)))2548__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x4)))
2190svuint8x4_t svreinterpret_u8(svfloat64x4_t op);2549svuint8x4_t svreinterpret_u8(svfloat64x4_t op);
2550__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x4)))
2551svmfloat8x4_t svreinterpret_mf8(svint8x4_t op);
2552__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x4)))
2553svmfloat8x4_t svreinterpret_mf8(svuint8x4_t op);
2554__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x4)))
2555svmfloat8x4_t svreinterpret_mf8(svmfloat8x4_t op);
2556__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x4)))
2557svmfloat8x4_t svreinterpret_mf8(svint16x4_t op);
2558__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x4)))
2559svmfloat8x4_t svreinterpret_mf8(svuint16x4_t op);
2560__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x4)))
2561svmfloat8x4_t svreinterpret_mf8(svint32x4_t op);
2562__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x4)))
2563svmfloat8x4_t svreinterpret_mf8(svuint32x4_t op);
2564__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x4)))
2565svmfloat8x4_t svreinterpret_mf8(svint64x4_t op);
2566__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x4)))
2567svmfloat8x4_t svreinterpret_mf8(svuint64x4_t op);
2568__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x4)))
2569svmfloat8x4_t svreinterpret_mf8(svfloat16x4_t op);
2570__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x4)))
2571svmfloat8x4_t svreinterpret_mf8(svbfloat16x4_t op);
2572__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x4)))
2573svmfloat8x4_t svreinterpret_mf8(svfloat32x4_t op);
2574__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x4)))
2575svmfloat8x4_t svreinterpret_mf8(svfloat64x4_t op);
2191__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x4)))2576__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x4)))
2192svint16x4_t svreinterpret_s16(svint8x4_t op);2577svint16x4_t svreinterpret_s16(svint8x4_t op);
2193__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x4)))2578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x4)))
2194svint16x4_t svreinterpret_s16(svuint8x4_t op);2579svint16x4_t svreinterpret_s16(svuint8x4_t op);
2580__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x4)))
2581svint16x4_t svreinterpret_s16(svmfloat8x4_t op);
2195__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x4)))2582__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x4)))
2196svint16x4_t svreinterpret_s16(svint16x4_t op);2583svint16x4_t svreinterpret_s16(svint16x4_t op);
2197__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x4)))2584__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x4)))
...@@ -2216,6 +2603,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_...@@ -2216,6 +2603,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
2216svuint16x4_t svreinterpret_u16(svint8x4_t op);2603svuint16x4_t svreinterpret_u16(svint8x4_t op);
2217__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x4)))2604__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x4)))
2218svuint16x4_t svreinterpret_u16(svuint8x4_t op);2605svuint16x4_t svreinterpret_u16(svuint8x4_t op);
2606__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x4)))
2607svuint16x4_t svreinterpret_u16(svmfloat8x4_t op);
2219__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x4)))2608__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x4)))
2220svuint16x4_t svreinterpret_u16(svint16x4_t op);2609svuint16x4_t svreinterpret_u16(svint16x4_t op);
2221__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x4)))2610__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x4)))
...@@ -2240,6 +2629,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_...@@ -2240,6 +2629,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
2240svint32x4_t svreinterpret_s32(svint8x4_t op);2629svint32x4_t svreinterpret_s32(svint8x4_t op);
2241__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x4)))2630__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x4)))
2242svint32x4_t svreinterpret_s32(svuint8x4_t op);2631svint32x4_t svreinterpret_s32(svuint8x4_t op);
2632__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x4)))
2633svint32x4_t svreinterpret_s32(svmfloat8x4_t op);
2243__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x4)))2634__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x4)))
2244svint32x4_t svreinterpret_s32(svint16x4_t op);2635svint32x4_t svreinterpret_s32(svint16x4_t op);
2245__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x4)))2636__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x4)))
...@@ -2264,6 +2655,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_...@@ -2264,6 +2655,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
2264svuint32x4_t svreinterpret_u32(svint8x4_t op);2655svuint32x4_t svreinterpret_u32(svint8x4_t op);
2265__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x4)))2656__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x4)))
2266svuint32x4_t svreinterpret_u32(svuint8x4_t op);2657svuint32x4_t svreinterpret_u32(svuint8x4_t op);
2658__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x4)))
2659svuint32x4_t svreinterpret_u32(svmfloat8x4_t op);
2267__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x4)))2660__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x4)))
2268svuint32x4_t svreinterpret_u32(svint16x4_t op);2661svuint32x4_t svreinterpret_u32(svint16x4_t op);
2269__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x4)))2662__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x4)))
...@@ -2288,6 +2681,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_...@@ -2288,6 +2681,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
2288svint64x4_t svreinterpret_s64(svint8x4_t op);2681svint64x4_t svreinterpret_s64(svint8x4_t op);
2289__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x4)))2682__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x4)))
2290svint64x4_t svreinterpret_s64(svuint8x4_t op);2683svint64x4_t svreinterpret_s64(svuint8x4_t op);
2684__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x4)))
2685svint64x4_t svreinterpret_s64(svmfloat8x4_t op);
2291__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x4)))2686__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x4)))
2292svint64x4_t svreinterpret_s64(svint16x4_t op);2687svint64x4_t svreinterpret_s64(svint16x4_t op);
2293__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x4)))2688__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x4)))
...@@ -2312,6 +2707,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_...@@ -2312,6 +2707,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
2312svuint64x4_t svreinterpret_u64(svint8x4_t op);2707svuint64x4_t svreinterpret_u64(svint8x4_t op);
2313__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x4)))2708__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x4)))
2314svuint64x4_t svreinterpret_u64(svuint8x4_t op);2709svuint64x4_t svreinterpret_u64(svuint8x4_t op);
2710__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x4)))
2711svuint64x4_t svreinterpret_u64(svmfloat8x4_t op);
2315__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x4)))2712__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x4)))
2316svuint64x4_t svreinterpret_u64(svint16x4_t op);2713svuint64x4_t svreinterpret_u64(svint16x4_t op);
2317__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x4)))2714__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x4)))
...@@ -2336,6 +2733,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_...@@ -2336,6 +2733,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
2336svfloat16x4_t svreinterpret_f16(svint8x4_t op);2733svfloat16x4_t svreinterpret_f16(svint8x4_t op);
2337__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x4)))2734__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x4)))
2338svfloat16x4_t svreinterpret_f16(svuint8x4_t op);2735svfloat16x4_t svreinterpret_f16(svuint8x4_t op);
2736__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x4)))
2737svfloat16x4_t svreinterpret_f16(svmfloat8x4_t op);
2339__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x4)))2738__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x4)))
2340svfloat16x4_t svreinterpret_f16(svint16x4_t op);2739svfloat16x4_t svreinterpret_f16(svint16x4_t op);
2341__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x4)))2740__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x4)))
...@@ -2360,6 +2759,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8...@@ -2360,6 +2759,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
2360svbfloat16x4_t svreinterpret_bf16(svint8x4_t op);2759svbfloat16x4_t svreinterpret_bf16(svint8x4_t op);
2361__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x4)))2760__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x4)))
2362svbfloat16x4_t svreinterpret_bf16(svuint8x4_t op);2761svbfloat16x4_t svreinterpret_bf16(svuint8x4_t op);
2762__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x4)))
2763svbfloat16x4_t svreinterpret_bf16(svmfloat8x4_t op);
2363__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x4)))2764__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x4)))
2364svbfloat16x4_t svreinterpret_bf16(svint16x4_t op);2765svbfloat16x4_t svreinterpret_bf16(svint16x4_t op);
2365__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x4)))2766__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x4)))
...@@ -2384,6 +2785,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_...@@ -2384,6 +2785,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
2384svfloat32x4_t svreinterpret_f32(svint8x4_t op);2785svfloat32x4_t svreinterpret_f32(svint8x4_t op);
2385__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x4)))2786__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x4)))
2386svfloat32x4_t svreinterpret_f32(svuint8x4_t op);2787svfloat32x4_t svreinterpret_f32(svuint8x4_t op);
2788__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x4)))
2789svfloat32x4_t svreinterpret_f32(svmfloat8x4_t op);
2387__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x4)))2790__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x4)))
2388svfloat32x4_t svreinterpret_f32(svint16x4_t op);2791svfloat32x4_t svreinterpret_f32(svint16x4_t op);
2389__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x4)))2792__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x4)))
...@@ -2408,6 +2811,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_...@@ -2408,6 +2811,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
2408svfloat64x4_t svreinterpret_f64(svint8x4_t op);2811svfloat64x4_t svreinterpret_f64(svint8x4_t op);
2409__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x4)))2812__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x4)))
2410svfloat64x4_t svreinterpret_f64(svuint8x4_t op);2813svfloat64x4_t svreinterpret_f64(svuint8x4_t op);
2814__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x4)))
2815svfloat64x4_t svreinterpret_f64(svmfloat8x4_t op);
2411__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x4)))2816__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x4)))
2412svfloat64x4_t svreinterpret_f64(svint16x4_t op);2817svfloat64x4_t svreinterpret_f64(svint16x4_t op);
2413__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x4)))2818__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x4)))
...@@ -3956,6 +4361,150 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq_s64_x4)))...@@ -3956,6 +4361,150 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq_s64_x4)))
3956svint64x4_t svzipq(svint64x4_t);4361svint64x4_t svzipq(svint64x4_t);
3957__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq_s16_x4)))4362__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq_s16_x4)))
3958svint16x4_t svzipq(svint16x4_t);4363svint16x4_t svzipq(svint16x4_t);
4364__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x2)))
4365svfloat64x2_t svamax_f64_x2(svfloat64x2_t, svfloat64x2_t);
4366__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x2)))
4367svfloat32x2_t svamax_f32_x2(svfloat32x2_t, svfloat32x2_t);
4368__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x2)))
4369svfloat16x2_t svamax_f16_x2(svfloat16x2_t, svfloat16x2_t);
4370__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x4)))
4371svfloat64x4_t svamax_f64_x4(svfloat64x4_t, svfloat64x4_t);
4372__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x4)))
4373svfloat32x4_t svamax_f32_x4(svfloat32x4_t, svfloat32x4_t);
4374__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x4)))
4375svfloat16x4_t svamax_f16_x4(svfloat16x4_t, svfloat16x4_t);
4376__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x2)))
4377svfloat64x2_t svamin_f64_x2(svfloat64x2_t, svfloat64x2_t);
4378__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x2)))
4379svfloat32x2_t svamin_f32_x2(svfloat32x2_t, svfloat32x2_t);
4380__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x2)))
4381svfloat16x2_t svamin_f16_x2(svfloat16x2_t, svfloat16x2_t);
4382__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x4)))
4383svfloat64x4_t svamin_f64_x4(svfloat64x4_t, svfloat64x4_t);
4384__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x4)))
4385svfloat32x4_t svamin_f32_x4(svfloat32x4_t, svfloat32x4_t);
4386__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x4)))
4387svfloat16x4_t svamin_f16_x4(svfloat16x4_t, svfloat16x4_t);
4388__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x2)))
4389svfloat64x2_t svamax(svfloat64x2_t, svfloat64x2_t);
4390__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x2)))
4391svfloat32x2_t svamax(svfloat32x2_t, svfloat32x2_t);
4392__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x2)))
4393svfloat16x2_t svamax(svfloat16x2_t, svfloat16x2_t);
4394__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x4)))
4395svfloat64x4_t svamax(svfloat64x4_t, svfloat64x4_t);
4396__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x4)))
4397svfloat32x4_t svamax(svfloat32x4_t, svfloat32x4_t);
4398__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x4)))
4399svfloat16x4_t svamax(svfloat16x4_t, svfloat16x4_t);
4400__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x2)))
4401svfloat64x2_t svamin(svfloat64x2_t, svfloat64x2_t);
4402__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x2)))
4403svfloat32x2_t svamin(svfloat32x2_t, svfloat32x2_t);
4404__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x2)))
4405svfloat16x2_t svamin(svfloat16x2_t, svfloat16x2_t);
4406__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x4)))
4407svfloat64x4_t svamin(svfloat64x4_t, svfloat64x4_t);
4408__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x4)))
4409svfloat32x4_t svamin(svfloat32x4_t, svfloat32x4_t);
4410__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x4)))
4411svfloat16x4_t svamin(svfloat16x4_t, svfloat16x4_t);
4412__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_x2_fpm)))
4413svbfloat16x2_t svcvt1_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4414__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_x2_fpm)))
4415svfloat16x2_t svcvt1_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4416__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_x2_fpm)))
4417svbfloat16x2_t svcvt2_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4418__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_x2_fpm)))
4419svfloat16x2_t svcvt2_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4420__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_bf16_x2_fpm)))
4421svmfloat8_t svcvt_mf8_bf16_x2_fpm(svbfloat16x2_t, fpm_t);
4422__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f16_x2_fpm)))
4423svmfloat8_t svcvt_mf8_f16_x2_fpm(svfloat16x2_t, fpm_t);
4424__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f32_x4_fpm)))
4425svmfloat8_t svcvt_mf8_f32_x4_fpm(svfloat32x4_t, fpm_t);
4426__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_bf16_mf8_x2_fpm)))
4427svbfloat16x2_t svcvtl1_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4428__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_f16_mf8_x2_fpm)))
4429svfloat16x2_t svcvtl1_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4430__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_bf16_mf8_x2_fpm)))
4431svbfloat16x2_t svcvtl2_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4432__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_f16_mf8_x2_fpm)))
4433svfloat16x2_t svcvtl2_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4434__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f32_x4_fpm)))
4435svmfloat8_t svcvtn_mf8_f32_x4_fpm(svfloat32x4_t, fpm_t);
4436__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x2)))
4437svfloat64x2_t svscale_single_f64_x2(svfloat64x2_t, svint64_t);
4438__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x2)))
4439svfloat32x2_t svscale_single_f32_x2(svfloat32x2_t, svint32_t);
4440__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x2)))
4441svfloat16x2_t svscale_single_f16_x2(svfloat16x2_t, svint16_t);
4442__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x4)))
4443svfloat64x4_t svscale_single_f64_x4(svfloat64x4_t, svint64_t);
4444__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x4)))
4445svfloat32x4_t svscale_single_f32_x4(svfloat32x4_t, svint32_t);
4446__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x4)))
4447svfloat16x4_t svscale_single_f16_x4(svfloat16x4_t, svint16_t);
4448__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x2)))
4449svfloat64x2_t svscale_f64_x2(svfloat64x2_t, svint64x2_t);
4450__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x2)))
4451svfloat32x2_t svscale_f32_x2(svfloat32x2_t, svint32x2_t);
4452__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x2)))
4453svfloat16x2_t svscale_f16_x2(svfloat16x2_t, svint16x2_t);
4454__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x4)))
4455svfloat64x4_t svscale_f64_x4(svfloat64x4_t, svint64x4_t);
4456__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x4)))
4457svfloat32x4_t svscale_f32_x4(svfloat32x4_t, svint32x4_t);
4458__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x4)))
4459svfloat16x4_t svscale_f16_x4(svfloat16x4_t, svint16x4_t);
4460__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_x2_fpm)))
4461svbfloat16x2_t svcvt1_bf16_x2_fpm(svmfloat8_t, fpm_t);
4462__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_x2_fpm)))
4463svfloat16x2_t svcvt1_f16_x2_fpm(svmfloat8_t, fpm_t);
4464__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_x2_fpm)))
4465svbfloat16x2_t svcvt2_bf16_x2_fpm(svmfloat8_t, fpm_t);
4466__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_x2_fpm)))
4467svfloat16x2_t svcvt2_f16_x2_fpm(svmfloat8_t, fpm_t);
4468__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_bf16_x2_fpm)))
4469svmfloat8_t svcvt_mf8_fpm(svbfloat16x2_t, fpm_t);
4470__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f16_x2_fpm)))
4471svmfloat8_t svcvt_mf8_fpm(svfloat16x2_t, fpm_t);
4472__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f32_x4_fpm)))
4473svmfloat8_t svcvt_mf8_fpm(svfloat32x4_t, fpm_t);
4474__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_bf16_mf8_x2_fpm)))
4475svbfloat16x2_t svcvtl1_bf16_x2_fpm(svmfloat8_t, fpm_t);
4476__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_f16_mf8_x2_fpm)))
4477svfloat16x2_t svcvtl1_f16_x2_fpm(svmfloat8_t, fpm_t);
4478__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_bf16_mf8_x2_fpm)))
4479svbfloat16x2_t svcvtl2_bf16_x2_fpm(svmfloat8_t, fpm_t);
4480__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_f16_mf8_x2_fpm)))
4481svfloat16x2_t svcvtl2_f16_x2_fpm(svmfloat8_t, fpm_t);
4482__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f32_x4_fpm)))
4483svmfloat8_t svcvtn_mf8_fpm(svfloat32x4_t, fpm_t);
4484__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x2)))
4485svfloat64x2_t svscale(svfloat64x2_t, svint64_t);
4486__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x2)))
4487svfloat32x2_t svscale(svfloat32x2_t, svint32_t);
4488__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x2)))
4489svfloat16x2_t svscale(svfloat16x2_t, svint16_t);
4490__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x4)))
4491svfloat64x4_t svscale(svfloat64x4_t, svint64_t);
4492__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x4)))
4493svfloat32x4_t svscale(svfloat32x4_t, svint32_t);
4494__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x4)))
4495svfloat16x4_t svscale(svfloat16x4_t, svint16_t);
4496__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x2)))
4497svfloat64x2_t svscale(svfloat64x2_t, svint64x2_t);
4498__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x2)))
4499svfloat32x2_t svscale(svfloat32x2_t, svint32x2_t);
4500__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x2)))
4501svfloat16x2_t svscale(svfloat16x2_t, svint16x2_t);
4502__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x4)))
4503svfloat64x4_t svscale(svfloat64x4_t, svint64x4_t);
4504__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x4)))
4505svfloat32x4_t svscale(svfloat32x4_t, svint32x4_t);
4506__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x4)))
4507svfloat16x4_t svscale(svfloat16x4_t, svint16x4_t);
3959__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_single_bf16_x2)))4508__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_single_bf16_x2)))
3960svbfloat16x2_t svclamp_single_bf16_x2(svbfloat16x2_t, svbfloat16_t, svbfloat16_t);4509svbfloat16x2_t svclamp_single_bf16_x2(svbfloat16x2_t, svbfloat16_t, svbfloat16_t);
3961__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_single_bf16_x4)))4510__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_single_bf16_x4)))
...@@ -4028,6 +4577,12 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x2)))...@@ -4028,6 +4577,12 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x2)))
4028svbfloat16x2_t svminnm(svbfloat16x2_t, svbfloat16x2_t);4577svbfloat16x2_t svminnm(svbfloat16x2_t, svbfloat16x2_t);
4029__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x4)))4578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x4)))
4030svbfloat16x4_t svminnm(svbfloat16x4_t, svbfloat16x4_t);4579svbfloat16x4_t svminnm(svbfloat16x4_t, svbfloat16x4_t);
4580__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
4581float64_t svadda_f64(svbool_t, float64_t, svfloat64_t);
4582__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
4583float32_t svadda_f32(svbool_t, float32_t, svfloat32_t);
4584__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
4585float16_t svadda_f16(svbool_t, float16_t, svfloat16_t);
4031__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u32base_u32offset)))4586__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u32base_u32offset)))
4032svuint32_t svadrb_u32base_u32offset(svuint32_t, svuint32_t);4587svuint32_t svadrb_u32base_u32offset(svuint32_t, svuint32_t);
4033__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u64base_u64offset)))4588__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u64base_u64offset)))
...@@ -5280,6 +5835,12 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtssel_f16)))...@@ -5280,6 +5835,12 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtssel_f16)))
5280svfloat16_t svtssel_f16(svfloat16_t, svuint16_t);5835svfloat16_t svtssel_f16(svfloat16_t, svuint16_t);
5281__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwrffr)))5836__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwrffr)))
5282void svwrffr(svbool_t);5837void svwrffr(svbool_t);
5838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
5839float64_t svadda(svbool_t, float64_t, svfloat64_t);
5840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
5841float32_t svadda(svbool_t, float32_t, svfloat32_t);
5842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
5843float16_t svadda(svbool_t, float16_t, svfloat16_t);
5283__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u32base_u32offset)))5844__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u32base_u32offset)))
5284svuint32_t svadrb_offset(svuint32_t, svuint32_t);5845svuint32_t svadrb_offset(svuint32_t, svuint32_t);
5285__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u64base_u64offset)))5846__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u64base_u64offset)))
...@@ -7948,6 +8509,406 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u6...@@ -7948,6 +8509,406 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u6
7948void svstnt1w_scatter_offset(svbool_t, int32_t *, svuint64_t, svint64_t);8509void svstnt1w_scatter_offset(svbool_t, int32_t *, svuint64_t, svint64_t);
7949__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))8510__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))
7950void svstnt1w_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint64_t);8511void svstnt1w_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint64_t);
8512__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8513svbfloat16_t svtbl2_bf16(svbfloat16x2_t, svuint16_t);
8514__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8515svbfloat16_t svtbx_bf16(svbfloat16_t, svbfloat16_t, svuint16_t);
8516__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8517svbool_t svwhilerw_bf16(bfloat16_t const *, bfloat16_t const *);
8518__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8519svbool_t svwhilewr_bf16(bfloat16_t const *, bfloat16_t const *);
8520__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8521svbfloat16_t svtbl2(svbfloat16x2_t, svuint16_t);
8522__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8523svbfloat16_t svtbx(svbfloat16_t, svbfloat16_t, svuint16_t);
8524__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8525svbool_t svwhilerw(bfloat16_t const *, bfloat16_t const *);
8526__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8527svbool_t svwhilewr(bfloat16_t const *, bfloat16_t const *);
8528__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
8529svfloat64_t svamax_n_f64_m(svbool_t, svfloat64_t, float64_t);
8530__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
8531svfloat32_t svamax_n_f32_m(svbool_t, svfloat32_t, float32_t);
8532__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
8533svfloat16_t svamax_n_f16_m(svbool_t, svfloat16_t, float16_t);
8534__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
8535svfloat64_t svamax_n_f64_x(svbool_t, svfloat64_t, float64_t);
8536__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
8537svfloat32_t svamax_n_f32_x(svbool_t, svfloat32_t, float32_t);
8538__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
8539svfloat16_t svamax_n_f16_x(svbool_t, svfloat16_t, float16_t);
8540__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
8541svfloat64_t svamax_n_f64_z(svbool_t, svfloat64_t, float64_t);
8542__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
8543svfloat32_t svamax_n_f32_z(svbool_t, svfloat32_t, float32_t);
8544__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
8545svfloat16_t svamax_n_f16_z(svbool_t, svfloat16_t, float16_t);
8546__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
8547svfloat64_t svamax_f64_m(svbool_t, svfloat64_t, svfloat64_t);
8548__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
8549svfloat32_t svamax_f32_m(svbool_t, svfloat32_t, svfloat32_t);
8550__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
8551svfloat16_t svamax_f16_m(svbool_t, svfloat16_t, svfloat16_t);
8552__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
8553svfloat64_t svamax_f64_x(svbool_t, svfloat64_t, svfloat64_t);
8554__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
8555svfloat32_t svamax_f32_x(svbool_t, svfloat32_t, svfloat32_t);
8556__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
8557svfloat16_t svamax_f16_x(svbool_t, svfloat16_t, svfloat16_t);
8558__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
8559svfloat64_t svamax_f64_z(svbool_t, svfloat64_t, svfloat64_t);
8560__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
8561svfloat32_t svamax_f32_z(svbool_t, svfloat32_t, svfloat32_t);
8562__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
8563svfloat16_t svamax_f16_z(svbool_t, svfloat16_t, svfloat16_t);
8564__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
8565svfloat64_t svamin_n_f64_m(svbool_t, svfloat64_t, float64_t);
8566__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
8567svfloat32_t svamin_n_f32_m(svbool_t, svfloat32_t, float32_t);
8568__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
8569svfloat16_t svamin_n_f16_m(svbool_t, svfloat16_t, float16_t);
8570__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
8571svfloat64_t svamin_n_f64_x(svbool_t, svfloat64_t, float64_t);
8572__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
8573svfloat32_t svamin_n_f32_x(svbool_t, svfloat32_t, float32_t);
8574__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
8575svfloat16_t svamin_n_f16_x(svbool_t, svfloat16_t, float16_t);
8576__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
8577svfloat64_t svamin_n_f64_z(svbool_t, svfloat64_t, float64_t);
8578__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
8579svfloat32_t svamin_n_f32_z(svbool_t, svfloat32_t, float32_t);
8580__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
8581svfloat16_t svamin_n_f16_z(svbool_t, svfloat16_t, float16_t);
8582__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
8583svfloat64_t svamin_f64_m(svbool_t, svfloat64_t, svfloat64_t);
8584__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
8585svfloat32_t svamin_f32_m(svbool_t, svfloat32_t, svfloat32_t);
8586__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
8587svfloat16_t svamin_f16_m(svbool_t, svfloat16_t, svfloat16_t);
8588__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
8589svfloat64_t svamin_f64_x(svbool_t, svfloat64_t, svfloat64_t);
8590__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
8591svfloat32_t svamin_f32_x(svbool_t, svfloat32_t, svfloat32_t);
8592__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
8593svfloat16_t svamin_f16_x(svbool_t, svfloat16_t, svfloat16_t);
8594__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
8595svfloat64_t svamin_f64_z(svbool_t, svfloat64_t, svfloat64_t);
8596__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
8597svfloat32_t svamin_f32_z(svbool_t, svfloat32_t, svfloat32_t);
8598__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
8599svfloat16_t svamin_f16_z(svbool_t, svfloat16_t, svfloat16_t);
8600__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
8601svfloat64_t svamax_m(svbool_t, svfloat64_t, float64_t);
8602__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
8603svfloat32_t svamax_m(svbool_t, svfloat32_t, float32_t);
8604__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
8605svfloat16_t svamax_m(svbool_t, svfloat16_t, float16_t);
8606__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
8607svfloat64_t svamax_x(svbool_t, svfloat64_t, float64_t);
8608__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
8609svfloat32_t svamax_x(svbool_t, svfloat32_t, float32_t);
8610__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
8611svfloat16_t svamax_x(svbool_t, svfloat16_t, float16_t);
8612__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
8613svfloat64_t svamax_z(svbool_t, svfloat64_t, float64_t);
8614__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
8615svfloat32_t svamax_z(svbool_t, svfloat32_t, float32_t);
8616__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
8617svfloat16_t svamax_z(svbool_t, svfloat16_t, float16_t);
8618__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
8619svfloat64_t svamax_m(svbool_t, svfloat64_t, svfloat64_t);
8620__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
8621svfloat32_t svamax_m(svbool_t, svfloat32_t, svfloat32_t);
8622__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
8623svfloat16_t svamax_m(svbool_t, svfloat16_t, svfloat16_t);
8624__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
8625svfloat64_t svamax_x(svbool_t, svfloat64_t, svfloat64_t);
8626__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
8627svfloat32_t svamax_x(svbool_t, svfloat32_t, svfloat32_t);
8628__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
8629svfloat16_t svamax_x(svbool_t, svfloat16_t, svfloat16_t);
8630__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
8631svfloat64_t svamax_z(svbool_t, svfloat64_t, svfloat64_t);
8632__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
8633svfloat32_t svamax_z(svbool_t, svfloat32_t, svfloat32_t);
8634__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
8635svfloat16_t svamax_z(svbool_t, svfloat16_t, svfloat16_t);
8636__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
8637svfloat64_t svamin_m(svbool_t, svfloat64_t, float64_t);
8638__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
8639svfloat32_t svamin_m(svbool_t, svfloat32_t, float32_t);
8640__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
8641svfloat16_t svamin_m(svbool_t, svfloat16_t, float16_t);
8642__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
8643svfloat64_t svamin_x(svbool_t, svfloat64_t, float64_t);
8644__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
8645svfloat32_t svamin_x(svbool_t, svfloat32_t, float32_t);
8646__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
8647svfloat16_t svamin_x(svbool_t, svfloat16_t, float16_t);
8648__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
8649svfloat64_t svamin_z(svbool_t, svfloat64_t, float64_t);
8650__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
8651svfloat32_t svamin_z(svbool_t, svfloat32_t, float32_t);
8652__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
8653svfloat16_t svamin_z(svbool_t, svfloat16_t, float16_t);
8654__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
8655svfloat64_t svamin_m(svbool_t, svfloat64_t, svfloat64_t);
8656__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
8657svfloat32_t svamin_m(svbool_t, svfloat32_t, svfloat32_t);
8658__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
8659svfloat16_t svamin_m(svbool_t, svfloat16_t, svfloat16_t);
8660__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
8661svfloat64_t svamin_x(svbool_t, svfloat64_t, svfloat64_t);
8662__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
8663svfloat32_t svamin_x(svbool_t, svfloat32_t, svfloat32_t);
8664__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
8665svfloat16_t svamin_x(svbool_t, svfloat16_t, svfloat16_t);
8666__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
8667svfloat64_t svamin_z(svbool_t, svfloat64_t, svfloat64_t);
8668__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
8669svfloat32_t svamin_z(svbool_t, svfloat32_t, svfloat32_t);
8670__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
8671svfloat16_t svamin_z(svbool_t, svfloat16_t, svfloat16_t);
8672__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
8673svfloat16_t svdot_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8674__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
8675svfloat16_t svdot_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8676__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
8677svfloat16_t svdot_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8678__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
8679svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8680__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
8681svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8682__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
8683svfloat16_t svdot_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8684__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
8685svfloat32_t svdot_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8686__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
8687svfloat32_t svdot_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8688__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
8689svfloat32_t svdot_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8690__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
8691svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8692__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
8693svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8694__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
8695svfloat32_t svdot_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8696__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
8697svfloat16_t svmlalb_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8698__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
8699svfloat16_t svmlalb_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8700__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
8701svfloat16_t svmlalb_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8702__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
8703svfloat32_t svmlallbb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8704__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
8705svfloat32_t svmlallbb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8706__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
8707svfloat32_t svmlallbb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8708__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
8709svfloat32_t svmlallbt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8710__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
8711svfloat32_t svmlallbt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8712__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
8713svfloat32_t svmlallbt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8714__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
8715svfloat32_t svmlalltb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8716__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
8717svfloat32_t svmlalltb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8718__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
8719svfloat32_t svmlalltb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8720__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
8721svfloat32_t svmlalltt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8722__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
8723svfloat32_t svmlalltt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8724__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
8725svfloat32_t svmlalltt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8726__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
8727svfloat16_t svmlalt_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8728__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
8729svfloat16_t svmlalt_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8730__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
8731svfloat16_t svmlalt_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8732__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
8733svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8734__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
8735svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8736__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
8737svfloat16_t svmlalb_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8738__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
8739svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8740__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
8741svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8742__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
8743svfloat32_t svmlallbb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8744__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
8745svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8746__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
8747svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8748__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
8749svfloat32_t svmlallbt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8750__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
8751svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8752__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
8753svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8754__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
8755svfloat32_t svmlalltb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8756__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
8757svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8758__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
8759svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8760__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
8761svfloat32_t svmlalltt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8762__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
8763svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8764__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
8765svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8766__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
8767svfloat16_t svmlalt_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8768__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
8769svbfloat16_t svcvt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8770__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
8771svfloat16_t svcvt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
8772__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
8773svbfloat16_t svcvt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8774__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
8775svfloat16_t svcvt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
8776__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
8777svbfloat16_t svcvtlt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8778__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
8779svfloat16_t svcvtlt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
8780__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
8781svbfloat16_t svcvtlt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8782__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
8783svfloat16_t svcvtlt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
8784__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
8785svmfloat8_t svcvtn_mf8_bf16_x2_fpm(svbfloat16x2_t, fpm_t);
8786__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
8787svmfloat8_t svcvtn_mf8_f16_x2_fpm(svfloat16x2_t, fpm_t);
8788__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
8789svmfloat8_t svcvtnb_mf8_f32_x2_fpm(svfloat32x2_t, fpm_t);
8790__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
8791svmfloat8_t svcvtnt_mf8_f32_x2_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
8792__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
8793svbfloat16_t svcvt1_bf16_fpm(svmfloat8_t, fpm_t);
8794__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
8795svfloat16_t svcvt1_f16_fpm(svmfloat8_t, fpm_t);
8796__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
8797svbfloat16_t svcvt2_bf16_fpm(svmfloat8_t, fpm_t);
8798__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
8799svfloat16_t svcvt2_f16_fpm(svmfloat8_t, fpm_t);
8800__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
8801svbfloat16_t svcvtlt1_bf16_fpm(svmfloat8_t, fpm_t);
8802__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
8803svfloat16_t svcvtlt1_f16_fpm(svmfloat8_t, fpm_t);
8804__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
8805svbfloat16_t svcvtlt2_bf16_fpm(svmfloat8_t, fpm_t);
8806__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
8807svfloat16_t svcvtlt2_f16_fpm(svmfloat8_t, fpm_t);
8808__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
8809svmfloat8_t svcvtn_mf8_fpm(svbfloat16x2_t, fpm_t);
8810__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
8811svmfloat8_t svcvtn_mf8_fpm(svfloat16x2_t, fpm_t);
8812__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
8813svmfloat8_t svcvtnb_mf8_fpm(svfloat32x2_t, fpm_t);
8814__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
8815svmfloat8_t svcvtnt_mf8_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
8816__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
8817svbfloat16_t svluti2_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
8818__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
8819svbfloat16_t svluti4_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
8820__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
8821svbfloat16_t svluti4_lane_bf16_x2(svbfloat16x2_t, svuint8_t, uint64_t);
8822__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
8823svbfloat16_t svluti2_lane(svbfloat16_t, svuint8_t, uint64_t);
8824__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
8825svbfloat16_t svluti4_lane(svbfloat16_t, svuint8_t, uint64_t);
8826__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
8827svbfloat16_t svluti4_lane(svbfloat16x2_t, svuint8_t, uint64_t);
8828__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
8829svuint8_t svluti2_lane_u8(svuint8_t, svuint8_t, uint64_t);
8830__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
8831svint8_t svluti2_lane_s8(svint8_t, svuint8_t, uint64_t);
8832__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
8833svuint16_t svluti2_lane_u16(svuint16_t, svuint8_t, uint64_t);
8834__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
8835svfloat16_t svluti2_lane_f16(svfloat16_t, svuint8_t, uint64_t);
8836__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
8837svint16_t svluti2_lane_s16(svint16_t, svuint8_t, uint64_t);
8838__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
8839svuint8_t svluti4_lane_u8(svuint8_t, svuint8_t, uint64_t);
8840__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
8841svint8_t svluti4_lane_s8(svint8_t, svuint8_t, uint64_t);
8842__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
8843svuint16_t svluti4_lane_u16(svuint16_t, svuint8_t, uint64_t);
8844__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
8845svfloat16_t svluti4_lane_f16(svfloat16_t, svuint8_t, uint64_t);
8846__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
8847svint16_t svluti4_lane_s16(svint16_t, svuint8_t, uint64_t);
8848__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
8849svuint16_t svluti4_lane_u16_x2(svuint16x2_t, svuint8_t, uint64_t);
8850__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
8851svfloat16_t svluti4_lane_f16_x2(svfloat16x2_t, svuint8_t, uint64_t);
8852__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
8853svint16_t svluti4_lane_s16_x2(svint16x2_t, svuint8_t, uint64_t);
8854__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
8855svuint8_t svluti2_lane(svuint8_t, svuint8_t, uint64_t);
8856__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
8857svint8_t svluti2_lane(svint8_t, svuint8_t, uint64_t);
8858__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
8859svuint16_t svluti2_lane(svuint16_t, svuint8_t, uint64_t);
8860__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
8861svfloat16_t svluti2_lane(svfloat16_t, svuint8_t, uint64_t);
8862__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
8863svint16_t svluti2_lane(svint16_t, svuint8_t, uint64_t);
8864__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
8865svuint8_t svluti4_lane(svuint8_t, svuint8_t, uint64_t);
8866__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
8867svint8_t svluti4_lane(svint8_t, svuint8_t, uint64_t);
8868__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
8869svuint16_t svluti4_lane(svuint16_t, svuint8_t, uint64_t);
8870__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
8871svfloat16_t svluti4_lane(svfloat16_t, svuint8_t, uint64_t);
8872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
8873svint16_t svluti4_lane(svint16_t, svuint8_t, uint64_t);
8874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
8875svuint16_t svluti4_lane(svuint16x2_t, svuint8_t, uint64_t);
8876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
8877svfloat16_t svluti4_lane(svfloat16x2_t, svuint8_t, uint64_t);
8878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
8879svint16_t svluti4_lane(svint16x2_t, svuint8_t, uint64_t);
8880__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8881svuint8_t svaesd_u8(svuint8_t, svuint8_t);
8882__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8883svuint8_t svaese_u8(svuint8_t, svuint8_t);
8884__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8885svuint8_t svaesimc_u8(svuint8_t);
8886__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8887svuint8_t svaesmc_u8(svuint8_t);
8888__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8889svuint64_t svpmullb_pair_n_u64(svuint64_t, uint64_t);
8890__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8891svuint64_t svpmullb_pair_u64(svuint64_t, svuint64_t);
8892__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8893svuint64_t svpmullt_pair_n_u64(svuint64_t, uint64_t);
8894__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8895svuint64_t svpmullt_pair_u64(svuint64_t, svuint64_t);
8896__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8897svuint8_t svaesd(svuint8_t, svuint8_t);
8898__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8899svuint8_t svaese(svuint8_t, svuint8_t);
8900__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8901svuint8_t svaesimc(svuint8_t);
8902__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8903svuint8_t svaesmc(svuint8_t);
8904__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8905svuint64_t svpmullb_pair(svuint64_t, uint64_t);
8906__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8907svuint64_t svpmullb_pair(svuint64_t, svuint64_t);
8908__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8909svuint64_t svpmullt_pair(svuint64_t, uint64_t);
8910__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8911svuint64_t svpmullt_pair(svuint64_t, svuint64_t);
7951__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))8912__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))
7952svbfloat16_t svadd_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);8913svbfloat16_t svadd_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
7953__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))8914__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))
...@@ -8180,54 +9141,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))...@@ -8180,54 +9141,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
8180svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t);9141svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t);
8181__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))9142__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
8182svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t);9143svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t);
8183__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8184svbfloat16_t svtbl2_bf16(svbfloat16x2_t, svuint16_t);
8185__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8186svbfloat16_t svtbx_bf16(svbfloat16_t, svbfloat16_t, svuint16_t);
8187__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8188svbool_t svwhilerw_bf16(bfloat16_t const *, bfloat16_t const *);
8189__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8190svbool_t svwhilewr_bf16(bfloat16_t const *, bfloat16_t const *);
8191__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8192svbfloat16_t svtbl2(svbfloat16x2_t, svuint16_t);
8193__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8194svbfloat16_t svtbx(svbfloat16_t, svbfloat16_t, svuint16_t);
8195__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8196svbool_t svwhilerw(bfloat16_t const *, bfloat16_t const *);
8197__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8198svbool_t svwhilewr(bfloat16_t const *, bfloat16_t const *);
8199__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8200svuint8_t svaesd_u8(svuint8_t, svuint8_t);
8201__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8202svuint8_t svaese_u8(svuint8_t, svuint8_t);
8203__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8204svuint8_t svaesimc_u8(svuint8_t);
8205__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8206svuint8_t svaesmc_u8(svuint8_t);
8207__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8208svuint64_t svpmullb_pair_n_u64(svuint64_t, uint64_t);
8209__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8210svuint64_t svpmullb_pair_u64(svuint64_t, svuint64_t);
8211__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8212svuint64_t svpmullt_pair_n_u64(svuint64_t, uint64_t);
8213__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8214svuint64_t svpmullt_pair_u64(svuint64_t, svuint64_t);
8215__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8216svuint8_t svaesd(svuint8_t, svuint8_t);
8217__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8218svuint8_t svaese(svuint8_t, svuint8_t);
8219__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8220svuint8_t svaesimc(svuint8_t);
8221__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8222svuint8_t svaesmc(svuint8_t);
8223__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8224svuint64_t svpmullb_pair(svuint64_t, uint64_t);
8225__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8226svuint64_t svpmullb_pair(svuint64_t, svuint64_t);
8227__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8228svuint64_t svpmullt_pair(svuint64_t, uint64_t);
8229__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8230svuint64_t svpmullt_pair(svuint64_t, svuint64_t);
8231__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))9144__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))
8232svuint8_t svbdep_n_u8(svuint8_t, uint8_t);9145svuint8_t svbdep_n_u8(svuint8_t, uint8_t);
8233__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))9146__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))
...@@ -8416,6 +9329,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s32)))...@@ -8416,6 +9329,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s32)))
8416svint32_t svextq_s32(svint32_t, svint32_t, int32_t);9329svint32_t svextq_s32(svint32_t, svint32_t, int32_t);
8417__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s64)))9330__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s64)))
8418svint64_t svextq_s64(svint64_t, svint64_t, int32_t);9331svint64_t svextq_s64(svint64_t, svint64_t, int32_t);
9332__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_mf8)))
9333svmfloat8_t svextq_mf8(svmfloat8_t, svmfloat8_t, int32_t);
8419__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s16)))9334__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s16)))
8420svint16_t svextq_s16(svint16_t, svint16_t, int32_t);9335svint16_t svextq_s16(svint16_t, svint16_t, int32_t);
8421__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))9336__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
...@@ -8831,17 +9746,17 @@ svuint32_t svpmov_u32_z(svbool_t);...@@ -8831,17 +9746,17 @@ svuint32_t svpmov_u32_z(svbool_t);
8831__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmov_s32_z)))9746__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmov_s32_z)))
8832svint32_t svpmov_s32_z(svbool_t);9747svint32_t svpmov_s32_z(svbool_t);
8833__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))9748__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
8834void svst1dq_u64(svbool_t, uint64_t const *, svuint64_t);9749void svst1dq_u64(svbool_t, uint64_t *, svuint64_t);
8835__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))9750__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
8836void svst1dq_f64(svbool_t, float64_t const *, svfloat64_t);9751void svst1dq_f64(svbool_t, float64_t *, svfloat64_t);
8837__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))9752__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
8838void svst1dq_s64(svbool_t, int64_t const *, svint64_t);9753void svst1dq_s64(svbool_t, int64_t *, svint64_t);
8839__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))9754__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
8840void svst1dq_vnum_u64(svbool_t, uint64_t const *, int64_t, svuint64_t);9755void svst1dq_vnum_u64(svbool_t, uint64_t *, int64_t, svuint64_t);
8841__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))9756__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
8842void svst1dq_vnum_f64(svbool_t, float64_t const *, int64_t, svfloat64_t);9757void svst1dq_vnum_f64(svbool_t, float64_t *, int64_t, svfloat64_t);
8843__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))9758__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
8844void svst1dq_vnum_s64(svbool_t, int64_t const *, int64_t, svint64_t);9759void svst1dq_vnum_s64(svbool_t, int64_t *, int64_t, svint64_t);
8845__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))9760__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
8846void svst1q_scatter_u64base_u8(svbool_t, svuint64_t, svuint8_t);9761void svst1q_scatter_u64base_u8(svbool_t, svuint64_t, svuint8_t);
8847__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))9762__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
...@@ -8910,6 +9825,26 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64ba...@@ -8910,6 +9825,26 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64ba
8910void svst1q_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);9825void svst1q_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
8911__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))9826__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
8912void svst1q_scatter_u64base_offset_s16(svbool_t, svuint64_t, int64_t, svint16_t);9827void svst1q_scatter_u64base_offset_s16(svbool_t, svuint64_t, int64_t, svint16_t);
9828__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
9829void svst1q_scatter_s64index_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
9830__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
9831void svst1q_scatter_s64index_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
9832__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
9833void svst1q_scatter_s64index_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
9834__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
9835void svst1q_scatter_s64index_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
9836__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
9837void svst1q_scatter_s64index_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
9838__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
9839void svst1q_scatter_s64index_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
9840__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
9841void svst1q_scatter_s64index_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
9842__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
9843void svst1q_scatter_s64index_s32(svbool_t, int32_t *, svint64_t, svint32_t);
9844__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
9845void svst1q_scatter_s64index_s64(svbool_t, int64_t *, svint64_t, svint64_t);
9846__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
9847void svst1q_scatter_s64index_s16(svbool_t, int16_t *, svint64_t, svint16_t);
8913__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))9848__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
8914void svst1q_scatter_u64index_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);9849void svst1q_scatter_u64index_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);
8915__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))9850__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
...@@ -8930,6 +9865,30 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64in...@@ -8930,6 +9865,30 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64in
8930void svst1q_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);9865void svst1q_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
8931__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))9866__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
8932void svst1q_scatter_u64index_s16(svbool_t, int16_t *, svuint64_t, svint16_t);9867void svst1q_scatter_u64index_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
9868__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
9869void svst1q_scatter_s64offset_u8(svbool_t, uint8_t *, svint64_t, svuint8_t);
9870__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
9871void svst1q_scatter_s64offset_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
9872__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
9873void svst1q_scatter_s64offset_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
9874__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
9875void svst1q_scatter_s64offset_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
9876__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
9877void svst1q_scatter_s64offset_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
9878__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
9879void svst1q_scatter_s64offset_s8(svbool_t, int8_t *, svint64_t, svint8_t);
9880__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
9881void svst1q_scatter_s64offset_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
9882__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
9883void svst1q_scatter_s64offset_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
9884__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
9885void svst1q_scatter_s64offset_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
9886__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
9887void svst1q_scatter_s64offset_s32(svbool_t, int32_t *, svint64_t, svint32_t);
9888__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
9889void svst1q_scatter_s64offset_s64(svbool_t, int64_t *, svint64_t, svint64_t);
9890__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
9891void svst1q_scatter_s64offset_s16(svbool_t, int16_t *, svint64_t, svint16_t);
8933__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))9892__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
8934void svst1q_scatter_u64offset_u8(svbool_t, uint8_t *, svuint64_t, svuint8_t);9893void svst1q_scatter_u64offset_u8(svbool_t, uint8_t *, svuint64_t, svuint8_t);
8935__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))9894__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
...@@ -8955,17 +9914,17 @@ void svst1q_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);...@@ -8955,17 +9914,17 @@ void svst1q_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
8955__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))9914__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
8956void svst1q_scatter_u64offset_s16(svbool_t, int16_t *, svuint64_t, svint16_t);9915void svst1q_scatter_u64offset_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
8957__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))9916__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
8958void svst1wq_u32(svbool_t, uint32_t const *, svuint32_t);9917void svst1wq_u32(svbool_t, uint32_t *, svuint32_t);
8959__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))9918__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
8960void svst1wq_f32(svbool_t, float32_t const *, svfloat32_t);9919void svst1wq_f32(svbool_t, float32_t *, svfloat32_t);
8961__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))9920__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
8962void svst1wq_s32(svbool_t, int32_t const *, svint32_t);9921void svst1wq_s32(svbool_t, int32_t *, svint32_t);
8963__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))9922__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
8964void svst1wq_vnum_u32(svbool_t, uint32_t const *, int64_t, svuint32_t);9923void svst1wq_vnum_u32(svbool_t, uint32_t *, int64_t, svuint32_t);
8965__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))9924__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
8966void svst1wq_vnum_f32(svbool_t, float32_t const *, int64_t, svfloat32_t);9925void svst1wq_vnum_f32(svbool_t, float32_t *, int64_t, svfloat32_t);
8967__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))9926__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
8968void svst1wq_vnum_s32(svbool_t, int32_t const *, int64_t, svint32_t);9927void svst1wq_vnum_s32(svbool_t, int32_t *, int64_t, svint32_t);
8969__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u8)))9928__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u8)))
8970void svst2q_u8(svbool_t, uint8_t const *, svuint8x2_t);9929void svst2q_u8(svbool_t, uint8_t const *, svuint8x2_t);
8971__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u32)))9930__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u32)))
...@@ -9132,6 +10091,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s32)))...@@ -9132,6 +10091,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s32)))
9132svint32_t svtblq_s32(svint32_t, svuint32_t);10091svint32_t svtblq_s32(svint32_t, svuint32_t);
9133__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s64)))10092__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s64)))
9134svint64_t svtblq_s64(svint64_t, svuint64_t);10093svint64_t svtblq_s64(svint64_t, svuint64_t);
10094__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_mf8)))
10095svmfloat8_t svtblq_mf8(svmfloat8_t, svuint8_t);
9135__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s16)))10096__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s16)))
9136svint16_t svtblq_s16(svint16_t, svuint16_t);10097svint16_t svtblq_s16(svint16_t, svuint16_t);
9137__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_u8)))10098__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_u8)))
...@@ -9156,6 +10117,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s32)))...@@ -9156,6 +10117,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s32)))
9156svint32_t svtbxq_s32(svint32_t, svint32_t, svuint32_t);10117svint32_t svtbxq_s32(svint32_t, svint32_t, svuint32_t);
9157__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s64)))10118__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s64)))
9158svint64_t svtbxq_s64(svint64_t, svint64_t, svuint64_t);10119svint64_t svtbxq_s64(svint64_t, svint64_t, svuint64_t);
10120__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_mf8)))
10121svmfloat8_t svtbxq_mf8(svmfloat8_t, svmfloat8_t, svuint8_t);
9159__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s16)))10122__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s16)))
9160svint16_t svtbxq_s16(svint16_t, svint16_t, svuint16_t);10123svint16_t svtbxq_s16(svint16_t, svint16_t, svuint16_t);
9161__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_u8)))10124__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_u8)))
...@@ -9180,6 +10143,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s32)))...@@ -9180,6 +10143,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s32)))
9180svint32_t svuzpq1_s32(svint32_t, svint32_t);10143svint32_t svuzpq1_s32(svint32_t, svint32_t);
9181__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s64)))10144__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s64)))
9182svint64_t svuzpq1_s64(svint64_t, svint64_t);10145svint64_t svuzpq1_s64(svint64_t, svint64_t);
10146__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_mf8)))
10147svmfloat8_t svuzpq1_mf8(svmfloat8_t, svmfloat8_t);
9183__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s16)))10148__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s16)))
9184svint16_t svuzpq1_s16(svint16_t, svint16_t);10149svint16_t svuzpq1_s16(svint16_t, svint16_t);
9185__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_u8)))10150__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_u8)))
...@@ -9204,6 +10169,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s32)))...@@ -9204,6 +10169,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s32)))
9204svint32_t svuzpq2_s32(svint32_t, svint32_t);10169svint32_t svuzpq2_s32(svint32_t, svint32_t);
9205__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s64)))10170__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s64)))
9206svint64_t svuzpq2_s64(svint64_t, svint64_t);10171svint64_t svuzpq2_s64(svint64_t, svint64_t);
10172__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_mf8)))
10173svmfloat8_t svuzpq2_mf8(svmfloat8_t, svmfloat8_t);
9207__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s16)))10174__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s16)))
9208svint16_t svuzpq2_s16(svint16_t, svint16_t);10175svint16_t svuzpq2_s16(svint16_t, svint16_t);
9209__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_u8)))10176__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_u8)))
...@@ -9228,6 +10195,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s32)))...@@ -9228,6 +10195,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s32)))
9228svint32_t svzipq1_s32(svint32_t, svint32_t);10195svint32_t svzipq1_s32(svint32_t, svint32_t);
9229__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s64)))10196__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s64)))
9230svint64_t svzipq1_s64(svint64_t, svint64_t);10197svint64_t svzipq1_s64(svint64_t, svint64_t);
10198__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_mf8)))
10199svmfloat8_t svzipq1_mf8(svmfloat8_t, svmfloat8_t);
9231__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s16)))10200__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s16)))
9232svint16_t svzipq1_s16(svint16_t, svint16_t);10201svint16_t svzipq1_s16(svint16_t, svint16_t);
9233__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_u8)))10202__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_u8)))
...@@ -9252,6 +10221,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s32)))...@@ -9252,6 +10221,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s32)))
9252svint32_t svzipq2_s32(svint32_t, svint32_t);10221svint32_t svzipq2_s32(svint32_t, svint32_t);
9253__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s64)))10222__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s64)))
9254svint64_t svzipq2_s64(svint64_t, svint64_t);10223svint64_t svzipq2_s64(svint64_t, svint64_t);
10224__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_mf8)))
10225svmfloat8_t svzipq2_mf8(svmfloat8_t, svmfloat8_t);
9255__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))10226__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))
9256svint16_t svzipq2_s16(svint16_t, svint16_t);10227svint16_t svzipq2_s16(svint16_t, svint16_t);
9257__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqv_u8)))10228__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqv_u8)))
...@@ -9330,6 +10301,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s32)))...@@ -9330,6 +10301,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s32)))
9330svint32_t svextq(svint32_t, svint32_t, int32_t);10301svint32_t svextq(svint32_t, svint32_t, int32_t);
9331__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s64)))10302__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s64)))
9332svint64_t svextq(svint64_t, svint64_t, int32_t);10303svint64_t svextq(svint64_t, svint64_t, int32_t);
10304__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_mf8)))
10305svmfloat8_t svextq(svmfloat8_t, svmfloat8_t, int32_t);
9333__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s16)))10306__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s16)))
9334svint16_t svextq(svint16_t, svint16_t, int32_t);10307svint16_t svextq(svint16_t, svint16_t, int32_t);
9335__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))10308__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
...@@ -9729,17 +10702,17 @@ svuint32_t svpmov_lane_m(svuint32_t, svbool_t, uint64_t);...@@ -9729,17 +10702,17 @@ svuint32_t svpmov_lane_m(svuint32_t, svbool_t, uint64_t);
9729__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmov_lane_s32_m)))10702__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmov_lane_s32_m)))
9730svint32_t svpmov_lane_m(svint32_t, svbool_t, uint64_t);10703svint32_t svpmov_lane_m(svint32_t, svbool_t, uint64_t);
9731__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))10704__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
9732void svst1dq(svbool_t, uint64_t const *, svuint64_t);10705void svst1dq(svbool_t, uint64_t *, svuint64_t);
9733__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))10706__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
9734void svst1dq(svbool_t, float64_t const *, svfloat64_t);10707void svst1dq(svbool_t, float64_t *, svfloat64_t);
9735__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))10708__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
9736void svst1dq(svbool_t, int64_t const *, svint64_t);10709void svst1dq(svbool_t, int64_t *, svint64_t);
9737__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))10710__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
9738void svst1dq_vnum(svbool_t, uint64_t const *, int64_t, svuint64_t);10711void svst1dq_vnum(svbool_t, uint64_t *, int64_t, svuint64_t);
9739__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))10712__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
9740void svst1dq_vnum(svbool_t, float64_t const *, int64_t, svfloat64_t);10713void svst1dq_vnum(svbool_t, float64_t *, int64_t, svfloat64_t);
9741__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))10714__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
9742void svst1dq_vnum(svbool_t, int64_t const *, int64_t, svint64_t);10715void svst1dq_vnum(svbool_t, int64_t *, int64_t, svint64_t);
9743__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))10716__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
9744void svst1q_scatter(svbool_t, svuint64_t, svuint8_t);10717void svst1q_scatter(svbool_t, svuint64_t, svuint8_t);
9745__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))10718__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
...@@ -9808,6 +10781,26 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64b...@@ -9808,6 +10781,26 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64b
9808void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);10781void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
9809__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))10782__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
9810void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint16_t);10783void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint16_t);
10784__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
10785void svst1q_scatter_index(svbool_t, uint32_t *, svint64_t, svuint32_t);
10786__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
10787void svst1q_scatter_index(svbool_t, uint64_t *, svint64_t, svuint64_t);
10788__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
10789void svst1q_scatter_index(svbool_t, uint16_t *, svint64_t, svuint16_t);
10790__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
10791void svst1q_scatter_index(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
10792__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
10793void svst1q_scatter_index(svbool_t, float64_t *, svint64_t, svfloat64_t);
10794__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
10795void svst1q_scatter_index(svbool_t, float32_t *, svint64_t, svfloat32_t);
10796__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
10797void svst1q_scatter_index(svbool_t, float16_t *, svint64_t, svfloat16_t);
10798__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
10799void svst1q_scatter_index(svbool_t, int32_t *, svint64_t, svint32_t);
10800__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
10801void svst1q_scatter_index(svbool_t, int64_t *, svint64_t, svint64_t);
10802__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
10803void svst1q_scatter_index(svbool_t, int16_t *, svint64_t, svint16_t);
9811__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))10804__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
9812void svst1q_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint32_t);10805void svst1q_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint32_t);
9813__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))10806__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
...@@ -9828,6 +10821,30 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64i...@@ -9828,6 +10821,30 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64i
9828void svst1q_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);10821void svst1q_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);
9829__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))10822__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
9830void svst1q_scatter_index(svbool_t, int16_t *, svuint64_t, svint16_t);10823void svst1q_scatter_index(svbool_t, int16_t *, svuint64_t, svint16_t);
10824__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
10825void svst1q_scatter_offset(svbool_t, uint8_t *, svint64_t, svuint8_t);
10826__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
10827void svst1q_scatter_offset(svbool_t, uint32_t *, svint64_t, svuint32_t);
10828__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
10829void svst1q_scatter_offset(svbool_t, uint64_t *, svint64_t, svuint64_t);
10830__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
10831void svst1q_scatter_offset(svbool_t, uint16_t *, svint64_t, svuint16_t);
10832__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
10833void svst1q_scatter_offset(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
10834__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
10835void svst1q_scatter_offset(svbool_t, int8_t *, svint64_t, svint8_t);
10836__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
10837void svst1q_scatter_offset(svbool_t, float64_t *, svint64_t, svfloat64_t);
10838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
10839void svst1q_scatter_offset(svbool_t, float32_t *, svint64_t, svfloat32_t);
10840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
10841void svst1q_scatter_offset(svbool_t, float16_t *, svint64_t, svfloat16_t);
10842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
10843void svst1q_scatter_offset(svbool_t, int32_t *, svint64_t, svint32_t);
10844__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
10845void svst1q_scatter_offset(svbool_t, int64_t *, svint64_t, svint64_t);
10846__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
10847void svst1q_scatter_offset(svbool_t, int16_t *, svint64_t, svint16_t);
9831__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))10848__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
9832void svst1q_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint8_t);10849void svst1q_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint8_t);
9833__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))10850__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
...@@ -9853,17 +10870,17 @@ void svst1q_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);...@@ -9853,17 +10870,17 @@ void svst1q_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);
9853__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))10870__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
9854void svst1q_scatter_offset(svbool_t, int16_t *, svuint64_t, svint16_t);10871void svst1q_scatter_offset(svbool_t, int16_t *, svuint64_t, svint16_t);
9855__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))10872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
9856void svst1wq(svbool_t, uint32_t const *, svuint32_t);10873void svst1wq(svbool_t, uint32_t *, svuint32_t);
9857__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))10874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
9858void svst1wq(svbool_t, float32_t const *, svfloat32_t);10875void svst1wq(svbool_t, float32_t *, svfloat32_t);
9859__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))10876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
9860void svst1wq(svbool_t, int32_t const *, svint32_t);10877void svst1wq(svbool_t, int32_t *, svint32_t);
9861__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))10878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
9862void svst1wq_vnum(svbool_t, uint32_t const *, int64_t, svuint32_t);10879void svst1wq_vnum(svbool_t, uint32_t *, int64_t, svuint32_t);
9863__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))10880__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
9864void svst1wq_vnum(svbool_t, float32_t const *, int64_t, svfloat32_t);10881void svst1wq_vnum(svbool_t, float32_t *, int64_t, svfloat32_t);
9865__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))10882__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
9866void svst1wq_vnum(svbool_t, int32_t const *, int64_t, svint32_t);10883void svst1wq_vnum(svbool_t, int32_t *, int64_t, svint32_t);
9867__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u8)))10884__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u8)))
9868void svst2q(svbool_t, uint8_t const *, svuint8x2_t);10885void svst2q(svbool_t, uint8_t const *, svuint8x2_t);
9869__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u32)))10886__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u32)))
...@@ -10030,6 +11047,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s32)))...@@ -10030,6 +11047,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s32)))
10030svint32_t svtblq(svint32_t, svuint32_t);11047svint32_t svtblq(svint32_t, svuint32_t);
10031__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s64)))11048__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s64)))
10032svint64_t svtblq(svint64_t, svuint64_t);11049svint64_t svtblq(svint64_t, svuint64_t);
11050__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_mf8)))
11051svmfloat8_t svtblq(svmfloat8_t, svuint8_t);
10033__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s16)))11052__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s16)))
10034svint16_t svtblq(svint16_t, svuint16_t);11053svint16_t svtblq(svint16_t, svuint16_t);
10035__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_u8)))11054__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_u8)))
...@@ -10054,6 +11073,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s32)))...@@ -10054,6 +11073,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s32)))
10054svint32_t svtbxq(svint32_t, svint32_t, svuint32_t);11073svint32_t svtbxq(svint32_t, svint32_t, svuint32_t);
10055__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s64)))11074__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s64)))
10056svint64_t svtbxq(svint64_t, svint64_t, svuint64_t);11075svint64_t svtbxq(svint64_t, svint64_t, svuint64_t);
11076__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_mf8)))
11077svmfloat8_t svtbxq(svmfloat8_t, svmfloat8_t, svuint8_t);
10057__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s16)))11078__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s16)))
10058svint16_t svtbxq(svint16_t, svint16_t, svuint16_t);11079svint16_t svtbxq(svint16_t, svint16_t, svuint16_t);
10059__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_u8)))11080__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_u8)))
...@@ -10078,6 +11099,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s32)))...@@ -10078,6 +11099,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s32)))
10078svint32_t svuzpq1(svint32_t, svint32_t);11099svint32_t svuzpq1(svint32_t, svint32_t);
10079__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s64)))11100__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s64)))
10080svint64_t svuzpq1(svint64_t, svint64_t);11101svint64_t svuzpq1(svint64_t, svint64_t);
11102__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_mf8)))
11103svmfloat8_t svuzpq1(svmfloat8_t, svmfloat8_t);
10081__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s16)))11104__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s16)))
10082svint16_t svuzpq1(svint16_t, svint16_t);11105svint16_t svuzpq1(svint16_t, svint16_t);
10083__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_u8)))11106__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_u8)))
...@@ -10102,6 +11125,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s32)))...@@ -10102,6 +11125,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s32)))
10102svint32_t svuzpq2(svint32_t, svint32_t);11125svint32_t svuzpq2(svint32_t, svint32_t);
10103__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s64)))11126__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s64)))
10104svint64_t svuzpq2(svint64_t, svint64_t);11127svint64_t svuzpq2(svint64_t, svint64_t);
11128__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_mf8)))
11129svmfloat8_t svuzpq2(svmfloat8_t, svmfloat8_t);
10105__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s16)))11130__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s16)))
10106svint16_t svuzpq2(svint16_t, svint16_t);11131svint16_t svuzpq2(svint16_t, svint16_t);
10107__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_u8)))11132__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_u8)))
...@@ -10126,6 +11151,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s32)))...@@ -10126,6 +11151,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s32)))
10126svint32_t svzipq1(svint32_t, svint32_t);11151svint32_t svzipq1(svint32_t, svint32_t);
10127__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s64)))11152__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s64)))
10128svint64_t svzipq1(svint64_t, svint64_t);11153svint64_t svzipq1(svint64_t, svint64_t);
11154__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_mf8)))
11155svmfloat8_t svzipq1(svmfloat8_t, svmfloat8_t);
10129__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s16)))11156__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s16)))
10130svint16_t svzipq1(svint16_t, svint16_t);11157svint16_t svzipq1(svint16_t, svint16_t);
10131__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_u8)))11158__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_u8)))
...@@ -10150,6 +11177,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s32)))...@@ -10150,6 +11177,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s32)))
10150svint32_t svzipq2(svint32_t, svint32_t);11177svint32_t svzipq2(svint32_t, svint32_t);
10151__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s64)))11178__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s64)))
10152svint64_t svzipq2(svint64_t, svint64_t);11179svint64_t svzipq2(svint64_t, svint64_t);
11180__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_mf8)))
11181svmfloat8_t svzipq2(svmfloat8_t, svmfloat8_t);
10153__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))11182__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))
10154svint16_t svzipq2(svint16_t, svint16_t);11183svint16_t svzipq2(svint16_t, svint16_t);
10155__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_bf16)))11184__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_bf16)))
...@@ -11522,6 +12551,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u8)))...@@ -11522,6 +12551,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u8)))
11522svuint8_t svdup_laneq_u8(svuint8_t, uint64_t);12551svuint8_t svdup_laneq_u8(svuint8_t, uint64_t);
11523__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_s8)))12552__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_s8)))
11524svint8_t svdup_laneq_s8(svint8_t, uint64_t);12553svint8_t svdup_laneq_s8(svint8_t, uint64_t);
12554__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_mf8)))
12555svmfloat8_t svdup_laneq_mf8(svmfloat8_t, uint64_t);
11525__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u64)))12556__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u64)))
11526svuint64_t svdup_laneq_u64(svuint64_t, uint64_t);12557svuint64_t svdup_laneq_u64(svuint64_t, uint64_t);
11527__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_f64)))12558__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_f64)))
...@@ -11544,6 +12575,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u8)))...@@ -11544,6 +12575,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u8)))
11544svuint8_t svdup_laneq(svuint8_t, uint64_t);12575svuint8_t svdup_laneq(svuint8_t, uint64_t);
11545__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_s8)))12576__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_s8)))
11546svint8_t svdup_laneq(svint8_t, uint64_t);12577svint8_t svdup_laneq(svint8_t, uint64_t);
12578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_mf8)))
12579svmfloat8_t svdup_laneq(svmfloat8_t, uint64_t);
11547__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u64)))12580__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u64)))
11548svuint64_t svdup_laneq(svuint64_t, uint64_t);12581svuint64_t svdup_laneq(svuint64_t, uint64_t);
11549__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_f64)))12582__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_f64)))
...@@ -18424,12 +19457,6 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s64_z)))...@@ -18424,12 +19457,6 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s64_z)))
18424svint64_t svadd_s64_z(svbool_t, svint64_t, svint64_t);19457svint64_t svadd_s64_z(svbool_t, svint64_t, svint64_t);
18425__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s16_z)))19458__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s16_z)))
18426svint16_t svadd_s16_z(svbool_t, svint16_t, svint16_t);19459svint16_t svadd_s16_z(svbool_t, svint16_t, svint16_t);
18427__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
18428float64_t svadda_f64(svbool_t, float64_t, svfloat64_t);
18429__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
18430float32_t svadda_f32(svbool_t, float32_t, svfloat32_t);
18431__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
18432float16_t svadda_f16(svbool_t, float16_t, svfloat16_t);
18433__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s8)))19460__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s8)))
18434int64_t svaddv_s8(svbool_t, svint8_t);19461int64_t svaddv_s8(svbool_t, svint8_t);
18435__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s32)))19462__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s32)))
...@@ -24810,12 +25837,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s64_z)))...@@ -24810,12 +25837,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s64_z)))
24810svint64_t svadd_z(svbool_t, svint64_t, svint64_t);25837svint64_t svadd_z(svbool_t, svint64_t, svint64_t);
24811__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s16_z)))25838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s16_z)))
24812svint16_t svadd_z(svbool_t, svint16_t, svint16_t);25839svint16_t svadd_z(svbool_t, svint16_t, svint16_t);
24813__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
24814float64_t svadda(svbool_t, float64_t, svfloat64_t);
24815__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
24816float32_t svadda(svbool_t, float32_t, svfloat32_t);
24817__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
24818float16_t svadda(svbool_t, float16_t, svfloat16_t);
24819__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s8)))25840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s8)))
24820int64_t svaddv(svbool_t, svint8_t);25841int64_t svaddv(svbool_t, svint8_t);
24821__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s32)))25842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s32)))
lib/include/arm_vector_types.h+87
...@@ -17,9 +17,62 @@...@@ -17,9 +17,62 @@
17typedef float float32_t;17typedef float float32_t;
18typedef __fp16 float16_t;18typedef __fp16 float16_t;
19#if defined(__aarch64__) || defined(__arm64ec__)19#if defined(__aarch64__) || defined(__arm64ec__)
20typedef __mfp8 mfloat8_t;
20typedef double float64_t;21typedef double float64_t;
21#endif22#endif
2223
24
25typedef uint64_t fpm_t;
26
27enum __ARM_FPM_FORMAT { __ARM_FPM_E5M2, __ARM_FPM_E4M3 };
28
29enum __ARM_FPM_OVERFLOW { __ARM_FPM_INFNAN, __ARM_FPM_SATURATE };
30
31static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
32__arm_fpm_init(void) {
33 return 0;
34}
35
36static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
37__arm_set_fpm_src1_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
38 return (__fpm & ~7ull) | (fpm_t)__format;
39}
40
41static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
42__arm_set_fpm_src2_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
43 return (__fpm & ~0x38ull) | ((fpm_t)__format << 3u);
44}
45
46static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
47__arm_set_fpm_dst_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
48 return (__fpm & ~0x1c0ull) | ((fpm_t)__format << 6u);
49}
50
51static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
52__arm_set_fpm_overflow_mul(fpm_t __fpm, enum __ARM_FPM_OVERFLOW __behaviour) {
53 return (__fpm & ~0x4000ull) | ((fpm_t)__behaviour << 14u);
54}
55
56static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
57__arm_set_fpm_overflow_cvt(fpm_t __fpm, enum __ARM_FPM_OVERFLOW __behaviour) {
58 return (__fpm & ~0x8000ull) | ((fpm_t)__behaviour << 15u);
59}
60
61static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
62__arm_set_fpm_lscale(fpm_t __fpm, uint64_t __scale) {
63 return (__fpm & ~0x7f0000ull) | (__scale << 16u);
64}
65
66static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
67__arm_set_fpm_nscale(fpm_t __fpm, int64_t __scale) {
68 return (__fpm & ~0xff000000ull) | (((fpm_t)__scale & 0xffu) << 24u);
69}
70
71static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
72__arm_set_fpm_lscale2(fpm_t __fpm, uint64_t __scale) {
73 return (uint32_t)__fpm | (__scale << 32u);
74}
75
23typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t;76typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t;
24typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t;77typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t;
25typedef __attribute__((neon_vector_type(4))) int16_t int16x4_t;78typedef __attribute__((neon_vector_type(4))) int16_t int16x4_t;
...@@ -36,6 +89,10 @@ typedef __attribute__((neon_vector_type(2))) uint32_t uint32x2_t;...@@ -36,6 +89,10 @@ typedef __attribute__((neon_vector_type(2))) uint32_t uint32x2_t;
36typedef __attribute__((neon_vector_type(4))) uint32_t uint32x4_t;89typedef __attribute__((neon_vector_type(4))) uint32_t uint32x4_t;
37typedef __attribute__((neon_vector_type(1))) uint64_t uint64x1_t;90typedef __attribute__((neon_vector_type(1))) uint64_t uint64x1_t;
38typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t;91typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t;
92#if defined(__aarch64__) || defined(__arm64ec__)
93typedef __attribute__((neon_vector_type(8))) mfloat8_t mfloat8x8_t;
94typedef __attribute__((neon_vector_type(16))) mfloat8_t mfloat8x16_t;
95#endif
39typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t;96typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t;
40typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t;97typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t;
41typedef __attribute__((neon_vector_type(2))) float32_t float32x2_t;98typedef __attribute__((neon_vector_type(2))) float32_t float32x2_t;
...@@ -109,6 +166,16 @@ typedef struct uint64x2x2_t {...@@ -109,6 +166,16 @@ typedef struct uint64x2x2_t {
109 uint64x2_t val[2];166 uint64x2_t val[2];
110} uint64x2x2_t;167} uint64x2x2_t;
111168
169#if defined(__aarch64__) || defined(__arm64ec__)
170typedef struct mfloat8x8x2_t {
171 mfloat8x8_t val[2];
172} mfloat8x8x2_t;
173
174typedef struct mfloat8x16x2_t {
175 mfloat8x16_t val[2];
176} mfloat8x16x2_t;
177
178#endif
112typedef struct float16x4x2_t {179typedef struct float16x4x2_t {
113 float16x4_t val[2];180 float16x4_t val[2];
114} float16x4x2_t;181} float16x4x2_t;
...@@ -199,6 +266,16 @@ typedef struct uint64x2x3_t {...@@ -199,6 +266,16 @@ typedef struct uint64x2x3_t {
199 uint64x2_t val[3];266 uint64x2_t val[3];
200} uint64x2x3_t;267} uint64x2x3_t;
201268
269#if defined(__aarch64__) || defined(__arm64ec__)
270typedef struct mfloat8x8x3_t {
271 mfloat8x8_t val[3];
272} mfloat8x8x3_t;
273
274typedef struct mfloat8x16x3_t {
275 mfloat8x16_t val[3];
276} mfloat8x16x3_t;
277
278#endif
202typedef struct float16x4x3_t {279typedef struct float16x4x3_t {
203 float16x4_t val[3];280 float16x4_t val[3];
204} float16x4x3_t;281} float16x4x3_t;
...@@ -289,6 +366,16 @@ typedef struct uint64x2x4_t {...@@ -289,6 +366,16 @@ typedef struct uint64x2x4_t {
289 uint64x2_t val[4];366 uint64x2_t val[4];
290} uint64x2x4_t;367} uint64x2x4_t;
291368
369#if defined(__aarch64__) || defined(__arm64ec__)
370typedef struct mfloat8x8x4_t {
371 mfloat8x8_t val[4];
372} mfloat8x8x4_t;
373
374typedef struct mfloat8x16x4_t {
375 mfloat8x16_t val[4];
376} mfloat8x16x4_t;
377
378#endif
292typedef struct float16x4x4_t {379typedef struct float16x4x4_t {
293 float16x4_t val[4];380 float16x4_t val[4];
294} float16x4x4_t;381} float16x4x4_t;
lib/include/avx10_2_512bf16intrin.h created+561
...@@ -0,0 +1,561 @@
1/*===----------- avx10_2_512bf16intrin.h - AVX10-BF16 intrinsics ---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512bf16intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2_512BF16INTRIN_H
17#define __AVX10_2_512BF16INTRIN_H
18
19/* Define the default attributes for the functions in this file. */
20typedef __bf16 __m512bh_u __attribute__((__vector_size__(64), __aligned__(1)));
21
22/* Define the default attributes for the functions in this file. */
23#define __DEFAULT_FN_ATTRS512 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
25 __min_vector_width__(512)))
26
27static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_setzero_pbh(void) {
28 return __builtin_bit_cast(__m512bh, _mm512_setzero_ps());
29}
30
31static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_undefined_pbh(void) {
32 return (__m512bh)__builtin_ia32_undef512();
33}
34
35static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_set1_pbh(__bf16 bf) {
36 return (__m512bh)(__v32bf){bf, bf, bf, bf, bf, bf, bf, bf, bf, bf, bf,
37 bf, bf, bf, bf, bf, bf, bf, bf, bf, bf, bf,
38 bf, bf, bf, bf, bf, bf, bf, bf, bf, bf};
39}
40
41static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_set_pbh(
42 __bf16 bf1, __bf16 bf2, __bf16 bf3, __bf16 bf4, __bf16 bf5, __bf16 bf6,
43 __bf16 bf7, __bf16 bf8, __bf16 bf9, __bf16 bf10, __bf16 bf11, __bf16 bf12,
44 __bf16 bf13, __bf16 bf14, __bf16 bf15, __bf16 bf16, __bf16 bf17,
45 __bf16 bf18, __bf16 bf19, __bf16 bf20, __bf16 bf21, __bf16 bf22,
46 __bf16 bf23, __bf16 bf24, __bf16 bf25, __bf16 bf26, __bf16 bf27,
47 __bf16 bf28, __bf16 bf29, __bf16 bf30, __bf16 bf31, __bf16 bf32) {
48 return (__m512bh)(__v32bf){bf32, bf31, bf30, bf29, bf28, bf27, bf26, bf25,
49 bf24, bf23, bf22, bf21, bf20, bf19, bf18, bf17,
50 bf16, bf15, bf14, bf13, bf12, bf11, bf10, bf9,
51 bf8, bf7, bf6, bf5, bf4, bf3, bf2, bf1};
52}
53
54#define _mm512_setr_pbh(bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8, bf9, bf10, \
55 bf11, bf12, bf13, bf14, bf15, bf16, bf17, bf18, bf19, \
56 bf20, bf21, bf22, bf23, bf24, bf25, bf26, bf27, bf28, \
57 bf29, bf30, bf31, bf32) \
58 _mm512_set_pbh((bf32), (bf31), (bf30), (bf29), (bf28), (bf27), (bf26), \
59 (bf25), (bf24), (bf23), (bf22), (bf21), (bf20), (bf19), \
60 (bf18), (bf17), (bf16), (bf15), (bf14), (bf13), (bf12), \
61 (bf11), (bf10), (bf9), (bf8), (bf7), (bf6), (bf5), (bf4), \
62 (bf3), (bf2), (bf1))
63
64static __inline__ __m512 __DEFAULT_FN_ATTRS512
65_mm512_castbf16_ps(__m512bh __a) {
66 return (__m512)__a;
67}
68
69static __inline__ __m512d __DEFAULT_FN_ATTRS512
70_mm512_castbf16_pd(__m512bh __a) {
71 return (__m512d)__a;
72}
73
74static __inline__ __m512i __DEFAULT_FN_ATTRS512
75_mm512_castbf16_si512(__m512bh __a) {
76 return (__m512i)__a;
77}
78
79static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_castps_pbh(__m512 __a) {
80 return (__m512bh)__a;
81}
82
83static __inline__ __m512bh __DEFAULT_FN_ATTRS512
84_mm512_castpd_pbh(__m512d __a) {
85 return (__m512bh)__a;
86}
87
88static __inline__ __m512bh __DEFAULT_FN_ATTRS512
89_mm512_castsi512_pbh(__m512i __a) {
90 return (__m512bh)__a;
91}
92
93static __inline__ __m128bh __DEFAULT_FN_ATTRS512
94_mm512_castbf16512_pbh128(__m512bh __a) {
95 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7);
96}
97
98static __inline__ __m256bh __DEFAULT_FN_ATTRS512
99_mm512_castbf16512_pbh256(__m512bh __a) {
100 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
101 12, 13, 14, 15);
102}
103
104static __inline__ __m512bh __DEFAULT_FN_ATTRS512
105_mm512_castbf16128_pbh512(__m128bh __a) {
106 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
107 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
108 -1, -1, -1, -1, -1, -1, -1, -1, -1);
109}
110
111static __inline__ __m512bh __DEFAULT_FN_ATTRS512
112_mm512_castbf16256_pbh512(__m256bh __a) {
113 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
114 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1,
115 -1, -1, -1, -1, -1, -1, -1, -1);
116}
117
118static __inline__ __m512bh __DEFAULT_FN_ATTRS512
119_mm512_zextbf16128_pbh512(__m128bh __a) {
120 return __builtin_shufflevector(
121 __a, (__v8bf)_mm_setzero_pbh(), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
122 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15);
123}
124
125static __inline__ __m512bh __DEFAULT_FN_ATTRS512
126_mm512_zextbf16256_pbh512(__m256bh __a) {
127 return __builtin_shufflevector(__a, (__v16bf)_mm256_setzero_pbh(), 0, 1, 2, 3,
128 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
129 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
130 29, 30, 31);
131}
132
133static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_abs_pbh(__m512bh __A) {
134 return (__m512bh)_mm512_and_epi32(_mm512_set1_epi32(0x7FFF7FFF),
135 (__m512i)__A);
136}
137
138static __inline__ __m512bh __DEFAULT_FN_ATTRS512
139_mm512_load_pbh(void const *__p) {
140 return *(const __m512bh *)__p;
141}
142
143static __inline__ __m512bh __DEFAULT_FN_ATTRS512
144_mm512_loadu_pbh(void const *__p) {
145 struct __loadu_pbh {
146 __m512bh_u __v;
147 } __attribute__((__packed__, __may_alias__));
148 return ((const struct __loadu_pbh *)__p)->__v;
149}
150
151static __inline__ void __DEFAULT_FN_ATTRS512 _mm512_store_pbh(void *__P,
152 __m512bh __A) {
153 *(__m512bh *)__P = __A;
154}
155
156static __inline__ void __DEFAULT_FN_ATTRS512 _mm512_storeu_pbh(void *__P,
157 __m512bh __A) {
158 struct __storeu_pbh {
159 __m512bh_u __v;
160 } __attribute__((__packed__, __may_alias__));
161 ((struct __storeu_pbh *)__P)->__v = __A;
162}
163
164static __inline__ __m512bh __DEFAULT_FN_ATTRS512
165_mm512_mask_blend_pbh(__mmask32 __U, __m512bh __A, __m512bh __W) {
166 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U, (__v32bf)__W,
167 (__v32bf)__A);
168}
169
170static __inline__ __m512bh __DEFAULT_FN_ATTRS512
171_mm512_permutex2var_pbh(__m512bh __A, __m512i __I, __m512bh __B) {
172 return (__m512bh)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I,
173 (__v32hi)__B);
174}
175
176static __inline__ __m512bh __DEFAULT_FN_ATTRS512
177_mm512_permutexvar_pbh(__m512i __A, __m512bh __B) {
178 return (__m512bh)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A);
179}
180
181static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_add_pbh(__m512bh __A,
182 __m512bh __B) {
183 return (__m512bh)((__v32bf)__A + (__v32bf)__B);
184}
185
186static __inline__ __m512bh __DEFAULT_FN_ATTRS512
187_mm512_mask_add_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
188 return (__m512bh)__builtin_ia32_selectpbf_512(
189 (__mmask32)__U, (__v32bf)_mm512_add_pbh(__A, __B), (__v32bf)__W);
190}
191
192static __inline__ __m512bh __DEFAULT_FN_ATTRS512
193_mm512_maskz_add_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
194 return (__m512bh)__builtin_ia32_selectpbf_512(
195 (__mmask32)__U, (__v32bf)_mm512_add_pbh(__A, __B),
196 (__v32bf)_mm512_setzero_pbh());
197}
198
199static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_sub_pbh(__m512bh __A,
200 __m512bh __B) {
201 return (__m512bh)((__v32bf)__A - (__v32bf)__B);
202}
203
204static __inline__ __m512bh __DEFAULT_FN_ATTRS512
205_mm512_mask_sub_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
206 return (__m512bh)__builtin_ia32_selectpbf_512(
207 (__mmask32)__U, (__v32bf)_mm512_sub_pbh(__A, __B), (__v32bf)__W);
208}
209
210static __inline__ __m512bh __DEFAULT_FN_ATTRS512
211_mm512_maskz_sub_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
212 return (__m512bh)__builtin_ia32_selectpbf_512(
213 (__mmask32)__U, (__v32bf)_mm512_sub_pbh(__A, __B),
214 (__v32bf)_mm512_setzero_pbh());
215}
216
217static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mul_pbh(__m512bh __A,
218 __m512bh __B) {
219 return (__m512bh)((__v32bf)__A * (__v32bf)__B);
220}
221
222static __inline__ __m512bh __DEFAULT_FN_ATTRS512
223_mm512_mask_mul_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
224 return (__m512bh)__builtin_ia32_selectpbf_512(
225 (__mmask32)__U, (__v32bf)_mm512_mul_pbh(__A, __B), (__v32bf)__W);
226}
227
228static __inline__ __m512bh __DEFAULT_FN_ATTRS512
229_mm512_maskz_mul_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
230 return (__m512bh)__builtin_ia32_selectpbf_512(
231 (__mmask32)__U, (__v32bf)_mm512_mul_pbh(__A, __B),
232 (__v32bf)_mm512_setzero_pbh());
233}
234
235static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_div_pbh(__m512bh __A,
236 __m512bh __B) {
237 return (__m512bh)((__v32bf)__A / (__v32bf)__B);
238}
239
240static __inline__ __m512bh __DEFAULT_FN_ATTRS512
241_mm512_mask_div_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
242 return (__m512bh)__builtin_ia32_selectpbf_512(
243 (__mmask32)__U, (__v32bf)_mm512_div_pbh(__A, __B), (__v32bf)__W);
244}
245
246static __inline__ __m512bh __DEFAULT_FN_ATTRS512
247_mm512_maskz_div_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
248 return (__m512bh)__builtin_ia32_selectpbf_512(
249 (__mmask32)__U, (__v32bf)_mm512_div_pbh(__A, __B),
250 (__v32bf)_mm512_setzero_pbh());
251}
252
253static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_max_pbh(__m512bh __A,
254 __m512bh __B) {
255 return (__m512bh)__builtin_ia32_vmaxbf16512((__v32bf)__A, (__v32bf)__B);
256}
257
258static __inline__ __m512bh __DEFAULT_FN_ATTRS512
259_mm512_mask_max_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
260 return (__m512bh)__builtin_ia32_selectpbf_512(
261 (__mmask32)__U, (__v32bf)_mm512_max_pbh(__A, __B), (__v32bf)__W);
262}
263
264static __inline__ __m512bh __DEFAULT_FN_ATTRS512
265_mm512_maskz_max_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
266 return (__m512bh)__builtin_ia32_selectpbf_512(
267 (__mmask32)__U, (__v32bf)_mm512_max_pbh(__A, __B),
268 (__v32bf)_mm512_setzero_pbh());
269}
270
271static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_min_pbh(__m512bh __A,
272 __m512bh __B) {
273 return (__m512bh)__builtin_ia32_vminbf16512((__v32bf)__A, (__v32bf)__B);
274}
275
276static __inline__ __m512bh __DEFAULT_FN_ATTRS512
277_mm512_mask_min_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
278 return (__m512bh)__builtin_ia32_selectpbf_512(
279 (__mmask32)__U, (__v32bf)_mm512_min_pbh(__A, __B), (__v32bf)__W);
280}
281
282static __inline__ __m512bh __DEFAULT_FN_ATTRS512
283_mm512_maskz_min_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
284 return (__m512bh)__builtin_ia32_selectpbf_512(
285 (__mmask32)__U, (__v32bf)_mm512_min_pbh(__A, __B),
286 (__v32bf)_mm512_setzero_pbh());
287}
288
289#define _mm512_cmp_pbh_mask(__A, __B, __P) \
290 ((__mmask32)__builtin_ia32_vcmpbf16512_mask((__v32bf)(__m512bh)(__A), \
291 (__v32bf)(__m512bh)(__B), \
292 (int)(__P), (__mmask32) - 1))
293
294#define _mm512_mask_cmp_pbh_mask(__U, __A, __B, __P) \
295 ((__mmask32)__builtin_ia32_vcmpbf16512_mask((__v32bf)(__m512bh)(__A), \
296 (__v32bf)(__m512bh)(__B), \
297 (int)(__P), (__mmask32)(__U)))
298
299#define _mm512_mask_fpclass_pbh_mask(__U, __A, imm) \
300 ((__mmask32)__builtin_ia32_vfpclassbf16512_mask( \
301 (__v32bf)(__m512bh)(__A), (int)(imm), (__mmask32)(__U)))
302
303#define _mm512_fpclass_pbh_mask(__A, imm) \
304 ((__mmask32)__builtin_ia32_vfpclassbf16512_mask( \
305 (__v32bf)(__m512bh)(__A), (int)(imm), (__mmask32) - 1))
306
307static __inline__ __m512bh __DEFAULT_FN_ATTRS512
308_mm512_scalef_pbh(__m512bh __A, __m512bh __B) {
309 return (__m512bh)__builtin_ia32_vscalefbf16512_mask(
310 (__v32bf)__A, (__v32bf)__B, (__v32bf)_mm512_undefined_pbh(),
311 (__mmask32)-1);
312}
313
314static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_scalef_pbh(
315 __m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
316 return (__m512bh)__builtin_ia32_vscalefbf16512_mask(
317 (__v32bf)__A, (__v32bf)__B, (__v32bf)__W, (__mmask32)__U);
318}
319
320static __inline__ __m512bh __DEFAULT_FN_ATTRS512
321_mm512_maskz_scalef_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
322 return (__m512bh)__builtin_ia32_vscalefbf16512_mask(
323 (__v32bf)__A, (__v32bf)__B, (__v32bf)_mm512_setzero_pbh(),
324 (__mmask32)__U);
325}
326
327static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_rcp_pbh(__m512bh __A) {
328 return (__m512bh)__builtin_ia32_vrcpbf16512_mask(
329 (__v32bf)__A, (__v32bf)_mm512_undefined_pbh(), (__mmask32)-1);
330}
331
332static __inline__ __m512bh __DEFAULT_FN_ATTRS512
333_mm512_mask_rcp_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
334 return (__m512bh)__builtin_ia32_vrcpbf16512_mask((__v32bf)__A, (__v32bf)__W,
335 (__mmask32)__U);
336}
337
338static __inline__ __m512bh __DEFAULT_FN_ATTRS512
339_mm512_maskz_rcp_pbh(__mmask32 __U, __m512bh __A) {
340 return (__m512bh)__builtin_ia32_vrcpbf16512_mask(
341 (__v32bf)__A, (__v32bf)_mm512_setzero_pbh(), (__mmask32)__U);
342}
343
344static __inline__ __m512bh __DEFAULT_FN_ATTRS512
345_mm512_getexp_pbh(__m512bh __A) {
346 return (__m512bh)__builtin_ia32_vgetexpbf16512_mask(
347 (__v32bf)__A, (__v32bf)_mm512_undefined_pbh(), (__mmask32)-1);
348}
349
350static __inline__ __m512bh __DEFAULT_FN_ATTRS512
351_mm512_mask_getexp_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
352 return (__m512bh)__builtin_ia32_vgetexpbf16512_mask(
353 (__v32bf)__A, (__v32bf)__W, (__mmask32)__U);
354}
355
356static __inline__ __m512bh __DEFAULT_FN_ATTRS512
357_mm512_maskz_getexp_pbh(__mmask32 __U, __m512bh __A) {
358 return (__m512bh)__builtin_ia32_vgetexpbf16512_mask(
359 (__v32bf)__A, (__v32bf)_mm512_setzero_pbh(), (__mmask32)__U);
360}
361
362static __inline__ __m512bh __DEFAULT_FN_ATTRS512
363_mm512_rsqrt_pbh(__m512bh __A) {
364 return (__m512bh)__builtin_ia32_vrsqrtbf16512_mask(
365 (__v32bf)__A, (__v32bf)_mm512_undefined_pbh(), (__mmask32)-1);
366}
367
368static __inline__ __m512bh __DEFAULT_FN_ATTRS512
369_mm512_mask_rsqrt_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
370 return (__m512bh)__builtin_ia32_vrsqrtbf16512_mask((__v32bf)__A, (__v32bf)__W,
371 (__mmask32)__U);
372}
373
374static __inline__ __m512bh __DEFAULT_FN_ATTRS512
375_mm512_maskz_rsqrt_pbh(__mmask32 __U, __m512bh __A) {
376 return (__m512bh)__builtin_ia32_vrsqrtbf16512_mask(
377 (__v32bf)__A, (__v32bf)_mm512_setzero_pbh(), (__mmask32)__U);
378}
379
380#define _mm512_reduce_pbh(__A, imm) \
381 ((__m512bh)__builtin_ia32_vreducebf16512_mask( \
382 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_undefined_pbh(), \
383 (__mmask32) - 1))
384
385#define _mm512_mask_reduce_pbh(__W, __U, __A, imm) \
386 ((__m512bh)__builtin_ia32_vreducebf16512_mask( \
387 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)(__m512bh)(__W), \
388 (__mmask32)(__U)))
389
390#define _mm512_maskz_reduce_pbh(__U, __A, imm) \
391 ((__m512bh)__builtin_ia32_vreducebf16512_mask( \
392 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_setzero_pbh(), \
393 (__mmask32)(__U)))
394
395#define _mm512_roundscale_pbh(__A, imm) \
396 ((__m512bh)__builtin_ia32_vrndscalebf16_mask( \
397 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_setzero_pbh(), \
398 (__mmask32) - 1))
399
400#define _mm512_mask_roundscale_pbh(__W, __U, __A, imm) \
401 ((__m512bh)__builtin_ia32_vrndscalebf16_mask( \
402 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)(__m512bh)(__W), \
403 (__mmask32)(__U)))
404
405#define _mm512_maskz_roundscale_pbh(__U, __A, imm) \
406 ((__m512bh)__builtin_ia32_vrndscalebf16_mask( \
407 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_setzero_pbh(), \
408 (__mmask32)(__U)))
409
410#define _mm512_getmant_pbh(__A, __B, __C) \
411 ((__m512bh)__builtin_ia32_vgetmantbf16512_mask( \
412 (__v32bf)(__m512bh)(__A), (int)(((__C) << 2) | (__B)), \
413 (__v32bf)_mm512_undefined_pbh(), (__mmask32) - 1))
414
415#define _mm512_mask_getmant_pbh(__W, __U, __A, __B, __C) \
416 ((__m512bh)__builtin_ia32_vgetmantbf16512_mask( \
417 (__v32bf)(__m512bh)(__A), (int)(((__C) << 2) | (__B)), \
418 (__v32bf)(__m512bh)(__W), (__mmask32)(__U)))
419
420#define _mm512_maskz_getmant_pbh(__U, __A, __B, __C) \
421 ((__m512bh)__builtin_ia32_vgetmantbf16512_mask( \
422 (__v32bf)(__m512bh)(__A), (int)(((__C) << 2) | (__B)), \
423 (__v32bf)_mm512_setzero_pbh(), (__mmask32)(__U)))
424
425static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_sqrt_pbh(__m512bh __A) {
426 return (__m512bh)__builtin_ia32_vsqrtbf16512((__v32bf)__A);
427}
428
429static __inline__ __m512bh __DEFAULT_FN_ATTRS512
430_mm512_mask_sqrt_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
431 return (__m512bh)__builtin_ia32_selectpbf_512(
432 (__mmask32)__U, (__v32bf)_mm512_sqrt_pbh(__A), (__v32bf)__W);
433}
434
435static __inline__ __m512bh __DEFAULT_FN_ATTRS512
436_mm512_maskz_sqrt_pbh(__mmask32 __U, __m512bh __A) {
437 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,
438 (__v32bf)_mm512_sqrt_pbh(__A),
439 (__v32bf)_mm512_setzero_pbh());
440}
441
442static __inline__ __m512bh __DEFAULT_FN_ATTRS512
443_mm512_fmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
444 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, (__v32bf)__B,
445 (__v32bf)__C);
446}
447
448static __inline__ __m512bh __DEFAULT_FN_ATTRS512
449_mm512_mask_fmadd_pbh(__m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
450 return (__m512bh)__builtin_ia32_selectpbf_512(
451 (__mmask32)__U,
452 _mm512_fmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__A);
453}
454
455static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fmadd_pbh(
456 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
457 return (__m512bh)__builtin_ia32_selectpbf_512(
458 (__mmask32)__U,
459 _mm512_fmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__C);
460}
461
462static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmadd_pbh(
463 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
464 return (__m512bh)__builtin_ia32_selectpbf_512(
465 (__mmask32)__U,
466 _mm512_fmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
467 (__v32bf)_mm512_setzero_pbh());
468}
469
470static __inline__ __m512bh __DEFAULT_FN_ATTRS512
471_mm512_fmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
472 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, (__v32bf)__B,
473 -(__v32bf)__C);
474}
475
476static __inline__ __m512bh __DEFAULT_FN_ATTRS512
477_mm512_mask_fmsub_pbh(__m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
478 return (__m512bh)__builtin_ia32_selectpbf_512(
479 (__mmask32)__U,
480 _mm512_fmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__A);
481}
482
483static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fmsub_pbh(
484 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
485 return (__m512bh)__builtin_ia32_selectpbf_512(
486 (__mmask32)__U,
487 _mm512_fmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__C);
488}
489
490static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmsub_pbh(
491 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
492 return (__m512bh)__builtin_ia32_selectpbf_512(
493 (__mmask32)__U,
494 _mm512_fmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
495 (__v32bf)_mm512_setzero_pbh());
496}
497
498static __inline__ __m512bh __DEFAULT_FN_ATTRS512
499_mm512_fnmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
500 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, -(__v32bf)__B,
501 (__v32bf)__C);
502}
503
504static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmadd_pbh(
505 __m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
506 return (__m512bh)__builtin_ia32_selectpbf_512(
507 (__mmask32)__U,
508 _mm512_fnmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
509 (__v32bf)__A);
510}
511
512static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fnmadd_pbh(
513 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
514 return (__m512bh)__builtin_ia32_selectpbf_512(
515 (__mmask32)__U,
516 _mm512_fnmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
517 (__v32bf)__C);
518}
519
520static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmadd_pbh(
521 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
522 return (__m512bh)__builtin_ia32_selectpbf_512(
523 (__mmask32)__U,
524 _mm512_fnmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
525 (__v32bf)_mm512_setzero_pbh());
526}
527
528static __inline__ __m512bh __DEFAULT_FN_ATTRS512
529_mm512_fnmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
530 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, -(__v32bf)__B,
531 -(__v32bf)__C);
532}
533
534static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmsub_pbh(
535 __m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
536 return (__m512bh)__builtin_ia32_selectpbf_512(
537 (__mmask32)__U,
538 _mm512_fnmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
539 (__v32bf)__A);
540}
541
542static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fnmsub_pbh(
543 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
544 return (__m512bh)__builtin_ia32_selectpbf_512(
545 (__mmask32)__U,
546 _mm512_fnmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
547 (__v32bf)__C);
548}
549
550static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmsub_pbh(
551 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
552 return (__m512bh)__builtin_ia32_selectpbf_512(
553 (__mmask32)__U,
554 _mm512_fnmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
555 (__v32bf)_mm512_setzero_pbh());
556}
557
558#undef __DEFAULT_FN_ATTRS512
559
560#endif
561#endif
lib/include/avx10_2_512convertintrin.h created+320
...@@ -0,0 +1,320 @@
1/*===--------- avx10_2_512convertintrin.h - AVX10_2_512CONVERT -------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512convertintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2_512CONVERTINTRIN_H
17#define __AVX10_2_512CONVERTINTRIN_H
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS512 \
21 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
22 __min_vector_width__(512)))
23
24static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvtx2ps_ph(__m512 __A,
25 __m512 __B) {
26 return (__m512h)__builtin_ia32_vcvt2ps2phx512_mask(
27 (__v16sf)__A, (__v16sf)__B, (__v32hf)_mm512_setzero_ph(), (__mmask32)(-1),
28 _MM_FROUND_CUR_DIRECTION);
29}
30
31static __inline__ __m512h __DEFAULT_FN_ATTRS512
32_mm512_mask_cvtx2ps_ph(__m512h __W, __mmask32 __U, __m512 __A, __m512 __B) {
33 return (__m512h)__builtin_ia32_vcvt2ps2phx512_mask(
34 (__v16sf)__A, (__v16sf)__B, (__v32hf)__W, (__mmask32)__U,
35 _MM_FROUND_CUR_DIRECTION);
36}
37
38static __inline__ __m512h __DEFAULT_FN_ATTRS512
39_mm512_maskz_cvtx2ps_ph(__mmask32 __U, __m512 __A, __m512 __B) {
40 return (__m512h)__builtin_ia32_vcvt2ps2phx512_mask(
41 (__v16sf)__A, (__v16sf)__B, (__v32hf)_mm512_setzero_ph(), (__mmask32)__U,
42 _MM_FROUND_CUR_DIRECTION);
43}
44
45#define _mm512_cvtx_round2ps_ph(A, B, R) \
46 ((__m512h)__builtin_ia32_vcvt2ps2phx512_mask( \
47 (__v16sf)(A), (__v16sf)(B), (__v32hf)_mm512_undefined_ph(), \
48 (__mmask32)(-1), (const int)(R)))
49
50#define _mm512_mask_cvtx_round2ps_ph(W, U, A, B, R) \
51 ((__m512h)__builtin_ia32_vcvt2ps2phx512_mask((__v16sf)(A), (__v16sf)(B), \
52 (__v32hf)(W), (__mmask32)(U), \
53 (const int)(R)))
54
55#define _mm512_maskz_cvtx_round2ps_ph(U, A, B, R) \
56 ((__m512h)__builtin_ia32_vcvt2ps2phx512_mask( \
57 (__v16sf)(A), (__v16sf)(B), (__v32hf)_mm512_setzero_ph(), \
58 (__mmask32)(U), (const int)(R)))
59
60static __inline__ __m256i __DEFAULT_FN_ATTRS512
61_mm512_cvtbiasph_bf8(__m512i __A, __m512h __B) {
62 return (__m256i)__builtin_ia32_vcvtbiasph2bf8_512_mask(
63 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
64 (__mmask32)-1);
65}
66
67static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiasph_bf8(
68 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
69 return (__m256i)__builtin_ia32_vcvtbiasph2bf8_512_mask(
70 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
71}
72
73static __inline__ __m256i __DEFAULT_FN_ATTRS512
74_mm512_maskz_cvtbiasph_bf8(__mmask32 __U, __m512i __A, __m512h __B) {
75 return (__m256i)__builtin_ia32_vcvtbiasph2bf8_512_mask(
76 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
77 (__mmask32)__U);
78}
79
80static __inline__ __m256i __DEFAULT_FN_ATTRS512
81_mm512_cvtbiassph_bf8(__m512i __A, __m512h __B) {
82 return (__m256i)__builtin_ia32_vcvtbiasph2bf8s_512_mask(
83 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
84 (__mmask32)-1);
85}
86
87static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiassph_bf8(
88 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
89 return (__m256i)__builtin_ia32_vcvtbiasph2bf8s_512_mask(
90 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
91}
92
93static __inline__ __m256i __DEFAULT_FN_ATTRS512
94_mm512_maskz_cvtbiassph_bf8(__mmask32 __U, __m512i __A, __m512h __B) {
95 return (__m256i)__builtin_ia32_vcvtbiasph2bf8s_512_mask(
96 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
97 (__mmask32)__U);
98}
99
100static __inline__ __m256i __DEFAULT_FN_ATTRS512
101_mm512_cvtbiasph_hf8(__m512i __A, __m512h __B) {
102 return (__m256i)__builtin_ia32_vcvtbiasph2hf8_512_mask(
103 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
104 (__mmask32)-1);
105}
106
107static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiasph_hf8(
108 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
109 return (__m256i)__builtin_ia32_vcvtbiasph2hf8_512_mask(
110 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
111}
112
113static __inline__ __m256i __DEFAULT_FN_ATTRS512
114_mm512_maskz_cvtbiasph_hf8(__mmask32 __U, __m512i __A, __m512h __B) {
115 return (__m256i)__builtin_ia32_vcvtbiasph2hf8_512_mask(
116 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
117 (__mmask32)__U);
118}
119
120static __inline__ __m256i __DEFAULT_FN_ATTRS512
121_mm512_cvtbiassph_hf8(__m512i __A, __m512h __B) {
122 return (__m256i)__builtin_ia32_vcvtbiasph2hf8s_512_mask(
123 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
124 (__mmask32)-1);
125}
126
127static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiassph_hf8(
128 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
129 return (__m256i)__builtin_ia32_vcvtbiasph2hf8s_512_mask(
130 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
131}
132
133static __inline__ __m256i __DEFAULT_FN_ATTRS512
134_mm512_maskz_cvtbiassph_hf8(__mmask32 __U, __m512i __A, __m512h __B) {
135 return (__m256i)__builtin_ia32_vcvtbiasph2hf8s_512_mask(
136 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
137 (__mmask32)__U);
138}
139
140static __inline__ __m512i __DEFAULT_FN_ATTRS512 _mm512_cvt2ph_bf8(__m512h __A,
141 __m512h __B) {
142 return (__m512i)__builtin_ia32_vcvt2ph2bf8_512((__v32hf)(__A),
143 (__v32hf)(__B));
144}
145
146static __inline__ __m512i __DEFAULT_FN_ATTRS512
147_mm512_mask_cvt2ph_bf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
148 return (__m512i)__builtin_ia32_selectb_512(
149 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_bf8(__A, __B), (__v64qi)__W);
150}
151
152static __inline__ __m512i __DEFAULT_FN_ATTRS512
153_mm512_maskz_cvt2ph_bf8(__mmask64 __U, __m512h __A, __m512h __B) {
154 return (__m512i)__builtin_ia32_selectb_512(
155 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_bf8(__A, __B),
156 (__v64qi)(__m512i)_mm512_setzero_si512());
157}
158
159static __inline__ __m512i __DEFAULT_FN_ATTRS512
160_mm512_cvts2ph_bf8(__m512h __A, __m512h __B) {
161 return (__m512i)__builtin_ia32_vcvt2ph2bf8s_512((__v32hf)(__A),
162 (__v32hf)(__B));
163}
164
165static __inline__ __m512i __DEFAULT_FN_ATTRS512
166_mm512_mask_cvts2ph_bf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
167 return (__m512i)__builtin_ia32_selectb_512(
168 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_bf8(__A, __B), (__v64qi)__W);
169}
170
171static __inline__ __m512i __DEFAULT_FN_ATTRS512
172_mm512_maskz_cvts2ph_bf8(__mmask64 __U, __m512h __A, __m512h __B) {
173 return (__m512i)__builtin_ia32_selectb_512(
174 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_bf8(__A, __B),
175 (__v64qi)(__m512i)_mm512_setzero_si512());
176}
177
178static __inline__ __m512i __DEFAULT_FN_ATTRS512 _mm512_cvt2ph_hf8(__m512h __A,
179 __m512h __B) {
180 return (__m512i)__builtin_ia32_vcvt2ph2hf8_512((__v32hf)(__A),
181 (__v32hf)(__B));
182}
183
184static __inline__ __m512i __DEFAULT_FN_ATTRS512
185_mm512_mask_cvt2ph_hf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
186 return (__m512i)__builtin_ia32_selectb_512(
187 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_hf8(__A, __B), (__v64qi)__W);
188}
189
190static __inline__ __m512i __DEFAULT_FN_ATTRS512
191_mm512_maskz_cvt2ph_hf8(__mmask64 __U, __m512h __A, __m512h __B) {
192 return (__m512i)__builtin_ia32_selectb_512(
193 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_hf8(__A, __B),
194 (__v64qi)(__m512i)_mm512_setzero_si512());
195}
196
197static __inline__ __m512i __DEFAULT_FN_ATTRS512
198_mm512_cvts2ph_hf8(__m512h __A, __m512h __B) {
199 return (__m512i)__builtin_ia32_vcvt2ph2hf8s_512((__v32hf)(__A),
200 (__v32hf)(__B));
201}
202
203static __inline__ __m512i __DEFAULT_FN_ATTRS512
204_mm512_mask_cvts2ph_hf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
205 return (__m512i)__builtin_ia32_selectb_512(
206 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_hf8(__A, __B), (__v64qi)__W);
207}
208
209static __inline__ __m512i __DEFAULT_FN_ATTRS512
210_mm512_maskz_cvts2ph_hf8(__mmask64 __U, __m512h __A, __m512h __B) {
211 return (__m512i)__builtin_ia32_selectb_512(
212 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_hf8(__A, __B),
213 (__v64qi)(__m512i)_mm512_setzero_si512());
214}
215
216static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvthf8(__m256i __A) {
217 return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
218 (__v32qi)__A, (__v32hf)(__m512h)_mm512_undefined_ph(), (__mmask32)-1);
219}
220
221static __inline__ __m512h __DEFAULT_FN_ATTRS512
222_mm512_mask_cvthf8(__m512h __W, __mmask32 __U, __m256i __A) {
223 return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
224 (__v32qi)__A, (__v32hf)(__m512h)__W, (__mmask32)__U);
225}
226
227static __inline__ __m512h __DEFAULT_FN_ATTRS512
228_mm512_maskz_cvthf8(__mmask32 __U, __m256i __A) {
229 return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
230 (__v32qi)__A, (__v32hf)(__m512h)_mm512_setzero_ph(), (__mmask32)__U);
231}
232
233static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtph_bf8(__m512h __A) {
234 return (__m256i)__builtin_ia32_vcvtph2bf8_512_mask(
235 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
236}
237
238static __inline__ __m256i __DEFAULT_FN_ATTRS512
239_mm512_mask_cvtph_bf8(__m256i __W, __mmask32 __U, __m512h __A) {
240 return (__m256i)__builtin_ia32_vcvtph2bf8_512_mask(
241 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
242}
243
244static __inline__ __m256i __DEFAULT_FN_ATTRS512
245_mm512_maskz_cvtph_bf8(__mmask32 __U, __m512h __A) {
246 return (__m256i)__builtin_ia32_vcvtph2bf8_512_mask(
247 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
248}
249
250static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtsph_bf8(__m512h __A) {
251 return (__m256i)__builtin_ia32_vcvtph2bf8s_512_mask(
252 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
253}
254
255static __inline__ __m256i __DEFAULT_FN_ATTRS512
256_mm512_mask_cvtsph_bf8(__m256i __W, __mmask32 __U, __m512h __A) {
257 return (__m256i)__builtin_ia32_vcvtph2bf8s_512_mask(
258 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
259}
260
261static __inline__ __m256i __DEFAULT_FN_ATTRS512
262_mm512_maskz_cvtsph_bf8(__mmask32 __U, __m512h __A) {
263 return (__m256i)__builtin_ia32_vcvtph2bf8s_512_mask(
264 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
265}
266
267static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtph_hf8(__m512h __A) {
268 return (__m256i)__builtin_ia32_vcvtph2hf8_512_mask(
269 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
270}
271
272static __inline__ __m256i __DEFAULT_FN_ATTRS512
273_mm512_mask_cvtph_hf8(__m256i __W, __mmask32 __U, __m512h __A) {
274 return (__m256i)__builtin_ia32_vcvtph2hf8_512_mask(
275 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
276}
277
278static __inline__ __m256i __DEFAULT_FN_ATTRS512
279_mm512_maskz_cvtph_hf8(__mmask32 __U, __m512h __A) {
280 return (__m256i)__builtin_ia32_vcvtph2hf8_512_mask(
281 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
282}
283
284static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtsph_hf8(__m512h __A) {
285 return (__m256i)__builtin_ia32_vcvtph2hf8s_512_mask(
286 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
287}
288
289static __inline__ __m256i __DEFAULT_FN_ATTRS512
290_mm512_mask_cvtsph_hf8(__m256i __W, __mmask32 __U, __m512h __A) {
291 return (__m256i)__builtin_ia32_vcvtph2hf8s_512_mask(
292 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
293}
294
295static __inline__ __m256i __DEFAULT_FN_ATTRS512
296_mm512_maskz_cvtsph_hf8(__mmask32 __U, __m512h __A) {
297 return (__m256i)__builtin_ia32_vcvtph2hf8s_512_mask(
298 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
299}
300
301static __inline __m512h __DEFAULT_FN_ATTRS512 _mm512_cvtbf8_ph(__m256i __A) {
302 return _mm512_castsi512_ph(_mm512_slli_epi16(_mm512_cvtepi8_epi16(__A), 8));
303}
304
305static __inline __m512h __DEFAULT_FN_ATTRS512
306_mm512_mask_cvtbf8_ph(__m512h __S, __mmask32 __U, __m256i __A) {
307 return _mm512_castsi512_ph(
308 _mm512_mask_slli_epi16((__m512i)__S, __U, _mm512_cvtepi8_epi16(__A), 8));
309}
310
311static __inline __m512h __DEFAULT_FN_ATTRS512
312_mm512_maskz_cvtbf8_ph(__mmask32 __U, __m256i __A) {
313 return _mm512_castsi512_ph(
314 _mm512_slli_epi16(_mm512_maskz_cvtepi8_epi16(__U, __A), 8));
315}
316
317#undef __DEFAULT_FN_ATTRS512
318
319#endif // __AVX10_2_512CONVERTINTRIN_H
320#endif // __SSE2__
lib/include/avx10_2_512minmaxintrin.h created+127
...@@ -0,0 +1,127 @@
1/*===---- avx10_2_512minmaxintrin.h - AVX10_2_512MINMAX intrinsics ---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512minmaxintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2_512MINMAXINTRIN_H
15#define __AVX10_2_512MINMAXINTRIN_H
16
17#define _mm512_minmax_pbh(A, B, C) \
18 ((__m512bh)__builtin_ia32_vminmaxbf16512((__v32bf)(__m512bh)(A), \
19 (__v32bf)(__m512bh)(A), (int)(C)))
20
21#define _mm512_mask_minmax_pbh(W, U, A, B, C) \
22 ((__m512bh)__builtin_ia32_selectpbf_512( \
23 (__mmask32)(U), \
24 (__v32bf)_mm512_minmax_pbh((__v32bf)(__m512bh)(A), \
25 (__v32bf)(__m512bh)(B), (int)(C)), \
26 (__v32bf)(__m512bh)(W)))
27
28#define _mm512_maskz_minmax_pbh(U, A, B, C) \
29 ((__m512bh)__builtin_ia32_selectpbf_512( \
30 (__mmask32)(U), \
31 (__v32bf)_mm512_minmax_pbh((__v32bf)(__m512bh)(A), \
32 (__v32bf)(__m512bh)(B), (int)(C)), \
33 (__v32bf) __builtin_bit_cast(__m512bh, _mm512_setzero_ps())))
34
35#define _mm512_minmax_pd(A, B, C) \
36 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
37 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
38 (__v8df)_mm512_undefined_pd(), (__mmask8)-1, \
39 _MM_FROUND_CUR_DIRECTION))
40
41#define _mm512_mask_minmax_pd(W, U, A, B, C) \
42 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
43 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
44 (__v8df)(__m512d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
45
46#define _mm512_maskz_minmax_pd(U, A, B, C) \
47 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
48 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
49 (__v8df)_mm512_setzero_pd(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
50
51#define _mm512_minmax_round_pd(A, B, C, R) \
52 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
53 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
54 (__v8df)_mm512_undefined_pd(), (__mmask8)-1, (int)(R)))
55
56#define _mm512_mask_minmax_round_pd(W, U, A, B, C, R) \
57 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
58 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
59 (__v8df)(__m512d)(W), (__mmask8)(U), (int)(R)))
60
61#define _mm512_maskz_minmax_round_pd(U, A, B, C, R) \
62 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
63 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
64 (__v8df)_mm512_setzero_pd(), (__mmask8)(U), (int)(R)))
65
66#define _mm512_minmax_ph(A, B, C) \
67 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
68 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
69 (__v32hf)_mm512_undefined_ph(), (__mmask32)-1, \
70 _MM_FROUND_CUR_DIRECTION))
71
72#define _mm512_mask_minmax_ph(W, U, A, B, C) \
73 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
74 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
75 (__v32hf)(__m512h)(W), (__mmask32)(U), _MM_FROUND_CUR_DIRECTION))
76
77#define _mm512_maskz_minmax_ph(U, A, B, C) \
78 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
79 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
80 (__v32hf)_mm512_setzero_ph(), (__mmask32)(U), _MM_FROUND_CUR_DIRECTION))
81
82#define _mm512_minmax_round_ph(A, B, C, R) \
83 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
84 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
85 (__v32hf)_mm512_undefined_ph(), (__mmask32)-1, (int)(R)))
86
87#define _mm512_mask_minmax_round_ph(W, U, A, B, C, R) \
88 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
89 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
90 (__v32hf)(__m512h)(W), (__mmask32)(U), (int)(R)))
91
92#define _mm512_maskz_minmax_round_ph(U, A, B, C, R) \
93 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
94 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
95 (__v32hf)_mm512_setzero_ph(), (__mmask32)(U), (int)(R)))
96
97#define _mm512_minmax_ps(A, B, C) \
98 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
99 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
100 (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, \
101 _MM_FROUND_CUR_DIRECTION))
102
103#define _mm512_mask_minmax_ps(W, U, A, B, C) \
104 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
105 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), (__v16sf)(W), \
106 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
107
108#define _mm512_maskz_minmax_ps(U, A, B, C) \
109 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
110 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
111 (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
112
113#define _mm512_minmax_round_ps(A, B, C, R) \
114 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
115 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
116 (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, (int)(R)))
117
118#define _mm512_mask_minmax_round_ps(W, U, A, B, C, R) \
119 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
120 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), (__v16sf)(W), \
121 (__mmask16)(U), (int)(R)))
122
123#define _mm512_maskz_minmax_round_ps(U, A, B, C, R) \
124 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
125 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
126 (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), (int)(R)))
127#endif // __AVX10_2_512MINMAXINTRIN_H
lib/include/avx10_2_512niintrin.h created+314
...@@ -0,0 +1,314 @@
1/*===---- avx10_2_512niintrin.h - AVX10.2-512 new instruction intrinsics ---===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512niintrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2_512NIINTRIN_H
17#define __AVX10_2_512NIINTRIN_H
18
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
21 __min_vector_width__(512)))
22
23/* VNNI FP16 */
24static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_dpph_ps(__m512 __W,
25 __m512h __A,
26 __m512h __B) {
27 return (__m512)__builtin_ia32_vdpphps512((__v16sf)__W, (__v32hf)__A,
28 (__v32hf)__B);
29}
30
31static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_mask_dpph_ps(__m512 __W,
32 __mmask16 __U,
33 __m512h __A,
34 __m512h __B) {
35 return (__m512)__builtin_ia32_selectps_512(
36 (__mmask16)__U, (__v16sf)_mm512_dpph_ps(__W, __A, __B), (__v16sf)__W);
37}
38
39static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_maskz_dpph_ps(__mmask16 __U,
40 __m512 __W,
41 __m512h __A,
42 __m512h __B) {
43 return (__m512)__builtin_ia32_selectps_512(
44 (__mmask16)__U, (__v16sf)_mm512_dpph_ps(__W, __A, __B),
45 (__v16sf)_mm512_setzero_ps());
46}
47
48/* VMPSADBW */
49#define _mm512_mpsadbw_epu8(A, B, imm) \
50 ((__m512i)__builtin_ia32_mpsadbw512((__v64qi)(__m512i)(A), \
51 (__v64qi)(__m512i)(B), (int)(imm)))
52
53#define _mm512_mask_mpsadbw_epu8(W, U, A, B, imm) \
54 ((__m512i)__builtin_ia32_selectw_512( \
55 (__mmask32)(U), (__v32hi)_mm512_mpsadbw_epu8((A), (B), (imm)), \
56 (__v32hi)(__m512i)(W)))
57
58#define _mm512_maskz_mpsadbw_epu8(U, A, B, imm) \
59 ((__m512i)__builtin_ia32_selectw_512( \
60 (__mmask32)(U), (__v32hi)_mm512_mpsadbw_epu8((A), (B), (imm)), \
61 (__v32hi)_mm512_setzero_si512()))
62
63/* VNNI INT8 */
64static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbssd_epi32(__m512i __W,
65 __m512i __A,
66 __m512i __B) {
67 return (__m512i)__builtin_ia32_vpdpbssd512((__v16si)__W, (__v16si)__A,
68 (__v16si)__B);
69}
70
71static __inline__ __m512i __DEFAULT_FN_ATTRS
72_mm512_mask_dpbssd_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
73 return (__m512i)__builtin_ia32_selectd_512(
74 __U, (__v16si)_mm512_dpbssd_epi32(__W, __A, __B), (__v16si)__W);
75}
76
77static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbssd_epi32(
78 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
79 return (__m512i)__builtin_ia32_selectd_512(
80 __U, (__v16si)_mm512_dpbssd_epi32(__W, __A, __B),
81 (__v16si)_mm512_setzero_si512());
82}
83
84static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbssds_epi32(__m512i __W,
85 __m512i __A,
86 __m512i __B) {
87 return (__m512i)__builtin_ia32_vpdpbssds512((__v16si)__W, (__v16si)__A,
88 (__v16si)__B);
89}
90
91static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbssds_epi32(
92 __m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
93 return (__m512i)__builtin_ia32_selectd_512(
94 __U, (__v16si)_mm512_dpbssds_epi32(__W, __A, __B), (__v16si)__W);
95}
96
97static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbssds_epi32(
98 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
99 return (__m512i)__builtin_ia32_selectd_512(
100 __U, (__v16si)_mm512_dpbssds_epi32(__W, __A, __B),
101 (__v16si)_mm512_setzero_si512());
102}
103
104static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbsud_epi32(__m512i __W,
105 __m512i __A,
106 __m512i __B) {
107 return (__m512i)__builtin_ia32_vpdpbsud512((__v16si)__W, (__v16si)__A,
108 (__v16si)__B);
109}
110
111static __inline__ __m512i __DEFAULT_FN_ATTRS
112_mm512_mask_dpbsud_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
113 return (__m512i)__builtin_ia32_selectd_512(
114 __U, (__v16si)_mm512_dpbsud_epi32(__W, __A, __B), (__v16si)__W);
115}
116
117static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbsud_epi32(
118 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
119 return (__m512i)__builtin_ia32_selectd_512(
120 __U, (__v16si)_mm512_dpbsud_epi32(__W, __A, __B),
121 (__v16si)_mm512_setzero_si512());
122}
123
124static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbsuds_epi32(__m512i __W,
125 __m512i __A,
126 __m512i __B) {
127 return (__m512i)__builtin_ia32_vpdpbsuds512((__v16si)__W, (__v16si)__A,
128 (__v16si)__B);
129}
130
131static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbsuds_epi32(
132 __m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
133 return (__m512i)__builtin_ia32_selectd_512(
134 __U, (__v16si)_mm512_dpbsuds_epi32(__W, __A, __B), (__v16si)__W);
135}
136
137static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbsuds_epi32(
138 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
139 return (__m512i)__builtin_ia32_selectd_512(
140 __U, (__v16si)_mm512_dpbsuds_epi32(__W, __A, __B),
141 (__v16si)_mm512_setzero_si512());
142}
143
144static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbuud_epi32(__m512i __W,
145 __m512i __A,
146 __m512i __B) {
147 return (__m512i)__builtin_ia32_vpdpbuud512((__v16si)__W, (__v16si)__A,
148 (__v16si)__B);
149}
150
151static __inline__ __m512i __DEFAULT_FN_ATTRS
152_mm512_mask_dpbuud_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
153 return (__m512i)__builtin_ia32_selectd_512(
154 __U, (__v16si)_mm512_dpbuud_epi32(__W, __A, __B), (__v16si)__W);
155}
156
157static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbuud_epi32(
158 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
159 return (__m512i)__builtin_ia32_selectd_512(
160 __U, (__v16si)_mm512_dpbuud_epi32(__W, __A, __B),
161 (__v16si)_mm512_setzero_si512());
162}
163
164static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbuuds_epi32(__m512i __W,
165 __m512i __A,
166 __m512i __B) {
167 return (__m512i)__builtin_ia32_vpdpbuuds512((__v16si)__W, (__v16si)__A,
168 (__v16si)__B);
169}
170
171static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbuuds_epi32(
172 __m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
173 return (__m512i)__builtin_ia32_selectd_512(
174 __U, (__v16si)_mm512_dpbuuds_epi32(__W, __A, __B), (__v16si)__W);
175}
176
177static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbuuds_epi32(
178 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
179 return (__m512i)__builtin_ia32_selectd_512(
180 __U, (__v16si)_mm512_dpbuuds_epi32(__W, __A, __B),
181 (__v16si)_mm512_setzero_si512());
182}
183
184/* VNNI INT16 */
185static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwsud_epi32(__m512i __A,
186 __m512i __B,
187 __m512i __C) {
188 return (__m512i)__builtin_ia32_vpdpwsud512((__v16si)__A, (__v16si)__B,
189 (__v16si)__C);
190}
191
192static __inline__ __m512i __DEFAULT_FN_ATTRS
193_mm512_mask_dpwsud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
194 return (__m512i)__builtin_ia32_selectd_512(
195 (__mmask16)__U, (__v16si)_mm512_dpwsud_epi32(__A, __B, __C),
196 (__v16si)__A);
197}
198
199static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsud_epi32(
200 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
201 return (__m512i)__builtin_ia32_selectd_512(
202 (__mmask16)__U, (__v16si)_mm512_dpwsud_epi32(__A, __B, __C),
203 (__v16si)_mm512_setzero_si512());
204}
205
206static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwsuds_epi32(__m512i __A,
207 __m512i __B,
208 __m512i __C) {
209 return (__m512i)__builtin_ia32_vpdpwsuds512((__v16si)__A, (__v16si)__B,
210 (__v16si)__C);
211}
212
213static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwsuds_epi32(
214 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
215 return (__m512i)__builtin_ia32_selectd_512(
216 (__mmask16)__U, (__v16si)_mm512_dpwsuds_epi32(__A, __B, __C),
217 (__v16si)__A);
218}
219
220static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsuds_epi32(
221 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
222 return (__m512i)__builtin_ia32_selectd_512(
223 (__mmask16)__U, (__v16si)_mm512_dpwsuds_epi32(__A, __B, __C),
224 (__v16si)_mm512_setzero_si512());
225}
226
227static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwusd_epi32(__m512i __A,
228 __m512i __B,
229 __m512i __C) {
230 return (__m512i)__builtin_ia32_vpdpwusd512((__v16si)__A, (__v16si)__B,
231 (__v16si)__C);
232}
233
234static __inline__ __m512i __DEFAULT_FN_ATTRS
235_mm512_mask_dpwusd_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
236 return (__m512i)__builtin_ia32_selectd_512(
237 (__mmask16)__U, (__v16si)_mm512_dpwusd_epi32(__A, __B, __C),
238 (__v16si)__A);
239}
240
241static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusd_epi32(
242 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
243 return (__m512i)__builtin_ia32_selectd_512(
244 (__mmask16)__U, (__v16si)_mm512_dpwusd_epi32(__A, __B, __C),
245 (__v16si)_mm512_setzero_si512());
246}
247
248static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwusds_epi32(__m512i __A,
249 __m512i __B,
250 __m512i __C) {
251 return (__m512i)__builtin_ia32_vpdpwusds512((__v16si)__A, (__v16si)__B,
252 (__v16si)__C);
253}
254
255static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwusds_epi32(
256 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
257 return (__m512i)__builtin_ia32_selectd_512(
258 (__mmask16)__U, (__v16si)_mm512_dpwusds_epi32(__A, __B, __C),
259 (__v16si)__A);
260}
261
262static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusds_epi32(
263 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
264 return (__m512i)__builtin_ia32_selectd_512(
265 (__mmask16)__U, (__v16si)_mm512_dpwusds_epi32(__A, __B, __C),
266 (__v16si)_mm512_setzero_si512());
267}
268
269static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwuud_epi32(__m512i __A,
270 __m512i __B,
271 __m512i __C) {
272 return (__m512i)__builtin_ia32_vpdpwuud512((__v16si)__A, (__v16si)__B,
273 (__v16si)__C);
274}
275
276static __inline__ __m512i __DEFAULT_FN_ATTRS
277_mm512_mask_dpwuud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
278 return (__m512i)__builtin_ia32_selectd_512(
279 (__mmask16)__U, (__v16si)_mm512_dpwuud_epi32(__A, __B, __C),
280 (__v16si)__A);
281}
282
283static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuud_epi32(
284 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
285 return (__m512i)__builtin_ia32_selectd_512(
286 (__mmask16)__U, (__v16si)_mm512_dpwuud_epi32(__A, __B, __C),
287 (__v16si)_mm512_setzero_si512());
288}
289
290static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwuuds_epi32(__m512i __A,
291 __m512i __B,
292 __m512i __C) {
293 return (__m512i)__builtin_ia32_vpdpwuuds512((__v16si)__A, (__v16si)__B,
294 (__v16si)__C);
295}
296
297static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwuuds_epi32(
298 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
299 return (__m512i)__builtin_ia32_selectd_512(
300 (__mmask16)__U, (__v16si)_mm512_dpwuuds_epi32(__A, __B, __C),
301 (__v16si)__A);
302}
303
304static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuuds_epi32(
305 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
306 return (__m512i)__builtin_ia32_selectd_512(
307 (__mmask16)__U, (__v16si)_mm512_dpwuuds_epi32(__A, __B, __C),
308 (__v16si)_mm512_setzero_si512());
309}
310
311#undef __DEFAULT_FN_ATTRS
312
313#endif /* __SSE2__ */
314#endif /* __AVX10_2_512NIINTRIN_H */
lib/include/avx10_2_512satcvtdsintrin.h created+303
...@@ -0,0 +1,303 @@
1/*===----- avx10_2_512satcvtdsintrin.h - AVX10_2_512SATCVTDS intrinsics ----===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512satcvtdsintrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __AVX10_2_512SATCVTDSINTRIN_H
15#define __AVX10_2_512SATCVTDSINTRIN_H
16
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
20 __min_vector_width__(512)))
21
22// 512 bit : Double -> Int
23static __inline__ __m256i __DEFAULT_FN_ATTRS _mm512_cvttspd_epi32(__m512d __A) {
24 return ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask(
25 (__v8df)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
26 _MM_FROUND_CUR_DIRECTION));
27}
28
29static __inline__ __m256i __DEFAULT_FN_ATTRS
30_mm512_mask_cvttspd_epi32(__m256i __W, __mmask8 __U, __m512d __A) {
31 return ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask(
32 (__v8df)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
33}
34
35static __inline__ __m256i __DEFAULT_FN_ATTRS
36_mm512_maskz_cvttspd_epi32(__mmask8 __U, __m512d __A) {
37 return ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask(
38 (__v8df)__A, (__v8si)_mm256_setzero_si256(), __U,
39 _MM_FROUND_CUR_DIRECTION));
40}
41
42#define _mm512_cvtts_roundpd_epi32(__A, __R) \
43 ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask( \
44 (__v8df)(__m512d)(__A), (__v8si)_mm256_undefined_si256(), \
45 (__mmask8) - 1, (const int)(__R)))
46
47#define _mm512_mask_cvtts_roundpd_epi32(__W, __U, __A, __R) \
48 ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask( \
49 (__v8df)(__m512d)(__A), (__v8si)(__m256i)(__W), (__mmask8)(__U), \
50 (const int)(__R)))
51
52#define _mm512_maskz_cvtts_roundpd_epi32(__U, __A, __R) \
53 ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask( \
54 (__v8df)(__m512d)(__A), (__v8si)_mm256_setzero_si256(), (__mmask8)(__U), \
55 (const int)(__R)))
56
57// 512 bit : Double -> uInt
58static __inline__ __m256i __DEFAULT_FN_ATTRS _mm512_cvttspd_epu32(__m512d __A) {
59 return ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask(
60 (__v8df)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
61 _MM_FROUND_CUR_DIRECTION));
62}
63
64static __inline__ __m256i __DEFAULT_FN_ATTRS
65_mm512_mask_cvttspd_epu32(__m256i __W, __mmask8 __U, __m512d __A) {
66 return ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask(
67 (__v8df)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
68}
69
70static __inline__ __m256i __DEFAULT_FN_ATTRS
71_mm512_maskz_cvttspd_epu32(__mmask8 __U, __m512d __A) {
72 return ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask(
73 (__v8df)__A, (__v8si)_mm256_setzero_si256(), __U,
74 _MM_FROUND_CUR_DIRECTION));
75}
76
77#define _mm512_cvtts_roundpd_epu32(__A, __R) \
78 ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask( \
79 (__v8df)(__m512d)(__A), (__v8si)_mm256_undefined_si256(), \
80 (__mmask8) - 1, (const int)(__R)))
81
82#define _mm512_mask_cvtts_roundpd_epu32(__W, __U, __A, __R) \
83 ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask( \
84 (__v8df)(__m512d)(__A), (__v8si)(__m256i)(__W), (__mmask8)(__U), \
85 (const int)(__R)))
86
87#define _mm512_maskz_cvtts_roundpd_epu32(__U, __A, __R) \
88 ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask( \
89 (__v8df)(__m512d)(__A), (__v8si)_mm256_setzero_si256(), (__mmask8)(__U), \
90 (const int)(__R)))
91
92// 512 bit : Double -> Long
93
94static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttspd_epi64(__m512d __A) {
95 return ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask(
96 (__v8df)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
97 _MM_FROUND_CUR_DIRECTION));
98}
99static __inline__ __m512i __DEFAULT_FN_ATTRS
100_mm512_mask_cvttspd_epi64(__m512i __W, __mmask8 __U, __m512d __A) {
101 return ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask(
102 (__v8df)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
103}
104static __inline__ __m512i __DEFAULT_FN_ATTRS
105_mm512_maskz_cvttspd_epi64(__mmask8 __U, __m512d __A) {
106 return ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask(
107 (__v8df)__A, (__v8di)_mm512_setzero_si512(), __U,
108 _MM_FROUND_CUR_DIRECTION));
109}
110
111#define _mm512_cvtts_roundpd_epi64(__A, __R) \
112 ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask( \
113 (__v8df)(__m512d)(__A), (__v8di)_mm512_undefined_epi32(), \
114 (__mmask8) - 1, (const int)(__R)))
115
116#define _mm512_mask_cvtts_roundpd_epi64(__W, __U, __A, __R) \
117 ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask( \
118 (__v8df)(__m512d)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
119 (const int)(__R)))
120
121#define _mm512_maskz_cvtts_roundpd_epi64(__U, __A, __R) \
122 ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask( \
123 (__v8df)(__m512d)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
124 (const int)(__R)))
125
126// 512 bit : Double -> ULong
127
128static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttspd_epu64(__m512d __A) {
129 return ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask(
130 (__v8df)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
131 _MM_FROUND_CUR_DIRECTION));
132}
133
134static __inline__ __m512i __DEFAULT_FN_ATTRS
135_mm512_mask_cvttspd_epu64(__m512i __W, __mmask8 __U, __m512d __A) {
136 return ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask(
137 (__v8df)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
138}
139
140static __inline__ __m512i __DEFAULT_FN_ATTRS
141_mm512_maskz_cvttspd_epu64(__mmask8 __U, __m512d __A) {
142 return ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask(
143 (__v8df)__A, (__v8di)_mm512_setzero_si512(), __U,
144 _MM_FROUND_CUR_DIRECTION));
145}
146
147#define _mm512_cvtts_roundpd_epu64(__A, __R) \
148 ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask( \
149 (__v8df)(__m512d)(__A), (__v8di)_mm512_undefined_epi32(), \
150 (__mmask8) - 1, (const int)(__R)))
151
152#define _mm512_mask_cvtts_roundpd_epu64(__W, __U, __A, __R) \
153 ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask( \
154 (__v8df)(__m512d)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
155 (const int)(__R)))
156
157#define _mm512_maskz_cvtts_roundpd_epu64(__U, __A, __R) \
158 ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask( \
159 (__v8df)(__m512d)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
160 (const int)(__R)))
161
162// 512 bit: Float -> int
163static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epi32(__m512 __A) {
164 return ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask(
165 (__v16sf)(__A), (__v16si)_mm512_undefined_epi32(), (__mmask16)-1,
166 _MM_FROUND_CUR_DIRECTION));
167}
168
169static __inline__ __m512i __DEFAULT_FN_ATTRS
170_mm512_mask_cvttsps_epi32(__m512i __W, __mmask16 __U, __m512 __A) {
171 return ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask(
172 (__v16sf)(__A), (__v16si)(__W), __U, _MM_FROUND_CUR_DIRECTION));
173}
174
175static __inline__ __m512i __DEFAULT_FN_ATTRS
176_mm512_maskz_cvttsps_epi32(__mmask16 __U, __m512 __A) {
177 return ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask(
178 (__v16sf)(__A), (__v16si)_mm512_setzero_si512(), __U,
179 _MM_FROUND_CUR_DIRECTION));
180}
181
182#define _mm512_cvtts_roundps_epi32(__A, __R) \
183 ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask( \
184 (__v16sf)(__m512)(__A), (__v16si)_mm512_undefined_epi32(), \
185 (__mmask16) - 1, (const int)(__R)))
186
187#define _mm512_mask_cvtts_roundps_epi32(__W, __U, __A, __R) \
188 ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask( \
189 (__v16sf)(__m512)(__A), (__v16si)(__m512i)(__W), (__mmask16)(__U), \
190 (const int)(__R)))
191
192#define _mm512_maskz_cvtts_roundps_epi32(__U, __A, __R) \
193 ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask( \
194 (__v16sf)(__m512)(__A), (__v16si)_mm512_setzero_si512(), \
195 (__mmask16)(__U), (const int)(__R)))
196
197// 512 bit: Float -> uint
198static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epu32(__m512 __A) {
199 return ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask(
200 (__v16sf)(__A), (__v16si)_mm512_undefined_epi32(), (__mmask16)-1,
201 _MM_FROUND_CUR_DIRECTION));
202}
203
204static __inline__ __m512i __DEFAULT_FN_ATTRS
205_mm512_mask_cvttsps_epu32(__m512i __W, __mmask16 __U, __m512 __A) {
206 return ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask(
207 (__v16sf)(__A), (__v16si)(__W), __U, _MM_FROUND_CUR_DIRECTION));
208}
209
210static __inline__ __m512i __DEFAULT_FN_ATTRS
211_mm512_maskz_cvttsps_epu32(__mmask16 __U, __m512 __A) {
212 return ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask(
213 (__v16sf)(__A), (__v16si)_mm512_setzero_si512(), __U,
214 _MM_FROUND_CUR_DIRECTION));
215}
216
217#define _mm512_cvtts_roundps_epu32(__A, __R) \
218 ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask( \
219 (__v16sf)(__m512)(__A), (__v16si)_mm512_undefined_epi32(), \
220 (__mmask16) - 1, (const int)(__R)))
221
222#define _mm512_mask_cvtts_roundps_epu32(__W, __U, __A, __R) \
223 ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask( \
224 (__v16sf)(__m512)(__A), (__v16si)(__m512i)(__W), (__mmask16)(__U), \
225 (const int)(__R)))
226
227#define _mm512_maskz_cvtts_roundps_epu32(__U, __A, __R) \
228 ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask( \
229 (__v16sf)(__m512)(__A), (__v16si)_mm512_setzero_si512(), \
230 (__mmask16)(__U), (const int)(__R)))
231
232// 512 bit : float -> long
233static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epi64(__m256 __A) {
234 return ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask(
235 (__v8sf)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
236 _MM_FROUND_CUR_DIRECTION));
237}
238
239static __inline__ __m512i __DEFAULT_FN_ATTRS
240_mm512_mask_cvttsps_epi64(__m512i __W, __mmask8 __U, __m256 __A) {
241 return ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask(
242 (__v8sf)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
243}
244
245static __inline__ __m512i __DEFAULT_FN_ATTRS
246_mm512_maskz_cvttsps_epi64(__mmask8 __U, __m256 __A) {
247 return ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask(
248 (__v8sf)__A, (__v8di)_mm512_setzero_si512(), __U,
249 _MM_FROUND_CUR_DIRECTION));
250}
251
252#define _mm512_cvtts_roundps_epi64(__A, __R) \
253 ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask( \
254 (__v8sf)(__m256)(__A), (__v8di)_mm512_undefined_epi32(), (__mmask8) - 1, \
255 (const int)(__R)))
256
257#define _mm512_mask_cvtts_roundps_epi64(__W, __U, __A, __R) \
258 ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask( \
259 (__v8sf)(__m256)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
260 (const int)(__R)))
261
262#define _mm512_maskz_cvtts_roundps_epi64(__U, __A, __R) \
263 ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask( \
264 (__v8sf)(__m256)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
265 (const int)(__R)))
266
267// 512 bit : float -> ulong
268static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epu64(__m256 __A) {
269 return ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask(
270 (__v8sf)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
271 _MM_FROUND_CUR_DIRECTION));
272}
273
274static __inline__ __m512i __DEFAULT_FN_ATTRS
275_mm512_mask_cvttsps_epu64(__m512i __W, __mmask8 __U, __m256 __A) {
276 return ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask(
277 (__v8sf)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
278}
279
280static __inline__ __m512i __DEFAULT_FN_ATTRS
281_mm512_maskz_cvttsps_epu64(__mmask8 __U, __m256 __A) {
282 return ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask(
283 (__v8sf)__A, (__v8di)_mm512_setzero_si512(), __U,
284 _MM_FROUND_CUR_DIRECTION));
285}
286
287#define _mm512_cvtts_roundps_epu64(__A, __R) \
288 ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask( \
289 (__v8sf)(__m256)(__A), (__v8di)_mm512_undefined_epi32(), (__mmask8) - 1, \
290 (const int)(__R)))
291
292#define _mm512_mask_cvtts_roundps_epu64(__W, __U, __A, __R) \
293 ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask( \
294 (__v8sf)(__m256)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
295 (const int)(__R)))
296
297#define _mm512_maskz_cvtts_roundps_epu64(__U, __A, __R) \
298 ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask( \
299 (__v8sf)(__m256)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
300 (const int)(__R)))
301
302#undef __DEFAULT_FN_ATTRS
303#endif // __AVX10_2_512SATCVTDSINTRIN_H
lib/include/avx10_2_512satcvtintrin.h created+301
...@@ -0,0 +1,301 @@
1/*===------ avx10_2_512satcvtintrin.h - AVX10_2_512SATCVT intrinsics -------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512satcvtintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2_512SATCVTINTRIN_H
15#define __AVX10_2_512SATCVTINTRIN_H
16
17#define _mm512_ipcvtbf16_epi8(A) \
18 ((__m512i)__builtin_ia32_vcvtbf162ibs512((__v32bf)(__m512bh)(A)))
19
20#define _mm512_mask_ipcvtbf16_epi8(W, U, A) \
21 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
22 (__v32hi)_mm512_ipcvtbf16_epi8(A), \
23 (__v32hi)(__m512i)(W)))
24
25#define _mm512_maskz_ipcvtbf16_epi8(U, A) \
26 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
27 (__v32hi)_mm512_ipcvtbf16_epi8(A), \
28 (__v32hi)_mm512_setzero_si512()))
29
30#define _mm512_ipcvtbf16_epu8(A) \
31 ((__m512i)__builtin_ia32_vcvtbf162iubs512((__v32bf)(__m512bh)(A)))
32
33#define _mm512_mask_ipcvtbf16_epu8(W, U, A) \
34 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
35 (__v32hi)_mm512_ipcvtbf16_epu8(A), \
36 (__v32hi)(__m512i)(W)))
37
38#define _mm512_maskz_ipcvtbf16_epu8(U, A) \
39 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
40 (__v32hi)_mm512_ipcvtbf16_epu8(A), \
41 (__v32hi)_mm512_setzero_si512()))
42
43#define _mm512_ipcvttbf16_epi8(A) \
44 ((__m512i)__builtin_ia32_vcvttbf162ibs512((__v32bf)(__m512bh)(A)))
45
46#define _mm512_mask_ipcvttbf16_epi8(W, U, A) \
47 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
48 (__v32hi)_mm512_ipcvttbf16_epi8(A), \
49 (__v32hi)(__m512i)(W)))
50
51#define _mm512_maskz_ipcvttbf16_epi8(U, A) \
52 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
53 (__v32hi)_mm512_ipcvttbf16_epi8(A), \
54 (__v32hi)_mm512_setzero_si512()))
55
56#define _mm512_ipcvttbf16_epu8(A) \
57 ((__m512i)__builtin_ia32_vcvttbf162iubs512((__v32bf)(__m512bh)(A)))
58
59#define _mm512_mask_ipcvttbf16_epu8(W, U, A) \
60 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
61 (__v32hi)_mm512_ipcvttbf16_epu8(A), \
62 (__v32hi)(__m512i)(W)))
63
64#define _mm512_maskz_ipcvttbf16_epu8(U, A) \
65 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
66 (__v32hi)_mm512_ipcvttbf16_epu8(A), \
67 (__v32hi)_mm512_setzero_si512()))
68
69#define _mm512_ipcvtph_epi8(A) \
70 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask( \
71 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
72 _MM_FROUND_CUR_DIRECTION))
73
74#define _mm512_mask_ipcvtph_epi8(W, U, A) \
75 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask((__v32hf)(__m512h)(A), \
76 (__v32hu)(W), (__mmask32)(U), \
77 _MM_FROUND_CUR_DIRECTION))
78
79#define _mm512_maskz_ipcvtph_epi8(U, A) \
80 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask( \
81 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
82 _MM_FROUND_CUR_DIRECTION))
83
84#define _mm512_ipcvt_roundph_epi8(A, R) \
85 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask((__v32hf)(__m512h)(A), \
86 (__v32hu)_mm512_setzero_si512(), \
87 (__mmask32)-1, (const int)R))
88
89#define _mm512_mask_ipcvt_roundph_epi8(W, U, A, R) \
90 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask( \
91 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), (const int)R))
92
93#define _mm512_maskz_ipcvt_roundph_epi8(U, A, R) \
94 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask((__v32hf)(__m512h)(A), \
95 (__v32hu)_mm512_setzero_si512(), \
96 (__mmask32)(U), (const int)R))
97
98#define _mm512_ipcvtph_epu8(A) \
99 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
100 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
101 _MM_FROUND_CUR_DIRECTION))
102
103#define _mm512_mask_ipcvtph_epu8(W, U, A) \
104 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask((__v32hf)(__m512h)(A), \
105 (__v32hu)(W), (__mmask32)(U), \
106 _MM_FROUND_CUR_DIRECTION))
107
108#define _mm512_maskz_ipcvtph_epu8(U, A) \
109 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
110 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
111 _MM_FROUND_CUR_DIRECTION))
112
113#define _mm512_ipcvt_roundph_epu8(A, R) \
114 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
115 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
116 (const int)R))
117
118#define _mm512_mask_ipcvt_roundph_epu8(W, U, A, R) \
119 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
120 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), (const int)R))
121
122#define _mm512_maskz_ipcvt_roundph_epu8(U, A, R) \
123 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
124 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
125 (const int)R))
126
127#define _mm512_ipcvtps_epi8(A) \
128 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask( \
129 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
130 _MM_FROUND_CUR_DIRECTION))
131
132#define _mm512_mask_ipcvtps_epi8(W, U, A) \
133 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask((__v16sf)(__m512)(A), \
134 (__v16su)(W), (__mmask16)(U), \
135 _MM_FROUND_CUR_DIRECTION))
136
137#define _mm512_maskz_ipcvtps_epi8(U, A) \
138 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask( \
139 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
140 _MM_FROUND_CUR_DIRECTION))
141
142#define _mm512_ipcvt_roundps_epi8(A, R) \
143 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask((__v16sf)(__m512)(A), \
144 (__v16su)_mm512_setzero_si512(), \
145 (__mmask16)-1, (const int)R))
146
147#define _mm512_mask_ipcvt_roundps_epi8(W, U, A, R) \
148 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask( \
149 (__v16sf)(__m512)(A), (__v16su)(W), (__mmask16)(U), (const int)R))
150
151#define _mm512_maskz_ipcvt_roundps_epi8(U, A, R) \
152 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask((__v16sf)(__m512)(A), \
153 (__v16su)_mm512_setzero_si512(), \
154 (__mmask16)(U), (const int)R))
155
156#define _mm512_ipcvtps_epu8(A) \
157 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
158 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
159 _MM_FROUND_CUR_DIRECTION))
160
161#define _mm512_mask_ipcvtps_epu8(W, U, A) \
162 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask((__v16sf)(__m512)(A), \
163 (__v16su)(W), (__mmask16)(U), \
164 _MM_FROUND_CUR_DIRECTION))
165
166#define _mm512_maskz_ipcvtps_epu8(U, A) \
167 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
168 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
169 _MM_FROUND_CUR_DIRECTION))
170
171#define _mm512_ipcvt_roundps_epu8(A, R) \
172 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
173 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
174 (const int)R))
175
176#define _mm512_mask_ipcvt_roundps_epu8(W, U, A, R) \
177 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
178 (__v16sf)(__m512)(A), (__v16su)(W), (__mmask16)(U), (const int)R))
179
180#define _mm512_maskz_ipcvt_roundps_epu8(U, A, R) \
181 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
182 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
183 (const int)R))
184
185#define _mm512_ipcvttph_epi8(A) \
186 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
187 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
188 _MM_FROUND_CUR_DIRECTION))
189
190#define _mm512_mask_ipcvttph_epi8(W, U, A) \
191 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask((__v32hf)(__m512h)(A), \
192 (__v32hu)(W), (__mmask32)(U), \
193 _MM_FROUND_CUR_DIRECTION))
194
195#define _mm512_maskz_ipcvttph_epi8(U, A) \
196 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
197 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
198 _MM_FROUND_CUR_DIRECTION))
199
200#define _mm512_ipcvtt_roundph_epi8(A, S) \
201 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
202 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
203 S))
204
205#define _mm512_mask_ipcvtt_roundph_epi8(W, U, A, S) \
206 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
207 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), S))
208
209#define _mm512_maskz_ipcvtt_roundph_epi8(U, A, S) \
210 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
211 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
212 S))
213
214#define _mm512_ipcvttph_epu8(A) \
215 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
216 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
217 _MM_FROUND_CUR_DIRECTION))
218
219#define _mm512_mask_ipcvttph_epu8(W, U, A) \
220 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask((__v32hf)(__m512h)(A), \
221 (__v32hu)(W), (__mmask32)(U), \
222 _MM_FROUND_CUR_DIRECTION))
223
224#define _mm512_maskz_ipcvttph_epu8(U, A) \
225 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
226 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
227 _MM_FROUND_CUR_DIRECTION))
228
229#define _mm512_ipcvtt_roundph_epu8(A, S) \
230 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
231 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
232 S))
233
234#define _mm512_mask_ipcvtt_roundph_epu8(W, U, A, S) \
235 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
236 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), S))
237
238#define _mm512_maskz_ipcvtt_roundph_epu8(U, A, S) \
239 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
240 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
241 S))
242
243#define _mm512_ipcvttps_epi8(A) \
244 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
245 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
246 _MM_FROUND_CUR_DIRECTION))
247
248#define _mm512_mask_ipcvttps_epi8(W, U, A) \
249 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask((__v16sf)(__m512h)(A), \
250 (__v16su)(W), (__mmask16)(U), \
251 _MM_FROUND_CUR_DIRECTION))
252
253#define _mm512_maskz_ipcvttps_epi8(U, A) \
254 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
255 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
256 _MM_FROUND_CUR_DIRECTION))
257
258#define _mm512_ipcvtt_roundps_epi8(A, S) \
259 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
260 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
261 S))
262
263#define _mm512_mask_ipcvtt_roundps_epi8(W, U, A, S) \
264 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
265 (__v16sf)(__m512h)(A), (__v16su)(W), (__mmask16)(U), S))
266
267#define _mm512_maskz_ipcvtt_roundps_epi8(U, A, S) \
268 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
269 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
270 S))
271
272#define _mm512_ipcvttps_epu8(A) \
273 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
274 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
275 _MM_FROUND_CUR_DIRECTION))
276
277#define _mm512_mask_ipcvttps_epu8(W, U, A) \
278 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask((__v16sf)(__m512h)(A), \
279 (__v16su)(W), (__mmask16)(U), \
280 _MM_FROUND_CUR_DIRECTION))
281
282#define _mm512_maskz_ipcvttps_epu8(U, A) \
283 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
284 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
285 _MM_FROUND_CUR_DIRECTION))
286
287#define _mm512_ipcvtt_roundps_epu8(A, S) \
288 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
289 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
290 S))
291
292#define _mm512_mask_ipcvtt_roundps_epu8(W, U, A, S) \
293 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
294 (__v16sf)(__m512h)(A), (__v16su)(W), (__mmask16)(U), S))
295
296#define _mm512_maskz_ipcvtt_roundps_epu8(U, A, S) \
297 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
298 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
299 S))
300
301#endif // __AVX10_2_512SATCVTINTRIN_H
lib/include/avx10_2bf16intrin.h created+1085
...@@ -0,0 +1,1085 @@
1/*===-------------- avx10_2bf16intrin.h - AVX10-BF16 intrinsics ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2bf16intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2BF16INTRIN_H
17#define __AVX10_2BF16INTRIN_H
18
19typedef __bf16 __m128bh_u __attribute__((__vector_size__(16), __aligned__(1)));
20typedef __bf16 __m256bh_u __attribute__((__vector_size__(32), __aligned__(1)));
21
22/* Define the default attributes for the functions in this file. */
23#define __DEFAULT_FN_ATTRS256 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
25 __min_vector_width__(256)))
26#define __DEFAULT_FN_ATTRS128 \
27 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
28 __min_vector_width__(128)))
29
30static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_setzero_pbh(void) {
31 return __builtin_bit_cast(__m256bh, _mm256_setzero_ps());
32}
33
34static __inline __m128bh __DEFAULT_FN_ATTRS128 _mm_setzero_pbh(void) {
35 return __builtin_bit_cast(__m128bh, _mm_setzero_ps());
36}
37
38static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_castbf16_ps(__m128bh __a) {
39 return (__m128)__a;
40}
41
42static __inline__ __m256 __DEFAULT_FN_ATTRS256
43_mm256_castbf16_ps(__m256bh __a) {
44 return (__m256)__a;
45}
46
47static __inline__ __m256d __DEFAULT_FN_ATTRS256
48_mm256_castbf16_pd(__m256bh __a) {
49 return (__m256d)__a;
50}
51
52static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_castbf16_pd(__m128bh __a) {
53 return (__m128d)__a;
54}
55
56static __inline__ __m128i __DEFAULT_FN_ATTRS128
57_mm_castbf16_si128(__m128bh __a) {
58 return (__m128i)__a;
59}
60
61static __inline__ __m256i __DEFAULT_FN_ATTRS256
62_mm256_castbf16_si256(__m256bh __a) {
63 return (__m256i)__a;
64}
65
66static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_castps_pbh(__m128 __a) {
67 return (__m128bh)__a;
68}
69
70static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_castps_pbh(__m256 __a) {
71 return (__m256bh)__a;
72}
73
74static __inline__ __bf16 __DEFAULT_FN_ATTRS128 _mm_cvtsbh_bf16(__m128bh __a) {
75 return __a[0];
76}
77
78static __inline__ __bf16 __DEFAULT_FN_ATTRS256
79_mm256_cvtsbh_bf16(__m256bh __a) {
80 return __a[0];
81}
82
83static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_castpd_pbh(__m128d __a) {
84 return (__m128bh)__a;
85}
86
87static __inline__ __m256bh __DEFAULT_FN_ATTRS256
88_mm256_castpd_pbh(__m256d __a) {
89 return (__m256bh)__a;
90}
91
92static __inline__ __m128bh __DEFAULT_FN_ATTRS128
93_mm_castsi128_pbh(__m128i __a) {
94 return (__m128bh)__a;
95}
96
97static __inline__ __m256bh __DEFAULT_FN_ATTRS256
98_mm256_castsi256_pbh(__m256i __a) {
99 return (__m256bh)__a;
100}
101
102static __inline__ __m128bh __DEFAULT_FN_ATTRS256
103_mm256_castbf16256_pbh128(__m256bh __a) {
104 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7);
105}
106
107static __inline__ __m256bh __DEFAULT_FN_ATTRS256
108_mm256_castbf16128_pbh256(__m128bh __a) {
109 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
110 -1, -1, -1, -1, -1);
111}
112
113static __inline__ __m256bh __DEFAULT_FN_ATTRS256
114_mm256_zextbf16128_pbh256(__m128bh __a) {
115 return __builtin_shufflevector(__a, (__v8bf)_mm_setzero_pbh(), 0, 1, 2, 3, 4,
116 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
117}
118
119static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_undefined_pbh(void) {
120 return (__m256bh)__builtin_ia32_undef256();
121}
122
123static __inline__ __m128bh __DEFAULT_FN_ATTRS128
124_mm_load_sbh(void const *__dp) {
125 __m128bh src = (__v8bf)_mm_setzero_pbh();
126 return (__m128bh)__builtin_ia32_loadsbf16128_mask((const __v8bf *)__dp, src,
127 1);
128}
129
130static __inline__ __m128bh __DEFAULT_FN_ATTRS128
131_mm_mask_load_sbh(__m128bh __W, __mmask8 __U, const void *__A) {
132 __m128bh src = (__v8bf)__builtin_shufflevector(
133 (__v8bf)__W, (__v8bf)_mm_setzero_pbh(), 0, 8, 8, 8, 8, 8, 8, 8);
134
135 return (__m128bh)__builtin_ia32_loadsbf16128_mask((const __v8bf *)__A, src,
136 __U & 1);
137}
138
139static __inline__ __m128bh __DEFAULT_FN_ATTRS128
140_mm_maskz_load_sbh(__mmask8 __U, const void *__A) {
141 return (__m128bh)__builtin_ia32_loadsbf16128_mask(
142 (const __v8bf *)__A, (__v8bf)_mm_setzero_pbh(), __U & 1);
143}
144
145static __inline__ __m256bh __DEFAULT_FN_ATTRS256
146_mm256_load_pbh(void const *__p) {
147 return *(const __m256bh *)__p;
148}
149
150static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_load_pbh(void const *__p) {
151 return *(const __m128bh *)__p;
152}
153
154static __inline__ __m256bh __DEFAULT_FN_ATTRS256
155_mm256_loadu_pbh(void const *__p) {
156 struct __loadu_pbh {
157 __m256bh_u __v;
158 } __attribute__((__packed__, __may_alias__));
159 return ((const struct __loadu_pbh *)__p)->__v;
160}
161
162static __inline__ __m128bh __DEFAULT_FN_ATTRS128
163_mm_loadu_pbh(void const *__p) {
164 struct __loadu_pbh {
165 __m128bh_u __v;
166 } __attribute__((__packed__, __may_alias__));
167 return ((const struct __loadu_pbh *)__p)->__v;
168}
169
170static __inline__ void __DEFAULT_FN_ATTRS128 _mm_store_sbh(void *__dp,
171 __m128bh __a) {
172 struct __mm_store_sbh_struct {
173 __bf16 __u;
174 } __attribute__((__packed__, __may_alias__));
175 ((struct __mm_store_sbh_struct *)__dp)->__u = __a[0];
176}
177
178static __inline__ void __DEFAULT_FN_ATTRS128 _mm_mask_store_sbh(void *__W,
179 __mmask8 __U,
180 __m128bh __A) {
181 __builtin_ia32_storesbf16128_mask((__v8bf *)__W, __A, __U & 1);
182}
183
184static __inline__ void __DEFAULT_FN_ATTRS256 _mm256_store_pbh(void *__P,
185 __m256bh __A) {
186 *(__m256bh *)__P = __A;
187}
188
189static __inline__ void __DEFAULT_FN_ATTRS128 _mm_store_pbh(void *__P,
190 __m128bh __A) {
191 *(__m128bh *)__P = __A;
192}
193
194static __inline__ void __DEFAULT_FN_ATTRS256 _mm256_storeu_pbh(void *__P,
195 __m256bh __A) {
196 struct __storeu_pbh {
197 __m256bh_u __v;
198 } __attribute__((__packed__, __may_alias__));
199 ((struct __storeu_pbh *)__P)->__v = __A;
200}
201
202static __inline__ void __DEFAULT_FN_ATTRS128 _mm_storeu_pbh(void *__P,
203 __m128bh __A) {
204 struct __storeu_pbh {
205 __m128bh_u __v;
206 } __attribute__((__packed__, __may_alias__));
207 ((struct __storeu_pbh *)__P)->__v = __A;
208}
209
210static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_move_sbh(__m128bh __a,
211 __m128bh __b) {
212 __a[0] = __b[0];
213 return __a;
214}
215
216static __inline__ __m128bh __DEFAULT_FN_ATTRS128
217_mm_mask_move_sbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
218 return __builtin_ia32_selectsbf_128(__U, _mm_move_sbh(__A, __B), __W);
219}
220
221static __inline__ __m128bh __DEFAULT_FN_ATTRS128
222_mm_maskz_move_sbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
223 return __builtin_ia32_selectsbf_128(__U, _mm_move_sbh(__A, __B),
224 _mm_setzero_pbh());
225}
226
227static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_undefined_pbh(void) {
228 return (__m128bh)__builtin_ia32_undef128();
229}
230
231static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_set_sbh(__bf16 bf) {
232 return (__v8bf)__builtin_shufflevector(
233 (__v8bf){bf, bf, bf, bf, bf, bf, bf, bf}, (__v8bf)_mm_setzero_pbh(), 0, 8,
234 8, 8, 8, 8, 8, 8);
235}
236
237static __inline __m128bh __DEFAULT_FN_ATTRS128 _mm_set1_pbh(__bf16 bf) {
238 return (__m128bh)(__v8bf){bf, bf, bf, bf, bf, bf, bf, bf};
239}
240
241static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_set1_pbh(__bf16 bf) {
242 return (__m256bh)(__v16bf){bf, bf, bf, bf, bf, bf, bf, bf,
243 bf, bf, bf, bf, bf, bf, bf, bf};
244}
245
246static __inline __m128bh __DEFAULT_FN_ATTRS128
247_mm_set_pbh(__bf16 bf1, __bf16 bf2, __bf16 bf3, __bf16 bf4, __bf16 bf5,
248 __bf16 bf6, __bf16 bf7, __bf16 bf8) {
249 return (__m128bh)(__v8bf){bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8};
250}
251
252static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_set_pbh(
253 __bf16 bf1, __bf16 bf2, __bf16 bf3, __bf16 bf4, __bf16 bf5, __bf16 bf6,
254 __bf16 bf7, __bf16 bf8, __bf16 bf9, __bf16 bf10, __bf16 bf11, __bf16 bf12,
255 __bf16 bf13, __bf16 bf14, __bf16 bf15, __bf16 bf16) {
256 return (__m256bh)(__v16bf){bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8,
257 bf9, bf10, bf11, bf12, bf13, bf14, bf15, bf16};
258}
259
260#define _mm_setr_pbh(bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8) \
261 _mm_set_pbh((bf8), (bf7), (bf6), (bf5), (bf4), (bf3), (bf2), (bf1))
262
263#define _mm256_setr_pbh(bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8, bf9, bf10, \
264 bf11, bf12, bf13, bf14, bf15, bf16) \
265 _mm256_set_pbh((bf16), (bf15), (bf14), (bf13), (bf12), (bf11), (bf10), \
266 (bf9), (bf8), (bf7), (bf6), (bf5), (bf4), (bf3), (bf2), \
267 (bf1))
268
269static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_abs_pbh(__m256bh __A) {
270 return (__m256bh)_mm256_and_epi32(_mm256_set1_epi32(0x7FFF7FFF),
271 (__m256i)__A);
272}
273
274static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_abs_pbh(__m128bh __A) {
275 return (__m128bh)_mm_and_epi32(_mm_set1_epi32(0x7FFF7FFF), (__m128i)__A);
276}
277
278static __inline__ __m128bh __DEFAULT_FN_ATTRS128
279_mm_mask_blend_pbh(__mmask8 __U, __m128bh __A, __m128bh __W) {
280 return (__m128bh)__builtin_ia32_selectpbf_128((__mmask8)__U, (__v8bf)__W,
281 (__v8bf)__A);
282}
283
284static __inline__ __m256bh __DEFAULT_FN_ATTRS256
285_mm256_mask_blend_pbh(__mmask16 __U, __m256bh __A, __m256bh __W) {
286 return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U, (__v16bf)__W,
287 (__v16bf)__A);
288}
289
290static __inline__ __m128bh __DEFAULT_FN_ATTRS128
291_mm_permutex2var_pbh(__m128bh __A, __m128i __I, __m128bh __B) {
292 return (__m128bh)__builtin_ia32_vpermi2varhi128((__v8hi)__A, (__v8hi)__I,
293 (__v8hi)__B);
294}
295
296static __inline__ __m256bh __DEFAULT_FN_ATTRS256
297_mm256_permutex2var_pbh(__m256bh __A, __m256i __I, __m256bh __B) {
298 return (__m256bh)__builtin_ia32_vpermi2varhi256((__v16hi)__A, (__v16hi)__I,
299 (__v16hi)__B);
300}
301
302static __inline__ __m128bh __DEFAULT_FN_ATTRS128
303_mm_permutexvar_pbh(__m128i __A, __m128bh __B) {
304 return (__m128bh)__builtin_ia32_permvarhi128((__v8hi)__B, (__v8hi)__A);
305}
306
307static __inline__ __m256bh __DEFAULT_FN_ATTRS256
308_mm256_permutexvar_pbh(__m256i __A, __m256bh __B) {
309 return (__m256bh)__builtin_ia32_permvarhi256((__v16hi)__B, (__v16hi)__A);
310}
311
312static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_add_pbh(__m256bh __A,
313 __m256bh __B) {
314 return (__m256bh)((__v16bf)__A + (__v16bf)__B);
315}
316
317static __inline__ __m256bh __DEFAULT_FN_ATTRS256
318_mm256_mask_add_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
319 return (__m256bh)__builtin_ia32_selectpbf_256(
320 (__mmask16)__U, (__v16bf)_mm256_add_pbh(__A, __B), (__v16bf)__W);
321}
322
323static __inline__ __m256bh __DEFAULT_FN_ATTRS256
324_mm256_maskz_add_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
325 return (__m256bh)__builtin_ia32_selectpbf_256(
326 (__mmask16)__U, (__v16bf)_mm256_add_pbh(__A, __B),
327 (__v16bf)_mm256_setzero_pbh());
328}
329
330static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_add_pbh(__m128bh __A,
331 __m128bh __B) {
332 return (__m128bh)((__v8bf)__A + (__v8bf)__B);
333}
334
335static __inline__ __m128bh __DEFAULT_FN_ATTRS128
336_mm_mask_add_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
337 return (__m128bh)__builtin_ia32_selectpbf_128(
338 (__mmask8)__U, (__v8bf)_mm_add_pbh(__A, __B), (__v8bf)__W);
339}
340
341static __inline__ __m128bh __DEFAULT_FN_ATTRS128
342_mm_maskz_add_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
343 return (__m128bh)__builtin_ia32_selectpbf_128(
344 (__mmask8)__U, (__v8bf)_mm_add_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
345}
346
347static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_sub_pbh(__m256bh __A,
348 __m256bh __B) {
349 return (__m256bh)((__v16bf)__A - (__v16bf)__B);
350}
351
352static __inline__ __m256bh __DEFAULT_FN_ATTRS256
353_mm256_mask_sub_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
354 return (__m256bh)__builtin_ia32_selectpbf_256(
355 (__mmask16)__U, (__v16bf)_mm256_sub_pbh(__A, __B), (__v16bf)__W);
356}
357
358static __inline__ __m256bh __DEFAULT_FN_ATTRS256
359_mm256_maskz_sub_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
360 return (__m256bh)__builtin_ia32_selectpbf_256(
361 (__mmask16)__U, (__v16bf)_mm256_sub_pbh(__A, __B),
362 (__v16bf)_mm256_setzero_pbh());
363}
364
365static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_sub_pbh(__m128bh __A,
366 __m128bh __B) {
367 return (__m128bh)((__v8bf)__A - (__v8bf)__B);
368}
369
370static __inline__ __m128bh __DEFAULT_FN_ATTRS128
371_mm_mask_sub_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
372 return (__m128bh)__builtin_ia32_selectpbf_128(
373 (__mmask8)__U, (__v8bf)_mm_sub_pbh(__A, __B), (__v8bf)__W);
374}
375
376static __inline__ __m128bh __DEFAULT_FN_ATTRS128
377_mm_maskz_sub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
378 return (__m128bh)__builtin_ia32_selectpbf_128(
379 (__mmask8)__U, (__v8bf)_mm_sub_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
380}
381
382static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mul_pbh(__m256bh __A,
383 __m256bh __B) {
384 return (__m256bh)((__v16bf)__A * (__v16bf)__B);
385}
386
387static __inline__ __m256bh __DEFAULT_FN_ATTRS256
388_mm256_mask_mul_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
389 return (__m256bh)__builtin_ia32_selectpbf_256(
390 (__mmask16)__U, (__v16bf)_mm256_mul_pbh(__A, __B), (__v16bf)__W);
391}
392
393static __inline__ __m256bh __DEFAULT_FN_ATTRS256
394_mm256_maskz_mul_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
395 return (__m256bh)__builtin_ia32_selectpbf_256(
396 (__mmask16)__U, (__v16bf)_mm256_mul_pbh(__A, __B),
397 (__v16bf)_mm256_setzero_pbh());
398}
399
400static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_mul_pbh(__m128bh __A,
401 __m128bh __B) {
402 return (__m128bh)((__v8bf)__A * (__v8bf)__B);
403}
404
405static __inline__ __m128bh __DEFAULT_FN_ATTRS128
406_mm_mask_mul_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
407 return (__m128bh)__builtin_ia32_selectpbf_128(
408 (__mmask8)__U, (__v8bf)_mm_mul_pbh(__A, __B), (__v8bf)__W);
409}
410
411static __inline__ __m128bh __DEFAULT_FN_ATTRS128
412_mm_maskz_mul_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
413 return (__m128bh)__builtin_ia32_selectpbf_128(
414 (__mmask8)__U, (__v8bf)_mm_mul_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
415}
416
417static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_div_pbh(__m256bh __A,
418 __m256bh __B) {
419 return (__m256bh)((__v16bf)__A / (__v16bf)__B);
420}
421
422static __inline__ __m256bh __DEFAULT_FN_ATTRS256
423_mm256_mask_div_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
424 return (__m256bh)__builtin_ia32_selectpbf_256(
425 (__mmask16)__U, (__v16bf)_mm256_div_pbh(__A, __B), (__v16bf)__W);
426}
427
428static __inline__ __m256bh __DEFAULT_FN_ATTRS256
429_mm256_maskz_div_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
430 return (__m256bh)__builtin_ia32_selectpbf_256(
431 (__mmask16)__U, (__v16bf)_mm256_div_pbh(__A, __B),
432 (__v16bf)_mm256_setzero_pbh());
433}
434
435static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_div_pbh(__m128bh __A,
436 __m128bh __B) {
437 return (__m128bh)((__v8bf)__A / (__v8bf)__B);
438}
439
440static __inline__ __m128bh __DEFAULT_FN_ATTRS128
441_mm_mask_div_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
442 return (__m128bh)__builtin_ia32_selectpbf_128(
443 (__mmask8)__U, (__v8bf)_mm_div_pbh(__A, __B), (__v8bf)__W);
444}
445
446static __inline__ __m128bh __DEFAULT_FN_ATTRS128
447_mm_maskz_div_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
448 return (__m128bh)__builtin_ia32_selectpbf_128(
449 (__mmask8)__U, (__v8bf)_mm_div_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
450}
451
452static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_max_pbh(__m256bh __A,
453 __m256bh __B) {
454 return (__m256bh)__builtin_ia32_vmaxbf16256((__v16bf)__A, (__v16bf)__B);
455}
456
457static __inline__ __m256bh __DEFAULT_FN_ATTRS256
458_mm256_mask_max_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
459 return (__m256bh)__builtin_ia32_selectpbf_256(
460 (__mmask16)__U, (__v16bf)_mm256_max_pbh(__A, __B), (__v16bf)__W);
461}
462
463static __inline__ __m256bh __DEFAULT_FN_ATTRS256
464_mm256_maskz_max_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
465 return (__m256bh)__builtin_ia32_selectpbf_256(
466 (__mmask16)__U, (__v16bf)_mm256_max_pbh(__A, __B),
467 (__v16bf)_mm256_setzero_pbh());
468}
469
470static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_max_pbh(__m128bh __A,
471 __m128bh __B) {
472 return (__m128bh)__builtin_ia32_vmaxbf16128((__v8bf)__A, (__v8bf)__B);
473}
474
475static __inline__ __m128bh __DEFAULT_FN_ATTRS128
476_mm_mask_max_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
477 return (__m128bh)__builtin_ia32_selectpbf_128(
478 (__mmask8)__U, (__v8bf)_mm_max_pbh(__A, __B), (__v8bf)__W);
479}
480
481static __inline__ __m128bh __DEFAULT_FN_ATTRS128
482_mm_maskz_max_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
483 return (__m128bh)__builtin_ia32_selectpbf_128(
484 (__mmask8)__U, (__v8bf)_mm_max_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
485}
486
487static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_min_pbh(__m256bh __A,
488 __m256bh __B) {
489 return (__m256bh)__builtin_ia32_vminbf16256((__v16bf)__A, (__v16bf)__B);
490}
491
492static __inline__ __m256bh __DEFAULT_FN_ATTRS256
493_mm256_mask_min_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
494 return (__m256bh)__builtin_ia32_selectpbf_256(
495 (__mmask16)__U, (__v16bf)_mm256_min_pbh(__A, __B), (__v16bf)__W);
496}
497
498static __inline__ __m256bh __DEFAULT_FN_ATTRS256
499_mm256_maskz_min_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
500 return (__m256bh)__builtin_ia32_selectpbf_256(
501 (__mmask16)__U, (__v16bf)_mm256_min_pbh(__A, __B),
502 (__v16bf)_mm256_setzero_pbh());
503}
504
505static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_min_pbh(__m128bh __A,
506 __m128bh __B) {
507 return (__m128bh)__builtin_ia32_vminbf16128((__v8bf)__A, (__v8bf)__B);
508}
509
510static __inline__ __m128bh __DEFAULT_FN_ATTRS128
511_mm_mask_min_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
512 return (__m128bh)__builtin_ia32_selectpbf_128(
513 (__mmask8)__U, (__v8bf)_mm_min_pbh(__A, __B), (__v8bf)__W);
514}
515
516static __inline__ __m128bh __DEFAULT_FN_ATTRS128
517_mm_maskz_min_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
518 return (__m128bh)__builtin_ia32_selectpbf_128(
519 (__mmask8)__U, (__v8bf)_mm_min_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
520}
521
522static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comieq_sbh(__m128bh A,
523 __m128bh B) {
524 return __builtin_ia32_vcomisbf16eq((__v8bf)A, (__v8bf)B);
525}
526
527static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comilt_sbh(__m128bh A,
528 __m128bh B) {
529 return __builtin_ia32_vcomisbf16lt((__v8bf)A, (__v8bf)B);
530}
531
532static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comile_sbh(__m128bh A,
533 __m128bh B) {
534 return __builtin_ia32_vcomisbf16le((__v8bf)A, (__v8bf)B);
535}
536
537static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comigt_sbh(__m128bh A,
538 __m128bh B) {
539 return __builtin_ia32_vcomisbf16gt((__v8bf)A, (__v8bf)B);
540}
541
542static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comige_sbh(__m128bh A,
543 __m128bh B) {
544 return __builtin_ia32_vcomisbf16ge((__v8bf)A, (__v8bf)B);
545}
546
547static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comineq_sbh(__m128bh A,
548 __m128bh B) {
549 return __builtin_ia32_vcomisbf16neq((__v8bf)A, (__v8bf)B);
550}
551
552#define _mm256_cmp_pbh_mask(__A, __B, __P) \
553 ((__mmask16)__builtin_ia32_vcmpbf16256_mask((__v16bf)(__m256bh)(__A), \
554 (__v16bf)(__m256bh)(__B), \
555 (int)(__P), (__mmask16) - 1))
556
557#define _mm256_mask_cmp_pbh_mask(__U, __A, __B, __P) \
558 ((__mmask16)__builtin_ia32_vcmpbf16256_mask((__v16bf)(__m256bh)(__A), \
559 (__v16bf)(__m256bh)(__B), \
560 (int)(__P), (__mmask16)(__U)))
561
562#define _mm_cmp_pbh_mask(__A, __B, __P) \
563 ((__mmask8)__builtin_ia32_vcmpbf16128_mask((__v8bf)(__m128bh)(__A), \
564 (__v8bf)(__m128bh)(__B), \
565 (int)(__P), (__mmask8) - 1))
566
567#define _mm_mask_cmp_pbh_mask(__U, __A, __B, __P) \
568 ((__mmask8)__builtin_ia32_vcmpbf16128_mask((__v8bf)(__m128bh)(__A), \
569 (__v8bf)(__m128bh)(__B), \
570 (int)(__P), (__mmask8)(__U)))
571
572#define _mm256_mask_fpclass_pbh_mask(__U, __A, imm) \
573 ((__mmask16)__builtin_ia32_vfpclassbf16256_mask( \
574 (__v16bf)(__m256bh)(__A), (int)(imm), (__mmask16)(__U)))
575
576#define _mm256_fpclass_pbh_mask(__A, imm) \
577 ((__mmask16)__builtin_ia32_vfpclassbf16256_mask( \
578 (__v16bf)(__m256bh)(__A), (int)(imm), (__mmask16) - 1))
579
580#define _mm_mask_fpclass_pbh_mask(__U, __A, imm) \
581 ((__mmask8)__builtin_ia32_vfpclassbf16128_mask((__v8bf)(__m128bh)(__A), \
582 (int)(imm), (__mmask8)(__U)))
583
584#define _mm_fpclass_pbh_mask(__A, imm) \
585 ((__mmask8)__builtin_ia32_vfpclassbf16128_mask((__v8bf)(__m128bh)(__A), \
586 (int)(imm), (__mmask8) - 1))
587
588static __inline__ __m256bh __DEFAULT_FN_ATTRS256
589_mm256_scalef_pbh(__m256bh __A, __m256bh __B) {
590 return (__m256bh)__builtin_ia32_vscalefbf16256_mask(
591 (__v16bf)__A, (__v16bf)__B, (__v16bf)_mm256_undefined_pbh(),
592 (__mmask16)-1);
593}
594
595static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_scalef_pbh(
596 __m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
597 return (__m256bh)__builtin_ia32_vscalefbf16256_mask(
598 (__v16bf)__A, (__v16bf)__B, (__v16bf)__W, (__mmask16)__U);
599}
600
601static __inline__ __m256bh __DEFAULT_FN_ATTRS256
602_mm256_maskz_scalef_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
603 return (__m256bh)__builtin_ia32_vscalefbf16256_mask(
604 (__v16bf)__A, (__v16bf)__B, (__v16bf)_mm256_setzero_pbh(),
605 (__mmask16)__U);
606}
607
608static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_scalef_pbh(__m128bh __A,
609 __m128bh __B) {
610 return (__m128bh)__builtin_ia32_vscalefbf16128_mask(
611 (__v8bf)__A, (__v8bf)__B, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
612}
613
614static __inline__ __m128bh __DEFAULT_FN_ATTRS128
615_mm_mask_scalef_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
616 return (__m128bh)__builtin_ia32_vscalefbf16128_mask(
617 (__v8bf)__A, (__v8bf)__B, (__v8bf)__W, (__mmask8)__U);
618}
619
620static __inline__ __m128bh __DEFAULT_FN_ATTRS128
621_mm_maskz_scalef_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
622 return (__m128bh)__builtin_ia32_vscalefbf16128_mask(
623 (__v8bf)__A, (__v8bf)__B, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
624}
625
626static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_rcp_pbh(__m256bh __A) {
627 return (__m256bh)__builtin_ia32_vrcpbf16256_mask(
628 (__v16bf)__A, (__v16bf)_mm256_undefined_pbh(), (__mmask16)-1);
629}
630
631static __inline__ __m256bh __DEFAULT_FN_ATTRS256
632_mm256_mask_rcp_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
633 return (__m256bh)__builtin_ia32_vrcpbf16256_mask((__v16bf)__A, (__v16bf)__W,
634 (__mmask16)__U);
635}
636
637static __inline__ __m256bh __DEFAULT_FN_ATTRS256
638_mm256_maskz_rcp_pbh(__mmask16 __U, __m256bh __A) {
639 return (__m256bh)__builtin_ia32_vrcpbf16256_mask(
640 (__v16bf)__A, (__v16bf)_mm256_setzero_pbh(), (__mmask16)__U);
641}
642
643static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_rcp_pbh(__m128bh __A) {
644 return (__m128bh)__builtin_ia32_vrcpbf16128_mask(
645 (__v8bf)__A, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
646}
647
648static __inline__ __m128bh __DEFAULT_FN_ATTRS128
649_mm_mask_rcp_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
650 return (__m128bh)__builtin_ia32_vrcpbf16128_mask((__v8bf)__A, (__v8bf)__W,
651 (__mmask8)__U);
652}
653
654static __inline__ __m128bh __DEFAULT_FN_ATTRS128
655_mm_maskz_rcp_pbh(__mmask8 __U, __m128bh __A) {
656 return (__m128bh)__builtin_ia32_vrcpbf16128_mask(
657 (__v8bf)__A, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
658}
659
660static __inline__ __m256bh __DEFAULT_FN_ATTRS256
661_mm256_getexp_pbh(__m256bh __A) {
662 return (__m256bh)__builtin_ia32_vgetexpbf16256_mask(
663 (__v16bf)__A, (__v16bf)_mm256_undefined_pbh(), (__mmask16)-1);
664}
665
666static __inline__ __m256bh __DEFAULT_FN_ATTRS256
667_mm256_mask_getexp_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
668 return (__m256bh)__builtin_ia32_vgetexpbf16256_mask(
669 (__v16bf)__A, (__v16bf)__W, (__mmask16)__U);
670}
671
672static __inline__ __m256bh __DEFAULT_FN_ATTRS256
673_mm256_maskz_getexp_pbh(__mmask16 __U, __m256bh __A) {
674 return (__m256bh)__builtin_ia32_vgetexpbf16256_mask(
675 (__v16bf)__A, (__v16bf)_mm256_setzero_pbh(), (__mmask16)__U);
676}
677
678static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_getexp_pbh(__m128bh __A) {
679 return (__m128bh)__builtin_ia32_vgetexpbf16128_mask(
680 (__v8bf)__A, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
681}
682
683static __inline__ __m128bh __DEFAULT_FN_ATTRS128
684_mm_mask_getexp_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
685 return (__m128bh)__builtin_ia32_vgetexpbf16128_mask((__v8bf)__A, (__v8bf)__W,
686 (__mmask8)__U);
687}
688
689static __inline__ __m128bh __DEFAULT_FN_ATTRS128
690_mm_maskz_getexp_pbh(__mmask8 __U, __m128bh __A) {
691 return (__m128bh)__builtin_ia32_vgetexpbf16128_mask(
692 (__v8bf)__A, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
693}
694
695static __inline__ __m256bh __DEFAULT_FN_ATTRS256
696_mm256_rsqrt_pbh(__m256bh __A) {
697 return (__m256bh)__builtin_ia32_vrsqrtbf16256_mask(
698 (__v16bf)__A, (__v16bf)_mm256_undefined_pbh(), (__mmask16)-1);
699}
700
701static __inline__ __m256bh __DEFAULT_FN_ATTRS256
702_mm256_mask_rsqrt_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
703 return (__m256bh)__builtin_ia32_vrsqrtbf16256_mask((__v16bf)__A, (__v16bf)__W,
704 (__mmask16)__U);
705}
706
707static __inline__ __m256bh __DEFAULT_FN_ATTRS256
708_mm256_maskz_rsqrt_pbh(__mmask16 __U, __m256bh __A) {
709 return (__m256bh)__builtin_ia32_vrsqrtbf16256_mask(
710 (__v16bf)__A, (__v16bf)_mm256_setzero_pbh(), (__mmask16)__U);
711}
712
713static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_rsqrt_pbh(__m128bh __A) {
714 return (__m128bh)__builtin_ia32_vrsqrtbf16128_mask(
715 (__v8bf)__A, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
716}
717
718static __inline__ __m128bh __DEFAULT_FN_ATTRS128
719_mm_mask_rsqrt_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
720 return (__m128bh)__builtin_ia32_vrsqrtbf16128_mask((__v8bf)__A, (__v8bf)__W,
721 (__mmask8)__U);
722}
723
724static __inline__ __m128bh __DEFAULT_FN_ATTRS128
725_mm_maskz_rsqrt_pbh(__mmask8 __U, __m128bh __A) {
726 return (__m128bh)__builtin_ia32_vrsqrtbf16128_mask(
727 (__v8bf)__A, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
728}
729
730#define _mm256_reduce_pbh(__A, imm) \
731 ((__m256bh)__builtin_ia32_vreducebf16256_mask( \
732 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_undefined_pbh(), \
733 (__mmask16) - 1))
734
735#define _mm256_mask_reduce_pbh(__W, __U, __A, imm) \
736 ((__m256bh)__builtin_ia32_vreducebf16256_mask( \
737 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)(__m256bh)(__W), \
738 (__mmask16)(__U)))
739
740#define _mm256_maskz_reduce_pbh(__U, __A, imm) \
741 ((__m256bh)__builtin_ia32_vreducebf16256_mask( \
742 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_setzero_pbh(), \
743 (__mmask16)(__U)))
744
745#define _mm_reduce_pbh(__A, imm) \
746 ((__m128bh)__builtin_ia32_vreducebf16128_mask( \
747 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_undefined_pbh(), \
748 (__mmask8) - 1))
749
750#define _mm_mask_reduce_pbh(__W, __U, __A, imm) \
751 ((__m128bh)__builtin_ia32_vreducebf16128_mask( \
752 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)(__m128bh)(__W), \
753 (__mmask8)(__U)))
754
755#define _mm_maskz_reduce_pbh(__U, __A, imm) \
756 ((__m128bh)__builtin_ia32_vreducebf16128_mask( \
757 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_setzero_pbh(), \
758 (__mmask8)(__U)))
759
760#define _mm256_roundscale_pbh(__A, imm) \
761 ((__m256bh)__builtin_ia32_vrndscalebf16_256_mask( \
762 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_setzero_pbh(), \
763 (__mmask16) - 1))
764
765#define _mm256_mask_roundscale_pbh(__W, __U, __A, imm) \
766 ((__m256bh)__builtin_ia32_vrndscalebf16_256_mask( \
767 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)(__m256bh)(__W), \
768 (__mmask16)(__U)))
769
770#define _mm256_maskz_roundscale_pbh(__U, __A, imm) \
771 ((__m256bh)__builtin_ia32_vrndscalebf16_256_mask( \
772 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_setzero_pbh(), \
773 (__mmask16)(__U)))
774
775#define _mm_roundscale_pbh(__A, imm) \
776 ((__m128bh)__builtin_ia32_vrndscalebf16_128_mask( \
777 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_setzero_pbh(), \
778 (__mmask8) - 1))
779
780#define _mm_mask_roundscale_pbh(__W, __U, __A, imm) \
781 ((__m128bh)__builtin_ia32_vrndscalebf16_128_mask( \
782 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)(__m128bh)(__W), \
783 (__mmask8)(__U)))
784
785#define _mm_maskz_roundscale_pbh(__U, __A, imm) \
786 ((__m128bh)__builtin_ia32_vrndscalebf16_128_mask( \
787 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_setzero_pbh(), \
788 (__mmask8)(__U)))
789
790#define _mm256_getmant_pbh(__A, __B, __C) \
791 ((__m256bh)__builtin_ia32_vgetmantbf16256_mask( \
792 (__v16bf)(__m256bh)(__A), (int)(((__C) << 2) | (__B)), \
793 (__v16bf)_mm256_undefined_pbh(), (__mmask16) - 1))
794
795#define _mm256_mask_getmant_pbh(__W, __U, __A, __B, __C) \
796 ((__m256bh)__builtin_ia32_vgetmantbf16256_mask( \
797 (__v16bf)(__m256bh)(__A), (int)(((__C) << 2) | (__B)), \
798 (__v16bf)(__m256bh)(__W), (__mmask16)(__U)))
799
800#define _mm256_maskz_getmant_pbh(__U, __A, __B, __C) \
801 ((__m256bh)__builtin_ia32_vgetmantbf16256_mask( \
802 (__v16bf)(__m256bh)(__A), (int)(((__C) << 2) | (__B)), \
803 (__v16bf)_mm256_setzero_pbh(), (__mmask16)(__U)))
804
805#define _mm_getmant_pbh(__A, __B, __C) \
806 ((__m128bh)__builtin_ia32_vgetmantbf16128_mask( \
807 (__v8bf)(__m128bh)(__A), (int)(((__C) << 2) | (__B)), \
808 (__v8bf)_mm_undefined_pbh(), (__mmask8) - 1))
809
810#define _mm_mask_getmant_pbh(__W, __U, __A, __B, __C) \
811 ((__m128bh)__builtin_ia32_vgetmantbf16128_mask( \
812 (__v8bf)(__m128bh)(__A), (int)(((__C) << 2) | (__B)), \
813 (__v8bf)(__m128bh)(__W), (__mmask8)(__U)))
814
815#define _mm_maskz_getmant_pbh(__U, __A, __B, __C) \
816 ((__m128bh)__builtin_ia32_vgetmantbf16128_mask( \
817 (__v8bf)(__m128bh)(__A), (int)(((__C) << 2) | (__B)), \
818 (__v8bf)_mm_setzero_pbh(), (__mmask8)(__U)))
819
820static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_sqrt_pbh(__m256bh __A) {
821 return (__m256bh)__builtin_ia32_vsqrtbf16256((__v16bf)__A);
822}
823
824static __inline__ __m256bh __DEFAULT_FN_ATTRS256
825_mm256_mask_sqrt_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
826 return (__m256bh)__builtin_ia32_selectpbf_256(
827 (__mmask16)__U, (__v16bf)_mm256_sqrt_pbh(__A), (__v16bf)__W);
828}
829
830static __inline__ __m256bh __DEFAULT_FN_ATTRS256
831_mm256_maskz_sqrt_pbh(__mmask16 __U, __m256bh __A) {
832 return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U,
833 (__v16bf)_mm256_sqrt_pbh(__A),
834 (__v16bf)_mm256_setzero_pbh());
835}
836
837static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_sqrt_pbh(__m128bh __A) {
838 return (__m128bh)__builtin_ia32_vsqrtbf16((__v8bf)__A);
839}
840
841static __inline__ __m128bh __DEFAULT_FN_ATTRS128
842_mm_mask_sqrt_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
843 return (__m128bh)__builtin_ia32_selectpbf_128(
844 (__mmask8)__U, (__v8bf)_mm_sqrt_pbh(__A), (__v8bf)__W);
845}
846
847static __inline__ __m128bh __DEFAULT_FN_ATTRS128
848_mm_maskz_sqrt_pbh(__mmask8 __U, __m128bh __A) {
849 return (__m128bh)__builtin_ia32_selectpbf_128(
850 (__mmask8)__U, (__v8bf)_mm_sqrt_pbh(__A), (__v8bf)_mm_setzero_pbh());
851}
852
853static __inline__ __m256bh __DEFAULT_FN_ATTRS256
854_mm256_fmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
855 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, (__v16bf)__B,
856 (__v16bf)__C);
857}
858
859static __inline__ __m256bh __DEFAULT_FN_ATTRS256
860_mm256_mask_fmadd_pbh(__m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
861 return (__m256bh)__builtin_ia32_selectpbf_256(
862 (__mmask16)__U,
863 _mm256_fmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__A);
864}
865
866static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fmadd_pbh(
867 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
868 return (__m256bh)__builtin_ia32_selectpbf_256(
869 (__mmask16)__U,
870 _mm256_fmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__C);
871}
872
873static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmadd_pbh(
874 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
875 return (__m256bh)__builtin_ia32_selectpbf_256(
876 (__mmask16)__U,
877 _mm256_fmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
878 (__v16bf)_mm256_setzero_pbh());
879}
880
881static __inline__ __m256bh __DEFAULT_FN_ATTRS256
882_mm256_fmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
883 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, (__v16bf)__B,
884 -(__v16bf)__C);
885}
886
887static __inline__ __m256bh __DEFAULT_FN_ATTRS256
888_mm256_mask_fmsub_pbh(__m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
889 return (__m256bh)__builtin_ia32_selectpbf_256(
890 (__mmask16)__U,
891 _mm256_fmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__A);
892}
893
894static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fmsub_pbh(
895 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
896 return (__m256bh)__builtin_ia32_selectpbf_256(
897 (__mmask16)__U,
898 _mm256_fmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__C);
899}
900
901static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmsub_pbh(
902 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
903 return (__m256bh)__builtin_ia32_selectpbf_256(
904 (__mmask16)__U,
905 _mm256_fmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
906 (__v16bf)_mm256_setzero_pbh());
907}
908
909static __inline__ __m256bh __DEFAULT_FN_ATTRS256
910_mm256_fnmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
911 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, -(__v16bf)__B,
912 (__v16bf)__C);
913}
914
915static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmadd_pbh(
916 __m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
917 return (__m256bh)__builtin_ia32_selectpbf_256(
918 (__mmask16)__U,
919 _mm256_fnmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
920 (__v16bf)__A);
921}
922
923static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fnmadd_pbh(
924 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
925 return (__m256bh)__builtin_ia32_selectpbf_256(
926 (__mmask16)__U,
927 _mm256_fnmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
928 (__v16bf)__C);
929}
930
931static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmadd_pbh(
932 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
933 return (__m256bh)__builtin_ia32_selectpbf_256(
934 (__mmask16)__U,
935 _mm256_fnmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
936 (__v16bf)_mm256_setzero_pbh());
937}
938
939static __inline__ __m256bh __DEFAULT_FN_ATTRS256
940_mm256_fnmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
941 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, -(__v16bf)__B,
942 -(__v16bf)__C);
943}
944
945static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmsub_pbh(
946 __m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
947 return (__m256bh)__builtin_ia32_selectpbf_256(
948 (__mmask16)__U,
949 _mm256_fnmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
950 (__v16bf)__A);
951}
952
953static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fnmsub_pbh(
954 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
955 return (__m256bh)__builtin_ia32_selectpbf_256(
956 (__mmask16)__U,
957 _mm256_fnmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
958 (__v16bf)__C);
959}
960
961static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmsub_pbh(
962 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
963 return (__m256bh)__builtin_ia32_selectpbf_256(
964 (__mmask16)__U,
965 _mm256_fnmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
966 (__v16bf)_mm256_setzero_pbh());
967}
968
969static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmadd_pbh(__m128bh __A,
970 __m128bh __B,
971 __m128bh __C) {
972 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, (__v8bf)__B,
973 (__v8bf)__C);
974}
975
976static __inline__ __m128bh __DEFAULT_FN_ATTRS128
977_mm_mask_fmadd_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
978 return (__m128bh)__builtin_ia32_selectpbf_128(
979 (__mmask8)__U, _mm_fmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
980 (__v8bf)__A);
981}
982
983static __inline__ __m128bh __DEFAULT_FN_ATTRS128
984_mm_mask3_fmadd_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
985 return (__m128bh)__builtin_ia32_selectpbf_128(
986 (__mmask8)__U, _mm_fmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
987 (__v8bf)__C);
988}
989
990static __inline__ __m128bh __DEFAULT_FN_ATTRS128
991_mm_maskz_fmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
992 return (__m128bh)__builtin_ia32_selectpbf_128(
993 (__mmask8)__U, _mm_fmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
994 (__v8bf)_mm_setzero_pbh());
995}
996
997static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmsub_pbh(__m128bh __A,
998 __m128bh __B,
999 __m128bh __C) {
1000 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, (__v8bf)__B,
1001 -(__v8bf)__C);
1002}
1003
1004static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1005_mm_mask_fmsub_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
1006 return (__m128bh)__builtin_ia32_selectpbf_128(
1007 (__mmask8)__U, _mm_fmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1008 (__v8bf)__A);
1009}
1010
1011static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1012_mm_mask3_fmsub_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
1013 return (__m128bh)__builtin_ia32_selectpbf_128(
1014 (__mmask8)__U, _mm_fmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1015 (__v8bf)__C);
1016}
1017
1018static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1019_mm_maskz_fmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1020 return (__m128bh)__builtin_ia32_selectpbf_128(
1021 (__mmask8)__U, _mm_fmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1022 (__v8bf)_mm_setzero_pbh());
1023}
1024
1025static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmadd_pbh(__m128bh __A,
1026 __m128bh __B,
1027 __m128bh __C) {
1028 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, -(__v8bf)__B,
1029 (__v8bf)__C);
1030}
1031
1032static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1033_mm_mask_fnmadd_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
1034 return (__m128bh)__builtin_ia32_selectpbf_128(
1035 (__mmask8)__U, _mm_fnmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1036 (__v8bf)__A);
1037}
1038
1039static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1040_mm_mask3_fnmadd_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
1041 return (__m128bh)__builtin_ia32_selectpbf_128(
1042 (__mmask8)__U, _mm_fnmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1043 (__v8bf)__C);
1044}
1045
1046static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1047_mm_maskz_fnmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1048 return (__m128bh)__builtin_ia32_selectpbf_128(
1049 (__mmask8)__U, _mm_fnmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1050 (__v8bf)_mm_setzero_pbh());
1051}
1052
1053static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmsub_pbh(__m128bh __A,
1054 __m128bh __B,
1055 __m128bh __C) {
1056 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, -(__v8bf)__B,
1057 -(__v8bf)__C);
1058}
1059
1060static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1061_mm_mask_fnmsub_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
1062 return (__m128bh)__builtin_ia32_selectpbf_128(
1063 (__mmask8)__U, _mm_fnmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1064 (__v8bf)__A);
1065}
1066
1067static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1068_mm_mask3_fnmsub_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
1069 return (__m128bh)__builtin_ia32_selectpbf_128(
1070 (__mmask8)__U, _mm_fnmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1071 (__v8bf)__C);
1072}
1073
1074static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1075_mm_maskz_fnmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1076 return (__m128bh)__builtin_ia32_selectpbf_128(
1077 (__mmask8)__U, _mm_fnmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1078 (__v8bf)_mm_setzero_pbh());
1079}
1080
1081#undef __DEFAULT_FN_ATTRS128
1082#undef __DEFAULT_FN_ATTRS256
1083
1084#endif
1085#endif
lib/include/avx10_2convertintrin.h created+590
...@@ -0,0 +1,590 @@
1/*===--------------- avx10_2convertintrin.h - AVX10_2CONVERT ---------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2convertintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2CONVERTINTRIN_H
17#define __AVX10_2CONVERTINTRIN_H
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS128 \
21 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
22 __min_vector_width__(128)))
23#define __DEFAULT_FN_ATTRS256 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
25 __min_vector_width__(256)))
26
27static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtx2ps_ph(__m128 __A,
28 __m128 __B) {
29 return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
30 (__v4sf)__A, (__v4sf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)(-1));
31}
32
33static __inline__ __m128h __DEFAULT_FN_ATTRS128
34_mm_mask_cvtx2ps_ph(__m128h __W, __mmask8 __U, __m128 __A, __m128 __B) {
35 return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
36 (__v4sf)__A, (__v4sf)__B, (__v8hf)__W, (__mmask8)__U);
37}
38
39static __inline__ __m128h __DEFAULT_FN_ATTRS128
40_mm_maskz_cvtx2ps_ph(__mmask8 __U, __m128 __A, __m128 __B) {
41 return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
42 (__v4sf)__A, (__v4sf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U);
43}
44
45static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvtx2ps_ph(__m256 __A,
46 __m256 __B) {
47 return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
48 (__v8sf)__A, (__v8sf)__B, (__v16hf)_mm256_setzero_ph(), (__mmask16)(-1),
49 _MM_FROUND_CUR_DIRECTION);
50}
51
52static __inline__ __m256h __DEFAULT_FN_ATTRS256
53_mm256_mask_cvtx2ps_ph(__m256h __W, __mmask16 __U, __m256 __A, __m256 __B) {
54 return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
55 (__v8sf)__A, (__v8sf)__B, (__v16hf)__W, (__mmask16)__U,
56 _MM_FROUND_CUR_DIRECTION);
57}
58
59static __inline__ __m256h __DEFAULT_FN_ATTRS256
60_mm256_maskz_cvtx2ps_ph(__mmask16 __U, __m256 __A, __m256 __B) {
61 return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
62 (__v8sf)__A, (__v8sf)__B, (__v16hf)_mm256_setzero_ph(), (__mmask16)__U,
63 _MM_FROUND_CUR_DIRECTION);
64}
65
66#define _mm256_cvtx_round2ps_ph(A, B, R) \
67 ((__m256h)__builtin_ia32_vcvt2ps2phx256_mask( \
68 (__v8sf)(A), (__v8sf)(B), (__v16hf)_mm256_undefined_ph(), \
69 (__mmask16)(-1), (const int)(R)))
70
71#define _mm256_mask_cvtx_round2ps_ph(W, U, A, B, R) \
72 ((__m256h)__builtin_ia32_vcvt2ps2phx256_mask( \
73 (__v8sf)(A), (__v8sf)(B), (__v16hf)(W), (__mmask16)(U), (const int)(R)))
74
75#define _mm256_maskz_cvtx_round2ps_ph(U, A, B, R) \
76 ((__m256h)__builtin_ia32_vcvt2ps2phx256_mask( \
77 (__v8sf)(A), (__v8sf)(B), (__v16hf)(_mm256_setzero_ph()), \
78 (__mmask16)(U), (const int)(R)))
79
80static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtbiasph_bf8(__m128i __A,
81 __m128h __B) {
82 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_128_mask(
83 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
84}
85
86static __inline__ __m128i __DEFAULT_FN_ATTRS128
87_mm_mask_cvtbiasph_bf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
88 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_128_mask(
89 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
90}
91
92static __inline__ __m128i __DEFAULT_FN_ATTRS128
93_mm_maskz_cvtbiasph_bf8(__mmask8 __U, __m128i __A, __m128h __B) {
94 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_128_mask(
95 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
96 (__mmask8)__U);
97}
98
99static __inline__ __m128i __DEFAULT_FN_ATTRS256
100_mm256_cvtbiasph_bf8(__m256i __A, __m256h __B) {
101 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_256_mask(
102 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
103 (__mmask16)-1);
104}
105
106static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiasph_bf8(
107 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
108 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_256_mask(
109 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
110}
111
112static __inline__ __m128i __DEFAULT_FN_ATTRS256
113_mm256_maskz_cvtbiasph_bf8(__mmask16 __U, __m256i __A, __m256h __B) {
114 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_256_mask(
115 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
116 (__mmask16)__U);
117}
118
119static __inline__ __m128i __DEFAULT_FN_ATTRS128
120_mm_cvtbiassph_bf8(__m128i __A, __m128h __B) {
121 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_128_mask(
122 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
123}
124
125static __inline__ __m128i __DEFAULT_FN_ATTRS128
126_mm_mask_cvtbiassph_bf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
127 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_128_mask(
128 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
129}
130
131static __inline__ __m128i __DEFAULT_FN_ATTRS128
132_mm_maskz_cvtbiassph_bf8(__mmask8 __U, __m128i __A, __m128h __B) {
133 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_128_mask(
134 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
135 (__mmask8)__U);
136}
137
138static __inline__ __m128i __DEFAULT_FN_ATTRS256
139_mm256_cvtbiassph_bf8(__m256i __A, __m256h __B) {
140 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_256_mask(
141 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
142 (__mmask16)-1);
143}
144
145static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiassph_bf8(
146 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
147 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_256_mask(
148 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
149}
150
151static __inline__ __m128i __DEFAULT_FN_ATTRS256
152_mm256_maskz_cvtbiassph_bf8(__mmask16 __U, __m256i __A, __m256h __B) {
153 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_256_mask(
154 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
155 (__mmask16)__U);
156}
157
158static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtbiasph_hf8(__m128i __A,
159 __m128h __B) {
160 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_128_mask(
161 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
162}
163
164static __inline__ __m128i __DEFAULT_FN_ATTRS128
165_mm_mask_cvtbiasph_hf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
166 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_128_mask(
167 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
168}
169
170static __inline__ __m128i __DEFAULT_FN_ATTRS128
171_mm_maskz_cvtbiasph_hf8(__mmask8 __U, __m128i __A, __m128h __B) {
172 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_128_mask(
173 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
174 (__mmask8)__U);
175}
176
177static __inline__ __m128i __DEFAULT_FN_ATTRS256
178_mm256_cvtbiasph_hf8(__m256i __A, __m256h __B) {
179 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_256_mask(
180 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
181 (__mmask16)-1);
182}
183
184static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiasph_hf8(
185 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
186 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_256_mask(
187 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
188}
189
190static __inline__ __m128i __DEFAULT_FN_ATTRS256
191_mm256_maskz_cvtbiasph_hf8(__mmask16 __U, __m256i __A, __m256h __B) {
192 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_256_mask(
193 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
194 (__mmask16)__U);
195}
196
197static __inline__ __m128i __DEFAULT_FN_ATTRS128
198_mm_cvtbiassph_hf8(__m128i __A, __m128h __B) {
199 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_128_mask(
200 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
201}
202
203static __inline__ __m128i __DEFAULT_FN_ATTRS128
204_mm_mask_cvtbiassph_hf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
205 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_128_mask(
206 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
207}
208
209static __inline__ __m128i __DEFAULT_FN_ATTRS128
210_mm_maskz_cvtbiassph_hf8(__mmask8 __U, __m128i __A, __m128h __B) {
211 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_128_mask(
212 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
213 (__mmask8)__U);
214}
215
216static __inline__ __m128i __DEFAULT_FN_ATTRS256
217_mm256_cvtbiassph_hf8(__m256i __A, __m256h __B) {
218 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_256_mask(
219 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
220 (__mmask16)-1);
221}
222
223static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiassph_hf8(
224 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
225 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_256_mask(
226 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
227}
228
229static __inline__ __m128i __DEFAULT_FN_ATTRS256
230_mm256_maskz_cvtbiassph_hf8(__mmask16 __U, __m256i __A, __m256h __B) {
231 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_256_mask(
232 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
233 (__mmask16)__U);
234}
235
236static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvt2ph_bf8(__m128h __A,
237 __m128h __B) {
238 return (__m128i)__builtin_ia32_vcvt2ph2bf8_128((__v8hf)(__A), (__v8hf)(__B));
239}
240
241static __inline__ __m128i __DEFAULT_FN_ATTRS128
242_mm_mask_cvt2ph_bf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
243 return (__m128i)__builtin_ia32_selectb_128(
244 (__mmask16)__U, (__v16qi)_mm_cvt2ph_bf8(__A, __B), (__v16qi)__W);
245}
246
247static __inline__ __m128i __DEFAULT_FN_ATTRS128
248_mm_maskz_cvt2ph_bf8(__mmask16 __U, __m128h __A, __m128h __B) {
249 return (__m128i)__builtin_ia32_selectb_128(
250 (__mmask16)__U, (__v16qi)_mm_cvt2ph_bf8(__A, __B),
251 (__v16qi)(__m128i)_mm_setzero_si128());
252}
253
254static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_cvt2ph_bf8(__m256h __A,
255 __m256h __B) {
256 return (__m256i)__builtin_ia32_vcvt2ph2bf8_256((__v16hf)(__A),
257 (__v16hf)(__B));
258}
259
260static __inline__ __m256i __DEFAULT_FN_ATTRS256
261_mm256_mask_cvt2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
262 return (__m256i)__builtin_ia32_selectb_256(
263 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B), (__v32qi)__W);
264}
265
266static __inline__ __m256i __DEFAULT_FN_ATTRS256
267_mm256_maskz_cvt2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
268 return (__m256i)__builtin_ia32_selectb_256(
269 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B),
270 (__v32qi)(__m256i)_mm256_setzero_si256());
271}
272
273static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvts2ph_bf8(__m128h __A,
274 __m128h __B) {
275 return (__m128i)__builtin_ia32_vcvt2ph2bf8s_128((__v8hf)(__A), (__v8hf)(__B));
276}
277
278static __inline__ __m128i __DEFAULT_FN_ATTRS128
279_mm_mask_cvts2ph_bf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
280 return (__m128i)__builtin_ia32_selectb_128(
281 (__mmask16)__U, (__v16qi)_mm_cvts2ph_bf8(__A, __B), (__v16qi)__W);
282}
283
284static __inline__ __m128i __DEFAULT_FN_ATTRS128
285_mm_maskz_cvts2ph_bf8(__mmask16 __U, __m128h __A, __m128h __B) {
286 return (__m128i)__builtin_ia32_selectb_128(
287 (__mmask16)__U, (__v16qi)_mm_cvts2ph_bf8(__A, __B),
288 (__v16qi)(__m128i)_mm_setzero_si128());
289}
290
291static __inline__ __m256i __DEFAULT_FN_ATTRS256
292_mm256_cvts2ph_bf8(__m256h __A, __m256h __B) {
293 return (__m256i)__builtin_ia32_vcvt2ph2bf8s_256((__v16hf)(__A),
294 (__v16hf)(__B));
295}
296
297static __inline__ __m256i __DEFAULT_FN_ATTRS256
298_mm256_mask_cvts2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
299 return (__m256i)__builtin_ia32_selectb_256(
300 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B), (__v32qi)__W);
301}
302
303static __inline__ __m256i __DEFAULT_FN_ATTRS256
304_mm256_maskz_cvts2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
305 return (__m256i)__builtin_ia32_selectb_256(
306 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B),
307 (__v32qi)(__m256i)_mm256_setzero_si256());
308}
309
310static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvt2ph_hf8(__m128h __A,
311 __m128h __B) {
312 return (__m128i)__builtin_ia32_vcvt2ph2hf8_128((__v8hf)(__A), (__v8hf)(__B));
313}
314
315static __inline__ __m128i __DEFAULT_FN_ATTRS128
316_mm_mask_cvt2ph_hf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
317 return (__m128i)__builtin_ia32_selectb_128(
318 (__mmask16)__U, (__v16qi)_mm_cvt2ph_hf8(__A, __B), (__v16qi)__W);
319}
320
321static __inline__ __m128i __DEFAULT_FN_ATTRS128
322_mm_maskz_cvt2ph_hf8(__mmask16 __U, __m128h __A, __m128h __B) {
323 return (__m128i)__builtin_ia32_selectb_128(
324 (__mmask16)__U, (__v16qi)_mm_cvt2ph_hf8(__A, __B),
325 (__v16qi)(__m128i)_mm_setzero_si128());
326}
327
328static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_cvt2ph_hf8(__m256h __A,
329 __m256h __B) {
330 return (__m256i)__builtin_ia32_vcvt2ph2hf8_256((__v16hf)(__A),
331 (__v16hf)(__B));
332}
333
334static __inline__ __m256i __DEFAULT_FN_ATTRS256
335_mm256_mask_cvt2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
336 return (__m256i)__builtin_ia32_selectb_256(
337 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B), (__v32qi)__W);
338}
339
340static __inline__ __m256i __DEFAULT_FN_ATTRS256
341_mm256_maskz_cvt2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
342 return (__m256i)__builtin_ia32_selectb_256(
343 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B),
344 (__v32qi)(__m256i)_mm256_setzero_si256());
345}
346
347static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvts2ph_hf8(__m128h __A,
348 __m128h __B) {
349 return (__m128i)__builtin_ia32_vcvt2ph2hf8s_128((__v8hf)(__A), (__v8hf)(__B));
350}
351
352static __inline__ __m128i __DEFAULT_FN_ATTRS128
353_mm_mask_cvts2ph_hf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
354 return (__m128i)__builtin_ia32_selectb_128(
355 (__mmask16)__U, (__v16qi)_mm_cvts2ph_hf8(__A, __B), (__v16qi)__W);
356}
357
358static __inline__ __m128i __DEFAULT_FN_ATTRS128
359_mm_maskz_cvts2ph_hf8(__mmask16 __U, __m128h __A, __m128h __B) {
360 return (__m128i)__builtin_ia32_selectb_128(
361 (__mmask16)__U, (__v16qi)_mm_cvts2ph_hf8(__A, __B),
362 (__v16qi)(__m128i)_mm_setzero_si128());
363}
364
365static __inline__ __m256i __DEFAULT_FN_ATTRS256
366_mm256_cvts2ph_hf8(__m256h __A, __m256h __B) {
367 return (__m256i)__builtin_ia32_vcvt2ph2hf8s_256((__v16hf)(__A),
368 (__v16hf)(__B));
369}
370
371static __inline__ __m256i __DEFAULT_FN_ATTRS256
372_mm256_mask_cvts2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
373 return (__m256i)__builtin_ia32_selectb_256(
374 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B), (__v32qi)__W);
375}
376
377static __inline__ __m256i __DEFAULT_FN_ATTRS256
378_mm256_maskz_cvts2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
379 return (__m256i)__builtin_ia32_selectb_256(
380 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B),
381 (__v32qi)(__m256i)_mm256_setzero_si256());
382}
383
384static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvthf8(__m128i __A) {
385 return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
386 (__v16qi)__A, (__v8hf)(__m128h)_mm_undefined_ph(), (__mmask8)-1);
387}
388
389static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_cvthf8(__m128h __W,
390 __mmask8 __U,
391 __m128i __A) {
392 return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
393 (__v16qi)__A, (__v8hf)(__m128h)__W, (__mmask8)__U);
394}
395
396static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_cvthf8(__mmask8 __U,
397 __m128i __A) {
398 return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
399 (__v16qi)__A, (__v8hf)(__m128h)_mm_setzero_ph(), (__mmask8)__U);
400}
401
402static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvthf8(__m128i __A) {
403 return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
404 (__v16qi)__A, (__v16hf)(__m256h)_mm256_undefined_ph(), (__mmask16)-1);
405}
406
407static __inline__ __m256h __DEFAULT_FN_ATTRS256
408_mm256_mask_cvthf8(__m256h __W, __mmask16 __U, __m128i __A) {
409 return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
410 (__v16qi)__A, (__v16hf)(__m256h)__W, (__mmask16)__U);
411}
412
413static __inline__ __m256h __DEFAULT_FN_ATTRS256
414_mm256_maskz_cvthf8(__mmask16 __U, __m128i __A) {
415 return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
416 (__v16qi)__A, (__v16hf)(__m256h)_mm256_setzero_ph(), (__mmask16)__U);
417}
418
419static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtph_bf8(__m128h __A) {
420 return (__m128i)__builtin_ia32_vcvtph2bf8_128_mask(
421 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
422}
423
424static __inline__ __m128i __DEFAULT_FN_ATTRS128
425_mm_mask_cvtph_bf8(__m128i __W, __mmask8 __U, __m128h __A) {
426 return (__m128i)__builtin_ia32_vcvtph2bf8_128_mask(
427 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
428}
429
430static __inline__ __m128i __DEFAULT_FN_ATTRS128
431_mm_maskz_cvtph_bf8(__mmask8 __U, __m128h __A) {
432 return (__m128i)__builtin_ia32_vcvtph2bf8_128_mask(
433 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
434}
435
436static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtph_bf8(__m256h __A) {
437 return (__m128i)__builtin_ia32_vcvtph2bf8_256_mask(
438 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
439}
440
441static __inline__ __m128i __DEFAULT_FN_ATTRS256
442_mm256_mask_cvtph_bf8(__m128i __W, __mmask16 __U, __m256h __A) {
443 return (__m128i)__builtin_ia32_vcvtph2bf8_256_mask(
444 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
445}
446
447static __inline__ __m128i __DEFAULT_FN_ATTRS256
448_mm256_maskz_cvtph_bf8(__mmask16 __U, __m256h __A) {
449 return (__m128i)__builtin_ia32_vcvtph2bf8_256_mask(
450 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
451}
452
453static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtsph_bf8(__m128h __A) {
454 return (__m128i)__builtin_ia32_vcvtph2bf8s_128_mask(
455 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
456}
457
458static __inline__ __m128i __DEFAULT_FN_ATTRS128
459_mm_mask_cvtsph_bf8(__m128i __W, __mmask8 __U, __m128h __A) {
460 return (__m128i)__builtin_ia32_vcvtph2bf8s_128_mask(
461 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
462}
463
464static __inline__ __m128i __DEFAULT_FN_ATTRS128
465_mm_maskz_cvtsph_bf8(__mmask8 __U, __m128h __A) {
466 return (__m128i)__builtin_ia32_vcvtph2bf8s_128_mask(
467 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
468}
469
470static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtsph_bf8(__m256h __A) {
471 return (__m128i)__builtin_ia32_vcvtph2bf8s_256_mask(
472 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
473}
474
475static __inline__ __m128i __DEFAULT_FN_ATTRS256
476_mm256_mask_cvtsph_bf8(__m128i __W, __mmask16 __U, __m256h __A) {
477 return (__m128i)__builtin_ia32_vcvtph2bf8s_256_mask(
478 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
479}
480
481static __inline__ __m128i __DEFAULT_FN_ATTRS256
482_mm256_maskz_cvtsph_bf8(__mmask16 __U, __m256h __A) {
483 return (__m128i)__builtin_ia32_vcvtph2bf8s_256_mask(
484 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
485}
486
487static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtph_hf8(__m128h __A) {
488 return (__m128i)__builtin_ia32_vcvtph2hf8_128_mask(
489 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
490}
491
492static __inline__ __m128i __DEFAULT_FN_ATTRS128
493_mm_mask_cvtph_hf8(__m128i __W, __mmask8 __U, __m128h __A) {
494 return (__m128i)__builtin_ia32_vcvtph2hf8_128_mask(
495 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
496}
497
498static __inline__ __m128i __DEFAULT_FN_ATTRS128
499_mm_maskz_cvtph_hf8(__mmask8 __U, __m128h __A) {
500 return (__m128i)__builtin_ia32_vcvtph2hf8_128_mask(
501 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
502}
503
504static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtph_hf8(__m256h __A) {
505 return (__m128i)__builtin_ia32_vcvtph2hf8_256_mask(
506 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
507}
508
509static __inline__ __m128i __DEFAULT_FN_ATTRS256
510_mm256_mask_cvtph_hf8(__m128i __W, __mmask16 __U, __m256h __A) {
511 return (__m128i)__builtin_ia32_vcvtph2hf8_256_mask(
512 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
513}
514
515static __inline__ __m128i __DEFAULT_FN_ATTRS256
516_mm256_maskz_cvtph_hf8(__mmask16 __U, __m256h __A) {
517 return (__m128i)__builtin_ia32_vcvtph2hf8_256_mask(
518 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
519}
520
521static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtsph_hf8(__m128h __A) {
522 return (__m128i)__builtin_ia32_vcvtph2hf8s_128_mask(
523 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
524}
525
526static __inline__ __m128i __DEFAULT_FN_ATTRS128
527_mm_mask_cvtsph_hf8(__m128i __W, __mmask8 __U, __m128h __A) {
528 return (__m128i)__builtin_ia32_vcvtph2hf8s_128_mask(
529 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
530}
531
532static __inline__ __m128i __DEFAULT_FN_ATTRS128
533_mm_maskz_cvtsph_hf8(__mmask8 __U, __m128h __A) {
534 return (__m128i)__builtin_ia32_vcvtph2hf8s_128_mask(
535 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
536}
537
538static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtsph_hf8(__m256h __A) {
539 return (__m128i)__builtin_ia32_vcvtph2hf8s_256_mask(
540 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
541}
542
543static __inline__ __m128i __DEFAULT_FN_ATTRS256
544_mm256_mask_cvtsph_hf8(__m128i __W, __mmask16 __U, __m256h __A) {
545 return (__m128i)__builtin_ia32_vcvtph2hf8s_256_mask(
546 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
547}
548
549static __inline__ __m128i __DEFAULT_FN_ATTRS256
550_mm256_maskz_cvtsph_hf8(__mmask16 __U, __m256h __A) {
551 return (__m128i)__builtin_ia32_vcvtph2hf8s_256_mask(
552 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
553}
554
555static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtbf8_ph(__m128i __A) {
556 return _mm_castsi128_ph(_mm_slli_epi16(_mm_cvtepi8_epi16(__A), 8));
557}
558
559static __inline__ __m128h __DEFAULT_FN_ATTRS128
560_mm_mask_cvtbf8_ph(__m128h __S, __mmask8 __U, __m128i __A) {
561 return _mm_castsi128_ph(
562 _mm_mask_slli_epi16((__m128i)__S, __U, _mm_cvtepi8_epi16(__A), 8));
563}
564
565static __inline__ __m128h __DEFAULT_FN_ATTRS128
566_mm_maskz_cvtbf8_ph(__mmask8 __U, __m128i __A) {
567 return _mm_castsi128_ph(_mm_slli_epi16(_mm_maskz_cvtepi8_epi16(__U, __A), 8));
568}
569
570static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvtbf8_ph(__m128i __A) {
571 return _mm256_castsi256_ph(_mm256_slli_epi16(_mm256_cvtepi8_epi16(__A), 8));
572}
573
574static __inline__ __m256h __DEFAULT_FN_ATTRS256
575_mm256_mask_cvtbf8_ph(__m256h __S, __mmask16 __U, __m128i __A) {
576 return _mm256_castsi256_ph(
577 _mm256_mask_slli_epi16((__m256i)__S, __U, _mm256_cvtepi8_epi16(__A), 8));
578}
579
580static __inline__ __m256h __DEFAULT_FN_ATTRS256
581_mm256_maskz_cvtbf8_ph(__mmask16 __U, __m128i __A) {
582 return _mm256_castsi256_ph(
583 _mm256_slli_epi16(_mm256_maskz_cvtepi8_epi16(__U, __A), 8));
584}
585
586#undef __DEFAULT_FN_ATTRS128
587#undef __DEFAULT_FN_ATTRS256
588
589#endif // __AVX10_2CONVERTINTRIN_H
590#endif // __SSE2__
lib/include/avx10_2copyintrin.h created+66
...@@ -0,0 +1,66 @@
1/*===---- avx10_2copyintrin.h - AVX10.2 Copy intrinsics -------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2copyintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2COPYINTRIN_H
15#define __AVX10_2COPYINTRIN_H
16
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS128 \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
20 __min_vector_width__(128)))
21
22/// Constructs a 128-bit integer vector, setting the lower 32 bits to the
23/// lower 32 bits of the parameter \a __A; the upper bits are zeoroed.
24///
25/// \code{.operation}
26/// result[31:0] := __A[31:0]
27/// result[MAX:32] := 0
28/// \endcode
29///
30/// \headerfile <immintrin.h>
31///
32/// This intrinsic corresponds to the <c> VMOVD </c> instruction.
33///
34/// \param __A
35/// A 128-bit integer vector.
36/// \returns A 128-bit integer vector. The lower 32 bits are copied from the
37/// parameter \a __A; the upper bits are zeroed.
38static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_move_epi32(__m128i __A) {
39 return (__m128i)__builtin_shufflevector(
40 (__v4si)__A, (__v4si)_mm_setzero_si128(), 0, 4, 4, 4);
41}
42
43/// Constructs a 128-bit integer vector, setting the lower 16 bits to the
44/// lower 16 bits of the parameter \a __A; the upper bits are zeoroed.
45///
46/// \code{.operation}
47/// result[15:0] := __A[15:0]
48/// result[MAX:16] := 0
49/// \endcode
50///
51/// \headerfile <immintrin.h>
52///
53/// This intrinsic corresponds to the <c> VMOVW </c> instruction.
54///
55/// \param __A
56/// A 128-bit integer vector.
57/// \returns A 128-bit integer vector. The lower 16 bits are copied from the
58/// parameter \a __A; the upper bits are zeroed.
59static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_move_epi16(__m128i __A) {
60 return (__m128i)__builtin_shufflevector(
61 (__v8hi)__A, (__v8hi)_mm_setzero_si128(), 0, 8, 8, 8, 8, 8, 8, 8);
62}
63
64#undef __DEFAULT_FN_ATTRS128
65
66#endif // __AVX10_2COPYINTRIN_H
lib/include/avx10_2minmaxintrin.h created+277
...@@ -0,0 +1,277 @@
1/*===-------- avx10_2minmaxintrin.h - AVX10_2MINMAX intrinsics -------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2minmaxintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2MINMAXINTRIN_H
15#define __AVX10_2MINMAXINTRIN_H
16
17#define _mm_minmax_pbh(A, B, C) \
18 ((__m128bh)__builtin_ia32_vminmaxbf16128((__m128bh)(__v8bf)(A), \
19 (__m128bh)(__v8bf)(B), (int)(C)))
20
21#define _mm_mask_minmax_pbh(W, U, A, B, C) \
22 ((__m128bh)__builtin_ia32_selectpbf_128( \
23 (__mmask8)(U), \
24 (__v8bf)_mm_minmax_pbh((__m128bh)(__v8bf)(A), (__m128bh)(__v8bf)(B), \
25 (int)(C)), \
26 (__v8bf)(W)))
27
28#define _mm_maskz_minmax_pbh(U, A, B, C) \
29 ((__m128bh)__builtin_ia32_selectpbf_128( \
30 (__mmask8)(U), \
31 (__v8bf)_mm_minmax_pbh((__m128bh)(__v8bf)(A), (__m128bh)(__v8bf)(B), \
32 (int)(C)), \
33 (__v8bf) __builtin_bit_cast(__m128bh, _mm_setzero_ps())))
34
35#define _mm256_minmax_pbh(A, B, C) \
36 ((__m256bh)__builtin_ia32_vminmaxbf16256((__m256bh)(__v16bf)(A), \
37 (__m256bh)(__v16bf)(B), (int)(C)))
38
39#define _mm256_mask_minmax_pbh(W, U, A, B, C) \
40 ((__m256bh)__builtin_ia32_selectpbf_256( \
41 (__mmask16)(U), \
42 (__v16bf)_mm256_minmax_pbh((__m256bh)(__v16bf)(A), \
43 (__m256bh)(__v16bf)(B), (int)(C)), \
44 (__v16bf)(W)))
45
46#define _mm256_maskz_minmax_pbh(U, A, B, C) \
47 ((__m256bh)__builtin_ia32_selectpbf_256( \
48 (__mmask16)(U), \
49 (__v16bf)_mm256_minmax_pbh((__m256bh)(__v16bf)(A), \
50 (__m256bh)(__v16bf)(B), (int)(C)), \
51 (__v16bf) __builtin_bit_cast(__m256bh, _mm256_setzero_ps())))
52
53#define _mm_minmax_pd(A, B, C) \
54 ((__m128d)__builtin_ia32_vminmaxpd128_mask( \
55 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
56 (__v2df)_mm_setzero_pd(), (__mmask8)-1))
57
58#define _mm_mask_minmax_pd(W, U, A, B, C) \
59 ((__m128d)__builtin_ia32_vminmaxpd128_mask( \
60 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
61 (__v2df)(__m128d)(W), (__mmask8)(U)))
62
63#define _mm_maskz_minmax_pd(U, A, B, C) \
64 ((__m128d)__builtin_ia32_vminmaxpd128_mask( \
65 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
66 (__v2df)_mm_setzero_pd(), (__mmask8)(U)))
67
68#define _mm256_minmax_pd(A, B, C) \
69 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
70 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
71 (__v4df)_mm256_setzero_pd(), (__mmask8)-1, _MM_FROUND_NO_EXC))
72
73#define _mm256_mask_minmax_pd(W, U, A, B, C) \
74 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
75 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
76 (__v4df)(__m256d)(W), (__mmask8)(U), _MM_FROUND_NO_EXC))
77
78#define _mm256_maskz_minmax_pd(U, A, B, C) \
79 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
80 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
81 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), _MM_FROUND_NO_EXC))
82
83#define _mm256_minmax_round_pd(A, B, C, R) \
84 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
85 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
86 (__v4df)_mm256_undefined_pd(), (__mmask8)-1, (int)(R)))
87
88#define _mm256_mask_minmax_round_pd(W, U, A, B, C, R) \
89 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
90 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
91 (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
92
93#define _mm256_maskz_minmax_round_pd(U, A, B, C, R) \
94 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
95 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
96 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
97
98#define _mm_minmax_ph(A, B, C) \
99 ((__m128h)__builtin_ia32_vminmaxph128_mask( \
100 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
101 (__v8hf)_mm_setzero_ph(), (__mmask8)-1))
102
103#define _mm_mask_minmax_ph(W, U, A, B, C) \
104 ((__m128h)__builtin_ia32_vminmaxph128_mask( \
105 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
106 (__v8hf)(__m128h)(W), (__mmask16)-1))
107
108#define _mm_maskz_minmax_ph(U, A, B, C) \
109 ((__m128h)__builtin_ia32_vminmaxph128_mask( \
110 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
111 (__v8hf)_mm_setzero_ph(), (__mmask8)(U)))
112
113#define _mm256_minmax_ph(A, B, C) \
114 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
115 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
116 (__v16hf)_mm256_setzero_ph(), (__mmask16)-1, _MM_FROUND_NO_EXC))
117
118#define _mm256_mask_minmax_ph(W, U, A, B, C) \
119 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
120 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
121 (__v16hf)(__m256h)(W), (__mmask16)(U), _MM_FROUND_NO_EXC))
122
123#define _mm256_maskz_minmax_ph(U, A, B, C) \
124 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
125 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
126 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), _MM_FROUND_NO_EXC))
127
128#define _mm256_minmax_round_ph(A, B, C, R) \
129 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
130 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
131 (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, (int)(R)))
132
133#define _mm256_mask_minmax_round_ph(W, U, A, B, C, R) \
134 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
135 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (C), \
136 (__v16hf)(__m256h)(W), (__mmask16)(U), (int)(R)))
137
138#define _mm256_maskz_minmax_round_ph(U, A, B, C, R) \
139 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
140 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
141 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
142
143#define _mm_minmax_ps(A, B, C) \
144 ((__m128)__builtin_ia32_vminmaxps128_mask( \
145 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
146 (__v4sf)_mm_setzero_ps(), (__mmask8)-1))
147
148#define _mm_mask_minmax_ps(W, U, A, B, C) \
149 ((__m128)__builtin_ia32_vminmaxps128_mask( \
150 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), (__v4sf)(__m128)(W), \
151 (__mmask8)(U)))
152
153#define _mm_maskz_minmax_ps(U, A, B, C) \
154 ((__m128)__builtin_ia32_vminmaxps128_mask( \
155 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
156 (__v4sf)_mm_setzero_ps(), (__mmask8)(U)))
157
158#define _mm256_minmax_ps(A, B, C) \
159 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
160 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
161 (__v8sf)_mm256_setzero_ps(), (__mmask8)-1, _MM_FROUND_NO_EXC))
162
163#define _mm256_mask_minmax_ps(W, U, A, B, C) \
164 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
165 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), (__v8sf)(__m256)(W), \
166 (__mmask8)(U), _MM_FROUND_NO_EXC))
167
168#define _mm256_maskz_minmax_ps(U, A, B, C) \
169 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
170 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
171 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), _MM_FROUND_NO_EXC))
172
173#define _mm256_minmax_round_ps(A, B, C, R) \
174 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
175 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
176 (__v8sf)_mm256_undefined_ps(), (__mmask8)-1, (int)(R)))
177
178#define _mm256_mask_minmax_round_ps(W, U, A, B, C, R) \
179 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
180 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), (__v8sf)(__m256)(W), \
181 (__mmask8)(U), (int)(R)))
182
183#define _mm256_maskz_minmax_round_ps(U, A, B, C, R) \
184 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
185 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
186 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
187
188#define _mm_minmax_sd(A, B, C) \
189 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
190 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
191 (__v2df)_mm_undefined_pd(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION))
192
193#define _mm_mask_minmax_sd(W, U, A, B, C) \
194 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
195 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
196 (__v2df)(__m128d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
197
198#define _mm_maskz_minmax_sd(U, A, B, C) \
199 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
200 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
201 (__v2df)_mm_setzero_pd(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
202
203#define _mm_minmax_round_sd(A, B, C, R) \
204 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
205 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
206 (__v2df)_mm_undefined_pd(), (__mmask8)-1, (int)(R)))
207
208#define _mm_mask_minmax_round_sd(W, U, A, B, C, R) \
209 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
210 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
211 (__v2df)(__m128d)(W), (__mmask8)(U), (int)(R)))
212
213#define _mm_maskz_minmax_round_sd(U, A, B, C, R) \
214 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
215 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
216 (__v2df)_mm_setzero_pd(), (__mmask8)(U), (int)(R)))
217
218#define _mm_minmax_sh(A, B, C) \
219 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
220 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
221 (__v8hf)_mm_undefined_ph(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION))
222
223#define _mm_mask_minmax_sh(W, U, A, B, C) \
224 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
225 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
226 (__v8hf)(__m128h)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
227
228#define _mm_maskz_minmax_sh(U, A, B, C) \
229 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
230 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
231 (__v8hf)_mm_setzero_ph(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
232
233#define _mm_minmax_round_sh(A, B, C, R) \
234 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
235 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
236 (__v8hf)_mm_undefined_ph(), (__mmask8)-1, (int)(R)))
237
238#define _mm_mask_minmax_round_sh(W, U, A, B, C, R) \
239 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
240 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
241 (__v8hf)(__m128h)(W), (__mmask8)(U), (int)(R)))
242
243#define _mm_maskz_minmax_round_sh(U, A, B, C, R) \
244 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
245 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
246 (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
247
248#define _mm_minmax_ss(A, B, C) \
249 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
250 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
251 (__v4sf)_mm_undefined_ps(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION))
252
253#define _mm_mask_minmax_ss(W, U, A, B, C) \
254 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
255 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), (__v4sf)(W), \
256 (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
257
258#define _mm_maskz_minmax_ss(U, A, B, C) \
259 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
260 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
261 (__v4sf)_mm_setzero_ps(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
262
263#define _mm_minmax_round_ss(A, B, C, R) \
264 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
265 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
266 (__v4sf)_mm_undefined_ps(), (__mmask8)-1, (int)(R)))
267
268#define _mm_mask_minmax_round_ss(W, U, A, B, C, R) \
269 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
270 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), (__v4sf)(W), \
271 (__mmask8)(U), (int)(R)))
272
273#define _mm_maskz_minmax_round_ss(U, A, B, C, R) \
274 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
275 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
276 (__v4sf)_mm_setzero_ps(), (__mmask8)(U), (int)(R)))
277#endif // __AVX10_2MINMAXINTRIN_H
lib/include/avx10_2niintrin.h created+2075
...@@ -0,0 +1,2075 @@
1/*===---- avx10_2niintrin.h - AVX10.2 new instruction intrinsics -----------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <avx10_2niintrin.h> directly; include <immintrin.h> instead."
11#endif
12
13#ifdef __SSE2__
14
15#ifndef __AVX10_2NIINTRIN_H
16#define __AVX10_2NIINTRIN_H
17
18#define __DEFAULT_FN_ATTRS128 \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
20 __min_vector_width__(128)))
21#define __DEFAULT_FN_ATTRS256 \
22 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
23 __min_vector_width__(256)))
24
25/* VNNI FP16 */
26static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_dpph_ps(__m128 __W,
27 __m128h __A,
28 __m128h __B) {
29 return (__m128)__builtin_ia32_vdpphps128((__v4sf)__W, (__v8hf)__A,
30 (__v8hf)__B);
31}
32
33static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_dpph_ps(__m128 __W,
34 __mmask8 __U,
35 __m128h __A,
36 __m128h __B) {
37 return (__m128)__builtin_ia32_selectps_128(
38 (__mmask8)__U, (__v4sf)_mm_dpph_ps(__W, __A, __B), (__v4sf)__W);
39}
40
41static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_dpph_ps(__mmask8 __U,
42 __m128 __W,
43 __m128h __A,
44 __m128h __B) {
45 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
46 (__v4sf)_mm_dpph_ps(__W, __A, __B),
47 (__v4sf)_mm_setzero_ps());
48}
49
50static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_dpph_ps(__m256 __W,
51 __m256h __A,
52 __m256h __B) {
53 return (__m256)__builtin_ia32_vdpphps256((__v8sf)__W, (__v16hf)__A,
54 (__v16hf)__B);
55}
56
57static __inline__ __m256 __DEFAULT_FN_ATTRS256
58_mm256_mask_dpph_ps(__m256 __W, __mmask8 __U, __m256h __A, __m256h __B) {
59 return (__m256)__builtin_ia32_selectps_256(
60 (__mmask8)__U, (__v8sf)_mm256_dpph_ps(__W, __A, __B), (__v8sf)__W);
61}
62
63static __inline__ __m256 __DEFAULT_FN_ATTRS256
64_mm256_maskz_dpph_ps(__mmask8 __U, __m256 __W, __m256h __A, __m256h __B) {
65 return (__m256)__builtin_ia32_selectps_256(
66 (__mmask8)__U, (__v8sf)_mm256_dpph_ps(__W, __A, __B),
67 (__v8sf)_mm256_setzero_ps());
68}
69
70/* VMPSADBW */
71#define _mm_mask_mpsadbw_epu8(W, U, A, B, imm) \
72 ((__m128i)__builtin_ia32_selectw_128( \
73 (__mmask8)(U), (__v8hi)_mm_mpsadbw_epu8((A), (B), (imm)), \
74 (__v8hi)(__m128i)(W)))
75
76#define _mm_maskz_mpsadbw_epu8(U, A, B, imm) \
77 ((__m128i)__builtin_ia32_selectw_128( \
78 (__mmask8)(U), (__v8hi)_mm_mpsadbw_epu8((A), (B), (imm)), \
79 (__v8hi)_mm_setzero_si128()))
80
81#define _mm256_mask_mpsadbw_epu8(W, U, A, B, imm) \
82 ((__m256i)__builtin_ia32_selectw_256( \
83 (__mmask16)(U), (__v16hi)_mm256_mpsadbw_epu8((A), (B), (imm)), \
84 (__v16hi)(__m256i)(W)))
85
86#define _mm256_maskz_mpsadbw_epu8(U, A, B, imm) \
87 ((__m256i)__builtin_ia32_selectw_256( \
88 (__mmask16)(U), (__v16hi)_mm256_mpsadbw_epu8((A), (B), (imm)), \
89 (__v16hi)_mm256_setzero_si256()))
90
91/* VNNI INT8 */
92static __inline__ __m128i __DEFAULT_FN_ATTRS128
93_mm_mask_dpbssd_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
94 return (__m128i)__builtin_ia32_selectd_128(
95 __U, (__v4si)_mm_dpbssd_epi32(__W, __A, __B), (__v4si)__W);
96}
97
98static __inline__ __m128i __DEFAULT_FN_ATTRS128
99_mm_maskz_dpbssd_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
100 return (__m128i)__builtin_ia32_selectd_128(
101 __U, (__v4si)_mm_dpbssd_epi32(__W, __A, __B),
102 (__v4si)_mm_setzero_si128());
103}
104
105static __inline__ __m256i __DEFAULT_FN_ATTRS256
106_mm256_mask_dpbssd_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
107 return (__m256i)__builtin_ia32_selectd_256(
108 __U, (__v8si)_mm256_dpbssd_epi32(__W, __A, __B), (__v8si)__W);
109}
110
111static __inline__ __m256i __DEFAULT_FN_ATTRS256
112_mm256_maskz_dpbssd_epi32(__mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
113 return (__m256i)__builtin_ia32_selectd_256(
114 __U, (__v8si)_mm256_dpbssd_epi32(__W, __A, __B),
115 (__v8si)_mm256_setzero_si256());
116}
117
118static __inline__ __m128i __DEFAULT_FN_ATTRS128
119_mm_mask_dpbssds_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
120 return (__m128i)__builtin_ia32_selectd_128(
121 __U, (__v4si)_mm_dpbssds_epi32(__W, __A, __B), (__v4si)__W);
122}
123
124static __inline__ __m128i __DEFAULT_FN_ATTRS128
125_mm_maskz_dpbssds_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
126 return (__m128i)__builtin_ia32_selectd_128(
127 __U, (__v4si)_mm_dpbssds_epi32(__W, __A, __B),
128 (__v4si)_mm_setzero_si128());
129}
130
131static __inline__ __m256i __DEFAULT_FN_ATTRS256
132_mm256_mask_dpbssds_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
133 return (__m256i)__builtin_ia32_selectd_256(
134 __U, (__v8si)_mm256_dpbssds_epi32(__W, __A, __B), (__v8si)__W);
135}
136
137static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbssds_epi32(
138 __mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
139 return (__m256i)__builtin_ia32_selectd_256(
140 __U, (__v8si)_mm256_dpbssds_epi32(__W, __A, __B),
141 (__v8si)_mm256_setzero_si256());
142}
143
144static __inline__ __m128i __DEFAULT_FN_ATTRS128
145_mm_mask_dpbsud_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
146 return (__m128i)__builtin_ia32_selectd_128(
147 __U, (__v4si)_mm_dpbsud_epi32(__W, __A, __B), (__v4si)__W);
148}
149
150static __inline__ __m128i __DEFAULT_FN_ATTRS128
151_mm_maskz_dpbsud_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
152 return (__m128i)__builtin_ia32_selectd_128(
153 __U, (__v4si)_mm_dpbsud_epi32(__W, __A, __B),
154 (__v4si)_mm_setzero_si128());
155}
156
157static __inline__ __m256i __DEFAULT_FN_ATTRS256
158_mm256_mask_dpbsud_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
159 return (__m256i)__builtin_ia32_selectd_256(
160 __U, (__v8si)_mm256_dpbsud_epi32(__W, __A, __B), (__v8si)__W);
161}
162
163static __inline__ __m256i __DEFAULT_FN_ATTRS256
164_mm256_maskz_dpbsud_epi32(__mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
165 return (__m256i)__builtin_ia32_selectd_256(
166 __U, (__v8si)_mm256_dpbsud_epi32(__W, __A, __B),
167 (__v8si)_mm256_setzero_si256());
168}
169
170static __inline__ __m128i __DEFAULT_FN_ATTRS128
171_mm_mask_dpbsuds_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
172 return (__m128i)__builtin_ia32_selectd_128(
173 __U, (__v4si)_mm_dpbsuds_epi32(__W, __A, __B), (__v4si)__W);
174}
175
176static __inline__ __m128i __DEFAULT_FN_ATTRS128
177_mm_maskz_dpbsuds_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
178 return (__m128i)__builtin_ia32_selectd_128(
179 __U, (__v4si)_mm_dpbsuds_epi32(__W, __A, __B),
180 (__v4si)_mm_setzero_si128());
181}
182
183static __inline__ __m256i __DEFAULT_FN_ATTRS256
184_mm256_mask_dpbsuds_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
185 return (__m256i)__builtin_ia32_selectd_256(
186 __U, (__v8si)_mm256_dpbsuds_epi32(__W, __A, __B), (__v8si)__W);
187}
188
189static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbsuds_epi32(
190 __mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
191 return (__m256i)__builtin_ia32_selectd_256(
192 __U, (__v8si)_mm256_dpbsuds_epi32(__W, __A, __B),
193 (__v8si)_mm256_setzero_si256());
194}
195
196static __inline__ __m128i __DEFAULT_FN_ATTRS128
197_mm_mask_dpbuud_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
198 return (__m128i)__builtin_ia32_selectd_128(
199 __U, (__v4si)_mm_dpbuud_epi32(__W, __A, __B), (__v4si)__W);
200}
201
202static __inline__ __m128i __DEFAULT_FN_ATTRS128
203_mm_maskz_dpbuud_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
204 return (__m128i)__builtin_ia32_selectd_128(
205 __U, (__v4si)_mm_dpbuud_epi32(__W, __A, __B),
206 (__v4si)_mm_setzero_si128());
207}
208
209static __inline__ __m256i __DEFAULT_FN_ATTRS256
210_mm256_mask_dpbuud_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
211 return (__m256i)__builtin_ia32_selectd_256(
212 __U, (__v8si)_mm256_dpbuud_epi32(__W, __A, __B), (__v8si)__W);
213}
214
215static __inline__ __m256i __DEFAULT_FN_ATTRS256
216_mm256_maskz_dpbuud_epi32(__mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
217 return (__m256i)__builtin_ia32_selectd_256(
218 __U, (__v8si)_mm256_dpbuud_epi32(__W, __A, __B),
219 (__v8si)_mm256_setzero_si256());
220}
221
222static __inline__ __m128i __DEFAULT_FN_ATTRS128
223_mm_mask_dpbuuds_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
224 return (__m128i)__builtin_ia32_selectd_128(
225 __U, (__v4si)_mm_dpbuuds_epi32(__W, __A, __B), (__v4si)__W);
226}
227
228static __inline__ __m128i __DEFAULT_FN_ATTRS128
229_mm_maskz_dpbuuds_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
230 return (__m128i)__builtin_ia32_selectd_128(
231 __U, (__v4si)_mm_dpbuuds_epi32(__W, __A, __B),
232 (__v4si)_mm_setzero_si128());
233}
234
235static __inline__ __m256i __DEFAULT_FN_ATTRS256
236_mm256_mask_dpbuuds_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
237 return (__m256i)__builtin_ia32_selectd_256(
238 __U, (__v8si)_mm256_dpbuuds_epi32(__W, __A, __B), (__v8si)__W);
239}
240
241static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbuuds_epi32(
242 __mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
243 return (__m256i)__builtin_ia32_selectd_256(
244 __U, (__v8si)_mm256_dpbuuds_epi32(__W, __A, __B),
245 (__v8si)_mm256_setzero_si256());
246}
247
248/* VNNI INT16 */
249static __inline__ __m128i __DEFAULT_FN_ATTRS128
250_mm_mask_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
251 return (__m128i)__builtin_ia32_selectd_128(
252 (__mmask8)__U, (__v4si)_mm_dpwsud_epi32(__A, __B, __C), (__v4si)__A);
253}
254
255static __inline__ __m128i __DEFAULT_FN_ATTRS128
256_mm_maskz_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
257 return (__m128i)__builtin_ia32_selectd_128(
258 (__mmask8)__U, (__v4si)_mm_dpwsud_epi32(__A, __B, __C),
259 (__v4si)_mm_setzero_si128());
260}
261
262static __inline__ __m256i __DEFAULT_FN_ATTRS256
263_mm256_mask_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
264 return (__m256i)__builtin_ia32_selectd_256(
265 (__mmask8)__U, (__v8si)_mm256_dpwsud_epi32(__A, __B, __C), (__v8si)__A);
266}
267
268static __inline__ __m256i __DEFAULT_FN_ATTRS256
269_mm256_maskz_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
270 return (__m256i)__builtin_ia32_selectd_256(
271 (__mmask8)__U, (__v8si)_mm256_dpwsud_epi32(__A, __B, __C),
272 (__v8si)_mm256_setzero_si256());
273}
274
275static __inline__ __m128i __DEFAULT_FN_ATTRS128
276_mm_mask_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
277 return (__m128i)__builtin_ia32_selectd_128(
278 (__mmask8)__U, (__v4si)_mm_dpwsuds_epi32(__A, __B, __C), (__v4si)__A);
279}
280
281static __inline__ __m128i __DEFAULT_FN_ATTRS128
282_mm_maskz_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
283 return (__m128i)__builtin_ia32_selectd_128(
284 (__mmask8)__U, (__v4si)_mm_dpwsuds_epi32(__A, __B, __C),
285 (__v4si)_mm_setzero_si128());
286}
287
288static __inline__ __m256i __DEFAULT_FN_ATTRS256
289_mm256_mask_dpwsuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
290 return (__m256i)__builtin_ia32_selectd_256(
291 (__mmask8)__U, (__v8si)_mm256_dpwsuds_epi32(__A, __B, __C), (__v8si)__A);
292}
293
294static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwsuds_epi32(
295 __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
296 return (__m256i)__builtin_ia32_selectd_256(
297 (__mmask8)__U, (__v8si)_mm256_dpwsuds_epi32(__A, __B, __C),
298 (__v8si)_mm256_setzero_si256());
299}
300
301static __inline__ __m128i __DEFAULT_FN_ATTRS128
302_mm_mask_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
303 return (__m128i)__builtin_ia32_selectd_128(
304 (__mmask8)__U, (__v4si)_mm_dpwusd_epi32(__A, __B, __C), (__v4si)__A);
305}
306
307static __inline__ __m128i __DEFAULT_FN_ATTRS128
308_mm_maskz_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
309 return (__m128i)__builtin_ia32_selectd_128(
310 (__mmask8)__U, (__v4si)_mm_dpwusd_epi32(__A, __B, __C),
311 (__v4si)_mm_setzero_si128());
312}
313
314static __inline__ __m256i __DEFAULT_FN_ATTRS256
315_mm256_mask_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
316 return (__m256i)__builtin_ia32_selectd_256(
317 (__mmask8)__U, (__v8si)_mm256_dpwusd_epi32(__A, __B, __C), (__v8si)__A);
318}
319
320static __inline__ __m256i __DEFAULT_FN_ATTRS256
321_mm256_maskz_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
322 return (__m256i)__builtin_ia32_selectd_256(
323 (__mmask8)__U, (__v8si)_mm256_dpwusd_epi32(__A, __B, __C),
324 (__v8si)_mm256_setzero_si256());
325}
326
327static __inline__ __m128i __DEFAULT_FN_ATTRS128
328_mm_mask_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
329 return (__m128i)__builtin_ia32_selectd_128(
330 (__mmask8)__U, (__v4si)_mm_dpwusds_epi32(__A, __B, __C), (__v4si)__A);
331}
332
333static __inline__ __m128i __DEFAULT_FN_ATTRS128
334_mm_maskz_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
335 return (__m128i)__builtin_ia32_selectd_128(
336 (__mmask8)__U, (__v4si)_mm_dpwusds_epi32(__A, __B, __C),
337 (__v4si)_mm_setzero_si128());
338}
339
340static __inline__ __m256i __DEFAULT_FN_ATTRS256
341_mm256_mask_dpwusds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
342 return (__m256i)__builtin_ia32_selectd_256(
343 (__mmask8)__U, (__v8si)_mm256_dpwusds_epi32(__A, __B, __C), (__v8si)__A);
344}
345
346static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwusds_epi32(
347 __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
348 return (__m256i)__builtin_ia32_selectd_256(
349 (__mmask8)__U, (__v8si)_mm256_dpwusds_epi32(__A, __B, __C),
350 (__v8si)_mm256_setzero_si256());
351}
352
353static __inline__ __m128i __DEFAULT_FN_ATTRS128
354_mm_mask_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
355 return (__m128i)__builtin_ia32_selectd_128(
356 (__mmask8)__U, (__v4si)_mm_dpwuud_epi32(__A, __B, __C), (__v4si)__A);
357}
358
359static __inline__ __m128i __DEFAULT_FN_ATTRS128
360_mm_maskz_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
361 return (__m128i)__builtin_ia32_selectd_128(
362 (__mmask8)__U, (__v4si)_mm_dpwuud_epi32(__A, __B, __C),
363 (__v4si)_mm_setzero_si128());
364}
365
366static __inline__ __m256i __DEFAULT_FN_ATTRS256
367_mm256_mask_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
368 return (__m256i)__builtin_ia32_selectd_256(
369 (__mmask8)__U, (__v8si)_mm256_dpwuud_epi32(__A, __B, __C), (__v8si)__A);
370}
371
372static __inline__ __m256i __DEFAULT_FN_ATTRS256
373_mm256_maskz_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
374 return (__m256i)__builtin_ia32_selectd_256(
375 (__mmask8)__U, (__v8si)_mm256_dpwuud_epi32(__A, __B, __C),
376 (__v8si)_mm256_setzero_si256());
377}
378
379static __inline__ __m128i __DEFAULT_FN_ATTRS128
380_mm_mask_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
381 return (__m128i)__builtin_ia32_selectd_128(
382 (__mmask8)__U, (__v4si)_mm_dpwuuds_epi32(__A, __B, __C), (__v4si)__A);
383}
384
385static __inline__ __m128i __DEFAULT_FN_ATTRS128
386_mm_maskz_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
387 return (__m128i)__builtin_ia32_selectd_128(
388 (__mmask8)__U, (__v4si)_mm_dpwuuds_epi32(__A, __B, __C),
389 (__v4si)_mm_setzero_si128());
390}
391
392static __inline__ __m256i __DEFAULT_FN_ATTRS256
393_mm256_mask_dpwuuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
394 return (__m256i)__builtin_ia32_selectd_256(
395 (__mmask8)__U, (__v8si)_mm256_dpwuuds_epi32(__A, __B, __C), (__v8si)__A);
396}
397
398static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwuuds_epi32(
399 __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
400 return (__m256i)__builtin_ia32_selectd_256(
401 (__mmask8)__U, (__v8si)_mm256_dpwuuds_epi32(__A, __B, __C),
402 (__v8si)_mm256_setzero_si256());
403}
404
405/* YMM Rounding */
406#define _mm256_add_round_pd(A, B, R) \
407 ((__m256d)__builtin_ia32_vaddpd256_round((__v4df)(__m256d)(A), \
408 (__v4df)(__m256d)(B), (int)(R)))
409
410#define _mm256_mask_add_round_pd(W, U, A, B, R) \
411 ((__m256d)__builtin_ia32_selectpd_256( \
412 (__mmask8)(U), (__v4df)_mm256_add_round_pd((A), (B), (R)), \
413 (__v4df)(__m256d)(W)))
414
415#define _mm256_maskz_add_round_pd(U, A, B, R) \
416 ((__m256d)__builtin_ia32_selectpd_256( \
417 (__mmask8)(U), (__v4df)_mm256_add_round_pd((A), (B), (R)), \
418 (__v4df)_mm256_setzero_pd()))
419
420#define _mm256_add_round_ph(A, B, R) \
421 ((__m256h)__builtin_ia32_vaddph256_round((__v16hf)(__m256h)(A), \
422 (__v16hf)(__m256h)(B), (int)(R)))
423
424#define _mm256_mask_add_round_ph(W, U, A, B, R) \
425 ((__m256h)__builtin_ia32_selectph_256( \
426 (__mmask16)(U), (__v16hf)_mm256_add_round_ph((A), (B), (R)), \
427 (__v16hf)(__m256h)(W)))
428
429#define _mm256_maskz_add_round_ph(U, A, B, R) \
430 ((__m256h)__builtin_ia32_selectph_256( \
431 (__mmask16)(U), (__v16hf)_mm256_add_round_ph((A), (B), (R)), \
432 (__v16hf)_mm256_setzero_ph()))
433
434#define _mm256_add_round_ps(A, B, R) \
435 ((__m256)__builtin_ia32_vaddps256_round((__v8sf)(__m256)(A), \
436 (__v8sf)(__m256)(B), (int)(R)))
437
438#define _mm256_mask_add_round_ps(W, U, A, B, R) \
439 ((__m256)__builtin_ia32_selectps_256( \
440 (__mmask8)(U), (__v8sf)_mm256_add_round_ps((A), (B), (R)), \
441 (__v8sf)(__m256)(W)))
442
443#define _mm256_maskz_add_round_ps(U, A, B, R) \
444 ((__m256)__builtin_ia32_selectps_256( \
445 (__mmask8)(U), (__v8sf)_mm256_add_round_ps((A), (B), (R)), \
446 (__v8sf)_mm256_setzero_ps()))
447
448#define _mm256_cmp_round_pd_mask(A, B, P, R) \
449 ((__mmask8)__builtin_ia32_vcmppd256_round_mask( \
450 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(P), (__mmask8)-1, \
451 (int)(R)))
452
453#define _mm256_mask_cmp_round_pd_mask(U, A, B, P, R) \
454 ((__mmask8)__builtin_ia32_vcmppd256_round_mask( \
455 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(P), (__mmask8)(U), \
456 (int)(R)))
457
458#define _mm256_cmp_round_ph_mask(A, B, P, R) \
459 ((__mmask16)__builtin_ia32_vcmpph256_round_mask( \
460 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(P), (__mmask16)-1, \
461 (int)(R)))
462
463#define _mm256_mask_cmp_round_ph_mask(U, A, B, P, R) \
464 ((__mmask16)__builtin_ia32_vcmpph256_round_mask( \
465 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(P), (__mmask16)(U), \
466 (int)(R)))
467
468#define _mm256_cmp_round_ps_mask(A, B, P, R) \
469 ((__mmask8)__builtin_ia32_vcmpps256_round_mask( \
470 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(P), (__mmask8)-1, \
471 (int)(R)))
472
473#define _mm256_mask_cmp_round_ps_mask(U, A, B, P, R) \
474 ((__mmask8)__builtin_ia32_vcmpps256_round_mask( \
475 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(P), (__mmask8)(U), \
476 (int)(R)))
477
478#define _mm256_cvt_roundepi32_ph(A, R) \
479 ((__m128h)__builtin_ia32_vcvtdq2ph256_round_mask( \
480 (__v8si)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
481
482#define _mm256_mask_cvt_roundepi32_ph(W, U, A, R) \
483 ((__m128h)__builtin_ia32_vcvtdq2ph256_round_mask((__v8si)(A), (__v8hf)(W), \
484 (__mmask8)(U), (int)(R)))
485
486#define _mm256_maskz_cvt_roundepi32_ph(U, A, R) \
487 ((__m128h)__builtin_ia32_vcvtdq2ph256_round_mask( \
488 (__v8si)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
489
490#define _mm256_cvt_roundepi32_ps(A, R) \
491 ((__m256)__builtin_ia32_vcvtdq2ps256_round_mask((__v8si)(__m256i)(A), \
492 (__v8sf)_mm256_setzero_ps(), \
493 (__mmask8)-1, (int)(R)))
494
495#define _mm256_mask_cvt_roundepi32_ps(W, U, A, R) \
496 ((__m256)__builtin_ia32_vcvtdq2ps256_round_mask( \
497 (__v8si)(__m256i)(A), (__v8sf)(__m256)(W), (__mmask8)(U), (int)(R)))
498
499#define _mm256_maskz_cvt_roundepi32_ps(U, A, R) \
500 ((__m256)__builtin_ia32_vcvtdq2ps256_round_mask((__v8si)(__m256i)(A), \
501 (__v8sf)_mm256_setzero_ps(), \
502 (__mmask8)(U), (int)(R)))
503
504#define _mm256_cvt_roundpd_epi32(A, R) \
505 ((__m128i)__builtin_ia32_vcvtpd2dq256_round_mask( \
506 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)-1, \
507 (int)(R)))
508
509#define _mm256_mask_cvt_roundpd_epi32(W, U, A, R) \
510 ((__m128i)__builtin_ia32_vcvtpd2dq256_round_mask( \
511 (__v4df)(__m256d)(A), (__v4si)(__m128i)(W), (__mmask8)(U), (int)(R)))
512
513#define _mm256_maskz_cvt_roundpd_epi32(U, A, R) \
514 ((__m128i)__builtin_ia32_vcvtpd2dq256_round_mask( \
515 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)(U), \
516 (int)(R)))
517
518#define _mm256_cvt_roundpd_ph(A, R) \
519 ((__m128h)__builtin_ia32_vcvtpd2ph256_round_mask( \
520 (__v4df)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
521
522#define _mm256_mask_cvt_roundpd_ph(W, U, A, R) \
523 ((__m128h)__builtin_ia32_vcvtpd2ph256_round_mask((__v4df)(A), (__v8hf)(W), \
524 (__mmask8)(U), (int)(R)))
525
526#define _mm256_maskz_cvt_roundpd_ph(U, A, R) \
527 ((__m128h)__builtin_ia32_vcvtpd2ph256_round_mask( \
528 (__v4df)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
529
530#define _mm256_cvt_roundpd_ps(A, R) \
531 ((__m128)__builtin_ia32_vcvtpd2ps256_round_mask( \
532 (__v4df)(__m256d)(A), (__v4sf)_mm_setzero_ps(), (__mmask8)-1, (int)(R)))
533
534#define _mm256_mask_cvt_roundpd_ps(W, U, A, R) \
535 ((__m128)__builtin_ia32_vcvtpd2ps256_round_mask( \
536 (__v4df)(__m256d)(A), (__v4sf)(__m128)(W), (__mmask8)(U), (int)(R)))
537
538#define _mm256_maskz_cvt_roundpd_ps(U, A, R) \
539 ((__m128)__builtin_ia32_vcvtpd2ps256_round_mask((__v4df)(__m256d)(A), \
540 (__v4sf)_mm_setzero_ps(), \
541 (__mmask8)(U), (int)(R)))
542
543#define _mm256_cvt_roundpd_epi64(A, R) \
544 ((__m256i)__builtin_ia32_vcvtpd2qq256_round_mask( \
545 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
546 (int)(R)))
547
548#define _mm256_mask_cvt_roundpd_epi64(W, U, A, R) \
549 ((__m256i)__builtin_ia32_vcvtpd2qq256_round_mask( \
550 (__v4df)(__m256d)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
551
552#define _mm256_maskz_cvt_roundpd_epi64(U, A, R) \
553 ((__m256i)__builtin_ia32_vcvtpd2qq256_round_mask( \
554 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
555 (int)(R)))
556
557#define _mm256_cvt_roundpd_epu32(A, R) \
558 ((__m128i)__builtin_ia32_vcvtpd2udq256_round_mask( \
559 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1, \
560 (int)(R)))
561
562#define _mm256_mask_cvt_roundpd_epu32(W, U, A, R) \
563 ((__m128i)__builtin_ia32_vcvtpd2udq256_round_mask( \
564 (__v4df)(__m256d)(A), (__v4su)(__m128i)(W), (__mmask8)(U), (int)(R)))
565
566#define _mm256_maskz_cvt_roundpd_epu32(U, A, R) \
567 ((__m128i)__builtin_ia32_vcvtpd2udq256_round_mask( \
568 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)(U), \
569 (int)(R)))
570
571#define _mm256_cvt_roundpd_epu64(A, R) \
572 ((__m256i)__builtin_ia32_vcvtpd2uqq256_round_mask( \
573 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
574 (int)(R)))
575
576#define _mm256_mask_cvt_roundpd_epu64(W, U, A, R) \
577 ((__m256i)__builtin_ia32_vcvtpd2uqq256_round_mask( \
578 (__v4df)(__m256d)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
579
580#define _mm256_maskz_cvt_roundpd_epu64(U, A, R) \
581 ((__m256i)__builtin_ia32_vcvtpd2uqq256_round_mask( \
582 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
583 (int)(R)))
584
585#define _mm256_cvt_roundph_epi32(A, R) \
586 ((__m256i)__builtin_ia32_vcvtph2dq256_round_mask( \
587 (__v8hf)(A), (__v8si)_mm256_undefined_si256(), (__mmask8)(-1), \
588 (int)(R)))
589
590#define _mm256_mask_cvt_roundph_epi32(W, U, A, R) \
591 ((__m256i)__builtin_ia32_vcvtph2dq256_round_mask((__v8hf)(A), (__v8si)(W), \
592 (__mmask8)(U), (int)(R)))
593
594#define _mm256_maskz_cvt_roundph_epi32(U, A, R) \
595 ((__m256i)__builtin_ia32_vcvtph2dq256_round_mask( \
596 (__v8hf)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
597
598#define _mm256_cvt_roundph_pd(A, R) \
599 ((__m256d)__builtin_ia32_vcvtph2pd256_round_mask( \
600 (__v8hf)(A), (__v4df)_mm256_undefined_pd(), (__mmask8)(-1), (int)(R)))
601
602#define _mm256_mask_cvt_roundph_pd(W, U, A, R) \
603 ((__m256d)__builtin_ia32_vcvtph2pd256_round_mask((__v8hf)(A), (__v4df)(W), \
604 (__mmask8)(U), (int)(R)))
605
606#define _mm256_maskz_cvt_roundph_pd(U, A, R) \
607 ((__m256d)__builtin_ia32_vcvtph2pd256_round_mask( \
608 (__v8hf)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
609
610#define _mm256_cvtx_roundph_ps(A, R) \
611 ((__m256)__builtin_ia32_vcvtph2psx256_round_mask( \
612 (__v8hf)(A), (__v8sf)_mm256_undefined_ps(), (__mmask8)(-1), (int)(R)))
613
614#define _mm256_mask_cvtx_roundph_ps(W, U, A, R) \
615 ((__m256)__builtin_ia32_vcvtph2psx256_round_mask((__v8hf)(A), (__v8sf)(W), \
616 (__mmask8)(U), (int)(R)))
617
618#define _mm256_maskz_cvtx_roundph_ps(U, A, R) \
619 ((__m256)__builtin_ia32_vcvtph2psx256_round_mask( \
620 (__v8hf)(A), (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
621
622#define _mm256_cvt_roundph_epi64(A, R) \
623 ((__m256i)__builtin_ia32_vcvtph2qq256_round_mask( \
624 (__v8hf)(A), (__v4di)_mm256_undefined_si256(), (__mmask8)(-1), \
625 (int)(R)))
626
627#define _mm256_mask_cvt_roundph_epi64(W, U, A, R) \
628 ((__m256i)__builtin_ia32_vcvtph2qq256_round_mask((__v8hf)(A), (__v4di)(W), \
629 (__mmask8)(U), (int)(R)))
630
631#define _mm256_maskz_cvt_roundph_epi64(U, A, R) \
632 ((__m256i)__builtin_ia32_vcvtph2qq256_round_mask( \
633 (__v8hf)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
634
635#define _mm256_cvt_roundph_epu32(A, R) \
636 ((__m256i)__builtin_ia32_vcvtph2udq256_round_mask( \
637 (__v8hf)(A), (__v8su)_mm256_undefined_si256(), (__mmask8)(-1), \
638 (int)(R)))
639
640#define _mm256_mask_cvt_roundph_epu32(W, U, A, R) \
641 ((__m256i)__builtin_ia32_vcvtph2udq256_round_mask((__v8hf)(A), (__v8su)(W), \
642 (__mmask8)(U), (int)(R)))
643
644#define _mm256_maskz_cvt_roundph_epu32(U, A, R) \
645 ((__m256i)__builtin_ia32_vcvtph2udq256_round_mask( \
646 (__v8hf)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
647
648#define _mm256_cvt_roundph_epu64(A, R) \
649 ((__m256i)__builtin_ia32_vcvtph2uqq256_round_mask( \
650 (__v8hf)(A), (__v4du)_mm256_undefined_si256(), (__mmask8)(-1), \
651 (int)(R)))
652
653#define _mm256_mask_cvt_roundph_epu64(W, U, A, R) \
654 ((__m256i)__builtin_ia32_vcvtph2uqq256_round_mask((__v8hf)(A), (__v4du)(W), \
655 (__mmask8)(U), (int)(R)))
656
657#define _mm256_maskz_cvt_roundph_epu64(U, A, R) \
658 ((__m256i)__builtin_ia32_vcvtph2uqq256_round_mask( \
659 (__v8hf)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
660
661#define _mm256_cvt_roundph_epu16(A, R) \
662 ((__m256i)__builtin_ia32_vcvtph2uw256_round_mask( \
663 (__v16hf)(A), (__v16hu)_mm256_undefined_si256(), (__mmask16)(-1), \
664 (int)(R)))
665
666#define _mm256_mask_cvt_roundph_epu16(W, U, A, R) \
667 ((__m256i)__builtin_ia32_vcvtph2uw256_round_mask((__v16hf)(A), (__v16hu)(W), \
668 (__mmask16)(U), (int)(R)))
669
670#define _mm256_maskz_cvt_roundph_epu16(U, A, R) \
671 ((__m256i)__builtin_ia32_vcvtph2uw256_round_mask( \
672 (__v16hf)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
673 (int)(R)))
674
675#define _mm256_cvt_roundph_epi16(A, R) \
676 ((__m256i)__builtin_ia32_vcvtph2w256_round_mask( \
677 (__v16hf)(A), (__v16hi)_mm256_undefined_si256(), (__mmask16)(-1), \
678 (int)(R)))
679
680#define _mm256_mask_cvt_roundph_epi16(W, U, A, R) \
681 ((__m256i)__builtin_ia32_vcvtph2w256_round_mask((__v16hf)(A), (__v16hi)(W), \
682 (__mmask16)(U), (int)(R)))
683
684#define _mm256_maskz_cvt_roundph_epi16(U, A, R) \
685 ((__m256i)__builtin_ia32_vcvtph2w256_round_mask( \
686 (__v16hf)(A), (__v16hi)_mm256_setzero_si256(), (__mmask16)(U), \
687 (int)(R)))
688
689#define _mm256_cvt_roundps_epi32(A, R) \
690 ((__m256i)__builtin_ia32_vcvtps2dq256_round_mask( \
691 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)-1, \
692 (int)(R)))
693
694#define _mm256_mask_cvt_roundps_epi32(W, U, A, R) \
695 ((__m256i)__builtin_ia32_vcvtps2dq256_round_mask( \
696 (__v8sf)(__m256)(A), (__v8si)(__m256i)(W), (__mmask8)(U), (int)(R)))
697
698#define _mm256_maskz_cvt_roundps_epi32(U, A, R) \
699 ((__m256i)__builtin_ia32_vcvtps2dq256_round_mask( \
700 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), \
701 (int)(R)))
702
703#define _mm256_cvt_roundps_pd(A, R) \
704 ((__m256d)__builtin_ia32_vcvtps2pd256_round_mask( \
705 (__v4sf)(__m128)(A), (__v4df)_mm256_undefined_pd(), (__mmask8)-1, \
706 (int)(R)))
707
708#define _mm256_mask_cvt_roundps_pd(W, U, A, R) \
709 ((__m256d)__builtin_ia32_vcvtps2pd256_round_mask( \
710 (__v4sf)(__m128)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
711
712#define _mm256_maskz_cvt_roundps_pd(U, A, R) \
713 ((__m256d)__builtin_ia32_vcvtps2pd256_round_mask( \
714 (__v4sf)(__m128)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
715 (int)(R)))
716
717#define _mm256_cvt_roundps_ph(A, I) \
718 ((__m128i)__builtin_ia32_vcvtps2ph256_mask((__v8sf)(__m256)(A), (int)(I), \
719 (__v8hi)_mm_undefined_si128(), \
720 (__mmask8)-1))
721
722/* FIXME: We may use these way in future.
723#define _mm256_cvt_roundps_ph(A, I) \
724 ((__m128i)__builtin_ia32_vcvtps2ph256_round_mask( \
725 (__v8sf)(__m256)(A), (int)(I), (__v8hi)_mm_undefined_si128(), \
726 (__mmask8)-1))
727#define _mm256_mask_cvt_roundps_ph(U, W, A, I) \
728 ((__m128i)__builtin_ia32_vcvtps2ph256_round_mask( \
729 (__v8sf)(__m256)(A), (int)(I), (__v8hi)(__m128i)(U), (__mmask8)(W)))
730#define _mm256_maskz_cvt_roundps_ph(W, A, I) \
731 ((__m128i)__builtin_ia32_vcvtps2ph256_round_mask( \
732 (__v8sf)(__m256)(A), (int)(I), (__v8hi)_mm_setzero_si128(), \
733 (__mmask8)(W))) */
734
735#define _mm256_cvtx_roundps_ph(A, R) \
736 ((__m128h)__builtin_ia32_vcvtps2phx256_round_mask( \
737 (__v8sf)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
738
739#define _mm256_mask_cvtx_roundps_ph(W, U, A, R) \
740 ((__m128h)__builtin_ia32_vcvtps2phx256_round_mask((__v8sf)(A), (__v8hf)(W), \
741 (__mmask8)(U), (int)(R)))
742
743#define _mm256_maskz_cvtx_roundps_ph(U, A, R) \
744 ((__m128h)__builtin_ia32_vcvtps2phx256_round_mask( \
745 (__v8sf)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
746
747#define _mm256_cvt_roundps_epi64(A, R) \
748 ((__m256i)__builtin_ia32_vcvtps2qq256_round_mask( \
749 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
750 (int)(R)))
751
752#define _mm256_mask_cvt_roundps_epi64(W, U, A, R) \
753 ((__m256i)__builtin_ia32_vcvtps2qq256_round_mask( \
754 (__v4sf)(__m128)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
755
756#define _mm256_maskz_cvt_roundps_epi64(U, A, R) \
757 ((__m256i)__builtin_ia32_vcvtps2qq256_round_mask( \
758 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
759 (int)(R)))
760
761#define _mm256_cvt_roundps_epu32(A, R) \
762 ((__m256i)__builtin_ia32_vcvtps2udq256_round_mask( \
763 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
764 (int)(R)))
765
766#define _mm256_mask_cvt_roundps_epu32(W, U, A, R) \
767 ((__m256i)__builtin_ia32_vcvtps2udq256_round_mask( \
768 (__v8sf)(__m256)(A), (__v8su)(__m256i)(W), (__mmask8)(U), (int)(R)))
769
770#define _mm256_maskz_cvt_roundps_epu32(U, A, R) \
771 ((__m256i)__builtin_ia32_vcvtps2udq256_round_mask( \
772 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), \
773 (int)(R)))
774
775#define _mm256_cvt_roundps_epu64(A, R) \
776 ((__m256i)__builtin_ia32_vcvtps2uqq256_round_mask( \
777 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
778 (int)(R)))
779
780#define _mm256_mask_cvt_roundps_epu64(W, U, A, R) \
781 ((__m256i)__builtin_ia32_vcvtps2uqq256_round_mask( \
782 (__v4sf)(__m128)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
783
784#define _mm256_maskz_cvt_roundps_epu64(U, A, R) \
785 ((__m256i)__builtin_ia32_vcvtps2uqq256_round_mask( \
786 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
787 (int)(R)))
788
789#define _mm256_cvt_roundepi64_pd(A, R) \
790 ((__m256d)__builtin_ia32_vcvtqq2pd256_round_mask( \
791 (__v4di)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)-1, \
792 (int)(R)))
793
794#define _mm256_mask_cvt_roundepi64_pd(W, U, A, R) \
795 ((__m256d)__builtin_ia32_vcvtqq2pd256_round_mask( \
796 (__v4di)(__m256i)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
797
798#define _mm256_maskz_cvt_roundepi64_pd(U, A, R) \
799 ((__m256d)__builtin_ia32_vcvtqq2pd256_round_mask( \
800 (__v4di)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
801 (int)(R)))
802
803#define _mm256_cvt_roundepi64_ph(A, R) \
804 ((__m128h)__builtin_ia32_vcvtqq2ph256_round_mask( \
805 (__v4di)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
806
807#define _mm256_mask_cvt_roundepi64_ph(W, U, A, R) \
808 ((__m128h)__builtin_ia32_vcvtqq2ph256_round_mask((__v4di)(A), (__v8hf)(W), \
809 (__mmask8)(U), (int)(R)))
810
811#define _mm256_maskz_cvt_roundepi64_ph(U, A, R) \
812 ((__m128h)__builtin_ia32_vcvtqq2ph256_round_mask( \
813 (__v4di)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
814
815#define _mm256_cvt_roundepi64_ps(A, R) \
816 ((__m128)__builtin_ia32_vcvtqq2ps256_round_mask( \
817 (__v4di)(__m256i)(A), (__v4sf)_mm_setzero_ps(), (__mmask8)-1, (int)(R)))
818
819#define _mm256_mask_cvt_roundepi64_ps(W, U, A, R) \
820 ((__m128)__builtin_ia32_vcvtqq2ps256_round_mask( \
821 (__v4di)(__m256i)(A), (__v4sf)(__m128)(W), (__mmask8)(U), (int)(R)))
822
823#define _mm256_maskz_cvt_roundepi64_ps(U, A, R) \
824 ((__m128)__builtin_ia32_vcvtqq2ps256_round_mask((__v4di)(__m256i)(A), \
825 (__v4sf)_mm_setzero_ps(), \
826 (__mmask8)(U), (int)(R)))
827
828#define _mm256_cvtt_roundpd_epi32(A, R) \
829 ((__m128i)__builtin_ia32_vcvttpd2dq256_round_mask( \
830 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)-1, \
831 (int)(R)))
832
833#define _mm256_mask_cvtt_roundpd_epi32(W, U, A, R) \
834 ((__m128i)__builtin_ia32_vcvttpd2dq256_round_mask( \
835 (__v4df)(__m256d)(A), (__v4si)(__m128i)(W), (__mmask8)(U), (int)(R)))
836
837#define _mm256_maskz_cvtt_roundpd_epi32(U, A, R) \
838 ((__m128i)__builtin_ia32_vcvttpd2dq256_round_mask( \
839 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)(U), \
840 (int)(R)))
841
842#define _mm256_cvtt_roundpd_epi64(A, R) \
843 ((__m256i)__builtin_ia32_vcvttpd2qq256_round_mask( \
844 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
845 (int)(R)))
846
847#define _mm256_mask_cvtt_roundpd_epi64(W, U, A, R) \
848 ((__m256i)__builtin_ia32_vcvttpd2qq256_round_mask( \
849 (__v4df)(__m256d)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
850
851#define _mm256_maskz_cvtt_roundpd_epi64(U, A, R) \
852 ((__m256i)__builtin_ia32_vcvttpd2qq256_round_mask( \
853 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
854 (int)(R)))
855
856#define _mm256_cvtt_roundpd_epu32(A, R) \
857 ((__m128i)__builtin_ia32_vcvttpd2udq256_round_mask( \
858 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1, \
859 (int)(R)))
860
861#define _mm256_mask_cvtt_roundpd_epu32(W, U, A, R) \
862 ((__m128i)__builtin_ia32_vcvttpd2udq256_round_mask( \
863 (__v4df)(__m256d)(A), (__v4su)(__m128i)(W), (__mmask8)(U), (int)(R)))
864
865#define _mm256_maskz_cvtt_roundpd_epu32(U, A, R) \
866 ((__m128i)__builtin_ia32_vcvttpd2udq256_round_mask( \
867 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)(U), \
868 (int)(R)))
869
870#define _mm256_cvtt_roundpd_epu64(A, R) \
871 ((__m256i)__builtin_ia32_vcvttpd2uqq256_round_mask( \
872 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
873 (int)(R)))
874
875#define _mm256_mask_cvtt_roundpd_epu64(W, U, A, R) \
876 ((__m256i)__builtin_ia32_vcvttpd2uqq256_round_mask( \
877 (__v4df)(__m256d)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
878
879#define _mm256_maskz_cvtt_roundpd_epu64(U, A, R) \
880 ((__m256i)__builtin_ia32_vcvttpd2uqq256_round_mask( \
881 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
882 (int)(R)))
883
884#define _mm256_cvtt_roundph_epi32(A, R) \
885 ((__m256i)__builtin_ia32_vcvttph2dq256_round_mask( \
886 (__v8hf)(A), (__v8si)_mm256_undefined_si256(), (__mmask8)(-1), \
887 (int)(R)))
888
889#define _mm256_mask_cvtt_roundph_epi32(W, U, A, R) \
890 ((__m256i)__builtin_ia32_vcvttph2dq256_round_mask((__v8hf)(A), (__v8si)(W), \
891 (__mmask8)(U), (int)(R)))
892
893#define _mm256_maskz_cvtt_roundph_epi32(U, A, R) \
894 ((__m256i)__builtin_ia32_vcvttph2dq256_round_mask( \
895 (__v8hf)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
896
897#define _mm256_cvtt_roundph_epi64(A, R) \
898 ((__m256i)__builtin_ia32_vcvttph2qq256_round_mask( \
899 (__v8hf)(A), (__v4di)_mm256_undefined_si256(), (__mmask8)(-1), \
900 (int)(R)))
901
902#define _mm256_mask_cvtt_roundph_epi64(W, U, A, R) \
903 ((__m256i)__builtin_ia32_vcvttph2qq256_round_mask((__v8hf)(A), (__v4di)(W), \
904 (__mmask8)(U), (int)(R)))
905
906#define _mm256_maskz_cvtt_roundph_epi64(U, A, R) \
907 ((__m256i)__builtin_ia32_vcvttph2qq256_round_mask( \
908 (__v8hf)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
909
910#define _mm256_cvtt_roundph_epu32(A, R) \
911 ((__m256i)__builtin_ia32_vcvttph2udq256_round_mask( \
912 (__v8hf)(A), (__v8su)_mm256_undefined_si256(), (__mmask8)(-1), \
913 (int)(R)))
914
915#define _mm256_mask_cvtt_roundph_epu32(W, U, A, R) \
916 ((__m256i)__builtin_ia32_vcvttph2udq256_round_mask((__v8hf)(A), (__v8su)(W), \
917 (__mmask8)(U), (int)(R)))
918
919#define _mm256_maskz_cvtt_roundph_epu32(U, A, R) \
920 ((__m256i)__builtin_ia32_vcvttph2udq256_round_mask( \
921 (__v8hf)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
922
923#define _mm256_cvtt_roundph_epu64(A, R) \
924 ((__m256i)__builtin_ia32_vcvttph2uqq256_round_mask( \
925 (__v8hf)(A), (__v4du)_mm256_undefined_si256(), (__mmask8)(-1), \
926 (int)(R)))
927
928#define _mm256_mask_cvtt_roundph_epu64(W, U, A, R) \
929 ((__m256i)__builtin_ia32_vcvttph2uqq256_round_mask((__v8hf)(A), (__v4du)(W), \
930 (__mmask8)(U), (int)(R)))
931
932#define _mm256_maskz_cvtt_roundph_epu64(U, A, R) \
933 ((__m256i)__builtin_ia32_vcvttph2uqq256_round_mask( \
934 (__v8hf)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
935
936#define _mm256_cvtt_roundph_epu16(A, R) \
937 ((__m256i)__builtin_ia32_vcvttph2uw256_round_mask( \
938 (__v16hf)(A), (__v16hu)_mm256_undefined_si256(), (__mmask16)(-1), \
939 (int)(R)))
940
941#define _mm256_mask_cvtt_roundph_epu16(W, U, A, R) \
942 ((__m256i)__builtin_ia32_vcvttph2uw256_round_mask( \
943 (__v16hf)(A), (__v16hu)(W), (__mmask16)(U), (int)(R)))
944
945#define _mm256_maskz_cvtt_roundph_epu16(U, A, R) \
946 ((__m256i)__builtin_ia32_vcvttph2uw256_round_mask( \
947 (__v16hf)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
948 (int)(R)))
949
950#define _mm256_cvtt_roundph_epi16(A, R) \
951 ((__m256i)__builtin_ia32_vcvttph2w256_round_mask( \
952 (__v16hf)(A), (__v16hi)_mm256_undefined_si256(), (__mmask16)(-1), \
953 (int)(R)))
954
955#define _mm256_mask_cvtt_roundph_epi16(W, U, A, R) \
956 ((__m256i)__builtin_ia32_vcvttph2w256_round_mask((__v16hf)(A), (__v16hi)(W), \
957 (__mmask16)(U), (int)(R)))
958
959#define _mm256_maskz_cvtt_roundph_epi16(U, A, R) \
960 ((__m256i)__builtin_ia32_vcvttph2w256_round_mask( \
961 (__v16hf)(A), (__v16hi)_mm256_setzero_si256(), (__mmask16)(U), \
962 (int)(R)))
963
964#define _mm256_cvtt_roundps_epi32(A, R) \
965 ((__m256i)__builtin_ia32_vcvttps2dq256_round_mask( \
966 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)-1, \
967 (int)(R)))
968
969#define _mm256_mask_cvtt_roundps_epi32(W, U, A, R) \
970 ((__m256i)__builtin_ia32_vcvttps2dq256_round_mask( \
971 (__v8sf)(__m256)(A), (__v8si)(__m256i)(W), (__mmask8)(U), (int)(R)))
972
973#define _mm256_maskz_cvtt_roundps_epi32(U, A, R) \
974 ((__m256i)__builtin_ia32_vcvttps2dq256_round_mask( \
975 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), \
976 (int)(R)))
977
978#define _mm256_cvtt_roundps_epi64(A, R) \
979 ((__m256i)__builtin_ia32_vcvttps2qq256_round_mask( \
980 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
981 (int)(R)))
982
983#define _mm256_mask_cvtt_roundps_epi64(W, U, A, R) \
984 ((__m256i)__builtin_ia32_vcvttps2qq256_round_mask( \
985 (__v4sf)(__m128)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
986
987#define _mm256_maskz_cvtt_roundps_epi64(U, A, R) \
988 ((__m256i)__builtin_ia32_vcvttps2qq256_round_mask( \
989 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
990 (int)(R)))
991
992#define _mm256_cvtt_roundps_epu32(A, R) \
993 ((__m256i)__builtin_ia32_vcvttps2udq256_round_mask( \
994 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
995 (int)(R)))
996
997#define _mm256_mask_cvtt_roundps_epu32(W, U, A, R) \
998 ((__m256i)__builtin_ia32_vcvttps2udq256_round_mask( \
999 (__v8sf)(__m256)(A), (__v8su)(__m256i)(W), (__mmask8)(U), (int)(R)))
1000
1001#define _mm256_maskz_cvtt_roundps_epu32(U, A, R) \
1002 ((__m256i)__builtin_ia32_vcvttps2udq256_round_mask( \
1003 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), \
1004 (int)(R)))
1005
1006#define _mm256_cvtt_roundps_epu64(A, R) \
1007 ((__m256i)__builtin_ia32_vcvttps2uqq256_round_mask( \
1008 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
1009 (int)(R)))
1010
1011#define _mm256_mask_cvtt_roundps_epu64(W, U, A, R) \
1012 ((__m256i)__builtin_ia32_vcvttps2uqq256_round_mask( \
1013 (__v4sf)(__m128)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
1014
1015#define _mm256_maskz_cvtt_roundps_epu64(U, A, R) \
1016 ((__m256i)__builtin_ia32_vcvttps2uqq256_round_mask( \
1017 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
1018 (int)(R)))
1019
1020#define _mm256_cvt_roundepu32_ph(A, R) \
1021 ((__m128h)__builtin_ia32_vcvtudq2ph256_round_mask( \
1022 (__v8su)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
1023
1024#define _mm256_mask_cvt_roundepu32_ph(W, U, A, R) \
1025 ((__m128h)__builtin_ia32_vcvtudq2ph256_round_mask((__v8su)(A), (__v8hf)(W), \
1026 (__mmask8)(U), (int)(R)))
1027
1028#define _mm256_maskz_cvt_roundepu32_ph(U, A, R) \
1029 ((__m128h)__builtin_ia32_vcvtudq2ph256_round_mask( \
1030 (__v8su)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
1031
1032#define _mm256_cvt_roundepu32_ps(A, R) \
1033 ((__m256)__builtin_ia32_vcvtudq2ps256_round_mask( \
1034 (__v8su)(__m256i)(A), (__v8sf)_mm256_setzero_ps(), (__mmask8)-1, \
1035 (int)(R)))
1036
1037#define _mm256_mask_cvt_roundepu32_ps(W, U, A, R) \
1038 ((__m256)__builtin_ia32_vcvtudq2ps256_round_mask( \
1039 (__v8su)(__m256i)(A), (__v8sf)(__m256)(W), (__mmask8)(U), (int)(R)))
1040
1041#define _mm256_maskz_cvt_roundepu32_ps(U, A, R) \
1042 ((__m256)__builtin_ia32_vcvtudq2ps256_round_mask( \
1043 (__v8su)(__m256i)(A), (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), \
1044 (int)(R)))
1045
1046#define _mm256_cvt_roundepu64_pd(A, R) \
1047 ((__m256d)__builtin_ia32_vcvtuqq2pd256_round_mask( \
1048 (__v4du)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)-1, \
1049 (int)(R)))
1050
1051#define _mm256_mask_cvt_roundepu64_pd(W, U, A, R) \
1052 ((__m256d)__builtin_ia32_vcvtuqq2pd256_round_mask( \
1053 (__v4du)(__m256i)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
1054
1055#define _mm256_maskz_cvt_roundepu64_pd(U, A, R) \
1056 ((__m256d)__builtin_ia32_vcvtuqq2pd256_round_mask( \
1057 (__v4du)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
1058 (int)(R)))
1059
1060#define _mm256_cvt_roundepu64_ph(A, R) \
1061 ((__m128h)__builtin_ia32_vcvtuqq2ph256_round_mask( \
1062 (__v4du)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
1063
1064#define _mm256_mask_cvt_roundepu64_ph(W, U, A, R) \
1065 ((__m128h)__builtin_ia32_vcvtuqq2ph256_round_mask((__v4du)(A), (__v8hf)(W), \
1066 (__mmask8)(U), (int)(R)))
1067
1068#define _mm256_maskz_cvt_roundepu64_ph(U, A, R) \
1069 ((__m128h)__builtin_ia32_vcvtuqq2ph256_round_mask( \
1070 (__v4du)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
1071
1072#define _mm256_cvt_roundepu64_ps(A, R) \
1073 ((__m128)__builtin_ia32_vcvtuqq2ps256_round_mask( \
1074 (__v4du)(__m256i)(A), (__v4sf)_mm_setzero_ps(), (__mmask8)-1, (int)(R)))
1075
1076#define _mm256_mask_cvt_roundepu64_ps(W, U, A, R) \
1077 ((__m128)__builtin_ia32_vcvtuqq2ps256_round_mask( \
1078 (__v4du)(__m256i)(A), (__v4sf)(__m128)(W), (__mmask8)(U), (int)(R)))
1079
1080#define _mm256_maskz_cvt_roundepu64_ps(U, A, R) \
1081 ((__m128)__builtin_ia32_vcvtuqq2ps256_round_mask((__v4du)(__m256i)(A), \
1082 (__v4sf)_mm_setzero_ps(), \
1083 (__mmask8)(U), (int)(R)))
1084
1085#define _mm256_cvt_roundepu16_ph(A, R) \
1086 ((__m256h)__builtin_ia32_vcvtuw2ph256_round_mask( \
1087 (__v16hu)(A), (__v16hf)_mm256_undefined_ph(), (__mmask16)(-1), \
1088 (int)(R)))
1089
1090#define _mm256_mask_cvt_roundepu16_ph(W, U, A, R) \
1091 ((__m256h)__builtin_ia32_vcvtuw2ph256_round_mask((__v16hu)(A), (__v16hf)(W), \
1092 (__mmask16)(U), (int)(R)))
1093
1094#define _mm256_maskz_cvt_roundepu16_ph(U, A, R) \
1095 ((__m256h)__builtin_ia32_vcvtuw2ph256_round_mask( \
1096 (__v16hu)(A), (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1097
1098#define _mm256_cvt_roundepi16_ph(A, R) \
1099 ((__m256h)__builtin_ia32_vcvtw2ph256_round_mask( \
1100 (__v16hi)(A), (__v16hf)_mm256_undefined_ph(), (__mmask16)(-1), \
1101 (int)(R)))
1102
1103#define _mm256_mask_cvt_roundepi16_ph(W, U, A, R) \
1104 ((__m256h)__builtin_ia32_vcvtw2ph256_round_mask((__v16hi)(A), (__v16hf)(W), \
1105 (__mmask16)(U), (int)(R)))
1106
1107#define _mm256_maskz_cvt_roundepi16_ph(U, A, R) \
1108 ((__m256h)__builtin_ia32_vcvtw2ph256_round_mask( \
1109 (__v16hi)(A), (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1110
1111#define _mm256_div_round_pd(A, B, R) \
1112 ((__m256d)__builtin_ia32_vdivpd256_round((__v4df)(__m256d)(A), \
1113 (__v4df)(__m256d)(B), (int)(R)))
1114
1115#define _mm256_mask_div_round_pd(W, U, A, B, R) \
1116 ((__m256d)__builtin_ia32_selectpd_256( \
1117 (__mmask8)(U), (__v4df)_mm256_div_round_pd((A), (B), (R)), \
1118 (__v4df)(__m256d)(W)))
1119
1120#define _mm256_maskz_div_round_pd(U, A, B, R) \
1121 ((__m256d)__builtin_ia32_selectpd_256( \
1122 (__mmask8)(U), (__v4df)_mm256_div_round_pd((A), (B), (R)), \
1123 (__v4df)_mm256_setzero_pd()))
1124
1125#define _mm256_div_round_ph(A, B, R) \
1126 ((__m256h)__builtin_ia32_vdivph256_round((__v16hf)(__m256h)(A), \
1127 (__v16hf)(__m256h)(B), (int)(R)))
1128
1129#define _mm256_mask_div_round_ph(W, U, A, B, R) \
1130 ((__m256h)__builtin_ia32_selectph_256( \
1131 (__mmask16)(U), (__v16hf)_mm256_div_round_ph((A), (B), (R)), \
1132 (__v16hf)(__m256h)(W)))
1133
1134#define _mm256_maskz_div_round_ph(U, A, B, R) \
1135 ((__m256h)__builtin_ia32_selectph_256( \
1136 (__mmask16)(U), (__v16hf)_mm256_div_round_ph((A), (B), (R)), \
1137 (__v16hf)_mm256_setzero_ph()))
1138
1139#define _mm256_div_round_ps(A, B, R) \
1140 ((__m256)__builtin_ia32_vdivps256_round((__v8sf)(__m256)(A), \
1141 (__v8sf)(__m256)(B), (int)(R)))
1142
1143#define _mm256_mask_div_round_ps(W, U, A, B, R) \
1144 ((__m256)__builtin_ia32_selectps_256( \
1145 (__mmask8)(U), (__v8sf)_mm256_div_round_ps((A), (B), (R)), \
1146 (__v8sf)(__m256)(W)))
1147
1148#define _mm256_maskz_div_round_ps(U, A, B, R) \
1149 ((__m256)__builtin_ia32_selectps_256( \
1150 (__mmask8)(U), (__v8sf)_mm256_div_round_ps((A), (B), (R)), \
1151 (__v8sf)_mm256_setzero_ps()))
1152
1153#define _mm256_fcmadd_round_pch(A, B, C, R) \
1154 ((__m256h)__builtin_ia32_vfcmaddcph256_round_mask3( \
1155 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1156 (__mmask8)-1, (int)(R)))
1157
1158#define _mm256_mask_fcmadd_round_pch(A, U, B, C, R) \
1159 ((__m256h)__builtin_ia32_vfcmaddcph256_round_mask( \
1160 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1161 (__mmask8)(U), (int)(R)))
1162
1163#define _mm256_mask3_fcmadd_round_pch(A, B, C, U, R) \
1164 ((__m256h)__builtin_ia32_vfcmaddcph256_round_mask3( \
1165 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1166 (__mmask8)(U), (int)(R)))
1167
1168#define _mm256_maskz_fcmadd_round_pch(U, A, B, C, R) \
1169 ((__m256h)__builtin_ia32_vfcmaddcph256_round_maskz( \
1170 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1171 (__mmask8)(U), (int)(R)))
1172
1173#define _mm256_cmul_round_pch(A, B, R) \
1174 ((__m256h)__builtin_ia32_vfcmulcph256_round_mask( \
1175 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1176 (__v8sf)(__m256h)_mm256_undefined_ph(), (__mmask8)-1, (int)(R)))
1177
1178#define _mm256_mask_cmul_round_pch(W, U, A, B, R) \
1179 ((__m256h)__builtin_ia32_vfcmulcph256_round_mask( \
1180 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(W), \
1181 (__mmask8)(U), (int)(R)))
1182
1183#define _mm256_maskz_cmul_round_pch(U, A, B, R) \
1184 ((__m256h)__builtin_ia32_vfcmulcph256_round_mask( \
1185 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1186 (__v8sf)(__m256h)_mm256_setzero_ph(), (__mmask8)(U), (int)(R)))
1187
1188#define _mm256_fixupimm_round_pd(A, B, C, imm, R) \
1189 ((__m256d)__builtin_ia32_vfixupimmpd256_round_mask( \
1190 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4di)(__m256i)(C), \
1191 (int)(imm), (__mmask8)-1, (int)(R)))
1192
1193#define _mm256_mask_fixupimm_round_pd(A, U, B, C, imm, R) \
1194 ((__m256d)__builtin_ia32_vfixupimmpd256_round_mask( \
1195 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4di)(__m256i)(C), \
1196 (int)(imm), (__mmask8)(U), (int)(R)))
1197
1198#define _mm256_maskz_fixupimm_round_pd(U, A, B, C, imm, R) \
1199 ((__m256d)__builtin_ia32_vfixupimmpd256_round_maskz( \
1200 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4di)(__m256i)(C), \
1201 (int)(imm), (__mmask8)(U), (int)(R)))
1202
1203#define _mm256_fixupimm_round_ps(A, B, C, imm, R) \
1204 ((__m256)__builtin_ia32_vfixupimmps256_round_mask( \
1205 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8si)(__m256i)(C), \
1206 (int)(imm), (__mmask8)-1, (int)(R)))
1207
1208#define _mm256_mask_fixupimm_round_ps(A, U, B, C, imm, R) \
1209 ((__m256)__builtin_ia32_vfixupimmps256_round_mask( \
1210 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8si)(__m256i)(C), \
1211 (int)(imm), (__mmask8)(U), (int)(R)))
1212
1213#define _mm256_maskz_fixupimm_round_ps(U, A, B, C, imm, R) \
1214 ((__m256)__builtin_ia32_vfixupimmps256_round_maskz( \
1215 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8si)(__m256i)(C), \
1216 (int)(imm), (__mmask8)(U), (int)(R)))
1217
1218#define _mm256_fmadd_round_pd(A, B, C, R) \
1219 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1220 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1221 (__mmask8)-1, (int)(R)))
1222
1223#define _mm256_mask_fmadd_round_pd(A, U, B, C, R) \
1224 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1225 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1226 (__mmask8)(U), (int)(R)))
1227
1228#define _mm256_mask3_fmadd_round_pd(A, B, C, U, R) \
1229 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask3( \
1230 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1231 (__mmask8)(U), (int)(R)))
1232
1233#define _mm256_maskz_fmadd_round_pd(U, A, B, C, R) \
1234 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1235 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1236 (__mmask8)(U), (int)(R)))
1237
1238#define _mm256_fmsub_round_pd(A, B, C, R) \
1239 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1240 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1241 (__mmask8)-1, (int)(R)))
1242
1243#define _mm256_mask_fmsub_round_pd(A, U, B, C, R) \
1244 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1245 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1246 (__mmask8)(U), (int)(R)))
1247
1248#define _mm256_maskz_fmsub_round_pd(U, A, B, C, R) \
1249 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1250 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1251 (__mmask8)(U), (int)(R)))
1252
1253#define _mm256_fnmadd_round_pd(A, B, C, R) \
1254 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1255 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1256 (__mmask8)-1, (int)(R)))
1257
1258#define _mm256_mask3_fnmadd_round_pd(A, B, C, U, R) \
1259 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask3( \
1260 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1261 (__mmask8)(U), (int)(R)))
1262
1263#define _mm256_maskz_fnmadd_round_pd(U, A, B, C, R) \
1264 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1265 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1266 (__mmask8)(U), (int)(R)))
1267
1268#define _mm256_fnmsub_round_pd(A, B, C, R) \
1269 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1270 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1271 (__mmask8)-1, (int)(R)))
1272
1273#define _mm256_maskz_fnmsub_round_pd(U, A, B, C, R) \
1274 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1275 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1276 (__mmask8)(U), (int)(R)))
1277
1278#define _mm256_fmadd_round_ph(A, B, C, R) \
1279 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1280 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1281 (__mmask16)-1, (int)(R)))
1282
1283#define _mm256_mask_fmadd_round_ph(A, U, B, C, R) \
1284 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1285 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1286 (__mmask16)(U), (int)(R)))
1287
1288#define _mm256_mask3_fmadd_round_ph(A, B, C, U, R) \
1289 ((__m256h)__builtin_ia32_vfmaddph256_round_mask3( \
1290 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1291 (__mmask16)(U), (int)(R)))
1292
1293#define _mm256_maskz_fmadd_round_ph(U, A, B, C, R) \
1294 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1295 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1296 (__mmask16)(U), (int)(R)))
1297
1298#define _mm256_fmsub_round_ph(A, B, C, R) \
1299 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1300 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1301 (__mmask16)-1, (int)(R)))
1302
1303#define _mm256_mask_fmsub_round_ph(A, U, B, C, R) \
1304 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1305 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1306 (__mmask16)(U), (int)(R)))
1307
1308#define _mm256_maskz_fmsub_round_ph(U, A, B, C, R) \
1309 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1310 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1311 (__mmask16)(U), (int)(R)))
1312
1313#define _mm256_fnmadd_round_ph(A, B, C, R) \
1314 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1315 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1316 (__mmask16)-1, (int)(R)))
1317
1318#define _mm256_mask3_fnmadd_round_ph(A, B, C, U, R) \
1319 ((__m256h)__builtin_ia32_vfmaddph256_round_mask3( \
1320 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1321 (__mmask16)(U), (int)(R)))
1322
1323#define _mm256_maskz_fnmadd_round_ph(U, A, B, C, R) \
1324 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1325 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1326 (__mmask16)(U), (int)(R)))
1327
1328#define _mm256_fnmsub_round_ph(A, B, C, R) \
1329 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1330 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1331 (__mmask16)-1, (int)(R)))
1332
1333#define _mm256_maskz_fnmsub_round_ph(U, A, B, C, R) \
1334 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1335 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1336 (__mmask16)(U), (int)(R)))
1337
1338#define _mm256_fmadd_round_ps(A, B, C, R) \
1339 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1340 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1341 (__mmask8)-1, (int)(R)))
1342
1343#define _mm256_mask_fmadd_round_ps(A, U, B, C, R) \
1344 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1345 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1346 (__mmask8)(U), (int)(R)))
1347
1348#define _mm256_mask3_fmadd_round_ps(A, B, C, U, R) \
1349 ((__m256)__builtin_ia32_vfmaddps256_round_mask3( \
1350 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1351 (__mmask8)(U), (int)(R)))
1352
1353#define _mm256_maskz_fmadd_round_ps(U, A, B, C, R) \
1354 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1355 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1356 (__mmask8)(U), (int)(R)))
1357
1358#define _mm256_fmsub_round_ps(A, B, C, R) \
1359 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1360 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1361 (__mmask8)-1, (int)(R)))
1362
1363#define _mm256_mask_fmsub_round_ps(A, U, B, C, R) \
1364 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1365 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1366 (__mmask8)(U), (int)(R)))
1367
1368#define _mm256_maskz_fmsub_round_ps(U, A, B, C, R) \
1369 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1370 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1371 (__mmask8)(U), (int)(R)))
1372
1373#define _mm256_fnmadd_round_ps(A, B, C, R) \
1374 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1375 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1376 (__mmask8)-1, (int)(R)))
1377
1378#define _mm256_mask3_fnmadd_round_ps(A, B, C, U, R) \
1379 ((__m256)__builtin_ia32_vfmaddps256_round_mask3( \
1380 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1381 (__mmask8)(U), (int)(R)))
1382
1383#define _mm256_maskz_fnmadd_round_ps(U, A, B, C, R) \
1384 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1385 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1386 (__mmask8)(U), (int)(R)))
1387
1388#define _mm256_fnmsub_round_ps(A, B, C, R) \
1389 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1390 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1391 (__mmask8)-1, (int)(R)))
1392
1393#define _mm256_maskz_fnmsub_round_ps(U, A, B, C, R) \
1394 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1395 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1396 (__mmask8)(U), (int)(R)))
1397
1398#define _mm256_fmadd_round_pch(A, B, C, R) \
1399 ((__m256h)__builtin_ia32_vfmaddcph256_round_mask3( \
1400 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1401 (__mmask8)-1, (int)(R)))
1402
1403#define _mm256_mask_fmadd_round_pch(A, U, B, C, R) \
1404 ((__m256h)__builtin_ia32_vfmaddcph256_round_mask( \
1405 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1406 (__mmask8)(U), (int)(R)))
1407
1408#define _mm256_mask3_fmadd_round_pch(A, B, C, U, R) \
1409 ((__m256h)__builtin_ia32_vfmaddcph256_round_mask3( \
1410 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1411 (__mmask8)(U), (int)(R)))
1412
1413#define _mm256_maskz_fmadd_round_pch(U, A, B, C, R) \
1414 ((__m256h)__builtin_ia32_vfmaddcph256_round_maskz( \
1415 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1416 (__mmask8)(U), (int)(R)))
1417
1418#define _mm256_fmaddsub_round_pd(A, B, C, R) \
1419 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1420 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1421 (__mmask8)-1, (int)(R)))
1422
1423#define _mm256_mask_fmaddsub_round_pd(A, U, B, C, R) \
1424 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1425 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1426 (__mmask8)(U), (int)(R)))
1427
1428#define _mm256_mask3_fmaddsub_round_pd(A, B, C, U, R) \
1429 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask3( \
1430 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1431 (__mmask8)(U), (int)(R)))
1432
1433#define _mm256_maskz_fmaddsub_round_pd(U, A, B, C, R) \
1434 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_maskz( \
1435 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1436 (__mmask8)(U), (int)(R)))
1437
1438#define _mm256_fmsubadd_round_pd(A, B, C, R) \
1439 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1440 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1441 (__mmask8)-1, (int)(R)))
1442
1443#define _mm256_mask_fmsubadd_round_pd(A, U, B, C, R) \
1444 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1445 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1446 (__mmask8)(U), (int)(R)))
1447
1448#define _mm256_maskz_fmsubadd_round_pd(U, A, B, C, R) \
1449 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_maskz( \
1450 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1451 (__mmask8)(U), (int)(R)))
1452
1453#define _mm256_fmaddsub_round_ph(A, B, C, R) \
1454 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1455 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1456 (__mmask16)-1, (int)(R)))
1457
1458#define _mm256_mask_fmaddsub_round_ph(A, U, B, C, R) \
1459 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1460 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1461 (__mmask16)(U), (int)(R)))
1462
1463#define _mm256_mask3_fmaddsub_round_ph(A, B, C, U, R) \
1464 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask3( \
1465 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1466 (__mmask16)(U), (int)(R)))
1467
1468#define _mm256_maskz_fmaddsub_round_ph(U, A, B, C, R) \
1469 ((__m256h)__builtin_ia32_vfmaddsubph256_round_maskz( \
1470 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1471 (__mmask16)(U), (int)(R)))
1472
1473#define _mm256_fmsubadd_round_ph(A, B, C, R) \
1474 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1475 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1476 (__mmask16)-1, (int)(R)))
1477
1478#define _mm256_mask_fmsubadd_round_ph(A, U, B, C, R) \
1479 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1480 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1481 (__mmask16)(U), (int)(R)))
1482
1483#define _mm256_maskz_fmsubadd_round_ph(U, A, B, C, R) \
1484 ((__m256h)__builtin_ia32_vfmaddsubph256_round_maskz( \
1485 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1486 (__mmask16)(U), (int)(R)))
1487
1488#define _mm256_fmaddsub_round_ps(A, B, C, R) \
1489 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1490 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1491 (__mmask8)-1, (int)(R)))
1492
1493#define _mm256_mask_fmaddsub_round_ps(A, U, B, C, R) \
1494 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1495 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1496 (__mmask8)(U), (int)(R)))
1497
1498#define _mm256_mask3_fmaddsub_round_ps(A, B, C, U, R) \
1499 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask3( \
1500 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1501 (__mmask8)(U), (int)(R)))
1502
1503#define _mm256_maskz_fmaddsub_round_ps(U, A, B, C, R) \
1504 ((__m256)__builtin_ia32_vfmaddsubps256_round_maskz( \
1505 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1506 (__mmask8)(U), (int)(R)))
1507
1508#define _mm256_fmsubadd_round_ps(A, B, C, R) \
1509 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1510 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1511 (__mmask8)-1, (int)(R)))
1512
1513#define _mm256_mask_fmsubadd_round_ps(A, U, B, C, R) \
1514 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1515 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1516 (__mmask8)(U), (int)(R)))
1517
1518#define _mm256_maskz_fmsubadd_round_ps(U, A, B, C, R) \
1519 ((__m256)__builtin_ia32_vfmaddsubps256_round_maskz( \
1520 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1521 (__mmask8)(U), (int)(R)))
1522#define _mm256_mask3_fmsub_round_pd(A, B, C, U, R) \
1523 ((__m256d)__builtin_ia32_vfmsubpd256_round_mask3( \
1524 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1525 (__mmask8)(U), (int)(R)))
1526
1527#define _mm256_mask3_fmsubadd_round_pd(A, B, C, U, R) \
1528 ((__m256d)__builtin_ia32_vfmsubaddpd256_round_mask3( \
1529 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1530 (__mmask8)(U), (int)(R)))
1531
1532#define _mm256_mask_fnmadd_round_pd(A, U, B, C, R) \
1533 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1534 (__v4df)(__m256d)(A), -(__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1535 (__mmask8)(U), (int)(R)))
1536
1537#define _mm256_mask_fnmsub_round_pd(A, U, B, C, R) \
1538 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1539 (__v4df)(__m256d)(A), -(__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1540 (__mmask8)(U), (int)(R)))
1541
1542#define _mm256_mask3_fnmsub_round_pd(A, B, C, U, R) \
1543 ((__m256d)__builtin_ia32_vfmsubpd256_round_mask3( \
1544 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1545 (__mmask8)(U), (int)(R)))
1546
1547#define _mm256_mask3_fmsub_round_ph(A, B, C, U, R) \
1548 ((__m256h)__builtin_ia32_vfmsubph256_round_mask3( \
1549 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1550 (__mmask16)(U), (int)(R)))
1551
1552#define _mm256_mask3_fmsubadd_round_ph(A, B, C, U, R) \
1553 ((__m256h)__builtin_ia32_vfmsubaddph256_round_mask3( \
1554 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1555 (__mmask16)(U), (int)(R)))
1556
1557#define _mm256_mask_fnmadd_round_ph(A, U, B, C, R) \
1558 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1559 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1560 (__mmask16)(U), (int)(R)))
1561
1562#define _mm256_mask_fnmsub_round_ph(A, U, B, C, R) \
1563 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1564 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1565 (__mmask16)(U), (int)(R)))
1566
1567#define _mm256_mask3_fnmsub_round_ph(A, B, C, U, R) \
1568 ((__m256h)__builtin_ia32_vfmsubph256_round_mask3( \
1569 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1570 (__mmask16)(U), (int)(R)))
1571
1572#define _mm256_mask3_fmsub_round_ps(A, B, C, U, R) \
1573 ((__m256)__builtin_ia32_vfmsubps256_round_mask3( \
1574 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1575 (__mmask8)(U), (int)(R)))
1576
1577#define _mm256_mask3_fmsubadd_round_ps(A, B, C, U, R) \
1578 ((__m256)__builtin_ia32_vfmsubaddps256_round_mask3( \
1579 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1580 (__mmask8)(U), (int)(R)))
1581
1582#define _mm256_mask_fnmadd_round_ps(A, U, B, C, R) \
1583 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1584 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1585 (__mmask8)(U), (int)(R)))
1586
1587#define _mm256_mask_fnmsub_round_ps(A, U, B, C, R) \
1588 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1589 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1590 (__mmask8)(U), (int)(R)))
1591
1592#define _mm256_mask3_fnmsub_round_ps(A, B, C, U, R) \
1593 ((__m256)__builtin_ia32_vfmsubps256_round_mask3( \
1594 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1595 (__mmask8)(U), (int)(R)))
1596
1597#define _mm256_mul_round_pch(A, B, R) \
1598 ((__m256h)__builtin_ia32_vfmulcph256_round_mask( \
1599 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1600 (__v8sf)(__m256h)_mm256_undefined_ph(), (__mmask8)-1, (int)(R)))
1601
1602#define _mm256_mask_mul_round_pch(W, U, A, B, R) \
1603 ((__m256h)__builtin_ia32_vfmulcph256_round_mask( \
1604 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(W), \
1605 (__mmask8)(U), (int)(R)))
1606
1607#define _mm256_maskz_mul_round_pch(U, A, B, R) \
1608 ((__m256h)__builtin_ia32_vfmulcph256_round_mask( \
1609 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1610 (__v8sf)(__m256h)_mm256_setzero_ph(), (__mmask8)(U), (int)(R)))
1611
1612#define _mm256_getexp_round_pd(A, R) \
1613 ((__m256d)__builtin_ia32_vgetexppd256_round_mask( \
1614 (__v4df)(__m256d)(A), (__v4df)_mm256_undefined_pd(), (__mmask8)-1, \
1615 (int)(R)))
1616
1617#define _mm256_mask_getexp_round_pd(W, U, A, R) \
1618 ((__m256d)__builtin_ia32_vgetexppd256_round_mask( \
1619 (__v4df)(__m256d)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
1620
1621#define _mm256_maskz_getexp_round_pd(U, A, R) \
1622 ((__m256d)__builtin_ia32_vgetexppd256_round_mask( \
1623 (__v4df)(__m256d)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
1624 (int)(R)))
1625
1626#define _mm256_getexp_round_ph(A, R) \
1627 ((__m256h)__builtin_ia32_vgetexpph256_round_mask( \
1628 (__v16hf)(__m256h)(A), (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, \
1629 (int)(R)))
1630
1631#define _mm256_mask_getexp_round_ph(W, U, A, R) \
1632 ((__m256h)__builtin_ia32_vgetexpph256_round_mask( \
1633 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(W), (__mmask16)(U), (int)(R)))
1634
1635#define _mm256_maskz_getexp_round_ph(U, A, R) \
1636 ((__m256h)__builtin_ia32_vgetexpph256_round_mask( \
1637 (__v16hf)(__m256h)(A), (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), \
1638 (int)(R)))
1639
1640#define _mm256_getexp_round_ps(A, R) \
1641 ((__m256)__builtin_ia32_vgetexpps256_round_mask( \
1642 (__v8sf)(__m256)(A), (__v8sf)_mm256_undefined_ps(), (__mmask8)-1, \
1643 (int)(R)))
1644
1645#define _mm256_mask_getexp_round_ps(W, U, A, R) \
1646 ((__m256)__builtin_ia32_vgetexpps256_round_mask( \
1647 (__v8sf)(__m256)(A), (__v8sf)(__m256)(W), (__mmask8)(U), (int)(R)))
1648
1649#define _mm256_maskz_getexp_round_ps(U, A, R) \
1650 ((__m256)__builtin_ia32_vgetexpps256_round_mask((__v8sf)(__m256)(A), \
1651 (__v8sf)_mm256_setzero_ps(), \
1652 (__mmask8)(U), (int)(R)))
1653
1654#define _mm256_getmant_round_pd(A, B, C, R) \
1655 ((__m256d)__builtin_ia32_vgetmantpd256_round_mask( \
1656 (__v4df)(__m256d)(A), (int)(((C) << 2) | (B)), \
1657 (__v4df)_mm256_undefined_pd(), (__mmask8)-1, (int)(R)))
1658
1659#define _mm256_mask_getmant_round_pd(W, U, A, B, C, R) \
1660 ((__m256d)__builtin_ia32_vgetmantpd256_round_mask( \
1661 (__v4df)(__m256d)(A), (int)(((C) << 2) | (B)), (__v4df)(__m256d)(W), \
1662 (__mmask8)(U), (int)(R)))
1663
1664#define _mm256_maskz_getmant_round_pd(U, A, B, C, R) \
1665 ((__m256d)__builtin_ia32_vgetmantpd256_round_mask( \
1666 (__v4df)(__m256d)(A), (int)(((C) << 2) | (B)), \
1667 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
1668
1669#define _mm256_getmant_round_ph(A, B, C, R) \
1670 ((__m256h)__builtin_ia32_vgetmantph256_round_mask( \
1671 (__v16hf)(__m256h)(A), (int)(((C) << 2) | (B)), \
1672 (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, (int)(R)))
1673
1674#define _mm256_mask_getmant_round_ph(W, U, A, B, C, R) \
1675 ((__m256h)__builtin_ia32_vgetmantph256_round_mask( \
1676 (__v16hf)(__m256h)(A), (int)(((C) << 2) | (B)), (__v16hf)(__m256h)(W), \
1677 (__mmask16)(U), (int)(R)))
1678
1679#define _mm256_maskz_getmant_round_ph(U, A, B, C, R) \
1680 ((__m256h)__builtin_ia32_vgetmantph256_round_mask( \
1681 (__v16hf)(__m256h)(A), (int)(((C) << 2) | (B)), \
1682 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1683
1684#define _mm256_getmant_round_ps(A, B, C, R) \
1685 ((__m256)__builtin_ia32_vgetmantps256_round_mask( \
1686 (__v8sf)(__m256)(A), (int)(((C) << 2) | (B)), \
1687 (__v8sf)_mm256_undefined_ps(), (__mmask8)-1, (int)(R)))
1688
1689#define _mm256_mask_getmant_round_ps(W, U, A, B, C, R) \
1690 ((__m256)__builtin_ia32_vgetmantps256_round_mask( \
1691 (__v8sf)(__m256)(A), (int)(((C) << 2) | (B)), (__v8sf)(__m256)(W), \
1692 (__mmask8)(U), (int)(R)))
1693
1694#define _mm256_maskz_getmant_round_ps(U, A, B, C, R) \
1695 ((__m256)__builtin_ia32_vgetmantps256_round_mask( \
1696 (__v8sf)(__m256)(A), (int)(((C) << 2) | (B)), \
1697 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
1698
1699#define _mm256_max_round_pd(A, B, R) \
1700 ((__m256d)__builtin_ia32_vmaxpd256_round((__v4df)(__m256d)(A), \
1701 (__v4df)(__m256d)(B), (int)(R)))
1702
1703#define _mm256_mask_max_round_pd(W, U, A, B, R) \
1704 ((__m256d)__builtin_ia32_selectpd_256( \
1705 (__mmask8)(U), (__v4df)_mm256_max_round_pd((A), (B), (R)), \
1706 (__v4df)(__m256d)(W)))
1707
1708#define _mm256_maskz_max_round_pd(U, A, B, R) \
1709 ((__m256d)__builtin_ia32_selectpd_256( \
1710 (__mmask8)(U), (__v4df)_mm256_max_round_pd((A), (B), (R)), \
1711 (__v4df)_mm256_setzero_pd()))
1712
1713#define _mm256_max_round_ph(A, B, R) \
1714 ((__m256h)__builtin_ia32_vmaxph256_round((__v16hf)(__m256h)(A), \
1715 (__v16hf)(__m256h)(B), (int)(R)))
1716
1717#define _mm256_mask_max_round_ph(W, U, A, B, R) \
1718 ((__m256h)__builtin_ia32_selectph_256( \
1719 (__mmask16)(U), (__v16hf)_mm256_max_round_ph((A), (B), (R)), \
1720 (__v16hf)(__m256h)(W)))
1721
1722#define _mm256_maskz_max_round_ph(U, A, B, R) \
1723 ((__m256h)__builtin_ia32_selectph_256( \
1724 (__mmask16)(U), (__v16hf)_mm256_max_round_ph((A), (B), (R)), \
1725 (__v16hf)_mm256_setzero_ph()))
1726
1727#define _mm256_max_round_ps(A, B, R) \
1728 ((__m256)__builtin_ia32_vmaxps256_round((__v8sf)(__m256)(A), \
1729 (__v8sf)(__m256)(B), (int)(R)))
1730
1731#define _mm256_mask_max_round_ps(W, U, A, B, R) \
1732 ((__m256)__builtin_ia32_selectps_256( \
1733 (__mmask8)(U), (__v8sf)_mm256_max_round_ps((A), (B), (R)), \
1734 (__v8sf)(__m256)(W)))
1735
1736#define _mm256_maskz_max_round_ps(U, A, B, R) \
1737 ((__m256)__builtin_ia32_selectps_256( \
1738 (__mmask8)(U), (__v8sf)_mm256_max_round_ps((A), (B), (R)), \
1739 (__v8sf)_mm256_setzero_ps()))
1740
1741#define _mm256_min_round_pd(A, B, R) \
1742 ((__m256d)__builtin_ia32_vminpd256_round((__v4df)(__m256d)(A), \
1743 (__v4df)(__m256d)(B), (int)(R)))
1744
1745#define _mm256_mask_min_round_pd(W, U, A, B, R) \
1746 ((__m256d)__builtin_ia32_selectpd_256( \
1747 (__mmask8)(U), (__v4df)_mm256_min_round_pd((A), (B), (R)), \
1748 (__v4df)(__m256d)(W)))
1749
1750#define _mm256_maskz_min_round_pd(U, A, B, R) \
1751 ((__m256d)__builtin_ia32_selectpd_256( \
1752 (__mmask8)(U), (__v4df)_mm256_min_round_pd((A), (B), (R)), \
1753 (__v4df)_mm256_setzero_pd()))
1754
1755#define _mm256_min_round_ph(A, B, R) \
1756 ((__m256h)__builtin_ia32_vminph256_round((__v16hf)(__m256h)(A), \
1757 (__v16hf)(__m256h)(B), (int)(R)))
1758
1759#define _mm256_mask_min_round_ph(W, U, A, B, R) \
1760 ((__m256h)__builtin_ia32_selectph_256( \
1761 (__mmask16)(U), (__v16hf)_mm256_min_round_ph((A), (B), (R)), \
1762 (__v16hf)(__m256h)(W)))
1763
1764#define _mm256_maskz_min_round_ph(U, A, B, R) \
1765 ((__m256h)__builtin_ia32_selectph_256( \
1766 (__mmask16)(U), (__v16hf)_mm256_min_round_ph((A), (B), (R)), \
1767 (__v16hf)_mm256_setzero_ph()))
1768
1769#define _mm256_min_round_ps(A, B, R) \
1770 ((__m256)__builtin_ia32_vminps256_round((__v8sf)(__m256)(A), \
1771 (__v8sf)(__m256)(B), (int)(R)))
1772
1773#define _mm256_mask_min_round_ps(W, U, A, B, R) \
1774 ((__m256)__builtin_ia32_selectps_256( \
1775 (__mmask8)(U), (__v8sf)_mm256_min_round_ps((A), (B), (R)), \
1776 (__v8sf)(__m256)(W)))
1777
1778#define _mm256_maskz_min_round_ps(U, A, B, R) \
1779 ((__m256)__builtin_ia32_selectps_256( \
1780 (__mmask8)(U), (__v8sf)_mm256_min_round_ps((A), (B), (R)), \
1781 (__v8sf)_mm256_setzero_ps()))
1782
1783#define _mm256_mul_round_pd(A, B, R) \
1784 ((__m256d)__builtin_ia32_vmulpd256_round((__v4df)(__m256d)(A), \
1785 (__v4df)(__m256d)(B), (int)(R)))
1786
1787#define _mm256_mask_mul_round_pd(W, U, A, B, R) \
1788 ((__m256d)__builtin_ia32_selectpd_256( \
1789 (__mmask8)(U), (__v4df)_mm256_mul_round_pd((A), (B), (R)), \
1790 (__v4df)(__m256d)(W)))
1791
1792#define _mm256_maskz_mul_round_pd(U, A, B, R) \
1793 ((__m256d)__builtin_ia32_selectpd_256( \
1794 (__mmask8)(U), (__v4df)_mm256_mul_round_pd((A), (B), (R)), \
1795 (__v4df)_mm256_setzero_pd()))
1796
1797#define _mm256_mul_round_ph(A, B, R) \
1798 ((__m256h)__builtin_ia32_vmulph256_round((__v16hf)(__m256h)(A), \
1799 (__v16hf)(__m256h)(B), (int)(R)))
1800
1801#define _mm256_mask_mul_round_ph(W, U, A, B, R) \
1802 ((__m256h)__builtin_ia32_selectph_256( \
1803 (__mmask16)(U), (__v16hf)_mm256_mul_round_ph((A), (B), (R)), \
1804 (__v16hf)(__m256h)(W)))
1805
1806#define _mm256_maskz_mul_round_ph(U, A, B, R) \
1807 ((__m256h)__builtin_ia32_selectph_256( \
1808 (__mmask16)(U), (__v16hf)_mm256_mul_round_ph((A), (B), (R)), \
1809 (__v16hf)_mm256_setzero_ph()))
1810
1811#define _mm256_mul_round_ps(A, B, R) \
1812 ((__m256)__builtin_ia32_vmulps256_round((__v8sf)(__m256)(A), \
1813 (__v8sf)(__m256)(B), (int)(R)))
1814
1815#define _mm256_mask_mul_round_ps(W, U, A, B, R) \
1816 ((__m256)__builtin_ia32_selectps_256( \
1817 (__mmask8)(U), (__v8sf)_mm256_mul_round_ps((A), (B), (R)), \
1818 (__v8sf)(__m256)(W)))
1819
1820#define _mm256_maskz_mul_round_ps(U, A, B, R) \
1821 ((__m256)__builtin_ia32_selectps_256( \
1822 (__mmask8)(U), (__v8sf)_mm256_mul_round_ps((A), (B), (R)), \
1823 (__v8sf)_mm256_setzero_ps()))
1824
1825#define _mm256_range_round_pd(A, B, C, R) \
1826 ((__m256d)__builtin_ia32_vrangepd256_round_mask( \
1827 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
1828 (__v4df)_mm256_setzero_pd(), (__mmask8)-1, (int)(R)))
1829
1830#define _mm256_mask_range_round_pd(W, U, A, B, C, R) \
1831 ((__m256d)__builtin_ia32_vrangepd256_round_mask( \
1832 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
1833 (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
1834
1835#define _mm256_maskz_range_round_pd(U, A, B, C, R) \
1836 ((__m256d)__builtin_ia32_vrangepd256_round_mask( \
1837 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
1838 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
1839
1840#define _mm256_range_round_ps(A, B, C, R) \
1841 ((__m256)__builtin_ia32_vrangeps256_round_mask( \
1842 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
1843 (__v8sf)_mm256_setzero_ps(), (__mmask8)-1, (int)(R)))
1844
1845#define _mm256_mask_range_round_ps(W, U, A, B, C, R) \
1846 ((__m256)__builtin_ia32_vrangeps256_round_mask( \
1847 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), (__v8sf)(__m256)(W), \
1848 (__mmask8)(U), (int)(R)))
1849
1850#define _mm256_maskz_range_round_ps(U, A, B, C, R) \
1851 ((__m256)__builtin_ia32_vrangeps256_round_mask( \
1852 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
1853 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
1854
1855#define _mm256_reduce_round_pd(A, B, R) \
1856 ((__m256d)__builtin_ia32_vreducepd256_round_mask( \
1857 (__v4df)(__m256d)(A), (int)(B), (__v4df)_mm256_setzero_pd(), \
1858 (__mmask8)-1, (int)(R)))
1859
1860#define _mm256_mask_reduce_round_pd(W, U, A, B, R) \
1861 ((__m256d)__builtin_ia32_vreducepd256_round_mask( \
1862 (__v4df)(__m256d)(A), (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U), \
1863 (int)(R)))
1864
1865#define _mm256_maskz_reduce_round_pd(U, A, B, R) \
1866 ((__m256d)__builtin_ia32_vreducepd256_round_mask( \
1867 (__v4df)(__m256d)(A), (int)(B), (__v4df)_mm256_setzero_pd(), \
1868 (__mmask8)(U), (int)(R)))
1869
1870#define _mm256_mask_reduce_round_ph(W, U, A, imm, R) \
1871 ((__m256h)__builtin_ia32_vreduceph256_round_mask( \
1872 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)(__m256h)(W), \
1873 (__mmask16)(U), (int)(R)))
1874
1875#define _mm256_maskz_reduce_round_ph(U, A, imm, R) \
1876 ((__m256h)__builtin_ia32_vreduceph256_round_mask( \
1877 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)_mm256_setzero_ph(), \
1878 (__mmask16)(U), (int)(R)))
1879
1880#define _mm256_reduce_round_ph(A, imm, R) \
1881 ((__m256h)__builtin_ia32_vreduceph256_round_mask( \
1882 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)_mm256_undefined_ph(), \
1883 (__mmask16)-1, (int)(R)))
1884
1885#define _mm256_reduce_round_ps(A, B, R) \
1886 ((__m256)__builtin_ia32_vreduceps256_round_mask( \
1887 (__v8sf)(__m256)(A), (int)(B), (__v8sf)_mm256_setzero_ps(), \
1888 (__mmask8)-1, (int)(R)))
1889
1890#define _mm256_mask_reduce_round_ps(W, U, A, B, R) \
1891 ((__m256)__builtin_ia32_vreduceps256_round_mask( \
1892 (__v8sf)(__m256)(A), (int)(B), (__v8sf)(__m256)(W), (__mmask8)(U), \
1893 (int)(R)))
1894
1895#define _mm256_maskz_reduce_round_ps(U, A, B, R) \
1896 ((__m256)__builtin_ia32_vreduceps256_round_mask( \
1897 (__v8sf)(__m256)(A), (int)(B), (__v8sf)_mm256_setzero_ps(), \
1898 (__mmask8)(U), (int)(R)))
1899
1900#define _mm256_roundscale_round_pd(A, imm, R) \
1901 ((__m256d)__builtin_ia32_vrndscalepd256_round_mask( \
1902 (__v4df)(__m256d)(A), (int)(imm), (__v4df)_mm256_undefined_pd(), \
1903 (__mmask8)-1, (int)(R)))
1904
1905#define _mm256_mask_roundscale_round_pd(A, B, C, imm, R) \
1906 ((__m256d)__builtin_ia32_vrndscalepd256_round_mask( \
1907 (__v4df)(__m256d)(C), (int)(imm), (__v4df)(__m256d)(A), (__mmask8)(B), \
1908 (int)(R)))
1909
1910#define _mm256_maskz_roundscale_round_pd(A, B, imm, R) \
1911 ((__m256d)__builtin_ia32_vrndscalepd256_round_mask( \
1912 (__v4df)(__m256d)(B), (int)(imm), (__v4df)_mm256_setzero_pd(), \
1913 (__mmask8)(A), (int)(R)))
1914
1915#define _mm256_roundscale_round_ph(A, imm, R) \
1916 ((__m256h)__builtin_ia32_vrndscaleph256_round_mask( \
1917 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)_mm256_undefined_ph(), \
1918 (__mmask16)-1, (int)(R)))
1919
1920#define _mm256_mask_roundscale_round_ph(A, B, C, imm, R) \
1921 ((__m256h)__builtin_ia32_vrndscaleph256_round_mask( \
1922 (__v16hf)(__m256h)(C), (int)(imm), (__v16hf)(__m256h)(A), \
1923 (__mmask16)(B), (int)(R)))
1924
1925#define _mm256_maskz_roundscale_round_ph(A, B, imm, R) \
1926 ((__m256h)__builtin_ia32_vrndscaleph256_round_mask( \
1927 (__v16hf)(__m256h)(B), (int)(imm), (__v16hf)_mm256_setzero_ph(), \
1928 (__mmask16)(A), (int)(R)))
1929
1930#define _mm256_roundscale_round_ps(A, imm, R) \
1931 ((__m256)__builtin_ia32_vrndscaleps256_round_mask( \
1932 (__v8sf)(__m256)(A), (int)(imm), (__v8sf)_mm256_undefined_ps(), \
1933 (__mmask8)-1, (int)(R)))
1934
1935#define _mm256_mask_roundscale_round_ps(A, B, C, imm, R) \
1936 ((__m256)__builtin_ia32_vrndscaleps256_round_mask( \
1937 (__v8sf)(__m256)(C), (int)(imm), (__v8sf)(__m256)(A), (__mmask8)(B), \
1938 (int)(R)))
1939
1940#define _mm256_maskz_roundscale_round_ps(A, B, imm, R) \
1941 ((__m256)__builtin_ia32_vrndscaleps256_round_mask( \
1942 (__v8sf)(__m256)(B), (int)(imm), (__v8sf)_mm256_setzero_ps(), \
1943 (__mmask8)(A), (int)(R)))
1944
1945#define _mm256_scalef_round_pd(A, B, R) \
1946 ((__m256d)__builtin_ia32_vscalefpd256_round_mask( \
1947 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), \
1948 (__v4df)_mm256_undefined_pd(), (__mmask8)-1, (int)(R)))
1949
1950#define _mm256_mask_scalef_round_pd(W, U, A, B, R) \
1951 ((__m256d)__builtin_ia32_vscalefpd256_round_mask( \
1952 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(W), \
1953 (__mmask8)(U), (int)(R)))
1954
1955#define _mm256_maskz_scalef_round_pd(U, A, B, R) \
1956 ((__m256d)__builtin_ia32_vscalefpd256_round_mask( \
1957 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)_mm256_setzero_pd(), \
1958 (__mmask8)(U), (int)(R)))
1959
1960#define _mm256_scalef_round_ph(A, B, R) \
1961 ((__m256h)__builtin_ia32_vscalefph256_round_mask( \
1962 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), \
1963 (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, (int)(R)))
1964
1965#define _mm256_mask_scalef_round_ph(W, U, A, B, R) \
1966 ((__m256h)__builtin_ia32_vscalefph256_round_mask( \
1967 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(W), \
1968 (__mmask16)(U), (int)(R)))
1969
1970#define _mm256_maskz_scalef_round_ph(U, A, B, R) \
1971 ((__m256h)__builtin_ia32_vscalefph256_round_mask( \
1972 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), \
1973 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1974
1975#define _mm256_scalef_round_ps(A, B, R) \
1976 ((__m256)__builtin_ia32_vscalefps256_round_mask( \
1977 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)_mm256_undefined_ps(), \
1978 (__mmask8)-1, (int)(R)))
1979
1980#define _mm256_mask_scalef_round_ps(W, U, A, B, R) \
1981 ((__m256)__builtin_ia32_vscalefps256_round_mask( \
1982 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(W), \
1983 (__mmask8)(U), (int)(R)))
1984
1985#define _mm256_maskz_scalef_round_ps(U, A, B, R) \
1986 ((__m256)__builtin_ia32_vscalefps256_round_mask( \
1987 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)_mm256_setzero_ps(), \
1988 (__mmask8)(U), (int)(R)))
1989
1990#define _mm256_sqrt_round_pd(A, R) \
1991 ((__m256d)__builtin_ia32_vsqrtpd256_round((__v4df)(__m256d)(A), (int)(R)))
1992
1993#define _mm256_mask_sqrt_round_pd(W, U, A, R) \
1994 ((__m256d)__builtin_ia32_selectpd_256( \
1995 (__mmask8)(U), (__v4df)_mm256_sqrt_round_pd((A), (R)), \
1996 (__v4df)(__m256d)(W)))
1997
1998#define _mm256_maskz_sqrt_round_pd(U, A, R) \
1999 ((__m256d)__builtin_ia32_selectpd_256( \
2000 (__mmask8)(U), (__v4df)_mm256_sqrt_round_pd((A), (R)), \
2001 (__v4df)_mm256_setzero_pd()))
2002
2003#define _mm256_sqrt_round_ph(A, R) \
2004 ((__m256h)__builtin_ia32_vsqrtph256_round((__v16hf)(__m256h)(A), (int)(R)))
2005
2006#define _mm256_mask_sqrt_round_ph(W, U, A, R) \
2007 ((__m256h)__builtin_ia32_selectph_256( \
2008 (__mmask16)(U), (__v16hf)_mm256_sqrt_round_ph((A), (R)), \
2009 (__v16hf)(__m256h)(W)))
2010
2011#define _mm256_maskz_sqrt_round_ph(U, A, R) \
2012 ((__m256h)__builtin_ia32_selectph_256( \
2013 (__mmask16)(U), (__v16hf)_mm256_sqrt_round_ph((A), (R)), \
2014 (__v16hf)_mm256_setzero_ph()))
2015
2016#define _mm256_sqrt_round_ps(A, R) \
2017 ((__m256)__builtin_ia32_vsqrtps256_round((__v8sf)(__m256)(A), (int)(R)))
2018
2019#define _mm256_mask_sqrt_round_ps(W, U, A, R) \
2020 ((__m256)__builtin_ia32_selectps_256((__mmask8)(U), \
2021 (__v8sf)_mm256_sqrt_round_ps((A), (R)), \
2022 (__v8sf)(__m256)(W)))
2023
2024#define _mm256_maskz_sqrt_round_ps(U, A, R) \
2025 ((__m256)__builtin_ia32_selectps_256((__mmask8)(U), \
2026 (__v8sf)_mm256_sqrt_round_ps((A), (R)), \
2027 (__v8sf)_mm256_setzero_ps()))
2028
2029#define _mm256_sub_round_pd(A, B, R) \
2030 ((__m256d)__builtin_ia32_vsubpd256_round((__v4df)(__m256d)(A), \
2031 (__v4df)(__m256d)(B), (int)(R)))
2032
2033#define _mm256_mask_sub_round_pd(W, U, A, B, R) \
2034 ((__m256d)__builtin_ia32_selectpd_256( \
2035 (__mmask8)(U), (__v4df)_mm256_sub_round_pd((A), (B), (R)), \
2036 (__v4df)(__m256d)(W)))
2037
2038#define _mm256_maskz_sub_round_pd(U, A, B, R) \
2039 ((__m256d)__builtin_ia32_selectpd_256( \
2040 (__mmask8)(U), (__v4df)_mm256_sub_round_pd((A), (B), (R)), \
2041 (__v4df)_mm256_setzero_pd()))
2042
2043#define _mm256_sub_round_ph(A, B, R) \
2044 ((__m256h)__builtin_ia32_vsubph256_round((__v16hf)(__m256h)(A), \
2045 (__v16hf)(__m256h)(B), (int)(R)))
2046
2047#define _mm256_mask_sub_round_ph(W, U, A, B, R) \
2048 ((__m256h)__builtin_ia32_selectph_256( \
2049 (__mmask16)(U), (__v16hf)_mm256_sub_round_ph((A), (B), (R)), \
2050 (__v16hf)(__m256h)(W)))
2051
2052#define _mm256_maskz_sub_round_ph(U, A, B, R) \
2053 ((__m256h)__builtin_ia32_selectph_256( \
2054 (__mmask16)(U), (__v16hf)_mm256_sub_round_ph((A), (B), (R)), \
2055 (__v16hf)_mm256_setzero_ph()))
2056
2057#define _mm256_sub_round_ps(A, B, R) \
2058 ((__m256)__builtin_ia32_vsubps256_round((__v8sf)(__m256)(A), \
2059 (__v8sf)(__m256)(B), (int)(R)))
2060
2061#define _mm256_mask_sub_round_ps(W, U, A, B, R) \
2062 ((__m256)__builtin_ia32_selectps_256( \
2063 (__mmask8)(U), (__v8sf)_mm256_sub_round_ps((A), (B), (R)), \
2064 (__v8sf)(__m256)(W)))
2065
2066#define _mm256_maskz_sub_round_ps(U, A, B, R) \
2067 ((__m256)__builtin_ia32_selectps_256( \
2068 (__mmask8)(U), (__v8sf)_mm256_sub_round_ps((A), (B), (R)), \
2069 (__v8sf)_mm256_setzero_ps()))
2070
2071#undef __DEFAULT_FN_ATTRS256
2072#undef __DEFAULT_FN_ATTRS128
2073
2074#endif /* __AVX10_2NIINTRIN_H */
2075#endif /* __SSE2__ */
lib/include/avx10_2satcvtdsintrin.h created+496
...@@ -0,0 +1,496 @@
1/*===----------- avx10_2satcvtdsintrin.h - AVX512SATCVTDS intrinsics --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <avx10_2satcvtdsintrin.h> directly; include <immintrin.h> instead."
13#endif // __IMMINTRIN_H
14
15#ifndef __AVX10_2SATCVTDSINTRIN_H
16#define __AVX10_2SATCVTDSINTRIN_H
17
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS256 \
20 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
21 __min_vector_width__(256)))
22
23#define __DEFAULT_FN_ATTRS128 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
25 __min_vector_width__(128)))
26
27#define _mm_cvtts_roundsd_i32(__A, __R) \
28 ((int)__builtin_ia32_vcvttsd2sis32((__v2df)(__m128)(__A), (const int)(__R)))
29
30#define _mm_cvtts_roundsd_si32(__A, __R) \
31 ((int)__builtin_ia32_vcvttsd2sis32((__v2df)(__m128d)(__A), (const int)(__R)))
32
33#define _mm_cvtts_roundsd_u32(__A, __R) \
34 ((unsigned int)__builtin_ia32_vcvttsd2usis32((__v2df)(__m128d)(__A), \
35 (const int)(__R)))
36
37#define _mm_cvtts_roundss_i32(__A, __R) \
38 ((int)__builtin_ia32_vcvttss2sis32((__v4sf)(__m128)(__A), (const int)(__R)))
39
40#define _mm_cvtts_roundss_si32(__A, __R) \
41 ((int)__builtin_ia32_vcvttss2sis32((__v4sf)(__m128)(__A), (const int)(__R)))
42
43#define _mm_cvtts_roundss_u32(__A, __R) \
44 ((unsigned int)__builtin_ia32_vcvttss2usis32((__v4sf)(__m128)(__A), \
45 (const int)(__R)))
46
47#ifdef __x86_64__
48#define _mm_cvtts_roundss_u64(__A, __R) \
49 ((unsigned long long)__builtin_ia32_vcvttss2usis64((__v4sf)(__m128)(__A), \
50 (const int)(__R)))
51
52#define _mm_cvtts_roundsd_u64(__A, __R) \
53 ((unsigned long long)__builtin_ia32_vcvttsd2usis64((__v2df)(__m128d)(__A), \
54 (const int)(__R)))
55
56#define _mm_cvtts_roundss_i64(__A, __R) \
57 ((long long)__builtin_ia32_vcvttss2sis64((__v4sf)(__m128)(__A), \
58 (const int)(__R)))
59
60#define _mm_cvtts_roundss_si64(__A, __R) \
61 ((long long)__builtin_ia32_vcvttss2sis64((__v4sf)(__m128)(__A), \
62 (const int)(__R)))
63
64#define _mm_cvtts_roundsd_si64(__A, __R) \
65 ((long long)__builtin_ia32_vcvttsd2sis64((__v2df)(__m128d)(__A), \
66 (const int)(__R)))
67
68#define _mm_cvtts_roundsd_i64(__A, __R) \
69 ((long long)__builtin_ia32_vcvttsd2sis64((__v2df)(__m128d)(__A), \
70 (const int)(__R)))
71#endif /* __x86_64__ */
72
73// 128 Bit : Double -> int
74static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epi32(__m128d __A) {
75 return ((__m128i)__builtin_ia32_vcvttpd2dqs128_mask(
76 (__v2df)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
77}
78
79static __inline__ __m128i __DEFAULT_FN_ATTRS128
80_mm_mask_cvttspd_epi32(__m128i __W, __mmask8 __U, __m128d __A) {
81 return ((__m128i)__builtin_ia32_vcvttpd2dqs128_mask((__v2df)__A, (__v4si)__W,
82 __U));
83}
84
85static __inline__ __m128i __DEFAULT_FN_ATTRS128
86_mm_maskz_cvttspd_epi32(__mmask16 __U, __m128d __A) {
87 return ((__m128i)__builtin_ia32_vcvttpd2dqs128_mask(
88 (__v2df)__A, (__v4si)(__m128i)_mm_setzero_si128(), __U));
89}
90
91// 256 Bit : Double -> int
92static __inline__ __m128i __DEFAULT_FN_ATTRS256
93_mm256_cvttspd_epi32(__m256d __A) {
94 return ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask(
95 (__v4df)__A, (__v4si)_mm_undefined_si128(), (__mmask8)-1,
96 _MM_FROUND_CUR_DIRECTION));
97}
98
99static __inline__ __m128i __DEFAULT_FN_ATTRS256
100_mm256_mask_cvttspd_epi32(__m128i __W, __mmask8 __U, __m256d __A) {
101 return ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask(
102 (__v4df)__A, (__v4si)__W, __U, _MM_FROUND_CUR_DIRECTION));
103}
104
105static __inline__ __m128i __DEFAULT_FN_ATTRS256
106_mm256_maskz_cvttspd_epi32(__mmask8 __U, __m256d __A) {
107 return ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask(
108 (__v4df)__A, (__v4si)_mm_setzero_si128(), __U, _MM_FROUND_CUR_DIRECTION));
109}
110
111#define _mm256_cvtts_roundpd_epi32(__A, __R) \
112 ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask( \
113 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_undefined_si128(), \
114 (__mmask8) - 1, (int)(__R)))
115
116#define _mm256_mask_cvtts_roundpd_epi32(__W, __U, __A, __R) \
117 ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask( \
118 (__v4df)(__m256d)__A, (__v4si)(__m128i)__W, (__mmask8)__U, (int)(__R)))
119
120#define _mm256_maskz_cvtts_roundpd_epi32(__U, __A, __R) \
121 ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask( \
122 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_setzero_si128(), \
123 (__mmask8)__U, (int)(__R)))
124
125// 128 Bit : Double -> uint
126static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epu32(__m128d __A) {
127 return ((__m128i)__builtin_ia32_vcvttpd2udqs128_mask(
128 (__v2df)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
129}
130
131static __inline__ __m128i __DEFAULT_FN_ATTRS128
132_mm_mask_cvttspd_epu32(__m128i __W, __mmask8 __U, __m128d __A) {
133 return ((__m128i)__builtin_ia32_vcvttpd2udqs128_mask(
134 (__v2df)__A, (__v4si)(__m128i)__W, (__mmask8)__U));
135}
136
137static __inline__ __m128i __DEFAULT_FN_ATTRS128
138_mm_maskz_cvttspd_epu32(__mmask8 __U, __m128d __A) {
139 return ((__m128i)__builtin_ia32_vcvttpd2udqs128_mask(
140 (__v2df)__A, (__v4si)(__m128i)_mm_setzero_si128(), __U));
141}
142
143// 256 Bit : Double -> uint
144static __inline__ __m128i __DEFAULT_FN_ATTRS256
145_mm256_cvttspd_epu32(__m256d __A) {
146 return ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask(
147 (__v4df)__A, (__v4si)_mm_undefined_si128(), (__mmask8)-1,
148 _MM_FROUND_CUR_DIRECTION));
149}
150
151static __inline__ __m128i __DEFAULT_FN_ATTRS256
152_mm256_mask_cvttspd_epu32(__m128i __W, __mmask8 __U, __m256d __A) {
153 return ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask(
154 (__v4df)__A, (__v4si)__W, __U, _MM_FROUND_CUR_DIRECTION));
155}
156
157static __inline__ __m128i __DEFAULT_FN_ATTRS256
158_mm256_maskz_cvttspd_epu32(__mmask8 __U, __m256d __A) {
159 return ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask(
160 (__v4df)__A, (__v4si)_mm_setzero_si128(), __U, _MM_FROUND_CUR_DIRECTION));
161}
162
163#define _mm256_cvtts_roundpd_epu32(__A, __R) \
164 ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask( \
165 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_undefined_si128(), \
166 (__mmask8) - 1, (int)(__R)))
167
168#define _mm256_mask_cvtts_roundpd_epu32(__W, __U, __A, __R) \
169 ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask( \
170 (__v4df)(__m256d)__A, (__v4si)(__m128i)__W, (__mmask8)__U, (int)(__R)))
171
172#define _mm256_maskz_cvtts_roundpd_epu32(__U, __A, __R) \
173 ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask( \
174 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_setzero_si128(), \
175 (__mmask8)__U, (int)(__R)))
176
177// 128 Bit : Double -> long
178static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epi64(__m128d __A) {
179 return ((__m128i)__builtin_ia32_vcvttpd2qqs128_mask(
180 (__v2df)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
181}
182
183static __inline__ __m128i __DEFAULT_FN_ATTRS128
184_mm_mask_cvttspd_epi64(__m128i __W, __mmask8 __U, __m128d __A) {
185 return ((__m128i)__builtin_ia32_vcvttpd2qqs128_mask((__v2df)__A, (__v2di)__W,
186 (__mmask8)__U));
187}
188
189static __inline__ __m128i __DEFAULT_FN_ATTRS128
190_mm_maskz_cvttspd_epi64(__mmask8 __U, __m128d __A) {
191 return ((__m128i)__builtin_ia32_vcvttpd2qqs128_mask(
192 (__v2df)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
193}
194
195// 256 Bit : Double -> long
196static __inline__ __m256i __DEFAULT_FN_ATTRS256
197_mm256_cvttspd_epi64(__m256d __A) {
198 return ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask(
199 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
200 _MM_FROUND_CUR_DIRECTION));
201}
202
203static __inline__ __m256i __DEFAULT_FN_ATTRS256
204_mm256_mask_cvttspd_epi64(__m256i __W, __mmask8 __U, __m256d __A) {
205 return ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask(
206 (__v4df)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
207}
208
209static __inline__ __m256i __DEFAULT_FN_ATTRS256
210_mm256_maskz_cvttspd_epi64(__mmask8 __U, __m256d __A) {
211 return ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask(
212 (__v4df)__A, (__v4di)_mm256_setzero_si256(), __U,
213 _MM_FROUND_CUR_DIRECTION));
214}
215
216#define _mm256_cvtts_roundpd_epi64(__A, __R) \
217 ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask( \
218 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
219 (int)__R))
220
221#define _mm256_mask_cvtts_roundpd_epi64(__W, __U, __A, __R) \
222 ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask((__v4df)__A, (__v4di)__W, \
223 (__mmask8)__U, (int)__R))
224
225#define _mm256_maskz_cvtts_roundpd_epi64(__U, __A, __R) \
226 ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask( \
227 (__v4df)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, (int)__R))
228
229// 128 Bit : Double -> ulong
230static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epu64(__m128d __A) {
231 return ((__m128i)__builtin_ia32_vcvttpd2uqqs128_mask(
232 (__v2df)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
233}
234
235static __inline__ __m128i __DEFAULT_FN_ATTRS128
236_mm_mask_cvttspd_epu64(__m128i __W, __mmask8 __U, __m128d __A) {
237 return ((__m128i)__builtin_ia32_vcvttpd2uqqs128_mask((__v2df)__A, (__v2di)__W,
238 (__mmask8)__U));
239}
240
241static __inline__ __m128i __DEFAULT_FN_ATTRS128
242_mm_maskz_cvttspd_epu64(__mmask8 __U, __m128d __A) {
243 return ((__m128i)__builtin_ia32_vcvttpd2uqqs128_mask(
244 (__v2df)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
245}
246
247// 256 Bit : Double -> ulong
248
249static __inline__ __m256i __DEFAULT_FN_ATTRS256
250_mm256_cvttspd_epu64(__m256d __A) {
251 return ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask(
252 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
253 _MM_FROUND_CUR_DIRECTION));
254}
255
256static __inline__ __m256i __DEFAULT_FN_ATTRS256
257_mm256_mask_cvttspd_epu64(__m256i __W, __mmask8 __U, __m256d __A) {
258 return ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask(
259 (__v4df)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
260}
261
262static __inline__ __m256i __DEFAULT_FN_ATTRS256
263_mm256_maskz_cvttspd_epu64(__mmask8 __U, __m256d __A) {
264 return ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask(
265 (__v4df)__A, (__v4di)_mm256_setzero_si256(), __U,
266 _MM_FROUND_CUR_DIRECTION));
267}
268
269#define _mm256_cvtts_roundpd_epu64(__A, __R) \
270 ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask( \
271 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
272 (int)__R))
273
274#define _mm256_mask_cvtts_roundpd_epu64(__W, __U, __A, __R) \
275 ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask( \
276 (__v4df)__A, (__v4di)__W, (__mmask8)__U, (int)__R))
277
278#define _mm256_maskz_cvtts_roundpd_epu64(__U, __A, __R) \
279 ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask( \
280 (__v4df)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, (int)__R))
281
282// 128 Bit : float -> int
283static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epi32(__m128 __A) {
284 return ((__m128i)__builtin_ia32_vcvttps2dqs128_mask(
285 (__v4sf)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
286}
287
288static __inline__ __m128i __DEFAULT_FN_ATTRS128
289_mm_mask_cvttsps_epi32(__m128i __W, __mmask8 __U, __m128 __A) {
290 return ((__m128i)__builtin_ia32_vcvttps2dqs128_mask((__v4sf)__A, (__v4si)__W,
291 (__mmask8)__U));
292}
293
294static __inline__ __m128i __DEFAULT_FN_ATTRS128
295_mm_maskz_cvttsps_epi32(__mmask8 __U, __m128 __A) {
296 return ((__m128i)__builtin_ia32_vcvttps2dqs128_mask(
297 (__v4sf)__A, (__v4si)(__m128i)_mm_setzero_si128(), (__mmask8)__U));
298}
299
300// 256 Bit : float -> int
301static __inline__ __m256i __DEFAULT_FN_ATTRS256
302_mm256_cvttsps_epi32(__m256 __A) {
303 return ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask(
304 (__v8sf)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
305 _MM_FROUND_CUR_DIRECTION));
306}
307
308static __inline__ __m256i __DEFAULT_FN_ATTRS256
309_mm256_mask_cvttsps_epi32(__m256i __W, __mmask8 __U, __m256 __A) {
310 return ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask(
311 (__v8sf)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
312}
313
314static __inline__ __m256i __DEFAULT_FN_ATTRS256
315_mm256_maskz_cvttsps_epi32(__mmask8 __U, __m256 __A) {
316 return ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask(
317 (__v8sf)__A, (__v8si)_mm256_setzero_si256(), __U,
318 _MM_FROUND_CUR_DIRECTION));
319}
320
321#define _mm256_cvtts_roundps_epi32(__A, __R) \
322 ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask( \
323 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_undefined_si256(), \
324 (__mmask8) - 1, (int)(__R)))
325
326#define _mm256_mask_cvtts_roundps_epi32(__W, __U, __A, __R) \
327 ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask( \
328 (__v8sf)(__m256)__A, (__v8si)(__m256i)__W, (__mmask8)__U, (int)(__R)))
329
330#define _mm256_maskz_cvtts_roundps_epi32(__U, __A, __R) \
331 ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask( \
332 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_setzero_si256(), \
333 (__mmask8)__U, (int)(__R)))
334
335// 128 Bit : float -> uint
336static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epu32(__m128 __A) {
337 return ((__m128i)__builtin_ia32_vcvttps2udqs128_mask(
338 (__v4sf)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
339}
340
341static __inline__ __m128i __DEFAULT_FN_ATTRS128
342_mm_mask_cvttsps_epu32(__m128i __W, __mmask8 __U, __m128 __A) {
343 return ((__m128i)__builtin_ia32_vcvttps2udqs128_mask((__v4sf)__A, (__v4si)__W,
344 (__mmask8)__U));
345}
346
347static __inline__ __m128i __DEFAULT_FN_ATTRS128
348_mm_maskz_cvttsps_epu32(__mmask8 __U, __m128 __A) {
349 return ((__m128i)__builtin_ia32_vcvttps2udqs128_mask(
350 (__v4sf)__A, (__v4si)_mm_setzero_si128(), (__mmask8)__U));
351}
352
353// 256 Bit : float -> uint
354
355static __inline__ __m256i __DEFAULT_FN_ATTRS256
356_mm256_cvttsps_epu32(__m256 __A) {
357 return ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask(
358 (__v8sf)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
359 _MM_FROUND_CUR_DIRECTION));
360}
361
362static __inline__ __m256i __DEFAULT_FN_ATTRS256
363_mm256_mask_cvttsps_epu32(__m256i __W, __mmask8 __U, __m256 __A) {
364 return ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask(
365 (__v8sf)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
366}
367
368static __inline__ __m256i __DEFAULT_FN_ATTRS256
369_mm256_maskz_cvttsps_epu32(__mmask8 __U, __m256 __A) {
370 return ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask(
371 (__v8sf)__A, (__v8si)_mm256_setzero_si256(), __U,
372 _MM_FROUND_CUR_DIRECTION));
373}
374
375#define _mm256_cvtts_roundps_epu32(__A, __R) \
376 ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask( \
377 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_undefined_si256(), \
378 (__mmask8) - 1, (int)(__R)))
379
380#define _mm256_mask_cvtts_roundps_epu32(__W, __U, __A, __R) \
381 ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask( \
382 (__v8sf)(__m256)__A, (__v8si)(__m256i)__W, (__mmask8)__U, (int)(__R)))
383
384#define _mm256_maskz_cvtts_roundps_epu32(__U, __A, __R) \
385 ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask( \
386 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_setzero_si256(), \
387 (__mmask8)__U, (int)(__R)))
388
389// 128 bit : float -> long
390static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epi64(__m128 __A) {
391 return ((__m128i)__builtin_ia32_vcvttps2qqs128_mask(
392 (__v4sf)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
393}
394
395static __inline__ __m128i __DEFAULT_FN_ATTRS128
396_mm_mask_cvttsps_epi64(__m128i __W, __mmask8 __U, __m128 __A) {
397 return ((__m128i)__builtin_ia32_vcvttps2qqs128_mask(
398 (__v4sf)__A, (__v2di)(__m128i)__W, (__mmask8)__U));
399}
400
401static __inline__ __m128i __DEFAULT_FN_ATTRS128
402_mm_maskz_cvttsps_epi64(__mmask8 __U, __m128 __A) {
403 return ((__m128i)__builtin_ia32_vcvttps2qqs128_mask(
404 (__v4sf)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
405}
406// 256 bit : float -> long
407
408static __inline__ __m256i __DEFAULT_FN_ATTRS256
409_mm256_cvttsps_epi64(__m128 __A) {
410 return ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask(
411 (__v4sf)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
412 _MM_FROUND_CUR_DIRECTION));
413}
414static __inline__ __m256i __DEFAULT_FN_ATTRS256
415_mm256_mask_cvttsps_epi64(__m256i __W, __mmask8 __U, __m128 __A) {
416 return ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask(
417 (__v4sf)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
418}
419
420static __inline__ __m256i __DEFAULT_FN_ATTRS256
421_mm256_maskz_cvttsps_epi64(__mmask8 __U, __m128 __A) {
422 return ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask(
423 (__v4sf)__A, (__v4di)_mm256_setzero_si256(), __U,
424 _MM_FROUND_CUR_DIRECTION));
425}
426
427#define _mm256_cvtts_roundps_epi64(__A, __R) \
428 ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask( \
429 (__v4sf)(__m128)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
430 (int)__R))
431
432#define _mm256_mask_cvtts_roundps_epi64(__W, __U, __A, __R) \
433 ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask( \
434 (__v4sf)(__m128)__A, (__v4di)__W, (__mmask8)__U, (int)__R))
435
436#define _mm256_maskz_cvtts_roundps_epi64(__U, __A, __R) \
437 ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask( \
438 (__v4sf)(__m128)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, \
439 (int)__R))
440
441// 128 bit : float -> ulong
442static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epu64(__m128 __A) {
443 return ((__m128i)__builtin_ia32_vcvttps2uqqs128_mask(
444 (__v4sf)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
445}
446
447static __inline__ __m128i __DEFAULT_FN_ATTRS128
448_mm_mask_cvttsps_epu64(__m128i __W, __mmask8 __U, __m128 __A) {
449 return ((__m128i)__builtin_ia32_vcvttps2uqqs128_mask(
450 (__v4sf)__A, (__v2di)(__m128i)__W, (__mmask8)__U));
451}
452
453static __inline__ __m128i __DEFAULT_FN_ATTRS128
454_mm_maskz_cvttsps_epu64(__mmask8 __U, __m128 __A) {
455 return ((__m128i)__builtin_ia32_vcvttps2uqqs128_mask(
456 (__v4sf)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
457}
458// 256 bit : float -> ulong
459
460static __inline__ __m256i __DEFAULT_FN_ATTRS256
461_mm256_cvttsps_epu64(__m128 __A) {
462 return ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask(
463 (__v4sf)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
464 _MM_FROUND_CUR_DIRECTION));
465}
466
467static __inline__ __m256i __DEFAULT_FN_ATTRS256
468_mm256_mask_cvttsps_epu64(__m256i __W, __mmask8 __U, __m128 __A) {
469 return ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask(
470 (__v4sf)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
471}
472
473static __inline__ __m256i __DEFAULT_FN_ATTRS256
474_mm256_maskz_cvttsps_epu64(__mmask8 __U, __m128 __A) {
475 return ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask(
476 (__v4sf)__A, (__v4di)_mm256_setzero_si256(), __U,
477 _MM_FROUND_CUR_DIRECTION));
478}
479
480#define _mm256_cvtts_roundps_epu64(__A, __R) \
481 ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask( \
482 (__v4sf)(__m128)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
483 (int)__R))
484
485#define _mm256_mask_cvtts_roundps_epu64(__W, __U, __A, __R) \
486 ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask( \
487 (__v4sf)(__m128)__A, (__v4di)__W, (__mmask8)__U, (int)__R))
488
489#define _mm256_maskz_cvtts_roundps_epu64(__U, __A, __R) \
490 ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask( \
491 (__v4sf)(__m128)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, \
492 (int)__R))
493
494#undef __DEFAULT_FN_ATTRS128
495#undef __DEFAULT_FN_ATTRS256
496#endif // __AVX10_2SATCVTDSINTRIN_H
lib/include/avx10_2satcvtintrin.h created+444
...@@ -0,0 +1,444 @@
1/*===----------- avx10_2satcvtintrin.h - AVX10_2SATCVT intrinsics ----------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2satcvtintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2SATCVTINTRIN_H
15#define __AVX10_2SATCVTINTRIN_H
16
17#define _mm_ipcvtbf16_epi8(A) \
18 ((__m128i)__builtin_ia32_vcvtbf162ibs128((__v8bf)(__m128bh)(A)))
19
20#define _mm_mask_ipcvtbf16_epi8(W, U, A) \
21 ((__m128i)__builtin_ia32_selectw_128( \
22 (__mmask8)(U), (__v8hi)_mm_ipcvtbf16_epi8(A), (__v8hi)(__m128i)(W)))
23
24#define _mm_maskz_ipcvtbf16_epi8(U, A) \
25 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
26 (__v8hi)_mm_ipcvtbf16_epi8(A), \
27 (__v8hi)_mm_setzero_si128()))
28
29#define _mm256_ipcvtbf16_epi8(A) \
30 ((__m256i)__builtin_ia32_vcvtbf162ibs256((__v16bf)(__m256bh)(A)))
31
32#define _mm256_mask_ipcvtbf16_epi8(W, U, A) \
33 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
34 (__v16hi)_mm256_ipcvtbf16_epi8(A), \
35 (__v16hi)(__m256i)(W)))
36
37#define _mm256_maskz_ipcvtbf16_epi8(U, A) \
38 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
39 (__v16hi)_mm256_ipcvtbf16_epi8(A), \
40 (__v16hi)_mm256_setzero_si256()))
41
42#define _mm_ipcvtbf16_epu8(A) \
43 ((__m128i)__builtin_ia32_vcvtbf162iubs128((__v8bf)(__m128bh)(A)))
44
45#define _mm_mask_ipcvtbf16_epu8(W, U, A) \
46 ((__m128i)__builtin_ia32_selectw_128( \
47 (__mmask8)(U), (__v8hi)_mm_ipcvtbf16_epu8(A), (__v8hi)(__m128i)(W)))
48
49#define _mm_maskz_ipcvtbf16_epu8(U, A) \
50 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
51 (__v8hi)_mm_ipcvtbf16_epu8(A), \
52 (__v8hi)_mm_setzero_si128()))
53
54#define _mm256_ipcvtbf16_epu8(A) \
55 ((__m256i)__builtin_ia32_vcvtbf162iubs256((__v16bf)(__m256bh)(A)))
56
57#define _mm256_mask_ipcvtbf16_epu8(W, U, A) \
58 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
59 (__v16hi)_mm256_ipcvtbf16_epu8(A), \
60 (__v16hi)(__m256i)(W)))
61
62#define _mm256_maskz_ipcvtbf16_epu8(U, A) \
63 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
64 (__v16hi)_mm256_ipcvtbf16_epu8(A), \
65 (__v16hi)_mm256_setzero_si256()))
66
67#define _mm_ipcvtph_epi8(A) \
68 ((__m128i)__builtin_ia32_vcvtph2ibs128_mask( \
69 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
70
71#define _mm_mask_ipcvtph_epi8(W, U, A) \
72 ((__m128i)__builtin_ia32_vcvtph2ibs128_mask((__v8hf)(__m128h)(A), \
73 (__v8hu)(W), (__mmask8)(U)))
74
75#define _mm_maskz_ipcvtph_epi8(U, A) \
76 ((__m128i)__builtin_ia32_vcvtph2ibs128_mask( \
77 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
78
79#define _mm256_ipcvtph_epi8(A) \
80 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask( \
81 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
82 _MM_FROUND_CUR_DIRECTION))
83
84#define _mm256_mask_ipcvtph_epi8(W, U, A) \
85 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask((__v16hf)(__m256h)(A), \
86 (__v16hu)(W), (__mmask16)(U), \
87 _MM_FROUND_CUR_DIRECTION))
88
89#define _mm256_maskz_ipcvtph_epi8(U, A) \
90 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask( \
91 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
92 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
93
94#define _mm256_ipcvt_roundph_epi8(A, R) \
95 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask((__v16hf)(__m256h)(A), \
96 (__v16hu)_mm256_setzero_si256(), \
97 (__mmask16)-1, (const int)R))
98
99#define _mm256_mask_ipcvt_roundph_epi8(W, U, A, R) \
100 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask( \
101 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
102
103#define _mm256_maskz_ipcvt_roundph_epi8(U, A, R) \
104 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask((__v16hf)(__m256h)(A), \
105 (__v16hu)_mm256_setzero_si256(), \
106 (__mmask16)(U), (const int)R))
107
108#define _mm_ipcvtph_epu8(A) \
109 ((__m128i)__builtin_ia32_vcvtph2iubs128_mask( \
110 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
111
112#define _mm_mask_ipcvtph_epu8(W, U, A) \
113 ((__m128i)__builtin_ia32_vcvtph2iubs128_mask((__v8hf)(__m128h)(A), \
114 (__v8hu)(W), (__mmask8)(U)))
115
116#define _mm_maskz_ipcvtph_epu8(U, A) \
117 ((__m128i)__builtin_ia32_vcvtph2iubs128_mask( \
118 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
119
120#define _mm256_ipcvtph_epu8(A) \
121 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
122 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
123 _MM_FROUND_CUR_DIRECTION))
124
125#define _mm256_mask_ipcvtph_epu8(W, U, A) \
126 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask((__v16hf)(__m256h)(A), \
127 (__v16hu)(W), (__mmask16)(U), \
128 _MM_FROUND_CUR_DIRECTION))
129
130#define _mm256_maskz_ipcvtph_epu8(U, A) \
131 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
132 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
133 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
134
135#define _mm256_ipcvt_roundph_epu8(A, R) \
136 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
137 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
138 (const int)R))
139
140#define _mm256_mask_ipcvt_roundph_epu8(W, U, A, R) \
141 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
142 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
143
144#define _mm256_maskz_ipcvt_roundph_epu8(U, A, R) \
145 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
146 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
147 (const int)R))
148
149#define _mm_ipcvtps_epi8(A) \
150 ((__m128i)__builtin_ia32_vcvtps2ibs128_mask( \
151 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
152
153#define _mm_mask_ipcvtps_epi8(W, U, A) \
154 ((__m128i)__builtin_ia32_vcvtps2ibs128_mask((__v4sf)(__m128)(A), \
155 (__v4su)(W), (__mmask8)(U)))
156
157#define _mm_maskz_ipcvtps_epi8(U, A) \
158 ((__m128i)__builtin_ia32_vcvtps2ibs128_mask( \
159 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
160
161#define _mm256_ipcvtps_epi8(A) \
162 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask( \
163 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
164 _MM_FROUND_CUR_DIRECTION))
165
166#define _mm256_mask_ipcvtps_epi8(W, U, A) \
167 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask((__v8sf)(__m256)(A), \
168 (__v8su)(W), (__mmask8)(U), \
169 _MM_FROUND_CUR_DIRECTION))
170
171#define _mm256_maskz_ipcvtps_epi8(U, A) \
172 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask( \
173 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
174 _MM_FROUND_CUR_DIRECTION))
175
176#define _mm256_ipcvt_roundps_epi8(A, R) \
177 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask((__v8sf)(__m256)(A), \
178 (__v8su)_mm256_setzero_si256(), \
179 (__mmask8)-1, (const int)R))
180
181#define _mm256_mask_ipcvt_roundps_epi8(W, U, A, R) \
182 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask( \
183 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
184
185#define _mm256_maskz_ipcvt_roundps_epi8(U, A, R) \
186 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask((__v8sf)(__m256)(A), \
187 (__v8su)_mm256_setzero_si256(), \
188 (__mmask8)(U), (const int)R))
189
190#define _mm_ipcvtps_epu8(A) \
191 ((__m128i)__builtin_ia32_vcvtps2iubs128_mask( \
192 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
193
194#define _mm_mask_ipcvtps_epu8(W, U, A) \
195 ((__m128i)__builtin_ia32_vcvtps2iubs128_mask((__v4sf)(__m128)(A), \
196 (__v4su)(W), (__mmask8)(U)))
197
198#define _mm_maskz_ipcvtps_epu8(U, A) \
199 ((__m128i)__builtin_ia32_vcvtps2iubs128_mask( \
200 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
201
202#define _mm256_ipcvtps_epu8(A) \
203 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask( \
204 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
205 _MM_FROUND_CUR_DIRECTION))
206
207#define _mm256_mask_ipcvtps_epu8(W, U, A) \
208 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask((__v8sf)(__m256)(A), \
209 (__v8su)(W), (__mmask8)(U), \
210 _MM_FROUND_CUR_DIRECTION))
211
212#define _mm256_maskz_ipcvtps_epu8(U, A) \
213 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask( \
214 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
215 _MM_FROUND_CUR_DIRECTION))
216
217#define _mm256_ipcvt_roundps_epu8(A, R) \
218 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask((__v8sf)(__m256)(A), \
219 (__v8su)_mm256_setzero_si256(), \
220 (__mmask8)-1, (const int)R))
221
222#define _mm256_mask_ipcvt_roundps_epu8(W, U, A, R) \
223 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask( \
224 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
225
226#define _mm256_maskz_ipcvt_roundps_epu8(U, A, R) \
227 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask((__v8sf)(__m256)(A), \
228 (__v8su)_mm256_setzero_si256(), \
229 (__mmask8)(U), (const int)R))
230
231#define _mm_ipcvttbf16_epi8(A) \
232 ((__m128i)__builtin_ia32_vcvttbf162ibs128((__v8bf)(__m128bh)(A)))
233
234#define _mm_mask_ipcvttbf16_epi8(W, U, A) \
235 ((__m128i)__builtin_ia32_selectw_128( \
236 (__mmask8)(U), (__v8hi)_mm_ipcvttbf16_epi8(A), (__v8hi)(__m128i)(W)))
237
238#define _mm_maskz_ipcvttbf16_epi8(U, A) \
239 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
240 (__v8hi)_mm_ipcvttbf16_epi8(A), \
241 (__v8hi)_mm_setzero_si128()))
242
243#define _mm256_ipcvttbf16_epi8(A) \
244 ((__m256i)__builtin_ia32_vcvttbf162ibs256((__v16bf)(__m256bh)(A)))
245
246#define _mm256_mask_ipcvttbf16_epi8(W, U, A) \
247 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
248 (__v16hi)_mm256_ipcvttbf16_epi8(A), \
249 (__v16hi)(__m256i)(W)))
250
251#define _mm256_maskz_ipcvttbf16_epi8(U, A) \
252 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
253 (__v16hi)_mm256_ipcvttbf16_epi8(A), \
254 (__v16hi)_mm256_setzero_si256()))
255
256#define _mm_ipcvttbf16_epu8(A) \
257 ((__m128i)__builtin_ia32_vcvttbf162iubs128((__v8bf)(__m128bh)(A)))
258
259#define _mm_mask_ipcvttbf16_epu8(W, U, A) \
260 ((__m128i)__builtin_ia32_selectw_128( \
261 (__mmask8)(U), (__v8hi)_mm_ipcvttbf16_epu8(A), (__v8hi)(__m128i)(W)))
262
263#define _mm_maskz_ipcvttbf16_epu8(U, A) \
264 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
265 (__v8hi)_mm_ipcvttbf16_epu8(A), \
266 (__v8hi)_mm_setzero_si128()))
267
268#define _mm256_ipcvttbf16_epu8(A) \
269 ((__m256i)__builtin_ia32_vcvttbf162iubs256((__v16bf)(__m256bh)(A)))
270
271#define _mm256_mask_ipcvttbf16_epu8(W, U, A) \
272 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
273 (__v16hi)_mm256_ipcvttbf16_epu8(A), \
274 (__v16hi)(__m256i)(W)))
275
276#define _mm256_maskz_ipcvttbf16_epu8(U, A) \
277 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
278 (__v16hi)_mm256_ipcvttbf16_epu8(A), \
279 (__v16hi)_mm256_setzero_si256()))
280
281#define _mm_ipcvttph_epi8(A) \
282 ((__m128i)__builtin_ia32_vcvttph2ibs128_mask( \
283 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
284
285#define _mm_mask_ipcvttph_epi8(W, U, A) \
286 ((__m128i)__builtin_ia32_vcvttph2ibs128_mask((__v8hf)(__m128h)(A), \
287 (__v8hu)(W), (__mmask8)(U)))
288
289#define _mm_maskz_ipcvttph_epi8(U, A) \
290 ((__m128i)__builtin_ia32_vcvttph2ibs128_mask( \
291 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
292
293#define _mm256_ipcvttph_epi8(A) \
294 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
295 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
296 _MM_FROUND_CUR_DIRECTION))
297
298#define _mm256_mask_ipcvttph_epi8(W, U, A) \
299 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask((__v16hf)(__m256h)(A), \
300 (__v16hu)(W), (__mmask16)(U), \
301 _MM_FROUND_CUR_DIRECTION))
302
303#define _mm256_maskz_ipcvttph_epi8(U, A) \
304 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
305 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
306 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
307
308#define _mm256_ipcvtt_roundph_epi8(A, R) \
309 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
310 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
311 (const int)R))
312
313#define _mm256_mask_ipcvtt_roundph_epi8(W, U, A, R) \
314 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
315 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
316
317#define _mm256_maskz_ipcvtt_roundph_epi8(U, A, R) \
318 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
319 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
320 (const int)R))
321
322#define _mm_ipcvttph_epu8(A) \
323 ((__m128i)__builtin_ia32_vcvttph2iubs128_mask( \
324 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
325
326#define _mm_mask_ipcvttph_epu8(W, U, A) \
327 ((__m128i)__builtin_ia32_vcvttph2iubs128_mask((__v8hf)(__m128h)(A), \
328 (__v8hu)(W), (__mmask8)(U)))
329
330#define _mm_maskz_ipcvttph_epu8(U, A) \
331 ((__m128i)__builtin_ia32_vcvttph2iubs128_mask( \
332 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
333
334#define _mm256_ipcvttph_epu8(A) \
335 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
336 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
337 _MM_FROUND_CUR_DIRECTION))
338
339#define _mm256_mask_ipcvttph_epu8(W, U, A) \
340 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask((__v16hf)(__m256h)(A), \
341 (__v16hu)(W), (__mmask16)(U), \
342 _MM_FROUND_CUR_DIRECTION))
343
344#define _mm256_maskz_ipcvttph_epu8(U, A) \
345 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
346 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
347 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
348
349#define _mm256_ipcvtt_roundph_epu8(A, R) \
350 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
351 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
352 (const int)R))
353
354#define _mm256_mask_ipcvtt_roundph_epu8(W, U, A, R) \
355 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
356 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
357
358#define _mm256_maskz_ipcvtt_roundph_epu8(U, A, R) \
359 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
360 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
361 (const int)R))
362
363#define _mm_ipcvttps_epi8(A) \
364 ((__m128i)__builtin_ia32_vcvttps2ibs128_mask( \
365 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
366
367#define _mm_mask_ipcvttps_epi8(W, U, A) \
368 ((__m128i)__builtin_ia32_vcvttps2ibs128_mask((__v4sf)(__m128)(A), \
369 (__v4su)(W), (__mmask8)(U)))
370
371#define _mm_maskz_ipcvttps_epi8(U, A) \
372 ((__m128i)__builtin_ia32_vcvttps2ibs128_mask( \
373 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
374
375#define _mm256_ipcvttps_epi8(A) \
376 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask( \
377 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
378 _MM_FROUND_CUR_DIRECTION))
379
380#define _mm256_mask_ipcvttps_epi8(W, U, A) \
381 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask((__v8sf)(__m256)(A), \
382 (__v8su)(W), (__mmask8)(U), \
383 _MM_FROUND_CUR_DIRECTION))
384
385#define _mm256_maskz_ipcvttps_epi8(U, A) \
386 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask( \
387 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
388 _MM_FROUND_CUR_DIRECTION))
389
390#define _mm256_ipcvtt_roundps_epi8(A, R) \
391 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask((__v8sf)(__m256)(A), \
392 (__v8su)_mm256_setzero_si256(), \
393 (__mmask8)-1, (const int)R))
394
395#define _mm256_mask_ipcvtt_roundps_epi8(W, U, A, R) \
396 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask( \
397 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
398
399#define _mm256_maskz_ipcvtt_roundps_epi8(U, A, R) \
400 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask((__v8sf)(__m256)(A), \
401 (__v8su)_mm256_setzero_si256(), \
402 (__mmask8)(U), (const int)R))
403
404#define _mm_ipcvttps_epu8(A) \
405 ((__m128i)__builtin_ia32_vcvttps2iubs128_mask( \
406 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
407
408#define _mm_mask_ipcvttps_epu8(W, U, A) \
409 ((__m128i)__builtin_ia32_vcvttps2iubs128_mask((__v4sf)(__m128)(A), \
410 (__v4su)(W), (__mmask8)(U)))
411
412#define _mm_maskz_ipcvttps_epu8(U, A) \
413 ((__m128i)__builtin_ia32_vcvttps2iubs128_mask( \
414 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
415
416#define _mm256_ipcvttps_epu8(A) \
417 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
418 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
419 _MM_FROUND_CUR_DIRECTION))
420
421#define _mm256_mask_ipcvttps_epu8(W, U, A) \
422 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask((__v8sf)(__m256)(A), \
423 (__v8su)(W), (__mmask8)(U), \
424 _MM_FROUND_CUR_DIRECTION))
425
426#define _mm256_maskz_ipcvttps_epu8(U, A) \
427 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
428 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
429 _MM_FROUND_CUR_DIRECTION))
430
431#define _mm256_ipcvtt_roundps_epu8(A, R) \
432 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
433 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
434 (const int)R))
435
436#define _mm256_mask_ipcvtt_roundps_epu8(W, U, A, R) \
437 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
438 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
439
440#define _mm256_maskz_ipcvtt_roundps_epu8(U, A, R) \
441 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
442 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), \
443 (const int)R))
444#endif // __AVX10_2SATCVTINTRIN_H
lib/include/avx2intrin.h+9
...@@ -15,12 +15,21 @@...@@ -15,12 +15,21 @@
15#define __AVX2INTRIN_H15#define __AVX2INTRIN_H
1616
17/* Define the default attributes for the functions in this file. */17/* Define the default attributes for the functions in this file. */
18#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
18#define __DEFAULT_FN_ATTRS256 \19#define __DEFAULT_FN_ATTRS256 \
19 __attribute__((__always_inline__, __nodebug__, \20 __attribute__((__always_inline__, __nodebug__, \
20 __target__("avx2,no-evex512"), __min_vector_width__(256)))21 __target__("avx2,no-evex512"), __min_vector_width__(256)))
21#define __DEFAULT_FN_ATTRS128 \22#define __DEFAULT_FN_ATTRS128 \
22 __attribute__((__always_inline__, __nodebug__, \23 __attribute__((__always_inline__, __nodebug__, \
23 __target__("avx2,no-evex512"), __min_vector_width__(128)))24 __target__("avx2,no-evex512"), __min_vector_width__(128)))
25#else
26#define __DEFAULT_FN_ATTRS256 \
27 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), \
28 __min_vector_width__(256)))
29#define __DEFAULT_FN_ATTRS128 \
30 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), \
31 __min_vector_width__(128)))
32#endif
2433
25/* SSE4 Multiple Packed Sums of Absolute Difference. */34/* SSE4 Multiple Packed Sums of Absolute Difference. */
26/// Computes sixteen sum of absolute difference (SAD) operations on sets of35/// Computes sixteen sum of absolute difference (SAD) operations on sets of
lib/include/avx512bitalgintrin.h+2-2
...@@ -23,7 +23,7 @@...@@ -23,7 +23,7 @@
23static __inline__ __m512i __DEFAULT_FN_ATTRS23static __inline__ __m512i __DEFAULT_FN_ATTRS
24_mm512_popcnt_epi16(__m512i __A)24_mm512_popcnt_epi16(__m512i __A)
25{25{
26 return (__m512i) __builtin_ia32_vpopcntw_512((__v32hi) __A);26 return (__m512i)__builtin_elementwise_popcount((__v32hu)__A);
27}27}
2828
29static __inline__ __m512i __DEFAULT_FN_ATTRS29static __inline__ __m512i __DEFAULT_FN_ATTRS
...@@ -45,7 +45,7 @@ _mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B)...@@ -45,7 +45,7 @@ _mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B)
45static __inline__ __m512i __DEFAULT_FN_ATTRS45static __inline__ __m512i __DEFAULT_FN_ATTRS
46_mm512_popcnt_epi8(__m512i __A)46_mm512_popcnt_epi8(__m512i __A)
47{47{
48 return (__m512i) __builtin_ia32_vpopcntb_512((__v64qi) __A);48 return (__m512i)__builtin_elementwise_popcount((__v64qu)__A);
49}49}
5050
51static __inline__ __m512i __DEFAULT_FN_ATTRS51static __inline__ __m512i __DEFAULT_FN_ATTRS
lib/include/avx512fintrin.h+22-14
...@@ -175,12 +175,21 @@ typedef enum...@@ -175,12 +175,21 @@ typedef enum
175 __attribute__((__always_inline__, __nodebug__, \175 __attribute__((__always_inline__, __nodebug__, \
176 __target__("avx512f,no-evex512")))176 __target__("avx512f,no-evex512")))
177177
178#if defined(__cplusplus) && (__cplusplus >= 201103L)
179#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
180#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
181#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
182#else
183#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
184#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
185#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
186#endif
187
178/* Create vectors with repeated elements */188/* Create vectors with repeated elements */
179189
180static __inline __m512i __DEFAULT_FN_ATTRS512190static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
181_mm512_setzero_si512(void)191_mm512_setzero_si512(void) {
182{192 return __extension__(__m512i)(__v8di){0, 0, 0, 0, 0, 0, 0, 0};
183 return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 };
184}193}
185194
186#define _mm512_setzero_epi32 _mm512_setzero_si512195#define _mm512_setzero_epi32 _mm512_setzero_si512
...@@ -256,20 +265,16 @@ _mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)...@@ -256,20 +265,16 @@ _mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)
256 (__v8di) _mm512_setzero_si512());265 (__v8di) _mm512_setzero_si512());
257}266}
258267
259268static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_setzero_ps(void) {
260static __inline __m512 __DEFAULT_FN_ATTRS512269 return __extension__(__m512){0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
261_mm512_setzero_ps(void)270 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
262{
263 return __extension__ (__m512){ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
264 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
265}271}
266272
267#define _mm512_setzero _mm512_setzero_ps273#define _mm512_setzero _mm512_setzero_ps
268274
269static __inline __m512d __DEFAULT_FN_ATTRS512275static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
270_mm512_setzero_pd(void)276_mm512_setzero_pd(void) {
271{277 return __extension__(__m512d){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
272 return __extension__ (__m512d){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
273}278}
274279
275static __inline __m512 __DEFAULT_FN_ATTRS512280static __inline __m512 __DEFAULT_FN_ATTRS512
...@@ -9775,5 +9780,8 @@ _mm512_cvtsi512_si32(__m512i __A) {...@@ -9775,5 +9780,8 @@ _mm512_cvtsi512_si32(__m512i __A) {
9775#undef __DEFAULT_FN_ATTRS5129780#undef __DEFAULT_FN_ATTRS512
9776#undef __DEFAULT_FN_ATTRS1289781#undef __DEFAULT_FN_ATTRS128
9777#undef __DEFAULT_FN_ATTRS9782#undef __DEFAULT_FN_ATTRS
9783#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
9784#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
9785#undef __DEFAULT_FN_ATTRS_CONSTEXPR
97789786
9779#endif /* __AVX512FINTRIN_H */9787#endif /* __AVX512FINTRIN_H */
lib/include/avx512vlbitalgintrin.h+4-4
...@@ -27,7 +27,7 @@...@@ -27,7 +27,7 @@
27static __inline__ __m256i __DEFAULT_FN_ATTRS25627static __inline__ __m256i __DEFAULT_FN_ATTRS256
28_mm256_popcnt_epi16(__m256i __A)28_mm256_popcnt_epi16(__m256i __A)
29{29{
30 return (__m256i) __builtin_ia32_vpopcntw_256((__v16hi) __A);30 return (__m256i)__builtin_elementwise_popcount((__v16hu)__A);
31}31}
3232
33static __inline__ __m256i __DEFAULT_FN_ATTRS25633static __inline__ __m256i __DEFAULT_FN_ATTRS256
...@@ -49,7 +49,7 @@ _mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B)...@@ -49,7 +49,7 @@ _mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B)
49static __inline__ __m128i __DEFAULT_FN_ATTRS12849static __inline__ __m128i __DEFAULT_FN_ATTRS128
50_mm_popcnt_epi16(__m128i __A)50_mm_popcnt_epi16(__m128i __A)
51{51{
52 return (__m128i) __builtin_ia32_vpopcntw_128((__v8hi) __A);52 return (__m128i)__builtin_elementwise_popcount((__v8hu)__A);
53}53}
5454
55static __inline__ __m128i __DEFAULT_FN_ATTRS12855static __inline__ __m128i __DEFAULT_FN_ATTRS128
...@@ -71,7 +71,7 @@ _mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B)...@@ -71,7 +71,7 @@ _mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B)
71static __inline__ __m256i __DEFAULT_FN_ATTRS25671static __inline__ __m256i __DEFAULT_FN_ATTRS256
72_mm256_popcnt_epi8(__m256i __A)72_mm256_popcnt_epi8(__m256i __A)
73{73{
74 return (__m256i) __builtin_ia32_vpopcntb_256((__v32qi) __A);74 return (__m256i)__builtin_elementwise_popcount((__v32qu)__A);
75}75}
7676
77static __inline__ __m256i __DEFAULT_FN_ATTRS25677static __inline__ __m256i __DEFAULT_FN_ATTRS256
...@@ -93,7 +93,7 @@ _mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B)...@@ -93,7 +93,7 @@ _mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B)
93static __inline__ __m128i __DEFAULT_FN_ATTRS12893static __inline__ __m128i __DEFAULT_FN_ATTRS128
94_mm_popcnt_epi8(__m128i __A)94_mm_popcnt_epi8(__m128i __A)
95{95{
96 return (__m128i) __builtin_ia32_vpopcntb_128((__v16qi) __A);96 return (__m128i)__builtin_elementwise_popcount((__v16qu)__A);
97}97}
9898
99static __inline__ __m128i __DEFAULT_FN_ATTRS12899static __inline__ __m128i __DEFAULT_FN_ATTRS128
lib/include/avx512vpopcntdqintrin.h+12-4
...@@ -21,8 +21,15 @@...@@ -21,8 +21,15 @@
21 __target__("avx512vpopcntdq,evex512"), \21 __target__("avx512vpopcntdq,evex512"), \
22 __min_vector_width__(512)))22 __min_vector_width__(512)))
2323
24static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi64(__m512i __A) {24#if defined(__cplusplus) && (__cplusplus >= 201103L)
25 return (__m512i)__builtin_ia32_vpopcntq_512((__v8di)__A);25#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
26#else
27#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
28#endif
29
30static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
31_mm512_popcnt_epi64(__m512i __A) {
32 return (__m512i)__builtin_elementwise_popcount((__v8du)__A);
26}33}
2734
28static __inline__ __m512i __DEFAULT_FN_ATTRS35static __inline__ __m512i __DEFAULT_FN_ATTRS
...@@ -36,8 +43,9 @@ _mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {...@@ -36,8 +43,9 @@ _mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
36 return _mm512_mask_popcnt_epi64((__m512i)_mm512_setzero_si512(), __U, __A);43 return _mm512_mask_popcnt_epi64((__m512i)_mm512_setzero_si512(), __U, __A);
37}44}
3845
39static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi32(__m512i __A) {46static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
40 return (__m512i)__builtin_ia32_vpopcntd_512((__v16si)__A);47_mm512_popcnt_epi32(__m512i __A) {
48 return (__m512i)__builtin_elementwise_popcount((__v16su)__A);
41}49}
4250
43static __inline__ __m512i __DEFAULT_FN_ATTRS51static __inline__ __m512i __DEFAULT_FN_ATTRS
lib/include/avx512vpopcntdqvlintrin.h+16-8
...@@ -25,9 +25,17 @@...@@ -25,9 +25,17 @@
25 __target__("avx512vpopcntdq,avx512vl,no-evex512"), \25 __target__("avx512vpopcntdq,avx512vl,no-evex512"), \
26 __min_vector_width__(256)))26 __min_vector_width__(256)))
2727
28static __inline__ __m128i __DEFAULT_FN_ATTRS12828#if defined(__cplusplus) && (__cplusplus >= 201103L)
29#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
30#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
31#else
32#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
33#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
34#endif
35
36static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
29_mm_popcnt_epi64(__m128i __A) {37_mm_popcnt_epi64(__m128i __A) {
30 return (__m128i)__builtin_ia32_vpopcntq_128((__v2di)__A);38 return (__m128i)__builtin_elementwise_popcount((__v2du)__A);
31}39}
3240
33static __inline__ __m128i __DEFAULT_FN_ATTRS12841static __inline__ __m128i __DEFAULT_FN_ATTRS128
...@@ -41,9 +49,9 @@ _mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {...@@ -41,9 +49,9 @@ _mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
41 return _mm_mask_popcnt_epi64((__m128i)_mm_setzero_si128(), __U, __A);49 return _mm_mask_popcnt_epi64((__m128i)_mm_setzero_si128(), __U, __A);
42}50}
4351
44static __inline__ __m128i __DEFAULT_FN_ATTRS12852static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
45_mm_popcnt_epi32(__m128i __A) {53_mm_popcnt_epi32(__m128i __A) {
46 return (__m128i)__builtin_ia32_vpopcntd_128((__v4si)__A);54 return (__m128i)__builtin_elementwise_popcount((__v4su)__A);
47}55}
4856
49static __inline__ __m128i __DEFAULT_FN_ATTRS12857static __inline__ __m128i __DEFAULT_FN_ATTRS128
...@@ -57,9 +65,9 @@ _mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {...@@ -57,9 +65,9 @@ _mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
57 return _mm_mask_popcnt_epi32((__m128i)_mm_setzero_si128(), __U, __A);65 return _mm_mask_popcnt_epi32((__m128i)_mm_setzero_si128(), __U, __A);
58}66}
5967
60static __inline__ __m256i __DEFAULT_FN_ATTRS25668static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
61_mm256_popcnt_epi64(__m256i __A) {69_mm256_popcnt_epi64(__m256i __A) {
62 return (__m256i)__builtin_ia32_vpopcntq_256((__v4di)__A);70 return (__m256i)__builtin_elementwise_popcount((__v4du)__A);
63}71}
6472
65static __inline__ __m256i __DEFAULT_FN_ATTRS25673static __inline__ __m256i __DEFAULT_FN_ATTRS256
...@@ -73,9 +81,9 @@ _mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {...@@ -73,9 +81,9 @@ _mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
73 return _mm256_mask_popcnt_epi64((__m256i)_mm256_setzero_si256(), __U, __A);81 return _mm256_mask_popcnt_epi64((__m256i)_mm256_setzero_si256(), __U, __A);
74}82}
7583
76static __inline__ __m256i __DEFAULT_FN_ATTRS25684static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
77_mm256_popcnt_epi32(__m256i __A) {85_mm256_popcnt_epi32(__m256i __A) {
78 return (__m256i)__builtin_ia32_vpopcntd_256((__v8si)__A);86 return (__m256i)__builtin_elementwise_popcount((__v8su)__A);
79}87}
8088
81static __inline__ __m256i __DEFAULT_FN_ATTRS25689static __inline__ __m256i __DEFAULT_FN_ATTRS256
lib/include/avxintrin.h+30-16
...@@ -50,12 +50,29 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));...@@ -50,12 +50,29 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
50#endif50#endif
5151
52/* Define the default attributes for the functions in this file. */52/* Define the default attributes for the functions in this file. */
53#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
53#define __DEFAULT_FN_ATTRS \54#define __DEFAULT_FN_ATTRS \
54 __attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \55 __attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \
55 __min_vector_width__(256)))56 __min_vector_width__(256)))
56#define __DEFAULT_FN_ATTRS128 \57#define __DEFAULT_FN_ATTRS128 \
57 __attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \58 __attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \
58 __min_vector_width__(128)))59 __min_vector_width__(128)))
60#else
61#define __DEFAULT_FN_ATTRS \
62 __attribute__((__always_inline__, __nodebug__, __target__("avx"), \
63 __min_vector_width__(256)))
64#define __DEFAULT_FN_ATTRS128 \
65 __attribute__((__always_inline__, __nodebug__, __target__("avx"), \
66 __min_vector_width__(128)))
67#endif
68
69#if defined(__cplusplus) && (__cplusplus >= 201103L)
70#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
71#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
72#else
73#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
74#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
75#endif
5976
60/* Arithmetic */77/* Arithmetic */
61/// Adds two 256-bit vectors of [4 x double].78/// Adds two 256-bit vectors of [4 x double].
...@@ -3689,7 +3706,7 @@ _mm256_undefined_si256(void)...@@ -3689,7 +3706,7 @@ _mm256_undefined_si256(void)
3689/// A double-precision floating-point value used to initialize bits [63:0]3706/// A double-precision floating-point value used to initialize bits [63:0]
3690/// of the result.3707/// of the result.
3691/// \returns An initialized 256-bit floating-point vector of [4 x double].3708/// \returns An initialized 256-bit floating-point vector of [4 x double].
3692static __inline __m256d __DEFAULT_FN_ATTRS3709static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
3693_mm256_set_pd(double __a, double __b, double __c, double __d)3710_mm256_set_pd(double __a, double __b, double __c, double __d)
3694{3711{
3695 return __extension__ (__m256d){ __d, __c, __b, __a };3712 return __extension__ (__m256d){ __d, __c, __b, __a };
...@@ -3728,7 +3745,7 @@ _mm256_set_pd(double __a, double __b, double __c, double __d)...@@ -3728,7 +3745,7 @@ _mm256_set_pd(double __a, double __b, double __c, double __d)
3728/// A single-precision floating-point value used to initialize bits [31:0]3745/// A single-precision floating-point value used to initialize bits [31:0]
3729/// of the result.3746/// of the result.
3730/// \returns An initialized 256-bit floating-point vector of [8 x float].3747/// \returns An initialized 256-bit floating-point vector of [8 x float].
3731static __inline __m256 __DEFAULT_FN_ATTRS3748static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
3732_mm256_set_ps(float __a, float __b, float __c, float __d,3749_mm256_set_ps(float __a, float __b, float __c, float __d,
3733 float __e, float __f, float __g, float __h)3750 float __e, float __f, float __g, float __h)
3734{3751{
...@@ -3955,7 +3972,7 @@ _mm256_set_epi64x(long long __a, long long __b, long long __c, long long __d)...@@ -3955,7 +3972,7 @@ _mm256_set_epi64x(long long __a, long long __b, long long __c, long long __d)
3955/// A double-precision floating-point value used to initialize bits [255:192]3972/// A double-precision floating-point value used to initialize bits [255:192]
3956/// of the result.3973/// of the result.
3957/// \returns An initialized 256-bit floating-point vector of [4 x double].3974/// \returns An initialized 256-bit floating-point vector of [4 x double].
3958static __inline __m256d __DEFAULT_FN_ATTRS3975static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
3959_mm256_setr_pd(double __a, double __b, double __c, double __d)3976_mm256_setr_pd(double __a, double __b, double __c, double __d)
3960{3977{
3961 return _mm256_set_pd(__d, __c, __b, __a);3978 return _mm256_set_pd(__d, __c, __b, __a);
...@@ -3995,7 +4012,7 @@ _mm256_setr_pd(double __a, double __b, double __c, double __d)...@@ -3995,7 +4012,7 @@ _mm256_setr_pd(double __a, double __b, double __c, double __d)
3995/// A single-precision floating-point value used to initialize bits [255:224]4012/// A single-precision floating-point value used to initialize bits [255:224]
3996/// of the result.4013/// of the result.
3997/// \returns An initialized 256-bit floating-point vector of [8 x float].4014/// \returns An initialized 256-bit floating-point vector of [8 x float].
3998static __inline __m256 __DEFAULT_FN_ATTRS4015static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
3999_mm256_setr_ps(float __a, float __b, float __c, float __d,4016_mm256_setr_ps(float __a, float __b, float __c, float __d,
4000 float __e, float __f, float __g, float __h)4017 float __e, float __f, float __g, float __h)
4001{4018{
...@@ -4212,7 +4229,7 @@ _mm256_setr_epi64x(long long __a, long long __b, long long __c, long long __d)...@@ -4212,7 +4229,7 @@ _mm256_setr_epi64x(long long __a, long long __b, long long __c, long long __d)
4212/// A double-precision floating-point value used to initialize each vector4229/// A double-precision floating-point value used to initialize each vector
4213/// element of the result.4230/// element of the result.
4214/// \returns An initialized 256-bit floating-point vector of [4 x double].4231/// \returns An initialized 256-bit floating-point vector of [4 x double].
4215static __inline __m256d __DEFAULT_FN_ATTRS4232static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
4216_mm256_set1_pd(double __w)4233_mm256_set1_pd(double __w)
4217{4234{
4218 return _mm256_set_pd(__w, __w, __w, __w);4235 return _mm256_set_pd(__w, __w, __w, __w);
...@@ -4231,7 +4248,7 @@ _mm256_set1_pd(double __w)...@@ -4231,7 +4248,7 @@ _mm256_set1_pd(double __w)
4231/// A single-precision floating-point value used to initialize each vector4248/// A single-precision floating-point value used to initialize each vector
4232/// element of the result.4249/// element of the result.
4233/// \returns An initialized 256-bit floating-point vector of [8 x float].4250/// \returns An initialized 256-bit floating-point vector of [8 x float].
4234static __inline __m256 __DEFAULT_FN_ATTRS4251static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
4235_mm256_set1_ps(float __w)4252_mm256_set1_ps(float __w)
4236{4253{
4237 return _mm256_set_ps(__w, __w, __w, __w, __w, __w, __w, __w);4254 return _mm256_set_ps(__w, __w, __w, __w, __w, __w, __w, __w);
...@@ -4322,10 +4339,8 @@ _mm256_set1_epi64x(long long __q)...@@ -4322,10 +4339,8 @@ _mm256_set1_epi64x(long long __q)
4322/// This intrinsic corresponds to the <c> VXORPS </c> instruction.4339/// This intrinsic corresponds to the <c> VXORPS </c> instruction.
4323///4340///
4324/// \returns A 256-bit vector of [4 x double] with all elements set to zero.4341/// \returns A 256-bit vector of [4 x double] with all elements set to zero.
4325static __inline __m256d __DEFAULT_FN_ATTRS4342static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_setzero_pd(void) {
4326_mm256_setzero_pd(void)4343 return __extension__(__m256d){0.0, 0.0, 0.0, 0.0};
4327{
4328 return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 };
4329}4344}
43304345
4331/// Constructs a 256-bit floating-point vector of [8 x float] with all4346/// Constructs a 256-bit floating-point vector of [8 x float] with all
...@@ -4336,9 +4351,7 @@ _mm256_setzero_pd(void)...@@ -4336,9 +4351,7 @@ _mm256_setzero_pd(void)
4336/// This intrinsic corresponds to the <c> VXORPS </c> instruction.4351/// This intrinsic corresponds to the <c> VXORPS </c> instruction.
4337///4352///
4338/// \returns A 256-bit vector of [8 x float] with all elements set to zero.4353/// \returns A 256-bit vector of [8 x float] with all elements set to zero.
4339static __inline __m256 __DEFAULT_FN_ATTRS4354static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_setzero_ps(void) {
4340_mm256_setzero_ps(void)
4341{
4342 return __extension__ (__m256){ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };4355 return __extension__ (__m256){ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
4343}4356}
43444357
...@@ -4349,9 +4362,8 @@ _mm256_setzero_ps(void)...@@ -4349,9 +4362,8 @@ _mm256_setzero_ps(void)
4349/// This intrinsic corresponds to the <c> VXORPS </c> instruction.4362/// This intrinsic corresponds to the <c> VXORPS </c> instruction.
4350///4363///
4351/// \returns A 256-bit integer vector initialized to zero.4364/// \returns A 256-bit integer vector initialized to zero.
4352static __inline __m256i __DEFAULT_FN_ATTRS4365static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4353_mm256_setzero_si256(void)4366_mm256_setzero_si256(void) {
4354{
4355 return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 };4367 return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 };
4356}4368}
43574369
...@@ -5121,6 +5133,8 @@ _mm256_storeu2_m128i(__m128i_u *__addr_hi, __m128i_u *__addr_lo, __m256i __a)...@@ -5121,6 +5133,8 @@ _mm256_storeu2_m128i(__m128i_u *__addr_hi, __m128i_u *__addr_lo, __m256i __a)
5121}5133}
51225134
5123#undef __DEFAULT_FN_ATTRS5135#undef __DEFAULT_FN_ATTRS
5136#undef __DEFAULT_FN_ATTRS_CONSTEXPR
5124#undef __DEFAULT_FN_ATTRS1285137#undef __DEFAULT_FN_ATTRS128
5138#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
51255139
5126#endif /* __AVXINTRIN_H */5140#endif /* __AVXINTRIN_H */
lib/include/avxvnniint16intrin.h+36-77
...@@ -15,14 +15,6 @@...@@ -15,14 +15,6 @@
15#ifndef __AVXVNNIINT16INTRIN_H15#ifndef __AVXVNNIINT16INTRIN_H
16#define __AVXVNNIINT16INTRIN_H16#define __AVXVNNIINT16INTRIN_H
1717
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS128 \
20 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \
21 __min_vector_width__(128)))
22#define __DEFAULT_FN_ATTRS256 \
23 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \
24 __min_vector_width__(256)))
25
26/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with18/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
27/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate19/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
28/// signed 16-bit results. Sum these 2 results with the corresponding20/// signed 16-bit results. Sum these 2 results with the corresponding
...@@ -53,12 +45,9 @@...@@ -53,12 +45,9 @@
53/// ENDFOR45/// ENDFOR
54/// dst[MAX:128] := 046/// dst[MAX:128] := 0
55/// \endcode47/// \endcode
56static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,48#define _mm_dpwsud_epi32(__W, __A, __B) \
57 __m128i __A,49 ((__m128i)__builtin_ia32_vpdpwsud128((__v4si)(__W), (__v4si)(__A), \
58 __m128i __B) {50 (__v4si)(__B)))
59 return (__m128i)__builtin_ia32_vpdpwsud128((__v4si)__W, (__v4si)__A,
60 (__v4si)__B);
61}
6251
63/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with52/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
64/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate53/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -90,11 +79,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,...@@ -90,11 +79,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,
90/// ENDFOR79/// ENDFOR
91/// dst[MAX:256] := 080/// dst[MAX:256] := 0
92/// \endcode81/// \endcode
93static __inline__ __m256i __DEFAULT_FN_ATTRS25682#define _mm256_dpwsud_epi32(__W, __A, __B) \
94_mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {83 ((__m256i)__builtin_ia32_vpdpwsud256((__v8si)(__W), (__v8si)(__A), \
95 return (__m256i)__builtin_ia32_vpdpwsud256((__v8si)__W, (__v8si)__A,84 (__v8si)(__B)))
96 (__v8si)__B);
97}
9885
99/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with86/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
100/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate87/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -127,12 +114,9 @@ _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -127,12 +114,9 @@ _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
127/// ENDFOR114/// ENDFOR
128/// dst[MAX:128] := 0115/// dst[MAX:128] := 0
129/// \endcode116/// \endcode
130static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,117#define _mm_dpwsuds_epi32(__W, __A, __B) \
131 __m128i __A,118 ((__m128i)__builtin_ia32_vpdpwsuds128((__v4si)(__W), (__v4si)(__A), \
132 __m128i __B) {119 (__v4si)(__B)))
133 return (__m128i)__builtin_ia32_vpdpwsuds128((__v4si)__W, (__v4si)__A,
134 (__v4si)__B);
135}
136120
137/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with121/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
138/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate122/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -165,11 +149,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,...@@ -165,11 +149,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,
165/// ENDFOR149/// ENDFOR
166/// dst[MAX:256] := 0150/// dst[MAX:256] := 0
167/// \endcode151/// \endcode
168static __inline__ __m256i __DEFAULT_FN_ATTRS256152#define _mm256_dpwsuds_epi32(__W, __A, __B) \
169_mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {153 ((__m256i)__builtin_ia32_vpdpwsuds256((__v8si)(__W), (__v8si)(__A), \
170 return (__m256i)__builtin_ia32_vpdpwsuds256((__v8si)__W, (__v8si)__A,154 (__v8si)(__B)))
171 (__v8si)__B);
172}
173155
174/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with156/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
175/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate157/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
...@@ -201,12 +183,9 @@ _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -201,12 +183,9 @@ _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
201/// ENDFOR183/// ENDFOR
202/// dst[MAX:128] := 0184/// dst[MAX:128] := 0
203/// \endcode185/// \endcode
204static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,186#define _mm_dpwusd_epi32(__W, __A, __B) \
205 __m128i __A,187 ((__m128i)__builtin_ia32_vpdpwusd128((__v4si)(__W), (__v4si)(__A), \
206 __m128i __B) {188 (__v4si)(__B)))
207 return (__m128i)__builtin_ia32_vpdpwusd128((__v4si)__W, (__v4si)__A,
208 (__v4si)__B);
209}
210189
211/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with190/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
212/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate191/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
...@@ -238,11 +217,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,...@@ -238,11 +217,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,
238/// ENDFOR217/// ENDFOR
239/// dst[MAX:256] := 0218/// dst[MAX:256] := 0
240/// \endcode219/// \endcode
241static __inline__ __m256i __DEFAULT_FN_ATTRS256220#define _mm256_dpwusd_epi32(__W, __A, __B) \
242_mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {221 ((__m256i)__builtin_ia32_vpdpwusd256((__v8si)(__W), (__v8si)(__A), \
243 return (__m256i)__builtin_ia32_vpdpwusd256((__v8si)__W, (__v8si)__A,222 (__v8si)(__B)))
244 (__v8si)__B);
245}
246223
247/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with224/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
248/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate225/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
...@@ -275,12 +252,9 @@ _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -275,12 +252,9 @@ _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {
275/// ENDFOR252/// ENDFOR
276/// dst[MAX:128] := 0253/// dst[MAX:128] := 0
277/// \endcode254/// \endcode
278static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,255#define _mm_dpwusds_epi32(__W, __A, __B) \
279 __m128i __A,256 ((__m128i)__builtin_ia32_vpdpwusds128((__v4si)(__W), (__v4si)(__A), \
280 __m128i __B) {257 (__v4si)(__B)))
281 return (__m128i)__builtin_ia32_vpdpwusds128((__v4si)__W, (__v4si)__A,
282 (__v4si)__B);
283}
284258
285/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with259/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
286/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate260/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
...@@ -313,11 +287,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,...@@ -313,11 +287,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,
313/// ENDFOR287/// ENDFOR
314/// dst[MAX:256] := 0288/// dst[MAX:256] := 0
315/// \endcode289/// \endcode
316static __inline__ __m256i __DEFAULT_FN_ATTRS256290#define _mm256_dpwusds_epi32(__W, __A, __B) \
317_mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {291 ((__m256i)__builtin_ia32_vpdpwusds256((__v8si)(__W), (__v8si)(__A), \
318 return (__m256i)__builtin_ia32_vpdpwusds256((__v8si)__W, (__v8si)__A,292 (__v8si)(__B)))
319 (__v8si)__B);
320}
321293
322/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with294/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
323/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate295/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -349,12 +321,9 @@ _mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -349,12 +321,9 @@ _mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {
349/// ENDFOR321/// ENDFOR
350/// dst[MAX:128] := 0322/// dst[MAX:128] := 0
351/// \endcode323/// \endcode
352static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,324#define _mm_dpwuud_epi32(__W, __A, __B) \
353 __m128i __A,325 ((__m128i)__builtin_ia32_vpdpwuud128((__v4si)(__W), (__v4si)(__A), \
354 __m128i __B) {326 (__v4si)(__B)))
355 return (__m128i)__builtin_ia32_vpdpwuud128((__v4si)__W, (__v4si)__A,
356 (__v4si)__B);
357}
358327
359/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with328/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
360/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate329/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -386,11 +355,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,...@@ -386,11 +355,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,
386/// ENDFOR355/// ENDFOR
387/// dst[MAX:256] := 0356/// dst[MAX:256] := 0
388/// \endcode357/// \endcode
389static __inline__ __m256i __DEFAULT_FN_ATTRS256358#define _mm256_dpwuud_epi32(__W, __A, __B) \
390_mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {359 ((__m256i)__builtin_ia32_vpdpwuud256((__v8si)(__W), (__v8si)(__A), \
391 return (__m256i)__builtin_ia32_vpdpwuud256((__v8si)__W, (__v8si)__A,360 (__v8si)(__B)))
392 (__v8si)__B);
393}
394361
395/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with362/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
396/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate363/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -423,12 +390,9 @@ _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -423,12 +390,9 @@ _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
423/// ENDFOR390/// ENDFOR
424/// dst[MAX:128] := 0391/// dst[MAX:128] := 0
425/// \endcode392/// \endcode
426static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,393#define _mm_dpwuuds_epi32(__W, __A, __B) \
427 __m128i __A,394 ((__m128i)__builtin_ia32_vpdpwuuds128((__v4si)(__W), (__v4si)(__A), \
428 __m128i __B) {395 (__v4si)(__B)))
429 return (__m128i)__builtin_ia32_vpdpwuuds128((__v4si)__W, (__v4si)__A,
430 (__v4si)__B);
431}
432396
433/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with397/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
434/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate398/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
...@@ -461,13 +425,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,...@@ -461,13 +425,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,
461/// ENDFOR425/// ENDFOR
462/// dst[MAX:256] := 0426/// dst[MAX:256] := 0
463/// \endcode427/// \endcode
464static __inline__ __m256i __DEFAULT_FN_ATTRS256428#define _mm256_dpwuuds_epi32(__W, __A, __B) \
465_mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {429 ((__m256i)__builtin_ia32_vpdpwuuds256((__v8si)(__W), (__v8si)(__A), \
466 return (__m256i)__builtin_ia32_vpdpwuuds256((__v8si)__W, (__v8si)__A,430 (__v8si)(__B)))
467 (__v8si)__B);
468}
469
470#undef __DEFAULT_FN_ATTRS128
471#undef __DEFAULT_FN_ATTRS256
472431
473#endif // __AVXVNNIINT16INTRIN_H432#endif // __AVXVNNIINT16INTRIN_H
lib/include/avxvnniint8intrin.h+36-77
...@@ -14,14 +14,6 @@...@@ -14,14 +14,6 @@
14#ifndef __AVXVNNIINT8INTRIN_H14#ifndef __AVXVNNIINT8INTRIN_H
15#define __AVXVNNIINT8INTRIN_H15#define __AVXVNNIINT8INTRIN_H
1616
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS256 \
19 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint8"), \
20 __min_vector_width__(256)))
21#define __DEFAULT_FN_ATTRS128 \
22 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint8"), \
23 __min_vector_width__(128)))
24
25/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with17/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
26/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate18/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
27/// signed 16-bit results. Sum these 4 results with the corresponding19/// signed 16-bit results. Sum these 4 results with the corresponding
...@@ -52,12 +44,9 @@...@@ -52,12 +44,9 @@
52/// ENDFOR44/// ENDFOR
53/// dst[MAX:128] := 045/// dst[MAX:128] := 0
54/// \endcode46/// \endcode
55static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssd_epi32(__m128i __W,47#define _mm_dpbssd_epi32(__W, __A, __B) \
56 __m128i __A,48 ((__m128i)__builtin_ia32_vpdpbssd128((__v4si)(__W), (__v4si)(__A), \
57 __m128i __B) {49 (__v4si)(__B)))
58 return (__m128i)__builtin_ia32_vpdpbssd128((__v4si)__W, (__v4si)__A,
59 (__v4si)__B);
60}
6150
62/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with51/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
63/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate52/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
...@@ -89,11 +78,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssd_epi32(__m128i __W,...@@ -89,11 +78,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssd_epi32(__m128i __W,
89/// ENDFOR78/// ENDFOR
90/// dst[MAX:256] := 079/// dst[MAX:256] := 0
91/// \endcode80/// \endcode
92static __inline__ __m256i __DEFAULT_FN_ATTRS25681#define _mm256_dpbssd_epi32(__W, __A, __B) \
93_mm256_dpbssd_epi32(__m256i __W, __m256i __A, __m256i __B) {82 ((__m256i)__builtin_ia32_vpdpbssd256((__v8si)(__W), (__v8si)(__A), \
94 return (__m256i)__builtin_ia32_vpdpbssd256((__v8si)__W, (__v8si)__A,83 (__v8si)(__B)))
95 (__v8si)__B);
96}
9784
98/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with85/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
99/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate86/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
...@@ -126,12 +113,9 @@ _mm256_dpbssd_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -126,12 +113,9 @@ _mm256_dpbssd_epi32(__m256i __W, __m256i __A, __m256i __B) {
126/// ENDFOR113/// ENDFOR
127/// dst[MAX:128] := 0114/// dst[MAX:128] := 0
128/// \endcode115/// \endcode
129static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssds_epi32(__m128i __W,116#define _mm_dpbssds_epi32(__W, __A, __B) \
130 __m128i __A,117 ((__m128i)__builtin_ia32_vpdpbssds128((__v4si)(__W), (__v4si)(__A), \
131 __m128i __B) {118 (__v4si)(__B)))
132 return (__m128i)__builtin_ia32_vpdpbssds128((__v4si)__W, (__v4si)__A,
133 (__v4si)__B);
134}
135119
136/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with120/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
137/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate121/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
...@@ -164,11 +148,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssds_epi32(__m128i __W,...@@ -164,11 +148,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssds_epi32(__m128i __W,
164/// ENDFOR148/// ENDFOR
165/// dst[MAX:256] := 0149/// dst[MAX:256] := 0
166/// \endcode150/// \endcode
167static __inline__ __m256i __DEFAULT_FN_ATTRS256151#define _mm256_dpbssds_epi32(__W, __A, __B) \
168_mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B) {152 ((__m256i)__builtin_ia32_vpdpbssds256((__v8si)(__W), (__v8si)(__A), \
169 return (__m256i)__builtin_ia32_vpdpbssds256((__v8si)__W, (__v8si)__A,153 (__v8si)(__B)))
170 (__v8si)__B);
171}
172154
173/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with155/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
174/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate156/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -200,12 +182,9 @@ _mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -200,12 +182,9 @@ _mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B) {
200/// ENDFOR182/// ENDFOR
201/// dst[MAX:128] := 0183/// dst[MAX:128] := 0
202/// \endcode184/// \endcode
203static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsud_epi32(__m128i __W,185#define _mm_dpbsud_epi32(__W, __A, __B) \
204 __m128i __A,186 ((__m128i)__builtin_ia32_vpdpbsud128((__v4si)(__W), (__v4si)(__A), \
205 __m128i __B) {187 (__v4si)(__B)))
206 return (__m128i)__builtin_ia32_vpdpbsud128((__v4si)__W, (__v4si)__A,
207 (__v4si)__B);
208}
209188
210/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with189/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
211/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate190/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -237,11 +216,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsud_epi32(__m128i __W,...@@ -237,11 +216,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsud_epi32(__m128i __W,
237/// ENDFOR216/// ENDFOR
238/// dst[MAX:256] := 0217/// dst[MAX:256] := 0
239/// \endcode218/// \endcode
240static __inline__ __m256i __DEFAULT_FN_ATTRS256219#define _mm256_dpbsud_epi32(__W, __A, __B) \
241_mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B) {220 ((__m256i)__builtin_ia32_vpdpbsud256((__v8si)(__W), (__v8si)(__A), \
242 return (__m256i)__builtin_ia32_vpdpbsud256((__v8si)__W, (__v8si)__A,221 (__v8si)(__B)))
243 (__v8si)__B);
244}
245222
246/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with223/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
247/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate224/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -274,12 +251,9 @@ _mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -274,12 +251,9 @@ _mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
274/// ENDFOR251/// ENDFOR
275/// dst[MAX:128] := 0252/// dst[MAX:128] := 0
276/// \endcode253/// \endcode
277static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsuds_epi32(__m128i __W,254#define _mm_dpbsuds_epi32(__W, __A, __B) \
278 __m128i __A,255 ((__m128i)__builtin_ia32_vpdpbsuds128((__v4si)(__W), (__v4si)(__A), \
279 __m128i __B) {256 (__v4si)(__B)))
280 return (__m128i)__builtin_ia32_vpdpbsuds128((__v4si)__W, (__v4si)__A,
281 (__v4si)__B);
282}
283257
284/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with258/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
285/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate259/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -312,11 +286,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsuds_epi32(__m128i __W,...@@ -312,11 +286,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsuds_epi32(__m128i __W,
312/// ENDFOR286/// ENDFOR
313/// dst[MAX:256] := 0287/// dst[MAX:256] := 0
314/// \endcode288/// \endcode
315static __inline__ __m256i __DEFAULT_FN_ATTRS256289#define _mm256_dpbsuds_epi32(__W, __A, __B) \
316_mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {290 ((__m256i)__builtin_ia32_vpdpbsuds256((__v8si)(__W), (__v8si)(__A), \
317 return (__m256i)__builtin_ia32_vpdpbsuds256((__v8si)__W, (__v8si)__A,291 (__v8si)(__B)))
318 (__v8si)__B);
319}
320292
321/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with293/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
322/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate294/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -348,12 +320,9 @@ _mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -348,12 +320,9 @@ _mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
348/// ENDFOR320/// ENDFOR
349/// dst[MAX:128] := 0321/// dst[MAX:128] := 0
350/// \endcode322/// \endcode
351static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuud_epi32(__m128i __W,323#define _mm_dpbuud_epi32(__W, __A, __B) \
352 __m128i __A,324 ((__m128i)__builtin_ia32_vpdpbuud128((__v4si)(__W), (__v4si)(__A), \
353 __m128i __B) {325 (__v4si)(__B)))
354 return (__m128i)__builtin_ia32_vpdpbuud128((__v4si)__W, (__v4si)__A,
355 (__v4si)__B);
356}
357326
358/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with327/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
359/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate328/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -385,11 +354,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuud_epi32(__m128i __W,...@@ -385,11 +354,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuud_epi32(__m128i __W,
385/// ENDFOR354/// ENDFOR
386/// dst[MAX:256] := 0355/// dst[MAX:256] := 0
387/// \endcode356/// \endcode
388static __inline__ __m256i __DEFAULT_FN_ATTRS256357#define _mm256_dpbuud_epi32(__W, __A, __B) \
389_mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B) {358 ((__m256i)__builtin_ia32_vpdpbuud256((__v8si)(__W), (__v8si)(__A), \
390 return (__m256i)__builtin_ia32_vpdpbuud256((__v8si)__W, (__v8si)__A,359 (__v8si)(__B)))
391 (__v8si)__B);
392}
393360
394/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with361/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
395/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate362/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
...@@ -422,14 +389,10 @@ _mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B) {...@@ -422,14 +389,10 @@ _mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
422/// ENDFOR389/// ENDFOR
423/// dst[MAX:128] := 0390/// dst[MAX:128] := 0
424/// \endcode391/// \endcode
425static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuuds_epi32(__m128i __W,392#define _mm_dpbuuds_epi32(__W, __A, __B) \
426 __m128i __A,393 ((__m128i)__builtin_ia32_vpdpbuuds128((__v4si)(__W), (__v4si)(__A), \
427 __m128i __B) {394 (__v4si)(__B)))
428 return (__m128i)__builtin_ia32_vpdpbuuds128((__v4si)__W, (__v4si)__A,
429 (__v4si)__B);
430}
431395
432/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
433/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate396/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
434/// signed 16-bit results. Sum these 4 results with the corresponding397/// signed 16-bit results. Sum these 4 results with the corresponding
435/// 32-bit integer in \a __W with signed saturation, and store the packed398/// 32-bit integer in \a __W with signed saturation, and store the packed
...@@ -460,12 +423,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuuds_epi32(__m128i __W,...@@ -460,12 +423,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuuds_epi32(__m128i __W,
460/// ENDFOR423/// ENDFOR
461/// dst[MAX:256] := 0424/// dst[MAX:256] := 0
462/// \endcode425/// \endcode
463static __inline__ __m256i __DEFAULT_FN_ATTRS256426#define _mm256_dpbuuds_epi32(__W, __A, __B) \
464_mm256_dpbuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {427 ((__m256i)__builtin_ia32_vpdpbuuds256((__v8si)(__W), (__v8si)(__A), \
465 return (__m256i)__builtin_ia32_vpdpbuuds256((__v8si)__W, (__v8si)__A,428 (__v8si)(__B)))
466 (__v8si)__B);
467}
468#undef __DEFAULT_FN_ATTRS128
469#undef __DEFAULT_FN_ATTRS256
470429
471#endif // __AVXVNNIINT8INTRIN_H430#endif // __AVXVNNIINT8INTRIN_H
lib/include/bmi2intrin.h+15-17
...@@ -15,7 +15,13 @@...@@ -15,7 +15,13 @@
15#define __BMI2INTRIN_H15#define __BMI2INTRIN_H
1616
17/* Define the default attributes for the functions in this file. */17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi2")))18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("bmi2"))) constexpr
21#else
22#define __DEFAULT_FN_ATTRS \
23 __attribute__((__always_inline__, __nodebug__, __target__("bmi2")))
24#endif
1925
20/// Copies the unsigned 32-bit integer \a __X and zeroes the upper bits26/// Copies the unsigned 32-bit integer \a __X and zeroes the upper bits
21/// starting at bit number \a __Y.27/// starting at bit number \a __Y.
...@@ -38,8 +44,7 @@...@@ -38,8 +44,7 @@
38/// The lower 8 bits specify the bit number of the lowest bit to zero.44/// The lower 8 bits specify the bit number of the lowest bit to zero.
39/// \returns The partially zeroed 32-bit value.45/// \returns The partially zeroed 32-bit value.
40static __inline__ unsigned int __DEFAULT_FN_ATTRS46static __inline__ unsigned int __DEFAULT_FN_ATTRS
41_bzhi_u32(unsigned int __X, unsigned int __Y)47_bzhi_u32(unsigned int __X, unsigned int __Y) {
42{
43 return __builtin_ia32_bzhi_si(__X, __Y);48 return __builtin_ia32_bzhi_si(__X, __Y);
44}49}
4550
...@@ -68,8 +73,7 @@ _bzhi_u32(unsigned int __X, unsigned int __Y)...@@ -68,8 +73,7 @@ _bzhi_u32(unsigned int __X, unsigned int __Y)
68/// The 32-bit mask specifying where to deposit source bits.73/// The 32-bit mask specifying where to deposit source bits.
69/// \returns The 32-bit result.74/// \returns The 32-bit result.
70static __inline__ unsigned int __DEFAULT_FN_ATTRS75static __inline__ unsigned int __DEFAULT_FN_ATTRS
71_pdep_u32(unsigned int __X, unsigned int __Y)76_pdep_u32(unsigned int __X, unsigned int __Y) {
72{
73 return __builtin_ia32_pdep_si(__X, __Y);77 return __builtin_ia32_pdep_si(__X, __Y);
74}78}
7579
...@@ -98,8 +102,7 @@ _pdep_u32(unsigned int __X, unsigned int __Y)...@@ -98,8 +102,7 @@ _pdep_u32(unsigned int __X, unsigned int __Y)
98/// The 32-bit mask specifying which source bits to extract.102/// The 32-bit mask specifying which source bits to extract.
99/// \returns The 32-bit result.103/// \returns The 32-bit result.
100static __inline__ unsigned int __DEFAULT_FN_ATTRS104static __inline__ unsigned int __DEFAULT_FN_ATTRS
101_pext_u32(unsigned int __X, unsigned int __Y)105_pext_u32(unsigned int __X, unsigned int __Y) {
102{
103 return __builtin_ia32_pext_si(__X, __Y);106 return __builtin_ia32_pext_si(__X, __Y);
104}107}
105108
...@@ -124,8 +127,7 @@ _pext_u32(unsigned int __X, unsigned int __Y)...@@ -124,8 +127,7 @@ _pext_u32(unsigned int __X, unsigned int __Y)
124/// A pointer to memory for storing the upper half of the product.127/// A pointer to memory for storing the upper half of the product.
125/// \returns The lower half of the product.128/// \returns The lower half of the product.
126static __inline__ unsigned int __DEFAULT_FN_ATTRS129static __inline__ unsigned int __DEFAULT_FN_ATTRS
127_mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)130_mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P) {
128{
129 unsigned long long __res = (unsigned long long) __X * __Y;131 unsigned long long __res = (unsigned long long) __X * __Y;
130 *__P = (unsigned int)(__res >> 32);132 *__P = (unsigned int)(__res >> 32);
131 return (unsigned int)__res;133 return (unsigned int)__res;
...@@ -154,8 +156,7 @@ _mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)...@@ -154,8 +156,7 @@ _mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)
154/// The lower 8 bits specify the bit number of the lowest bit to zero.156/// The lower 8 bits specify the bit number of the lowest bit to zero.
155/// \returns The partially zeroed 64-bit value.157/// \returns The partially zeroed 64-bit value.
156static __inline__ unsigned long long __DEFAULT_FN_ATTRS158static __inline__ unsigned long long __DEFAULT_FN_ATTRS
157_bzhi_u64(unsigned long long __X, unsigned long long __Y)159_bzhi_u64(unsigned long long __X, unsigned long long __Y) {
158{
159 return __builtin_ia32_bzhi_di(__X, __Y);160 return __builtin_ia32_bzhi_di(__X, __Y);
160}161}
161162
...@@ -184,8 +185,7 @@ _bzhi_u64(unsigned long long __X, unsigned long long __Y)...@@ -184,8 +185,7 @@ _bzhi_u64(unsigned long long __X, unsigned long long __Y)
184/// The 64-bit mask specifying where to deposit source bits.185/// The 64-bit mask specifying where to deposit source bits.
185/// \returns The 64-bit result.186/// \returns The 64-bit result.
186static __inline__ unsigned long long __DEFAULT_FN_ATTRS187static __inline__ unsigned long long __DEFAULT_FN_ATTRS
187_pdep_u64(unsigned long long __X, unsigned long long __Y)188_pdep_u64(unsigned long long __X, unsigned long long __Y) {
188{
189 return __builtin_ia32_pdep_di(__X, __Y);189 return __builtin_ia32_pdep_di(__X, __Y);
190}190}
191191
...@@ -214,8 +214,7 @@ _pdep_u64(unsigned long long __X, unsigned long long __Y)...@@ -214,8 +214,7 @@ _pdep_u64(unsigned long long __X, unsigned long long __Y)
214/// The 64-bit mask specifying which source bits to extract.214/// The 64-bit mask specifying which source bits to extract.
215/// \returns The 64-bit result.215/// \returns The 64-bit result.
216static __inline__ unsigned long long __DEFAULT_FN_ATTRS216static __inline__ unsigned long long __DEFAULT_FN_ATTRS
217_pext_u64(unsigned long long __X, unsigned long long __Y)217_pext_u64(unsigned long long __X, unsigned long long __Y) {
218{
219 return __builtin_ia32_pext_di(__X, __Y);218 return __builtin_ia32_pext_di(__X, __Y);
220}219}
221220
...@@ -241,8 +240,7 @@ _pext_u64(unsigned long long __X, unsigned long long __Y)...@@ -241,8 +240,7 @@ _pext_u64(unsigned long long __X, unsigned long long __Y)
241/// \returns The lower half of the product.240/// \returns The lower half of the product.
242static __inline__ unsigned long long __DEFAULT_FN_ATTRS241static __inline__ unsigned long long __DEFAULT_FN_ATTRS
243_mulx_u64 (unsigned long long __X, unsigned long long __Y,242_mulx_u64 (unsigned long long __X, unsigned long long __Y,
244 unsigned long long *__P)243 unsigned long long *__P) {
245{
246 unsigned __int128 __res = (unsigned __int128) __X * __Y;244 unsigned __int128 __res = (unsigned __int128) __X * __Y;
247 *__P = (unsigned long long) (__res >> 64);245 *__P = (unsigned long long) (__res >> 64);
248 return (unsigned long long) __res;246 return (unsigned long long) __res;
lib/include/bmiintrin.h+31-37
...@@ -17,7 +17,12 @@...@@ -17,7 +17,12 @@
17/* Allow using the tzcnt intrinsics even for non-BMI targets. Since the TZCNT17/* Allow using the tzcnt intrinsics even for non-BMI targets. Since the TZCNT
18 instruction behaves as BSF on non-BMI targets, there is code that expects18 instruction behaves as BSF on non-BMI targets, there is code that expects
19 to use it as a potentially faster version of BSF. */19 to use it as a potentially faster version of BSF. */
20#if defined(__cplusplus) && (__cplusplus >= 201103L)
21#define __RELAXED_FN_ATTRS \
22 __attribute__((__always_inline__, __nodebug__)) constexpr
23#else
20#define __RELAXED_FN_ATTRS __attribute__((__always_inline__, __nodebug__))24#define __RELAXED_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
25#endif
2126
22/// Counts the number of trailing zero bits in the operand.27/// Counts the number of trailing zero bits in the operand.
23///28///
...@@ -31,8 +36,7 @@...@@ -31,8 +36,7 @@
31/// bits in the operand.36/// bits in the operand.
32/// \see _tzcnt_u1637/// \see _tzcnt_u16
33static __inline__ unsigned short __RELAXED_FN_ATTRS38static __inline__ unsigned short __RELAXED_FN_ATTRS
34__tzcnt_u16(unsigned short __X)39__tzcnt_u16(unsigned short __X) {
35{
36 return __builtin_ia32_tzcnt_u16(__X);40 return __builtin_ia32_tzcnt_u16(__X);
37}41}
3842
...@@ -65,8 +69,7 @@ __tzcnt_u16(unsigned short __X)...@@ -65,8 +69,7 @@ __tzcnt_u16(unsigned short __X)
65/// bits in the operand.69/// bits in the operand.
66/// \see { _mm_tzcnt_32 _tzcnt_u32 }70/// \see { _mm_tzcnt_32 _tzcnt_u32 }
67static __inline__ unsigned int __RELAXED_FN_ATTRS71static __inline__ unsigned int __RELAXED_FN_ATTRS
68__tzcnt_u32(unsigned int __X)72__tzcnt_u32(unsigned int __X) {
69{
70 return __builtin_ia32_tzcnt_u32(__X);73 return __builtin_ia32_tzcnt_u32(__X);
71}74}
7275
...@@ -82,8 +85,7 @@ __tzcnt_u32(unsigned int __X)...@@ -82,8 +85,7 @@ __tzcnt_u32(unsigned int __X)
82/// the operand.85/// the operand.
83/// \see { __tzcnt_u32 _tzcnt_u32 }86/// \see { __tzcnt_u32 _tzcnt_u32 }
84static __inline__ int __RELAXED_FN_ATTRS87static __inline__ int __RELAXED_FN_ATTRS
85_mm_tzcnt_32(unsigned int __X)88_mm_tzcnt_32(unsigned int __X) {
86{
87 return (int)__builtin_ia32_tzcnt_u32(__X);89 return (int)__builtin_ia32_tzcnt_u32(__X);
88}90}
8991
...@@ -118,8 +120,7 @@ _mm_tzcnt_32(unsigned int __X)...@@ -118,8 +120,7 @@ _mm_tzcnt_32(unsigned int __X)
118/// bits in the operand.120/// bits in the operand.
119/// \see { _mm_tzcnt_64 _tzcnt_u64 }121/// \see { _mm_tzcnt_64 _tzcnt_u64 }
120static __inline__ unsigned long long __RELAXED_FN_ATTRS122static __inline__ unsigned long long __RELAXED_FN_ATTRS
121__tzcnt_u64(unsigned long long __X)123__tzcnt_u64(unsigned long long __X) {
122{
123 return __builtin_ia32_tzcnt_u64(__X);124 return __builtin_ia32_tzcnt_u64(__X);
124}125}
125126
...@@ -135,8 +136,7 @@ __tzcnt_u64(unsigned long long __X)...@@ -135,8 +136,7 @@ __tzcnt_u64(unsigned long long __X)
135/// the operand.136/// the operand.
136/// \see { __tzcnt_u64 _tzcnt_u64 }137/// \see { __tzcnt_u64 _tzcnt_u64 }
137static __inline__ long long __RELAXED_FN_ATTRS138static __inline__ long long __RELAXED_FN_ATTRS
138_mm_tzcnt_64(unsigned long long __X)139_mm_tzcnt_64(unsigned long long __X) {
139{
140 return (long long)__builtin_ia32_tzcnt_u64(__X);140 return (long long)__builtin_ia32_tzcnt_u64(__X);
141}141}
142142
...@@ -164,7 +164,13 @@ _mm_tzcnt_64(unsigned long long __X)...@@ -164,7 +164,13 @@ _mm_tzcnt_64(unsigned long long __X)
164#if !defined(__SCE__) || __has_feature(modules) || defined(__BMI__)164#if !defined(__SCE__) || __has_feature(modules) || defined(__BMI__)
165165
166/* Define the default attributes for the functions in this file. */166/* Define the default attributes for the functions in this file. */
167#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi")))167#if defined(__cplusplus) && (__cplusplus >= 201103L)
168#define __DEFAULT_FN_ATTRS \
169 __attribute__((__always_inline__, __nodebug__, __target__("bmi"))) constexpr
170#else
171#define __DEFAULT_FN_ATTRS \
172 __attribute__((__always_inline__, __nodebug__, __target__("bmi")))
173#endif
168174
169/// Performs a bitwise AND of the second operand with the one's175/// Performs a bitwise AND of the second operand with the one's
170/// complement of the first operand.176/// complement of the first operand.
...@@ -181,8 +187,7 @@ _mm_tzcnt_64(unsigned long long __X)...@@ -181,8 +187,7 @@ _mm_tzcnt_64(unsigned long long __X)
181/// operand with the one's complement of the first operand.187/// operand with the one's complement of the first operand.
182/// \see _andn_u32188/// \see _andn_u32
183static __inline__ unsigned int __DEFAULT_FN_ATTRS189static __inline__ unsigned int __DEFAULT_FN_ATTRS
184__andn_u32(unsigned int __X, unsigned int __Y)190__andn_u32(unsigned int __X, unsigned int __Y) {
185{
186 return ~__X & __Y;191 return ~__X & __Y;
187}192}
188193
...@@ -224,8 +229,7 @@ __andn_u32(unsigned int __X, unsigned int __Y)...@@ -224,8 +229,7 @@ __andn_u32(unsigned int __X, unsigned int __Y)
224/// extracted bits.229/// extracted bits.
225/// \see _bextr_u32230/// \see _bextr_u32
226static __inline__ unsigned int __DEFAULT_FN_ATTRS231static __inline__ unsigned int __DEFAULT_FN_ATTRS
227__bextr_u32(unsigned int __X, unsigned int __Y)232__bextr_u32(unsigned int __X, unsigned int __Y) {
228{
229 return __builtin_ia32_bextr_u32(__X, __Y);233 return __builtin_ia32_bextr_u32(__X, __Y);
230}234}
231235
...@@ -249,9 +253,8 @@ __bextr_u32(unsigned int __X, unsigned int __Y)...@@ -249,9 +253,8 @@ __bextr_u32(unsigned int __X, unsigned int __Y)
249/// extracted bits.253/// extracted bits.
250/// \see __bextr_u32254/// \see __bextr_u32
251static __inline__ unsigned int __DEFAULT_FN_ATTRS255static __inline__ unsigned int __DEFAULT_FN_ATTRS
252_bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z)256_bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z) {
253{257 return __builtin_ia32_bextr_u32(__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
254 return __builtin_ia32_bextr_u32 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
255}258}
256259
257/* Intel-specified, single-leading-underscore version of BEXTR2 */260/* Intel-specified, single-leading-underscore version of BEXTR2 */
...@@ -289,8 +292,7 @@ _bextr2_u32(unsigned int __X, unsigned int __Y) {...@@ -289,8 +292,7 @@ _bextr2_u32(unsigned int __X, unsigned int __Y) {
289/// the source operand.292/// the source operand.
290/// \see _blsi_u32293/// \see _blsi_u32
291static __inline__ unsigned int __DEFAULT_FN_ATTRS294static __inline__ unsigned int __DEFAULT_FN_ATTRS
292__blsi_u32(unsigned int __X)295__blsi_u32(unsigned int __X) {
293{
294 return __X & -__X;296 return __X & -__X;
295}297}
296298
...@@ -325,8 +327,7 @@ __blsi_u32(unsigned int __X)...@@ -325,8 +327,7 @@ __blsi_u32(unsigned int __X)
325/// \returns An unsigned integer containing the newly created mask.327/// \returns An unsigned integer containing the newly created mask.
326/// \see _blsmsk_u32328/// \see _blsmsk_u32
327static __inline__ unsigned int __DEFAULT_FN_ATTRS329static __inline__ unsigned int __DEFAULT_FN_ATTRS
328__blsmsk_u32(unsigned int __X)330__blsmsk_u32(unsigned int __X) {
329{
330 return __X ^ (__X - 1);331 return __X ^ (__X - 1);
331}332}
332333
...@@ -361,8 +362,7 @@ __blsmsk_u32(unsigned int __X)...@@ -361,8 +362,7 @@ __blsmsk_u32(unsigned int __X)
361/// operand.362/// operand.
362/// \see _blsr_u32363/// \see _blsr_u32
363static __inline__ unsigned int __DEFAULT_FN_ATTRS364static __inline__ unsigned int __DEFAULT_FN_ATTRS
364__blsr_u32(unsigned int __X)365__blsr_u32(unsigned int __X) {
365{
366 return __X & (__X - 1);366 return __X & (__X - 1);
367}367}
368368
...@@ -401,8 +401,7 @@ __blsr_u32(unsigned int __X)...@@ -401,8 +401,7 @@ __blsr_u32(unsigned int __X)
401/// operand with the one's complement of the first operand.401/// operand with the one's complement of the first operand.
402/// \see _andn_u64402/// \see _andn_u64
403static __inline__ unsigned long long __DEFAULT_FN_ATTRS403static __inline__ unsigned long long __DEFAULT_FN_ATTRS
404__andn_u64 (unsigned long long __X, unsigned long long __Y)404__andn_u64 (unsigned long long __X, unsigned long long __Y) {
405{
406 return ~__X & __Y;405 return ~__X & __Y;
407}406}
408407
...@@ -445,8 +444,7 @@ __andn_u64 (unsigned long long __X, unsigned long long __Y)...@@ -445,8 +444,7 @@ __andn_u64 (unsigned long long __X, unsigned long long __Y)
445/// extracted bits.444/// extracted bits.
446/// \see _bextr_u64445/// \see _bextr_u64
447static __inline__ unsigned long long __DEFAULT_FN_ATTRS446static __inline__ unsigned long long __DEFAULT_FN_ATTRS
448__bextr_u64(unsigned long long __X, unsigned long long __Y)447__bextr_u64(unsigned long long __X, unsigned long long __Y) {
449{
450 return __builtin_ia32_bextr_u64(__X, __Y);448 return __builtin_ia32_bextr_u64(__X, __Y);
451}449}
452450
...@@ -470,9 +468,8 @@ __bextr_u64(unsigned long long __X, unsigned long long __Y)...@@ -470,9 +468,8 @@ __bextr_u64(unsigned long long __X, unsigned long long __Y)
470/// extracted bits.468/// extracted bits.
471/// \see __bextr_u64469/// \see __bextr_u64
472static __inline__ unsigned long long __DEFAULT_FN_ATTRS470static __inline__ unsigned long long __DEFAULT_FN_ATTRS
473_bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z)471_bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z) {
474{472 return __builtin_ia32_bextr_u64(__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
475 return __builtin_ia32_bextr_u64 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
476}473}
477474
478/* Intel-specified, single-leading-underscore version of BEXTR2 */475/* Intel-specified, single-leading-underscore version of BEXTR2 */
...@@ -510,8 +507,7 @@ _bextr2_u64(unsigned long long __X, unsigned long long __Y) {...@@ -510,8 +507,7 @@ _bextr2_u64(unsigned long long __X, unsigned long long __Y) {
510/// bits from the source operand.507/// bits from the source operand.
511/// \see _blsi_u64508/// \see _blsi_u64
512static __inline__ unsigned long long __DEFAULT_FN_ATTRS509static __inline__ unsigned long long __DEFAULT_FN_ATTRS
513__blsi_u64(unsigned long long __X)510__blsi_u64(unsigned long long __X) {
514{
515 return __X & -__X;511 return __X & -__X;
516}512}
517513
...@@ -546,8 +542,7 @@ __blsi_u64(unsigned long long __X)...@@ -546,8 +542,7 @@ __blsi_u64(unsigned long long __X)
546/// \returns An unsigned 64-bit integer containing the newly created mask.542/// \returns An unsigned 64-bit integer containing the newly created mask.
547/// \see _blsmsk_u64543/// \see _blsmsk_u64
548static __inline__ unsigned long long __DEFAULT_FN_ATTRS544static __inline__ unsigned long long __DEFAULT_FN_ATTRS
549__blsmsk_u64(unsigned long long __X)545__blsmsk_u64(unsigned long long __X) {
550{
551 return __X ^ (__X - 1);546 return __X ^ (__X - 1);
552}547}
553548
...@@ -582,8 +577,7 @@ __blsmsk_u64(unsigned long long __X)...@@ -582,8 +577,7 @@ __blsmsk_u64(unsigned long long __X)
582/// source operand.577/// source operand.
583/// \see _blsr_u64578/// \see _blsr_u64
584static __inline__ unsigned long long __DEFAULT_FN_ATTRS579static __inline__ unsigned long long __DEFAULT_FN_ATTRS
585__blsr_u64(unsigned long long __X)580__blsr_u64(unsigned long long __X) {
586{
587 return __X & (__X - 1);581 return __X & (__X - 1);
588}582}
589583
lib/include/cmpccxaddintrin.h+1-1
...@@ -63,7 +63,7 @@ typedef enum {...@@ -63,7 +63,7 @@ typedef enum {
63 (int)(__D))))63 (int)(__D))))
6464
65#define _cmpccxadd_epi64(__A, __B, __C, __D) \65#define _cmpccxadd_epi64(__A, __B, __C, __D) \
66 ((long long)(__builtin_ia32_cmpccxadd64((void *)(__A), (long long)(__B), \66 ((long long)(__builtin_ia32_cmpccxadd64((__A), (long long)(__B), \
67 (long long)(__C), (int)(__D))))67 (long long)(__C), (int)(__D))))
6868
69#endif // __x86_64__69#endif // __x86_64__
lib/include/cpuid.h+12-11
...@@ -187,17 +187,18 @@...@@ -187,17 +187,18 @@
187#define bit_ENQCMD 0x20000000187#define bit_ENQCMD 0x20000000
188188
189/* Features in %edx for leaf 7 sub-leaf 0 */189/* Features in %edx for leaf 7 sub-leaf 0 */
190#define bit_AVX5124VNNIW 0x00000004190#define bit_AVX5124VNNIW 0x00000004
191#define bit_AVX5124FMAPS 0x00000008191#define bit_AVX5124FMAPS 0x00000008
192#define bit_UINTR 0x00000020192#define bit_UINTR 0x00000020
193#define bit_SERIALIZE 0x00004000193#define bit_AVX512VP2INTERSECT 0x00000100
194#define bit_TSXLDTRK 0x00010000194#define bit_SERIALIZE 0x00004000
195#define bit_PCONFIG 0x00040000195#define bit_TSXLDTRK 0x00010000
196#define bit_IBT 0x00100000196#define bit_PCONFIG 0x00040000
197#define bit_AMXBF16 0x00400000197#define bit_IBT 0x00100000
198#define bit_AVX512FP16 0x00800000198#define bit_AMXBF16 0x00400000
199#define bit_AMXTILE 0x01000000199#define bit_AVX512FP16 0x00800000
200#define bit_AMXINT8 0x02000000200#define bit_AMXTILE 0x01000000
201#define bit_AMXINT8 0x02000000
201202
202/* Features in %eax for leaf 7 sub-leaf 1 */203/* Features in %eax for leaf 7 sub-leaf 1 */
203#define bit_SHA512 0x00000001204#define bit_SHA512 0x00000001
lib/include/emmintrin.h+141-107
...@@ -49,12 +49,27 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));...@@ -49,12 +49,27 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
49#endif49#endif
5050
51/* Define the default attributes for the functions in this file. */51/* Define the default attributes for the functions in this file. */
52#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
52#define __DEFAULT_FN_ATTRS \53#define __DEFAULT_FN_ATTRS \
53 __attribute__((__always_inline__, __nodebug__, \54 __attribute__((__always_inline__, __nodebug__, \
54 __target__("sse2,no-evex512"), __min_vector_width__(128)))55 __target__("sse2,no-evex512"), __min_vector_width__(128)))
55#define __DEFAULT_FN_ATTRS_MMX \56#else
56 __attribute__((__always_inline__, __nodebug__, \57#define __DEFAULT_FN_ATTRS \
57 __target__("mmx,sse2,no-evex512"), __min_vector_width__(64)))58 __attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
59 __min_vector_width__(128)))
60#endif
61
62#if defined(__cplusplus) && (__cplusplus >= 201103L)
63#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
64#else
65#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
66#endif
67
68#define __trunc64(x) \
69 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
70#define __anyext128(x) \
71 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
72 1, -1, -1)
5873
59/// Adds lower double-precision values in both operands and returns the74/// Adds lower double-precision values in both operands and returns the
60/// sum in the lower 64 bits of the result. The upper 64 bits of the result75/// sum in the lower 64 bits of the result. The upper 64 bits of the result
...@@ -71,8 +86,8 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));...@@ -71,8 +86,8 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
71/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the86/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
72/// sum of the lower 64 bits of both operands. The upper 64 bits are copied87/// sum of the lower 64 bits of both operands. The upper 64 bits are copied
73/// from the upper 64 bits of the first source operand.88/// from the upper 64 bits of the first source operand.
74static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_sd(__m128d __a,89static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_sd(__m128d __a,
75 __m128d __b) {90 __m128d __b) {
76 __a[0] += __b[0];91 __a[0] += __b[0];
77 return __a;92 return __a;
78}93}
...@@ -89,8 +104,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_sd(__m128d __a,...@@ -89,8 +104,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_sd(__m128d __a,
89/// A 128-bit vector of [2 x double] containing one of the source operands.104/// A 128-bit vector of [2 x double] containing one of the source operands.
90/// \returns A 128-bit vector of [2 x double] containing the sums of both105/// \returns A 128-bit vector of [2 x double] containing the sums of both
91/// operands.106/// operands.
92static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_pd(__m128d __a,107static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_pd(__m128d __a,
93 __m128d __b) {108 __m128d __b) {
94 return (__m128d)((__v2df)__a + (__v2df)__b);109 return (__m128d)((__v2df)__a + (__v2df)__b);
95}110}
96111
...@@ -111,8 +126,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_pd(__m128d __a,...@@ -111,8 +126,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_pd(__m128d __a,
111/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the126/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
112/// difference of the lower 64 bits of both operands. The upper 64 bits are127/// difference of the lower 64 bits of both operands. The upper 64 bits are
113/// copied from the upper 64 bits of the first source operand.128/// copied from the upper 64 bits of the first source operand.
114static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_sd(__m128d __a,129static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_sd(__m128d __a,
115 __m128d __b) {130 __m128d __b) {
116 __a[0] -= __b[0];131 __a[0] -= __b[0];
117 return __a;132 return __a;
118}133}
...@@ -129,8 +144,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_sd(__m128d __a,...@@ -129,8 +144,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_sd(__m128d __a,
129/// A 128-bit vector of [2 x double] containing the subtrahend.144/// A 128-bit vector of [2 x double] containing the subtrahend.
130/// \returns A 128-bit vector of [2 x double] containing the differences between145/// \returns A 128-bit vector of [2 x double] containing the differences between
131/// both operands.146/// both operands.
132static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_pd(__m128d __a,147static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_pd(__m128d __a,
133 __m128d __b) {148 __m128d __b) {
134 return (__m128d)((__v2df)__a - (__v2df)__b);149 return (__m128d)((__v2df)__a - (__v2df)__b);
135}150}
136151
...@@ -150,8 +165,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_pd(__m128d __a,...@@ -150,8 +165,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_pd(__m128d __a,
150/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the165/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
151/// product of the lower 64 bits of both operands. The upper 64 bits are166/// product of the lower 64 bits of both operands. The upper 64 bits are
152/// copied from the upper 64 bits of the first source operand.167/// copied from the upper 64 bits of the first source operand.
153static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_sd(__m128d __a,168static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_mul_sd(__m128d __a,
154 __m128d __b) {169 __m128d __b) {
155 __a[0] *= __b[0];170 __a[0] *= __b[0];
156 return __a;171 return __a;
157}172}
...@@ -168,8 +183,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_sd(__m128d __a,...@@ -168,8 +183,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_sd(__m128d __a,
168/// A 128-bit vector of [2 x double] containing one of the operands.183/// A 128-bit vector of [2 x double] containing one of the operands.
169/// \returns A 128-bit vector of [2 x double] containing the products of both184/// \returns A 128-bit vector of [2 x double] containing the products of both
170/// operands.185/// operands.
171static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_pd(__m128d __a,186static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_mul_pd(__m128d __a,
172 __m128d __b) {187 __m128d __b) {
173 return (__m128d)((__v2df)__a * (__v2df)__b);188 return (__m128d)((__v2df)__a * (__v2df)__b);
174}189}
175190
...@@ -190,8 +205,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_pd(__m128d __a,...@@ -190,8 +205,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_pd(__m128d __a,
190/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the205/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
191/// quotient of the lower 64 bits of both operands. The upper 64 bits are206/// quotient of the lower 64 bits of both operands. The upper 64 bits are
192/// copied from the upper 64 bits of the first source operand.207/// copied from the upper 64 bits of the first source operand.
193static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_sd(__m128d __a,208static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_div_sd(__m128d __a,
194 __m128d __b) {209 __m128d __b) {
195 __a[0] /= __b[0];210 __a[0] /= __b[0];
196 return __a;211 return __a;
197}212}
...@@ -209,8 +224,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_sd(__m128d __a,...@@ -209,8 +224,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_sd(__m128d __a,
209/// A 128-bit vector of [2 x double] containing the divisor.224/// A 128-bit vector of [2 x double] containing the divisor.
210/// \returns A 128-bit vector of [2 x double] containing the quotients of both225/// \returns A 128-bit vector of [2 x double] containing the quotients of both
211/// operands.226/// operands.
212static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_pd(__m128d __a,227static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_div_pd(__m128d __a,
213 __m128d __b) {228 __m128d __b) {
214 return (__m128d)((__v2df)__a / (__v2df)__b);229 return (__m128d)((__v2df)__a / (__v2df)__b);
215}230}
216231
...@@ -358,8 +373,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_pd(__m128d __a,...@@ -358,8 +373,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_pd(__m128d __a,
358/// A 128-bit vector of [2 x double] containing one of the source operands.373/// A 128-bit vector of [2 x double] containing one of the source operands.
359/// \returns A 128-bit vector of [2 x double] containing the bitwise AND of the374/// \returns A 128-bit vector of [2 x double] containing the bitwise AND of the
360/// values between both operands.375/// values between both operands.
361static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_and_pd(__m128d __a,376static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_and_pd(__m128d __a,
362 __m128d __b) {377 __m128d __b) {
363 return (__m128d)((__v2du)__a & (__v2du)__b);378 return (__m128d)((__v2du)__a & (__v2du)__b);
364}379}
365380
...@@ -378,8 +393,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_and_pd(__m128d __a,...@@ -378,8 +393,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_and_pd(__m128d __a,
378/// \returns A 128-bit vector of [2 x double] containing the bitwise AND of the393/// \returns A 128-bit vector of [2 x double] containing the bitwise AND of the
379/// values in the second operand and the one's complement of the first394/// values in the second operand and the one's complement of the first
380/// operand.395/// operand.
381static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_andnot_pd(__m128d __a,396static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
382 __m128d __b) {397_mm_andnot_pd(__m128d __a, __m128d __b) {
383 return (__m128d)(~(__v2du)__a & (__v2du)__b);398 return (__m128d)(~(__v2du)__a & (__v2du)__b);
384}399}
385400
...@@ -395,8 +410,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_andnot_pd(__m128d __a,...@@ -395,8 +410,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_andnot_pd(__m128d __a,
395/// A 128-bit vector of [2 x double] containing one of the source operands.410/// A 128-bit vector of [2 x double] containing one of the source operands.
396/// \returns A 128-bit vector of [2 x double] containing the bitwise OR of the411/// \returns A 128-bit vector of [2 x double] containing the bitwise OR of the
397/// values between both operands.412/// values between both operands.
398static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_or_pd(__m128d __a,413static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_or_pd(__m128d __a,
399 __m128d __b) {414 __m128d __b) {
400 return (__m128d)((__v2du)__a | (__v2du)__b);415 return (__m128d)((__v2du)__a | (__v2du)__b);
401}416}
402417
...@@ -412,8 +427,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_or_pd(__m128d __a,...@@ -412,8 +427,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_or_pd(__m128d __a,
412/// A 128-bit vector of [2 x double] containing one of the source operands.427/// A 128-bit vector of [2 x double] containing one of the source operands.
413/// \returns A 128-bit vector of [2 x double] containing the bitwise XOR of the428/// \returns A 128-bit vector of [2 x double] containing the bitwise XOR of the
414/// values between both operands.429/// values between both operands.
415static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_xor_pd(__m128d __a,430static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_xor_pd(__m128d __a,
416 __m128d __b) {431 __m128d __b) {
417 return (__m128d)((__v2du)__a ^ (__v2du)__b);432 return (__m128d)((__v2du)__a ^ (__v2du)__b);
418}433}
419434
...@@ -1291,7 +1306,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtpd_ps(__m128d __a) {...@@ -1291,7 +1306,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtpd_ps(__m128d __a) {
1291/// floating-point elements are converted to double-precision values. The1306/// floating-point elements are converted to double-precision values. The
1292/// upper two elements are unused.1307/// upper two elements are unused.
1293/// \returns A 128-bit vector of [2 x double] containing the converted values.1308/// \returns A 128-bit vector of [2 x double] containing the converted values.
1294static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtps_pd(__m128 __a) {1309static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1310_mm_cvtps_pd(__m128 __a) {
1295 return (__m128d) __builtin_convertvector(1311 return (__m128d) __builtin_convertvector(
1296 __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1), __v2df);1312 __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1), __v2df);
1297}1313}
...@@ -1312,7 +1328,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtps_pd(__m128 __a) {...@@ -1312,7 +1328,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtps_pd(__m128 __a) {
1312///1328///
1313/// The upper two elements are unused.1329/// The upper two elements are unused.
1314/// \returns A 128-bit vector of [2 x double] containing the converted values.1330/// \returns A 128-bit vector of [2 x double] containing the converted values.
1315static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtepi32_pd(__m128i __a) {1331static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1332_mm_cvtepi32_pd(__m128i __a) {
1316 return (__m128d) __builtin_convertvector(1333 return (__m128d) __builtin_convertvector(
1317 __builtin_shufflevector((__v4si)__a, (__v4si)__a, 0, 1), __v2df);1334 __builtin_shufflevector((__v4si)__a, (__v4si)__a, 0, 1), __v2df);
1318}1335}
...@@ -1398,8 +1415,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtsd_ss(__m128 __a,...@@ -1398,8 +1415,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtsd_ss(__m128 __a,
1398/// \returns A 128-bit vector of [2 x double]. The lower 64 bits contain the1415/// \returns A 128-bit vector of [2 x double]. The lower 64 bits contain the
1399/// converted value from the second parameter. The upper 64 bits are copied1416/// converted value from the second parameter. The upper 64 bits are copied
1400/// from the upper 64 bits of the first parameter.1417/// from the upper 64 bits of the first parameter.
1401static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi32_sd(__m128d __a,1418static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1402 int __b) {1419_mm_cvtsi32_sd(__m128d __a, int __b) {
1403 __a[0] = __b;1420 __a[0] = __b;
1404 return __a;1421 return __a;
1405}1422}
...@@ -1423,8 +1440,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi32_sd(__m128d __a,...@@ -1423,8 +1440,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi32_sd(__m128d __a,
1423/// \returns A 128-bit vector of [2 x double]. The lower 64 bits contain the1440/// \returns A 128-bit vector of [2 x double]. The lower 64 bits contain the
1424/// converted value from the second parameter. The upper 64 bits are copied1441/// converted value from the second parameter. The upper 64 bits are copied
1425/// from the upper 64 bits of the first parameter.1442/// from the upper 64 bits of the first parameter.
1426static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtss_sd(__m128d __a,1443static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1427 __m128 __b) {1444_mm_cvtss_sd(__m128d __a, __m128 __b) {
1428 __a[0] = __b[0];1445 __a[0] = __b[0];
1429 return __a;1446 return __a;
1430}1447}
...@@ -1486,8 +1503,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_cvttsd_si32(__m128d __a) {...@@ -1486,8 +1503,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_cvttsd_si32(__m128d __a) {
1486/// \param __a1503/// \param __a
1487/// A 128-bit vector of [2 x double].1504/// A 128-bit vector of [2 x double].
1488/// \returns A 64-bit vector of [2 x i32] containing the converted values.1505/// \returns A 64-bit vector of [2 x i32] containing the converted values.
1489static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvtpd_pi32(__m128d __a) {1506static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_cvtpd_pi32(__m128d __a) {
1490 return (__m64)__builtin_ia32_cvtpd2pi((__v2df)__a);1507 return __trunc64(__builtin_ia32_cvtpd2dq((__v2df)__a));
1491}1508}
14921509
1493/// Converts the two double-precision floating-point elements of a1510/// Converts the two double-precision floating-point elements of a
...@@ -1505,8 +1522,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvtpd_pi32(__m128d __a) {...@@ -1505,8 +1522,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvtpd_pi32(__m128d __a) {
1505/// \param __a1522/// \param __a
1506/// A 128-bit vector of [2 x double].1523/// A 128-bit vector of [2 x double].
1507/// \returns A 64-bit vector of [2 x i32] containing the converted values.1524/// \returns A 64-bit vector of [2 x i32] containing the converted values.
1508static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvttpd_pi32(__m128d __a) {1525static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_cvttpd_pi32(__m128d __a) {
1509 return (__m64)__builtin_ia32_cvttpd2pi((__v2df)__a);1526 return __trunc64(__builtin_ia32_cvttpd2dq((__v2df)__a));
1510}1527}
15111528
1512/// Converts the two signed 32-bit integer elements of a 64-bit vector of1529/// Converts the two signed 32-bit integer elements of a 64-bit vector of
...@@ -1520,8 +1537,9 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvttpd_pi32(__m128d __a) {...@@ -1520,8 +1537,9 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvttpd_pi32(__m128d __a) {
1520/// \param __a1537/// \param __a
1521/// A 64-bit vector of [2 x i32].1538/// A 64-bit vector of [2 x i32].
1522/// \returns A 128-bit vector of [2 x double] containing the converted values.1539/// \returns A 128-bit vector of [2 x double] containing the converted values.
1523static __inline__ __m128d __DEFAULT_FN_ATTRS_MMX _mm_cvtpi32_pd(__m64 __a) {1540static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1524 return __builtin_ia32_cvtpi2pd((__v2si)__a);1541_mm_cvtpi32_pd(__m64 __a) {
1542 return (__m128d) __builtin_convertvector((__v2si)__a, __v2df);
1525}1543}
15261544
1527/// Returns the low-order element of a 128-bit vector of [2 x double] as1545/// Returns the low-order element of a 128-bit vector of [2 x double] as
...@@ -1535,7 +1553,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS_MMX _mm_cvtpi32_pd(__m64 __a) {...@@ -1535,7 +1553,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS_MMX _mm_cvtpi32_pd(__m64 __a) {
1535/// A 128-bit vector of [2 x double]. The lower 64 bits are returned.1553/// A 128-bit vector of [2 x double]. The lower 64 bits are returned.
1536/// \returns A double-precision floating-point value copied from the lower 641554/// \returns A double-precision floating-point value copied from the lower 64
1537/// bits of \a __a.1555/// bits of \a __a.
1538static __inline__ double __DEFAULT_FN_ATTRS _mm_cvtsd_f64(__m128d __a) {1556static __inline__ double __DEFAULT_FN_ATTRS_CONSTEXPR
1557_mm_cvtsd_f64(__m128d __a) {
1539 return __a[0];1558 return __a[0];
1540}1559}
15411560
...@@ -1770,7 +1789,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_undefined_pd(void) {...@@ -1770,7 +1789,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_undefined_pd(void) {
1770/// \returns An initialized 128-bit floating-point vector of [2 x double]. The1789/// \returns An initialized 128-bit floating-point vector of [2 x double]. The
1771/// lower 64 bits contain the value of the parameter. The upper 64 bits are1790/// lower 64 bits contain the value of the parameter. The upper 64 bits are
1772/// set to zero.1791/// set to zero.
1773static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_sd(double __w) {1792static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_sd(double __w) {
1774 return __extension__(__m128d){__w, 0.0};1793 return __extension__(__m128d){__w, 0.0};
1775}1794}
17761795
...@@ -1786,7 +1805,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_sd(double __w) {...@@ -1786,7 +1805,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_sd(double __w) {
1786/// A double-precision floating-point value used to initialize each vector1805/// A double-precision floating-point value used to initialize each vector
1787/// element of the result.1806/// element of the result.
1788/// \returns An initialized 128-bit floating-point vector of [2 x double].1807/// \returns An initialized 128-bit floating-point vector of [2 x double].
1789static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set1_pd(double __w) {1808static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_pd(double __w) {
1790 return __extension__(__m128d){__w, __w};1809 return __extension__(__m128d){__w, __w};
1791}1810}
17921811
...@@ -1802,7 +1821,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set1_pd(double __w) {...@@ -1802,7 +1821,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set1_pd(double __w) {
1802/// A double-precision floating-point value used to initialize each vector1821/// A double-precision floating-point value used to initialize each vector
1803/// element of the result.1822/// element of the result.
1804/// \returns An initialized 128-bit floating-point vector of [2 x double].1823/// \returns An initialized 128-bit floating-point vector of [2 x double].
1805static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd1(double __w) {1824static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_pd1(double __w) {
1806 return _mm_set1_pd(__w);1825 return _mm_set1_pd(__w);
1807}1826}
18081827
...@@ -1820,8 +1839,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd1(double __w) {...@@ -1820,8 +1839,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd1(double __w) {
1820/// A double-precision floating-point value used to initialize the lower 641839/// A double-precision floating-point value used to initialize the lower 64
1821/// bits of the result.1840/// bits of the result.
1822/// \returns An initialized 128-bit floating-point vector of [2 x double].1841/// \returns An initialized 128-bit floating-point vector of [2 x double].
1823static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd(double __w,1842static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_pd(double __w,
1824 double __x) {1843 double __x) {
1825 return __extension__(__m128d){__x, __w};1844 return __extension__(__m128d){__x, __w};
1826}1845}
18271846
...@@ -1840,8 +1859,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd(double __w,...@@ -1840,8 +1859,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd(double __w,
1840/// A double-precision floating-point value used to initialize the upper 641859/// A double-precision floating-point value used to initialize the upper 64
1841/// bits of the result.1860/// bits of the result.
1842/// \returns An initialized 128-bit floating-point vector of [2 x double].1861/// \returns An initialized 128-bit floating-point vector of [2 x double].
1843static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setr_pd(double __w,1862static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_setr_pd(double __w,
1844 double __x) {1863 double __x) {
1845 return __extension__(__m128d){__w, __x};1864 return __extension__(__m128d){__w, __x};
1846}1865}
18471866
...@@ -1854,7 +1873,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setr_pd(double __w,...@@ -1854,7 +1873,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setr_pd(double __w,
1854///1873///
1855/// \returns An initialized 128-bit floating-point vector of [2 x double] with1874/// \returns An initialized 128-bit floating-point vector of [2 x double] with
1856/// all elements set to zero.1875/// all elements set to zero.
1857static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setzero_pd(void) {1876static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_setzero_pd(void) {
1858 return __extension__(__m128d){0.0, 0.0};1877 return __extension__(__m128d){0.0, 0.0};
1859}1878}
18601879
...@@ -1873,8 +1892,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setzero_pd(void) {...@@ -1873,8 +1892,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setzero_pd(void) {
1873/// A 128-bit vector of [2 x double]. The lower 64 bits are written to the1892/// A 128-bit vector of [2 x double]. The lower 64 bits are written to the
1874/// lower 64 bits of the result.1893/// lower 64 bits of the result.
1875/// \returns A 128-bit vector of [2 x double] containing the moved values.1894/// \returns A 128-bit vector of [2 x double] containing the moved values.
1876static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_move_sd(__m128d __a,1895static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1877 __m128d __b) {1896_mm_move_sd(__m128d __a, __m128d __b) {
1878 __a[0] = __b[0];1897 __a[0] = __b[0];
1879 return __a;1898 return __a;
1880}1899}
...@@ -2091,8 +2110,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi16(__m128i __a,...@@ -2091,8 +2110,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi16(__m128i __a,
2091/// A 128-bit vector of [4 x i32].2110/// A 128-bit vector of [4 x i32].
2092/// \returns A 128-bit vector of [4 x i32] containing the sums of both2111/// \returns A 128-bit vector of [4 x i32] containing the sums of both
2093/// parameters.2112/// parameters.
2094static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi32(__m128i __a,2113static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2095 __m128i __b) {2114_mm_add_epi32(__m128i __a, __m128i __b) {
2096 return (__m128i)((__v4su)__a + (__v4su)__b);2115 return (__m128i)((__v4su)__a + (__v4su)__b);
2097}2116}
20982117
...@@ -2108,9 +2127,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi32(__m128i __a,...@@ -2108,9 +2127,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi32(__m128i __a,
2108/// \param __b2127/// \param __b
2109/// A 64-bit integer.2128/// A 64-bit integer.
2110/// \returns A 64-bit integer containing the sum of both parameters.2129/// \returns A 64-bit integer containing the sum of both parameters.
2111static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_add_si64(__m64 __a,2130static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_add_si64(__m64 __a, __m64 __b) {
2112 __m64 __b) {2131 return (__m64)(((unsigned long long)__a) + ((unsigned long long)__b));
2113 return (__m64)__builtin_ia32_paddq((__v1di)__a, (__v1di)__b);
2114}2132}
21152133
2116/// Adds the corresponding elements of two 128-bit vectors of [2 x i64],2134/// Adds the corresponding elements of two 128-bit vectors of [2 x i64],
...@@ -2129,8 +2147,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_add_si64(__m64 __a,...@@ -2129,8 +2147,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_add_si64(__m64 __a,
2129/// A 128-bit vector of [2 x i64].2147/// A 128-bit vector of [2 x i64].
2130/// \returns A 128-bit vector of [2 x i64] containing the sums of both2148/// \returns A 128-bit vector of [2 x i64] containing the sums of both
2131/// parameters.2149/// parameters.
2132static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi64(__m128i __a,2150static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2133 __m128i __b) {2151_mm_add_epi64(__m128i __a, __m128i __b) {
2134 return (__m128i)((__v2du)__a + (__v2du)__b);2152 return (__m128i)((__v2du)__a + (__v2du)__b);
2135}2153}
21362154
...@@ -2431,9 +2449,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,...@@ -2431,9 +2449,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,
2431/// \param __b2449/// \param __b
2432/// A 64-bit integer containing one of the source operands.2450/// A 64-bit integer containing one of the source operands.
2433/// \returns A 64-bit integer vector containing the product of both operands.2451/// \returns A 64-bit integer vector containing the product of both operands.
2434static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_mul_su32(__m64 __a,2452static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_mul_su32(__m64 __a, __m64 __b) {
2435 __m64 __b) {2453 return __trunc64(__builtin_ia32_pmuludq128((__v4si)__anyext128(__a),
2436 return __builtin_ia32_pmuludq((__v2si)__a, (__v2si)__b);2454 (__v4si)__anyext128(__b)));
2437}2455}
24382456
2439/// Multiplies 32-bit unsigned integer values contained in the lower2457/// Multiplies 32-bit unsigned integer values contained in the lower
...@@ -2521,8 +2539,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi16(__m128i __a,...@@ -2521,8 +2539,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi16(__m128i __a,
2521/// A 128-bit integer vector containing the subtrahends.2539/// A 128-bit integer vector containing the subtrahends.
2522/// \returns A 128-bit integer vector containing the differences of the values2540/// \returns A 128-bit integer vector containing the differences of the values
2523/// in the operands.2541/// in the operands.
2524static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi32(__m128i __a,2542static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2525 __m128i __b) {2543_mm_sub_epi32(__m128i __a, __m128i __b) {
2526 return (__m128i)((__v4su)__a - (__v4su)__b);2544 return (__m128i)((__v4su)__a - (__v4su)__b);
2527}2545}
25282546
...@@ -2539,9 +2557,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi32(__m128i __a,...@@ -2539,9 +2557,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi32(__m128i __a,
2539/// A 64-bit integer vector containing the subtrahend.2557/// A 64-bit integer vector containing the subtrahend.
2540/// \returns A 64-bit integer vector containing the difference of the values in2558/// \returns A 64-bit integer vector containing the difference of the values in
2541/// the operands.2559/// the operands.
2542static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_sub_si64(__m64 __a,2560static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_sub_si64(__m64 __a, __m64 __b) {
2543 __m64 __b) {2561 return (__m64)((unsigned long long)__a - (unsigned long long)__b);
2544 return (__m64)__builtin_ia32_psubq((__v1di)__a, (__v1di)__b);
2545}2562}
25462563
2547/// Subtracts the corresponding elements of two [2 x i64] vectors.2564/// Subtracts the corresponding elements of two [2 x i64] vectors.
...@@ -2556,8 +2573,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_sub_si64(__m64 __a,...@@ -2556,8 +2573,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_sub_si64(__m64 __a,
2556/// A 128-bit integer vector containing the subtrahends.2573/// A 128-bit integer vector containing the subtrahends.
2557/// \returns A 128-bit integer vector containing the differences of the values2574/// \returns A 128-bit integer vector containing the differences of the values
2558/// in the operands.2575/// in the operands.
2559static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi64(__m128i __a,2576static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2560 __m128i __b) {2577_mm_sub_epi64(__m128i __a, __m128i __b) {
2561 return (__m128i)((__v2du)__a - (__v2du)__b);2578 return (__m128i)((__v2du)__a - (__v2du)__b);
2562}2579}
25632580
...@@ -3255,8 +3272,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi32(__m128i __a,...@@ -3255,8 +3272,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi32(__m128i __a,
3255/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the3272/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
3256/// converted value of the second operand. The upper 64 bits are copied from3273/// converted value of the second operand. The upper 64 bits are copied from
3257/// the upper 64 bits of the first operand.3274/// the upper 64 bits of the first operand.
3258static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi64_sd(__m128d __a,3275static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
3259 long long __b) {3276_mm_cvtsi64_sd(__m128d __a, long long __b) {
3260 __a[0] = __b;3277 __a[0] = __b;
3261 return __a;3278 return __a;
3262}3279}
...@@ -3310,7 +3327,8 @@ static __inline__ long long __DEFAULT_FN_ATTRS _mm_cvttsd_si64(__m128d __a) {...@@ -3310,7 +3327,8 @@ static __inline__ long long __DEFAULT_FN_ATTRS _mm_cvttsd_si64(__m128d __a) {
3310/// \param __a3327/// \param __a
3311/// A 128-bit integer vector.3328/// A 128-bit integer vector.
3312/// \returns A 128-bit vector of [4 x float] containing the converted values.3329/// \returns A 128-bit vector of [4 x float] containing the converted values.
3313static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtepi32_ps(__m128i __a) {3330static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
3331_mm_cvtepi32_ps(__m128i __a) {
3314 return (__m128) __builtin_convertvector((__v4si)__a, __v4sf);3332 return (__m128) __builtin_convertvector((__v4si)__a, __v4sf);
3315}3333}
33163334
...@@ -3494,8 +3512,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_undefined_si128(void) {...@@ -3494,8 +3512,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_undefined_si128(void) {
3494/// destination vector of [2 x i64].3512/// destination vector of [2 x i64].
3495/// \returns An initialized 128-bit vector of [2 x i64] containing the values3513/// \returns An initialized 128-bit vector of [2 x i64] containing the values
3496/// provided in the operands.3514/// provided in the operands.
3497static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,3515static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3498 long long __q0) {3516_mm_set_epi64x(long long __q1, long long __q0) {
3499 return __extension__(__m128i)(__v2di){__q0, __q1};3517 return __extension__(__m128i)(__v2di){__q0, __q1};
3500}3518}
35013519
...@@ -3515,9 +3533,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,...@@ -3515,9 +3533,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,
3515/// destination vector of [2 x i64].3533/// destination vector of [2 x i64].
3516/// \returns An initialized 128-bit vector of [2 x i64] containing the values3534/// \returns An initialized 128-bit vector of [2 x i64] containing the values
3517/// provided in the operands.3535/// provided in the operands.
3518static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,3536static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3519 __m64 __q0) {3537_mm_set_epi64(__m64 __q1, __m64 __q0) {
3520 return _mm_set_epi64x((long long)__q1, (long long)__q0);3538 return _mm_set_epi64x((long long)__q1[0], (long long)__q0[0]);
3521}3539}
35223540
3523/// Initializes the 32-bit values in a 128-bit vector of [4 x i32] with3541/// Initializes the 32-bit values in a 128-bit vector of [4 x i32] with
...@@ -3542,8 +3560,10 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,...@@ -3542,8 +3560,10 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,
3542/// vector.3560/// vector.
3543/// \returns An initialized 128-bit vector of [4 x i32] containing the values3561/// \returns An initialized 128-bit vector of [4 x i32] containing the values
3544/// provided in the operands.3562/// provided in the operands.
3545static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,3563static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_epi32(int __i3,
3546 int __i1, int __i0) {3564 int __i2,
3565 int __i1,
3566 int __i0) {
3547 return __extension__(__m128i)(__v4si){__i0, __i1, __i2, __i3};3567 return __extension__(__m128i)(__v4si){__i0, __i1, __i2, __i3};
3548}3568}
35493569
...@@ -3581,7 +3601,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,...@@ -3581,7 +3601,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,
3581/// vector.3601/// vector.
3582/// \returns An initialized 128-bit vector of [8 x i16] containing the values3602/// \returns An initialized 128-bit vector of [8 x i16] containing the values
3583/// provided in the operands.3603/// provided in the operands.
3584static __inline__ __m128i __DEFAULT_FN_ATTRS3604static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3585_mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,3605_mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,
3586 short __w2, short __w1, short __w0) {3606 short __w2, short __w1, short __w0) {
3587 return __extension__(__m128i)(__v8hi){__w0, __w1, __w2, __w3,3607 return __extension__(__m128i)(__v8hi){__w0, __w1, __w2, __w3,
...@@ -3630,7 +3650,7 @@ _mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,...@@ -3630,7 +3650,7 @@ _mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,
3630/// Initializes bits [7:0] of the destination vector.3650/// Initializes bits [7:0] of the destination vector.
3631/// \returns An initialized 128-bit vector of [16 x i8] containing the values3651/// \returns An initialized 128-bit vector of [16 x i8] containing the values
3632/// provided in the operands.3652/// provided in the operands.
3633static __inline__ __m128i __DEFAULT_FN_ATTRS3653static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3634_mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,3654_mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,
3635 char __b10, char __b9, char __b8, char __b7, char __b6, char __b5,3655 char __b10, char __b9, char __b8, char __b7, char __b6, char __b5,
3636 char __b4, char __b3, char __b2, char __b1, char __b0) {3656 char __b4, char __b3, char __b2, char __b1, char __b0) {
...@@ -3652,7 +3672,8 @@ _mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,...@@ -3652,7 +3672,8 @@ _mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,
3652/// vector.3672/// vector.
3653/// \returns An initialized 128-bit integer vector of [2 x i64] with both3673/// \returns An initialized 128-bit integer vector of [2 x i64] with both
3654/// elements containing the value provided in the operand.3674/// elements containing the value provided in the operand.
3655static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {3675static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3676_mm_set1_epi64x(long long __q) {
3656 return _mm_set_epi64x(__q, __q);3677 return _mm_set_epi64x(__q, __q);
3657}3678}
36583679
...@@ -3669,7 +3690,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {...@@ -3669,7 +3690,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {
3669/// vector.3690/// vector.
3670/// \returns An initialized 128-bit vector of [2 x i64] with all elements3691/// \returns An initialized 128-bit vector of [2 x i64] with all elements
3671/// containing the value provided in the operand.3692/// containing the value provided in the operand.
3672static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {3693static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3694_mm_set1_epi64(__m64 __q) {
3673 return _mm_set_epi64(__q, __q);3695 return _mm_set_epi64(__q, __q);
3674}3696}
36753697
...@@ -3686,7 +3708,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {...@@ -3686,7 +3708,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {
3686/// vector.3708/// vector.
3687/// \returns An initialized 128-bit vector of [4 x i32] with all elements3709/// \returns An initialized 128-bit vector of [4 x i32] with all elements
3688/// containing the value provided in the operand.3710/// containing the value provided in the operand.
3689static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {3711static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_epi32(int __i) {
3690 return _mm_set_epi32(__i, __i, __i, __i);3712 return _mm_set_epi32(__i, __i, __i, __i);
3691}3713}
36923714
...@@ -3703,7 +3725,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {...@@ -3703,7 +3725,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {
3703/// vector.3725/// vector.
3704/// \returns An initialized 128-bit vector of [8 x i16] with all elements3726/// \returns An initialized 128-bit vector of [8 x i16] with all elements
3705/// containing the value provided in the operand.3727/// containing the value provided in the operand.
3706static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {3728static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3729_mm_set1_epi16(short __w) {
3707 return _mm_set_epi16(__w, __w, __w, __w, __w, __w, __w, __w);3730 return _mm_set_epi16(__w, __w, __w, __w, __w, __w, __w, __w);
3708}3731}
37093732
...@@ -3720,7 +3743,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {...@@ -3720,7 +3743,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {
3720/// vector.3743/// vector.
3721/// \returns An initialized 128-bit vector of [16 x i8] with all elements3744/// \returns An initialized 128-bit vector of [16 x i8] with all elements
3722/// containing the value provided in the operand.3745/// containing the value provided in the operand.
3723static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {3746static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_epi8(char __b) {
3724 return _mm_set_epi8(__b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b,3747 return _mm_set_epi8(__b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b,
3725 __b, __b, __b, __b, __b);3748 __b, __b, __b, __b, __b);
3726}3749}
...@@ -3739,8 +3762,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {...@@ -3739,8 +3762,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {
3739/// A 64-bit integral value used to initialize the upper 64 bits of the3762/// A 64-bit integral value used to initialize the upper 64 bits of the
3740/// result.3763/// result.
3741/// \returns An initialized 128-bit integer vector.3764/// \returns An initialized 128-bit integer vector.
3742static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,3765static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3743 __m64 __q1) {3766_mm_setr_epi64(__m64 __q0, __m64 __q1) {
3744 return _mm_set_epi64(__q1, __q0);3767 return _mm_set_epi64(__q1, __q0);
3745}3768}
37463769
...@@ -3761,9 +3784,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,...@@ -3761,9 +3784,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,
3761/// \param __i33784/// \param __i3
3762/// A 32-bit integral value used to initialize bits [127:96] of the result.3785/// A 32-bit integral value used to initialize bits [127:96] of the result.
3763/// \returns An initialized 128-bit integer vector.3786/// \returns An initialized 128-bit integer vector.
3764static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,3787static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3765 int __i2,3788_mm_setr_epi32(int __i0, int __i1, int __i2, int __i3) {
3766 int __i3) {
3767 return _mm_set_epi32(__i3, __i2, __i1, __i0);3789 return _mm_set_epi32(__i3, __i2, __i1, __i0);
3768}3790}
37693791
...@@ -3792,7 +3814,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,...@@ -3792,7 +3814,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,
3792/// \param __w73814/// \param __w7
3793/// A 16-bit integral value used to initialize bits [127:112] of the result.3815/// A 16-bit integral value used to initialize bits [127:112] of the result.
3794/// \returns An initialized 128-bit integer vector.3816/// \returns An initialized 128-bit integer vector.
3795static __inline__ __m128i __DEFAULT_FN_ATTRS3817static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3796_mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,3818_mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,
3797 short __w5, short __w6, short __w7) {3819 short __w5, short __w6, short __w7) {
3798 return _mm_set_epi16(__w7, __w6, __w5, __w4, __w3, __w2, __w1, __w0);3820 return _mm_set_epi16(__w7, __w6, __w5, __w4, __w3, __w2, __w1, __w0);
...@@ -3839,7 +3861,7 @@ _mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,...@@ -3839,7 +3861,7 @@ _mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,
3839/// \param __b153861/// \param __b15
3840/// An 8-bit integral value used to initialize bits [127:120] of the result.3862/// An 8-bit integral value used to initialize bits [127:120] of the result.
3841/// \returns An initialized 128-bit integer vector.3863/// \returns An initialized 128-bit integer vector.
3842static __inline__ __m128i __DEFAULT_FN_ATTRS3864static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3843_mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,3865_mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
3844 char __b6, char __b7, char __b8, char __b9, char __b10,3866 char __b6, char __b7, char __b8, char __b9, char __b10,
3845 char __b11, char __b12, char __b13, char __b14, char __b15) {3867 char __b11, char __b12, char __b13, char __b14, char __b15) {
...@@ -3855,7 +3877,7 @@ _mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,...@@ -3855,7 +3877,7 @@ _mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
3855///3877///
3856/// \returns An initialized 128-bit integer vector with all elements set to3878/// \returns An initialized 128-bit integer vector with all elements set to
3857/// zero.3879/// zero.
3858static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setzero_si128(void) {3880static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_setzero_si128(void) {
3859 return __extension__(__m128i)(__v2di){0LL, 0LL};3881 return __extension__(__m128i)(__v2di){0LL, 0LL};
3860}3882}
38613883
...@@ -4588,7 +4610,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi64(__m128i __a,...@@ -4588,7 +4610,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi64(__m128i __a,
4588/// A 128-bit integer vector operand. The lower 64 bits are moved to the4610/// A 128-bit integer vector operand. The lower 64 bits are moved to the
4589/// destination.4611/// destination.
4590/// \returns A 64-bit integer containing the lower 64 bits of the parameter.4612/// \returns A 64-bit integer containing the lower 64 bits of the parameter.
4591static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_movepi64_pi64(__m128i __a) {4613static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR
4614_mm_movepi64_pi64(__m128i __a) {
4592 return (__m64)__a[0];4615 return (__m64)__a[0];
4593}4616}
45944617
...@@ -4603,8 +4626,9 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_movepi64_pi64(__m128i __a) {...@@ -4603,8 +4626,9 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_movepi64_pi64(__m128i __a) {
4603/// A 64-bit value.4626/// A 64-bit value.
4604/// \returns A 128-bit integer vector. The lower 64 bits contain the value from4627/// \returns A 128-bit integer vector. The lower 64 bits contain the value from
4605/// the operand. The upper 64 bits are assigned zeros.4628/// the operand. The upper 64 bits are assigned zeros.
4606static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_movpi64_epi64(__m64 __a) {4629static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4607 return __extension__(__m128i)(__v2di){(long long)__a, 0};4630_mm_movpi64_epi64(__m64 __a) {
4631 return __builtin_shufflevector((__v1di)__a, _mm_setzero_si64(), 0, 1);
4608}4632}
46094633
4610/// Moves the lower 64 bits of a 128-bit integer vector to a 128-bit4634/// Moves the lower 64 bits of a 128-bit integer vector to a 128-bit
...@@ -4619,7 +4643,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_movpi64_epi64(__m64 __a) {...@@ -4619,7 +4643,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_movpi64_epi64(__m64 __a) {
4619/// destination.4643/// destination.
4620/// \returns A 128-bit integer vector. The lower 64 bits contain the value from4644/// \returns A 128-bit integer vector. The lower 64 bits contain the value from
4621/// the operand. The upper 64 bits are assigned zeros.4645/// the operand. The upper 64 bits are assigned zeros.
4622static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_move_epi64(__m128i __a) {4646static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4647_mm_move_epi64(__m128i __a) {
4623 return __builtin_shufflevector((__v2di)__a, _mm_setzero_si128(), 0, 2);4648 return __builtin_shufflevector((__v2di)__a, _mm_setzero_si128(), 0, 2);
4624}4649}
46254650
...@@ -4638,8 +4663,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_move_epi64(__m128i __a) {...@@ -4638,8 +4663,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_move_epi64(__m128i __a) {
4638/// A 128-bit vector of [2 x double]. \n4663/// A 128-bit vector of [2 x double]. \n
4639/// Bits [127:64] are written to bits [127:64] of the destination.4664/// Bits [127:64] are written to bits [127:64] of the destination.
4640/// \returns A 128-bit vector of [2 x double] containing the interleaved values.4665/// \returns A 128-bit vector of [2 x double] containing the interleaved values.
4641static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpackhi_pd(__m128d __a,4666static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4642 __m128d __b) {4667_mm_unpackhi_pd(__m128d __a, __m128d __b) {
4643 return __builtin_shufflevector((__v2df)__a, (__v2df)__b, 1, 2 + 1);4668 return __builtin_shufflevector((__v2df)__a, (__v2df)__b, 1, 2 + 1);
4644}4669}
46454670
...@@ -4658,8 +4683,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpackhi_pd(__m128d __a,...@@ -4658,8 +4683,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpackhi_pd(__m128d __a,
4658/// A 128-bit vector of [2 x double]. \n4683/// A 128-bit vector of [2 x double]. \n
4659/// Bits [63:0] are written to bits [127:64] of the destination.4684/// Bits [63:0] are written to bits [127:64] of the destination.
4660/// \returns A 128-bit vector of [2 x double] containing the interleaved values.4685/// \returns A 128-bit vector of [2 x double] containing the interleaved values.
4661static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpacklo_pd(__m128d __a,4686static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4662 __m128d __b) {4687_mm_unpacklo_pd(__m128d __a, __m128d __b) {
4663 return __builtin_shufflevector((__v2df)__a, (__v2df)__b, 0, 2 + 0);4688 return __builtin_shufflevector((__v2df)__a, (__v2df)__b, 0, 2 + 0);
4664}4689}
46654690
...@@ -4722,7 +4747,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_pd(__m128d __a) {...@@ -4722,7 +4747,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_pd(__m128d __a) {
4722/// A 128-bit floating-point vector of [2 x double].4747/// A 128-bit floating-point vector of [2 x double].
4723/// \returns A 128-bit floating-point vector of [4 x float] containing the same4748/// \returns A 128-bit floating-point vector of [4 x float] containing the same
4724/// bitwise pattern as the parameter.4749/// bitwise pattern as the parameter.
4725static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castpd_ps(__m128d __a) {4750static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
4751_mm_castpd_ps(__m128d __a) {
4726 return (__m128)__a;4752 return (__m128)__a;
4727}4753}
47284754
...@@ -4737,7 +4763,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castpd_ps(__m128d __a) {...@@ -4737,7 +4763,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castpd_ps(__m128d __a) {
4737/// A 128-bit floating-point vector of [2 x double].4763/// A 128-bit floating-point vector of [2 x double].
4738/// \returns A 128-bit integer vector containing the same bitwise pattern as the4764/// \returns A 128-bit integer vector containing the same bitwise pattern as the
4739/// parameter.4765/// parameter.
4740static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castpd_si128(__m128d __a) {4766static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4767_mm_castpd_si128(__m128d __a) {
4741 return (__m128i)__a;4768 return (__m128i)__a;
4742}4769}
47434770
...@@ -4752,7 +4779,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castpd_si128(__m128d __a) {...@@ -4752,7 +4779,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castpd_si128(__m128d __a) {
4752/// A 128-bit floating-point vector of [4 x float].4779/// A 128-bit floating-point vector of [4 x float].
4753/// \returns A 128-bit floating-point vector of [2 x double] containing the same4780/// \returns A 128-bit floating-point vector of [2 x double] containing the same
4754/// bitwise pattern as the parameter.4781/// bitwise pattern as the parameter.
4755static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castps_pd(__m128 __a) {4782static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4783_mm_castps_pd(__m128 __a) {
4756 return (__m128d)__a;4784 return (__m128d)__a;
4757}4785}
47584786
...@@ -4767,7 +4795,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castps_pd(__m128 __a) {...@@ -4767,7 +4795,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castps_pd(__m128 __a) {
4767/// A 128-bit floating-point vector of [4 x float].4795/// A 128-bit floating-point vector of [4 x float].
4768/// \returns A 128-bit integer vector containing the same bitwise pattern as the4796/// \returns A 128-bit integer vector containing the same bitwise pattern as the
4769/// parameter.4797/// parameter.
4770static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castps_si128(__m128 __a) {4798static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4799_mm_castps_si128(__m128 __a) {
4771 return (__m128i)__a;4800 return (__m128i)__a;
4772}4801}
47734802
...@@ -4782,7 +4811,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castps_si128(__m128 __a) {...@@ -4782,7 +4811,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castps_si128(__m128 __a) {
4782/// A 128-bit integer vector.4811/// A 128-bit integer vector.
4783/// \returns A 128-bit floating-point vector of [4 x float] containing the same4812/// \returns A 128-bit floating-point vector of [4 x float] containing the same
4784/// bitwise pattern as the parameter.4813/// bitwise pattern as the parameter.
4785static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castsi128_ps(__m128i __a) {4814static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
4815_mm_castsi128_ps(__m128i __a) {
4786 return (__m128)__a;4816 return (__m128)__a;
4787}4817}
47884818
...@@ -4797,7 +4827,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castsi128_ps(__m128i __a) {...@@ -4797,7 +4827,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castsi128_ps(__m128i __a) {
4797/// A 128-bit integer vector.4827/// A 128-bit integer vector.
4798/// \returns A 128-bit floating-point vector of [2 x double] containing the same4828/// \returns A 128-bit floating-point vector of [2 x double] containing the same
4799/// bitwise pattern as the parameter.4829/// bitwise pattern as the parameter.
4800static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castsi128_pd(__m128i __a) {4830static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4831_mm_castsi128_pd(__m128i __a) {
4801 return (__m128d)__a;4832 return (__m128d)__a;
4802}4833}
48034834
...@@ -4889,8 +4920,11 @@ void _mm_pause(void);...@@ -4889,8 +4920,11 @@ void _mm_pause(void);
4889#if defined(__cplusplus)4920#if defined(__cplusplus)
4890} // extern "C"4921} // extern "C"
4891#endif4922#endif
4923
4924#undef __anyext128
4925#undef __trunc64
4892#undef __DEFAULT_FN_ATTRS4926#undef __DEFAULT_FN_ATTRS
4893#undef __DEFAULT_FN_ATTRS_MMX4927#undef __DEFAULT_FN_ATTRS_CONSTEXPR
48944928
4895#define _MM_SHUFFLE2(x, y) (((x) << 1) | (y))4929#define _MM_SHUFFLE2(x, y) (((x) << 1) | (y))
48964930
lib/include/gfniintrin.h+32-10
...@@ -14,6 +14,7 @@...@@ -14,6 +14,7 @@
14#ifndef __GFNIINTRIN_H14#ifndef __GFNIINTRIN_H
15#define __GFNIINTRIN_H15#define __GFNIINTRIN_H
1616
17#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
17/* Default attributes for simple form (no masking). */18/* Default attributes for simple form (no masking). */
18#define __DEFAULT_FN_ATTRS \19#define __DEFAULT_FN_ATTRS \
19 __attribute__((__always_inline__, __nodebug__, \20 __attribute__((__always_inline__, __nodebug__, \
...@@ -25,26 +26,47 @@...@@ -25,26 +26,47 @@
25 __target__("avx,gfni,no-evex512"), \26 __target__("avx,gfni,no-evex512"), \
26 __min_vector_width__(256)))27 __min_vector_width__(256)))
2728
28/* Default attributes for ZMM unmasked forms. */29/* Default attributes for VLX masked forms. */
29#define __DEFAULT_FN_ATTRS_Z \30#define __DEFAULT_FN_ATTRS_VL128 \
30 __attribute__((__always_inline__, __nodebug__, \31 __attribute__((__always_inline__, __nodebug__, \
31 __target__("avx512f,evex512,gfni"), \32 __target__("avx512bw,avx512vl,gfni,no-evex512"), \
32 __min_vector_width__(512)))33 __min_vector_width__(128)))
33/* Default attributes for ZMM masked forms. */34#define __DEFAULT_FN_ATTRS_VL256 \
34#define __DEFAULT_FN_ATTRS_Z_MASK \
35 __attribute__((__always_inline__, __nodebug__, \35 __attribute__((__always_inline__, __nodebug__, \
36 __target__("avx512bw,evex512,gfni"), \36 __target__("avx512bw,avx512vl,gfni,no-evex512"), \
37 __min_vector_width__(512)))37 __min_vector_width__(256)))
38#else
39/* Default attributes for simple form (no masking). */
40#define __DEFAULT_FN_ATTRS \
41 __attribute__((__always_inline__, __nodebug__, __target__("gfni"), \
42 __min_vector_width__(128)))
43
44/* Default attributes for YMM unmasked form. */
45#define __DEFAULT_FN_ATTRS_Y \
46 __attribute__((__always_inline__, __nodebug__, __target__("avx,gfni"), \
47 __min_vector_width__(256)))
3848
39/* Default attributes for VLX masked forms. */49/* Default attributes for VLX masked forms. */
40#define __DEFAULT_FN_ATTRS_VL128 \50#define __DEFAULT_FN_ATTRS_VL128 \
41 __attribute__((__always_inline__, __nodebug__, \51 __attribute__((__always_inline__, __nodebug__, \
42 __target__("avx512bw,avx512vl,gfni,no-evex512"), \52 __target__("avx512bw,avx512vl,gfni"), \
43 __min_vector_width__(128)))53 __min_vector_width__(128)))
44#define __DEFAULT_FN_ATTRS_VL256 \54#define __DEFAULT_FN_ATTRS_VL256 \
45 __attribute__((__always_inline__, __nodebug__, \55 __attribute__((__always_inline__, __nodebug__, \
46 __target__("avx512bw,avx512vl,gfni,no-evex512"), \56 __target__("avx512bw,avx512vl,gfni"), \
47 __min_vector_width__(256)))57 __min_vector_width__(256)))
58#endif
59
60/* Default attributes for ZMM unmasked forms. */
61#define __DEFAULT_FN_ATTRS_Z \
62 __attribute__((__always_inline__, __nodebug__, \
63 __target__("avx512f,evex512,gfni"), \
64 __min_vector_width__(512)))
65/* Default attributes for ZMM masked forms. */
66#define __DEFAULT_FN_ATTRS_Z_MASK \
67 __attribute__((__always_inline__, __nodebug__, \
68 __target__("avx512bw,evex512,gfni"), \
69 __min_vector_width__(512)))
4870
49#define _mm_gf2p8affineinv_epi64_epi8(A, B, I) \71#define _mm_gf2p8affineinv_epi64_epi8(A, B, I) \
50 ((__m128i)__builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \72 ((__m128i)__builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \
lib/include/hexagon_types.h+8-4
...@@ -1,7 +1,11 @@...@@ -1,7 +1,11 @@
1/******************************************************************************/1//===----------------------------------------------------------------------===//
2/* (c) 2020 Qualcomm Innovation Center, Inc. All rights reserved. */2//
3/* */3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4/******************************************************************************/4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
5#ifndef HEXAGON_TYPES_H9#ifndef HEXAGON_TYPES_H
6#define HEXAGON_TYPES_H10#define HEXAGON_TYPES_H
711
lib/include/hvx_hexagon_protos.h+427
...@@ -5178,6 +5178,433 @@...@@ -5178,6 +5178,433 @@
5178#define Q6_Vuh_vmpy_VuhVuh_rs16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhvs)(Vu,Vv)5178#define Q6_Vuh_vmpy_VuhVuh_rs16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhvs)(Vu,Vv)
5179#endif /* __HEXAGON_ARCH___ >= 69 */5179#endif /* __HEXAGON_ARCH___ >= 69 */
51805180
5181#if __HVX_ARCH__ >= 73
5182/* ==========================================================================
5183 Assembly Syntax: Vdd32.sf=vadd(Vu32.bf,Vv32.bf)
5184 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu,
5185 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5186 ========================================================================== */
5187
5188#define Q6_Wsf_vadd_VbfVbf(Vu, Vv) \
5189 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_bf)(Vu, Vv)
5190#endif /* __HEXAGON_ARCH___ >= 73 */
5191
5192#if __HVX_ARCH__ >= 73
5193/* ==========================================================================
5194 Assembly Syntax: Vd32.h=Vu32.hf
5195 C Intrinsic Prototype: HVX_Vector Q6_Vh_equals_Vhf(HVX_Vector Vu)
5196 Instruction Type: CVI_VS
5197 Execution Slots: SLOT0123
5198 ========================================================================== */
5199
5200#define Q6_Vh_equals_Vhf(Vu) \
5201 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_h_hf)(Vu)
5202#endif /* __HEXAGON_ARCH___ >= 73 */
5203
5204#if __HVX_ARCH__ >= 73
5205/* ==========================================================================
5206 Assembly Syntax: Vd32.hf=Vu32.h
5207 C Intrinsic Prototype: HVX_Vector Q6_Vhf_equals_Vh(HVX_Vector Vu)
5208 Instruction Type: CVI_VS
5209 Execution Slots: SLOT0123
5210 ========================================================================== */
5211
5212#define Q6_Vhf_equals_Vh(Vu) \
5213 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_h)(Vu)
5214#endif /* __HEXAGON_ARCH___ >= 73 */
5215
5216#if __HVX_ARCH__ >= 73
5217/* ==========================================================================
5218 Assembly Syntax: Vd32.sf=Vu32.w
5219 C Intrinsic Prototype: HVX_Vector Q6_Vsf_equals_Vw(HVX_Vector Vu)
5220 Instruction Type: CVI_VS
5221 Execution Slots: SLOT0123
5222 ========================================================================== */
5223
5224#define Q6_Vsf_equals_Vw(Vu) \
5225 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_sf_w)(Vu)
5226#endif /* __HEXAGON_ARCH___ >= 73 */
5227
5228#if __HVX_ARCH__ >= 73
5229/* ==========================================================================
5230 Assembly Syntax: Vd32.w=Vu32.sf
5231 C Intrinsic Prototype: HVX_Vector Q6_Vw_equals_Vsf(HVX_Vector Vu)
5232 Instruction Type: CVI_VS
5233 Execution Slots: SLOT0123
5234 ========================================================================== */
5235
5236#define Q6_Vw_equals_Vsf(Vu) \
5237 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_w_sf)(Vu)
5238#endif /* __HEXAGON_ARCH___ >= 73 */
5239
5240#if __HVX_ARCH__ >= 73
5241/* ==========================================================================
5242 Assembly Syntax: Vd32.bf=vcvt(Vu32.sf,Vv32.sf)
5243 C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu,
5244 HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5245 ========================================================================== */
5246
5247#define Q6_Vbf_vcvt_VsfVsf(Vu, Vv) \
5248 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_bf_sf)(Vu, Vv)
5249#endif /* __HEXAGON_ARCH___ >= 73 */
5250
5251#if __HVX_ARCH__ >= 73
5252/* ==========================================================================
5253 Assembly Syntax: Qd4=vcmp.gt(Vu32.bf,Vv32.bf)
5254 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbfVbf(HVX_Vector Vu,
5255 HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
5256 ========================================================================== */
5257
5258#define Q6_Q_vcmp_gt_VbfVbf(Vu, Vv) \
5259 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5260 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf)(Vu, Vv)), -1)
5261#endif /* __HEXAGON_ARCH___ >= 73 */
5262
5263#if __HVX_ARCH__ >= 73
5264/* ==========================================================================
5265 Assembly Syntax: Qx4&=vcmp.gt(Vu32.bf,Vv32.bf)
5266 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbfVbf(HVX_VectorPred
5267 Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
5268 Slots: SLOT0123
5269 ========================================================================== */
5270
5271#define Q6_Q_vcmp_gtand_QVbfVbf(Qx, Vu, Vv) \
5272 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5273 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf_and)( \
5274 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5275 Vv)), \
5276 -1)
5277#endif /* __HEXAGON_ARCH___ >= 73 */
5278
5279#if __HVX_ARCH__ >= 73
5280/* ==========================================================================
5281 Assembly Syntax: Qx4|=vcmp.gt(Vu32.bf,Vv32.bf)
5282 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbfVbf(HVX_VectorPred
5283 Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
5284 Slots: SLOT0123
5285 ========================================================================== */
5286
5287#define Q6_Q_vcmp_gtor_QVbfVbf(Qx, Vu, Vv) \
5288 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5289 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf_or)( \
5290 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5291 Vv)), \
5292 -1)
5293#endif /* __HEXAGON_ARCH___ >= 73 */
5294
5295#if __HVX_ARCH__ >= 73
5296/* ==========================================================================
5297 Assembly Syntax: Qx4^=vcmp.gt(Vu32.bf,Vv32.bf)
5298 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbfVbf(HVX_VectorPred
5299 Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
5300 Slots: SLOT0123
5301 ========================================================================== */
5302
5303#define Q6_Q_vcmp_gtxacc_QVbfVbf(Qx, Vu, Vv) \
5304 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5305 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf_xor)( \
5306 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5307 Vv)), \
5308 -1)
5309#endif /* __HEXAGON_ARCH___ >= 73 */
5310
5311#if __HVX_ARCH__ >= 73
5312/* ==========================================================================
5313 Assembly Syntax: Vd32.bf=vmax(Vu32.bf,Vv32.bf)
5314 C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu,
5315 HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23
5316 ========================================================================== */
5317
5318#define Q6_Vbf_vmax_VbfVbf(Vu, Vv) \
5319 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_bf)(Vu, Vv)
5320#endif /* __HEXAGON_ARCH___ >= 73 */
5321
5322#if __HVX_ARCH__ >= 73
5323/* ==========================================================================
5324 Assembly Syntax: Vd32.bf=vmin(Vu32.bf,Vv32.bf)
5325 C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu,
5326 HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23
5327 ========================================================================== */
5328
5329#define Q6_Vbf_vmin_VbfVbf(Vu, Vv) \
5330 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_bf)(Vu, Vv)
5331#endif /* __HEXAGON_ARCH___ >= 73 */
5332
5333#if __HVX_ARCH__ >= 73
5334/* ==========================================================================
5335 Assembly Syntax: Vdd32.sf=vmpy(Vu32.bf,Vv32.bf)
5336 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu,
5337 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5338 ========================================================================== */
5339
5340#define Q6_Wsf_vmpy_VbfVbf(Vu, Vv) \
5341 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_bf)(Vu, Vv)
5342#endif /* __HEXAGON_ARCH___ >= 73 */
5343
5344#if __HVX_ARCH__ >= 73
5345/* ==========================================================================
5346 Assembly Syntax: Vxx32.sf+=vmpy(Vu32.bf,Vv32.bf)
5347 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair
5348 Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution
5349 Slots: SLOT23
5350 ========================================================================== */
5351
5352#define Q6_Wsf_vmpyacc_WsfVbfVbf(Vxx, Vu, Vv) \
5353 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_bf_acc)(Vxx, Vu, Vv)
5354#endif /* __HEXAGON_ARCH___ >= 73 */
5355
5356#if __HVX_ARCH__ >= 73
5357/* ==========================================================================
5358 Assembly Syntax: Vdd32.sf=vsub(Vu32.bf,Vv32.bf)
5359 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu,
5360 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5361 ========================================================================== */
5362
5363#define Q6_Wsf_vsub_VbfVbf(Vu, Vv) \
5364 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_bf)(Vu, Vv)
5365#endif /* __HEXAGON_ARCH___ >= 73 */
5366
5367#if __HVX_ARCH__ >= 79
5368/* ==========================================================================
5369 Assembly Syntax: Vd32=vgetqfext(Vu32.x,Rt32)
5370 C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfext_VR(HVX_Vector Vu, Word32 Rt)
5371 Instruction Type: CVI_VX
5372 Execution Slots: SLOT23
5373 ========================================================================== */
5374
5375#define Q6_V_vgetqfext_VR(Vu, Rt) \
5376 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_get_qfext)(Vu, Rt)
5377#endif /* __HEXAGON_ARCH___ >= 79 */
5378
5379#if __HVX_ARCH__ >= 79
5380/* ==========================================================================
5381 Assembly Syntax: Vx32|=vgetqfext(Vu32.x,Rt32)
5382 C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfextor_VVR(HVX_Vector Vx,
5383 HVX_Vector Vu, Word32 Rt) Instruction Type: CVI_VX Execution Slots:
5384 SLOT23
5385 ========================================================================== */
5386
5387#define Q6_V_vgetqfextor_VVR(Vx, Vu, Rt) \
5388 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_get_qfext_oracc)(Vx, Vu, Rt)
5389#endif /* __HEXAGON_ARCH___ >= 79 */
5390
5391#if __HVX_ARCH__ >= 79
5392/* ==========================================================================
5393 Assembly Syntax: Vd32.x=vsetqfext(Vu32,Rt32)
5394 C Intrinsic Prototype: HVX_Vector Q6_V_vsetqfext_VR(HVX_Vector Vu, Word32 Rt)
5395 Instruction Type: CVI_VX
5396 Execution Slots: SLOT23
5397 ========================================================================== */
5398
5399#define Q6_V_vsetqfext_VR(Vu, Rt) \
5400 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_set_qfext)(Vu, Rt)
5401#endif /* __HEXAGON_ARCH___ >= 79 */
5402
5403#if __HVX_ARCH__ >= 79
5404/* ==========================================================================
5405 Assembly Syntax: Vd32.f8=vabs(Vu32.f8)
5406 C Intrinsic Prototype: HVX_Vector Q6_V_vabs_V(HVX_Vector Vu)
5407 Instruction Type: CVI_VX_LATE
5408 Execution Slots: SLOT23
5409 ========================================================================== */
5410
5411#define Q6_V_vabs_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_f8)(Vu)
5412#endif /* __HEXAGON_ARCH___ >= 79 */
5413
5414#if __HVX_ARCH__ >= 79
5415/* ==========================================================================
5416 Assembly Syntax: Vdd32.hf=vadd(Vu32.f8,Vv32.f8)
5417 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu,
5418 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5419 ========================================================================== */
5420
5421#define Q6_Whf_vadd_VV(Vu, Vv) \
5422 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf_f8)(Vu, Vv)
5423#endif /* __HEXAGON_ARCH___ >= 79 */
5424
5425#if __HVX_ARCH__ >= 79
5426/* ==========================================================================
5427 Assembly Syntax: Vd32.b=vcvt2(Vu32.hf,Vv32.hf)
5428 C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu,
5429 HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5430 ========================================================================== */
5431
5432#define Q6_Vb_vcvt2_VhfVhf(Vu, Vv) \
5433 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_b_hf)(Vu, Vv)
5434#endif /* __HEXAGON_ARCH___ >= 79 */
5435
5436#if __HVX_ARCH__ >= 79
5437/* ==========================================================================
5438 Assembly Syntax: Vdd32.hf=vcvt2(Vu32.b)
5439 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt2_Vb(HVX_Vector Vu)
5440 Instruction Type: CVI_VX_DV
5441 Execution Slots: SLOT23
5442 ========================================================================== */
5443
5444#define Q6_Whf_vcvt2_Vb(Vu) \
5445 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_hf_b)(Vu)
5446#endif /* __HEXAGON_ARCH___ >= 79 */
5447
5448#if __HVX_ARCH__ >= 79
5449/* ==========================================================================
5450 Assembly Syntax: Vdd32.hf=vcvt2(Vu32.ub)
5451 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt2_Vub(HVX_Vector Vu)
5452 Instruction Type: CVI_VX_DV
5453 Execution Slots: SLOT23
5454 ========================================================================== */
5455
5456#define Q6_Whf_vcvt2_Vub(Vu) \
5457 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_hf_ub)(Vu)
5458#endif /* __HEXAGON_ARCH___ >= 79 */
5459
5460#if __HVX_ARCH__ >= 79
5461/* ==========================================================================
5462 Assembly Syntax: Vd32.ub=vcvt2(Vu32.hf,Vv32.hf)
5463 C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu,
5464 HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5465 ========================================================================== */
5466
5467#define Q6_Vub_vcvt2_VhfVhf(Vu, Vv) \
5468 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_ub_hf)(Vu, Vv)
5469#endif /* __HEXAGON_ARCH___ >= 79 */
5470
5471#if __HVX_ARCH__ >= 79
5472/* ==========================================================================
5473 Assembly Syntax: Vd32.f8=vcvt(Vu32.hf,Vv32.hf)
5474 C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector
5475 Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5476 ========================================================================== */
5477
5478#define Q6_V_vcvt_VhfVhf(Vu, Vv) \
5479 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_f8_hf)(Vu, Vv)
5480#endif /* __HEXAGON_ARCH___ >= 79 */
5481
5482#if __HVX_ARCH__ >= 79
5483/* ==========================================================================
5484 Assembly Syntax: Vdd32.hf=vcvt(Vu32.f8)
5485 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_V(HVX_Vector Vu)
5486 Instruction Type: CVI_VX_DV
5487 Execution Slots: SLOT23
5488 ========================================================================== */
5489
5490#define Q6_Whf_vcvt_V(Vu) \
5491 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_f8)(Vu)
5492#endif /* __HEXAGON_ARCH___ >= 79 */
5493
5494#if __HVX_ARCH__ >= 79
5495/* ==========================================================================
5496 Assembly Syntax: Vd32.f8=vfmax(Vu32.f8,Vv32.f8)
5497 C Intrinsic Prototype: HVX_Vector Q6_V_vfmax_VV(HVX_Vector Vu, HVX_Vector Vv)
5498 Instruction Type: CVI_VX_LATE
5499 Execution Slots: SLOT23
5500 ========================================================================== */
5501
5502#define Q6_V_vfmax_VV(Vu, Vv) \
5503 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_f8)(Vu, Vv)
5504#endif /* __HEXAGON_ARCH___ >= 79 */
5505
5506#if __HVX_ARCH__ >= 79
5507/* ==========================================================================
5508 Assembly Syntax: Vd32.f8=vfmin(Vu32.f8,Vv32.f8)
5509 C Intrinsic Prototype: HVX_Vector Q6_V_vfmin_VV(HVX_Vector Vu, HVX_Vector Vv)
5510 Instruction Type: CVI_VX_LATE
5511 Execution Slots: SLOT23
5512 ========================================================================== */
5513
5514#define Q6_V_vfmin_VV(Vu, Vv) \
5515 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_f8)(Vu, Vv)
5516#endif /* __HEXAGON_ARCH___ >= 79 */
5517
5518#if __HVX_ARCH__ >= 79
5519/* ==========================================================================
5520 Assembly Syntax: Vd32.f8=vfneg(Vu32.f8)
5521 C Intrinsic Prototype: HVX_Vector Q6_V_vfneg_V(HVX_Vector Vu)
5522 Instruction Type: CVI_VX_LATE
5523 Execution Slots: SLOT23
5524 ========================================================================== */
5525
5526#define Q6_V_vfneg_V(Vu) \
5527 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_f8)(Vu)
5528#endif /* __HEXAGON_ARCH___ >= 79 */
5529
5530#if __HVX_ARCH__ >= 79
5531/* ==========================================================================
5532 Assembly Syntax: Vd32=vmerge(Vu32.x,Vv32.w)
5533 C Intrinsic Prototype: HVX_Vector Q6_V_vmerge_VVw(HVX_Vector Vu, HVX_Vector
5534 Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
5535 ========================================================================== */
5536
5537#define Q6_V_vmerge_VVw(Vu, Vv) \
5538 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmerge_qf)(Vu, Vv)
5539#endif /* __HEXAGON_ARCH___ >= 79 */
5540
5541#if __HVX_ARCH__ >= 79
5542/* ==========================================================================
5543 Assembly Syntax: Vdd32.hf=vmpy(Vu32.f8,Vv32.f8)
5544 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu,
5545 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5546 ========================================================================== */
5547
5548#define Q6_Whf_vmpy_VV(Vu, Vv) \
5549 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_f8)(Vu, Vv)
5550#endif /* __HEXAGON_ARCH___ >= 79 */
5551
5552#if __HVX_ARCH__ >= 79
5553/* ==========================================================================
5554 Assembly Syntax: Vxx32.hf+=vmpy(Vu32.f8,Vv32.f8)
5555 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair
5556 Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution
5557 Slots: SLOT23
5558 ========================================================================== */
5559
5560#define Q6_Whf_vmpyacc_WhfVV(Vxx, Vu, Vv) \
5561 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_f8_acc)(Vxx, Vu, Vv)
5562#endif /* __HEXAGON_ARCH___ >= 79 */
5563
5564#if __HVX_ARCH__ >= 79
5565/* ==========================================================================
5566 Assembly Syntax: Vd32.qf16=vmpy(Vu32.hf,Rt32.hf)
5567 C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfRhf(HVX_Vector Vu, Word32
5568 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5569 ========================================================================== */
5570
5571#define Q6_Vqf16_vmpy_VhfRhf(Vu, Rt) \
5572 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_hf)(Vu, Rt)
5573#endif /* __HEXAGON_ARCH___ >= 79 */
5574
5575#if __HVX_ARCH__ >= 79
5576/* ==========================================================================
5577 Assembly Syntax: Vd32.qf16=vmpy(Vu32.qf16,Rt32.hf)
5578 C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Rhf(HVX_Vector Vu,
5579 Word32 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5580 ========================================================================== */
5581
5582#define Q6_Vqf16_vmpy_Vqf16Rhf(Vu, Rt) \
5583 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_qf16)(Vu, Rt)
5584#endif /* __HEXAGON_ARCH___ >= 79 */
5585
5586#if __HVX_ARCH__ >= 79
5587/* ==========================================================================
5588 Assembly Syntax: Vd32.qf32=vmpy(Vu32.sf,Rt32.sf)
5589 C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfRsf(HVX_Vector Vu, Word32
5590 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5591 ========================================================================== */
5592
5593#define Q6_Vqf32_vmpy_VsfRsf(Vu, Rt) \
5594 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_sf)(Vu, Rt)
5595#endif /* __HEXAGON_ARCH___ >= 79 */
5596
5597#if __HVX_ARCH__ >= 79
5598/* ==========================================================================
5599 Assembly Syntax: Vdd32.hf=vsub(Vu32.f8,Vv32.f8)
5600 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu,
5601 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5602 ========================================================================== */
5603
5604#define Q6_Whf_vsub_VV(Vu, Vv) \
5605 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_f8)(Vu, Vv)
5606#endif /* __HEXAGON_ARCH___ >= 79 */
5607
5181#endif /* __HVX__ */5608#endif /* __HVX__ */
51825609
5183#endif5610#endif
lib/include/immintrin.h+87-3
...@@ -605,6 +605,20 @@ _storebe_i64(void * __P, long long __D) {...@@ -605,6 +605,20 @@ _storebe_i64(void * __P, long long __D) {
605#include <movdirintrin.h>605#include <movdirintrin.h>
606#endif606#endif
607607
608#if !defined(__SCE__) || __has_feature(modules) || defined(__MOVRS__)
609#include <movrsintrin.h>
610#endif
611
612#if !defined(__SCE__) || __has_feature(modules) || \
613 (defined(__AVX10_2__) && defined(__MOVRS__))
614#include <movrs_avx10_2intrin.h>
615#endif
616
617#if !defined(__SCE__) || __has_feature(modules) || \
618 (defined(__AVX10_2_512__) && defined(__MOVRS__))
619#include <movrs_avx10_2_512intrin.h>
620#endif
621
608#if !defined(__SCE__) || __has_feature(modules) || defined(__PCONFIG__)622#if !defined(__SCE__) || __has_feature(modules) || defined(__PCONFIG__)
609#include <pconfigintrin.h>623#include <pconfigintrin.h>
610#endif624#endif
...@@ -620,9 +634,6 @@ _storebe_i64(void * __P, long long __D) {...@@ -620,9 +634,6 @@ _storebe_i64(void * __P, long long __D) {
620#if !defined(__SCE__) || __has_feature(modules) || defined(__INVPCID__)634#if !defined(__SCE__) || __has_feature(modules) || defined(__INVPCID__)
621#include <invpcidintrin.h>635#include <invpcidintrin.h>
622#endif636#endif
623#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP16__)
624#include <amxfp16intrin.h>
625#endif
626637
627#if !defined(__SCE__) || __has_feature(modules) || defined(__KL__) || \638#if !defined(__SCE__) || __has_feature(modules) || defined(__KL__) || \
628 defined(__WIDEKL__)639 defined(__WIDEKL__)
...@@ -634,10 +645,59 @@ _storebe_i64(void * __P, long long __D) {...@@ -634,10 +645,59 @@ _storebe_i64(void * __P, long long __D) {
634#include <amxintrin.h>645#include <amxintrin.h>
635#endif646#endif
636647
648#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP16__)
649#include <amxfp16intrin.h>
650#endif
651
637#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_COMPLEX__)652#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_COMPLEX__)
638#include <amxcomplexintrin.h>653#include <amxcomplexintrin.h>
639#endif654#endif
640655
656#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP8__)
657#include <amxfp8intrin.h>
658#endif
659
660#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TRANSPOSE__)
661#include <amxtransposeintrin.h>
662#endif
663
664#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_MOVRS__)
665#include <amxmovrsintrin.h>
666#endif
667
668#if !defined(__SCE__) || __has_feature(modules) || \
669 (defined(__AMX_MOVRS__) && defined(__AMX_TRANSPOSE__))
670#include <amxmovrstransposeintrin.h>
671#endif
672
673#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_AVX512__)
674#include <amxavx512intrin.h>
675#endif
676
677#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TF32__)
678#include <amxtf32intrin.h>
679#endif
680
681#if !defined(__SCE__) || __has_feature(modules) || \
682 (defined(__AMX_TF32__) && defined(__AMX_TRANSPOSE__))
683#include <amxtf32transposeintrin.h>
684#endif
685
686#if !defined(__SCE__) || __has_feature(modules) || \
687 (defined(__AMX_BF16__) && defined(__AMX_TRANSPOSE__))
688#include <amxbf16transposeintrin.h>
689#endif
690
691#if !defined(__SCE__) || __has_feature(modules) || \
692 (defined(__AMX_FP16__) && defined(__AMX_TRANSPOSE__))
693#include <amxfp16transposeintrin.h>
694#endif
695
696#if !defined(__SCE__) || __has_feature(modules) || \
697 (defined(__AMX_COMPLEX__) && defined(__AMX_TRANSPOSE__))
698#include <amxcomplextransposeintrin.h>
699#endif
700
641#if !defined(__SCE__) || __has_feature(modules) || \701#if !defined(__SCE__) || __has_feature(modules) || \
642 defined(__AVX512VP2INTERSECT__)702 defined(__AVX512VP2INTERSECT__)
643#include <avx512vp2intersectintrin.h>703#include <avx512vp2intersectintrin.h>
...@@ -648,6 +708,30 @@ _storebe_i64(void * __P, long long __D) {...@@ -648,6 +708,30 @@ _storebe_i64(void * __P, long long __D) {
648#include <avx512vlvp2intersectintrin.h>708#include <avx512vlvp2intersectintrin.h>
649#endif709#endif
650710
711#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX10_2__)
712#include <avx10_2bf16intrin.h>
713#include <avx10_2convertintrin.h>
714#include <avx10_2copyintrin.h>
715#include <avx10_2minmaxintrin.h>
716#include <avx10_2niintrin.h>
717#include <avx10_2satcvtdsintrin.h>
718#include <avx10_2satcvtintrin.h>
719#endif
720
721#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX10_2_512__)
722#include <avx10_2_512bf16intrin.h>
723#include <avx10_2_512convertintrin.h>
724#include <avx10_2_512minmaxintrin.h>
725#include <avx10_2_512niintrin.h>
726#include <avx10_2_512satcvtdsintrin.h>
727#include <avx10_2_512satcvtintrin.h>
728#endif
729
730#if !defined(__SCE__) || __has_feature(modules) || \
731 (defined(__AVX10_2_512__) && defined(__SM4__))
732#include <sm4evexintrin.h>
733#endif
734
651#if !defined(__SCE__) || __has_feature(modules) || defined(__ENQCMD__)735#if !defined(__SCE__) || __has_feature(modules) || defined(__ENQCMD__)
652#include <enqcmdintrin.h>736#include <enqcmdintrin.h>
653#endif737#endif
lib/include/intrin.h+20-10
...@@ -94,8 +94,8 @@ void __outwordstring(unsigned short, unsigned short *, unsigned long);...@@ -94,8 +94,8 @@ void __outwordstring(unsigned short, unsigned short *, unsigned long);
94unsigned long __readcr0(void);94unsigned long __readcr0(void);
95unsigned long __readcr2(void);95unsigned long __readcr2(void);
96unsigned __LPTRINT_TYPE__ __readcr3(void);96unsigned __LPTRINT_TYPE__ __readcr3(void);
97unsigned long __readcr4(void);97unsigned __LPTRINT_TYPE__ __readcr4(void);
98unsigned long __readcr8(void);98unsigned __int64 __readcr8(void);
99unsigned int __readdr(unsigned int);99unsigned int __readdr(unsigned int);
100#ifdef __i386__100#ifdef __i386__
101unsigned char __readfsbyte(unsigned long);101unsigned char __readfsbyte(unsigned long);
...@@ -124,8 +124,8 @@ void __vmx_vmptrst(unsigned __int64 *);...@@ -124,8 +124,8 @@ void __vmx_vmptrst(unsigned __int64 *);
124void __wbinvd(void);124void __wbinvd(void);
125void __writecr0(unsigned int);125void __writecr0(unsigned int);
126void __writecr3(unsigned __INTPTR_TYPE__);126void __writecr3(unsigned __INTPTR_TYPE__);
127void __writecr4(unsigned int);127void __writecr4(unsigned __INTPTR_TYPE__);
128void __writecr8(unsigned int);128void __writecr8(unsigned __int64);
129void __writedr(unsigned int, unsigned int);129void __writedr(unsigned int, unsigned int);
130void __writefsbyte(unsigned long, unsigned char);130void __writefsbyte(unsigned long, unsigned char);
131void __writefsdword(unsigned long, unsigned long);131void __writefsdword(unsigned long, unsigned long);
...@@ -330,33 +330,33 @@ static __inline__ void __DEFAULT_FN_ATTRS __halt(void) {...@@ -330,33 +330,33 @@ static __inline__ void __DEFAULT_FN_ATTRS __halt(void) {
330 __asm__ volatile("hlt");330 __asm__ volatile("hlt");
331}331}
332332
333static inline unsigned char __inbyte(unsigned short port) {333static __inline__ unsigned char __inbyte(unsigned short port) {
334 unsigned char ret;334 unsigned char ret;
335 __asm__ __volatile__("inb %w1, %b0" : "=a"(ret) : "Nd"(port));335 __asm__ __volatile__("inb %w1, %b0" : "=a"(ret) : "Nd"(port));
336 return ret;336 return ret;
337}337}
338338
339static inline unsigned short __inword(unsigned short port) {339static __inline__ unsigned short __inword(unsigned short port) {
340 unsigned short ret;340 unsigned short ret;
341 __asm__ __volatile__("inw %w1, %w0" : "=a"(ret) : "Nd"(port));341 __asm__ __volatile__("inw %w1, %w0" : "=a"(ret) : "Nd"(port));
342 return ret;342 return ret;
343}343}
344344
345static inline unsigned long __indword(unsigned short port) {345static __inline__ unsigned long __indword(unsigned short port) {
346 unsigned long ret;346 unsigned long ret;
347 __asm__ __volatile__("inl %w1, %k0" : "=a"(ret) : "Nd"(port));347 __asm__ __volatile__("inl %w1, %k0" : "=a"(ret) : "Nd"(port));
348 return ret;348 return ret;
349}349}
350350
351static inline void __outbyte(unsigned short port, unsigned char data) {351static __inline__ void __outbyte(unsigned short port, unsigned char data) {
352 __asm__ __volatile__("outb %b0, %w1" : : "a"(data), "Nd"(port));352 __asm__ __volatile__("outb %b0, %w1" : : "a"(data), "Nd"(port));
353}353}
354354
355static inline void __outword(unsigned short port, unsigned short data) {355static __inline__ void __outword(unsigned short port, unsigned short data) {
356 __asm__ __volatile__("outw %w0, %w1" : : "a"(data), "Nd"(port));356 __asm__ __volatile__("outw %w0, %w1" : : "a"(data), "Nd"(port));
357}357}
358358
359static inline void __outdword(unsigned short port, unsigned long data) {359static __inline__ void __outdword(unsigned short port, unsigned long data) {
360 __asm__ __volatile__("outl %k0, %w1" : : "a"(data), "Nd"(port));360 __asm__ __volatile__("outl %k0, %w1" : : "a"(data), "Nd"(port));
361}361}
362#endif362#endif
...@@ -396,6 +396,16 @@ unsigned short __readx18word(unsigned long offset);...@@ -396,6 +396,16 @@ unsigned short __readx18word(unsigned long offset);
396unsigned long __readx18dword(unsigned long offset);396unsigned long __readx18dword(unsigned long offset);
397unsigned __int64 __readx18qword(unsigned long offset);397unsigned __int64 __readx18qword(unsigned long offset);
398398
399void __addx18byte(unsigned long offset, unsigned char data);
400void __addx18word(unsigned long offset, unsigned short data);
401void __addx18dword(unsigned long offset, unsigned long data);
402void __addx18qword(unsigned long offset, unsigned __int64 data);
403
404void __incx18byte(unsigned long offset);
405void __incx18word(unsigned long offset);
406void __incx18dword(unsigned long offset);
407void __incx18qword(unsigned long offset);
408
399double _CopyDoubleFromInt64(__int64);409double _CopyDoubleFromInt64(__int64);
400float _CopyFloatFromInt32(__int32);410float _CopyFloatFromInt32(__int32);
401__int32 _CopyInt32FromFloat(float);411__int32 _CopyInt32FromFloat(float);
lib/include/intrin0.h+10-1
...@@ -44,7 +44,7 @@ unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination,...@@ -44,7 +44,7 @@ unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination,
44 __int64 *_ComparandResult);44 __int64 *_ComparandResult);
45#endif45#endif
4646
47#ifdef __x86_64__ && !defined(__arm64ec__)47#if defined(__x86_64__) && !defined(__arm64ec__)
48unsigned __int64 _umul128(unsigned __int64, unsigned __int64,48unsigned __int64 _umul128(unsigned __int64, unsigned __int64,
49 unsigned __int64 *);49 unsigned __int64 *);
50unsigned __int64 __shiftleft128(unsigned __int64 _LowPart,50unsigned __int64 __shiftleft128(unsigned __int64 _LowPart,
...@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);...@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
207__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);207__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
208__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);208__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
209__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);209__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
210void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
211void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
212void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
210213
211/*----------------------------------------------------------------------------*\214/*----------------------------------------------------------------------------*\
212|* Interlocked Compare Exchange215|* Interlocked Compare Exchange
...@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,...@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
237__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,240__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
238 __int64 _Exchange,241 __int64 _Exchange,
239 __int64 _Comparand);242 __int64 _Comparand);
243void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
244 void *_Exchange, void *_Comparand);
245void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
246 void *_Exchange, void *_Comparand);
247void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
248 void *_Exchange, void *_Comparand);
240#endif249#endif
241250
242#ifdef __cplusplus251#ifdef __cplusplus
lib/include/larchintrin.h+22-8
...@@ -228,17 +228,31 @@ extern __inline void...@@ -228,17 +228,31 @@ extern __inline void
228 ((void)__builtin_loongarch_ldpte_d((long int)(_1), (_2)))228 ((void)__builtin_loongarch_ldpte_d((long int)(_1), (_2)))
229#endif229#endif
230230
231#define __frecipe_s(/*float*/ _1) \231#ifdef __loongarch_frecipe
232 (float)__builtin_loongarch_frecipe_s((float)_1)232extern __inline float
233 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
234 __frecipe_s(float _1) {
235 return __builtin_loongarch_frecipe_s(_1);
236}
233237
234#define __frecipe_d(/*double*/ _1) \238extern __inline double
235 (double)__builtin_loongarch_frecipe_d((double)_1)239 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
240 __frecipe_d(double _1) {
241 return __builtin_loongarch_frecipe_d(_1);
242}
236243
237#define __frsqrte_s(/*float*/ _1) \244extern __inline float
238 (float)__builtin_loongarch_frsqrte_s((float)_1)245 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
246 __frsqrte_s(float _1) {
247 return __builtin_loongarch_frsqrte_s(_1);
248}
239249
240#define __frsqrte_d(/*double*/ _1) \250extern __inline double
241 (double)__builtin_loongarch_frsqrte_d((double)_1)251 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
252 __frsqrte_d(double _1) {
253 return __builtin_loongarch_frsqrte_d(_1);
254}
255#endif
242256
243#ifdef __cplusplus257#ifdef __cplusplus
244}258}
lib/include/lasxintrin.h+27-25
...@@ -1726,18 +1726,6 @@ extern __inline...@@ -1726,18 +1726,6 @@ extern __inline
1726 return (__m256d)__builtin_lasx_xvfrecip_d((v4f64)_1);1726 return (__m256d)__builtin_lasx_xvfrecip_d((v4f64)_1);
1727}1727}
17281728
1729extern __inline
1730 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
1731 __lasx_xvfrecipe_s(__m256 _1) {
1732 return (__m256)__builtin_lasx_xvfrecipe_s((v8f32)_1);
1733}
1734
1735extern __inline
1736 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
1737 __lasx_xvfrecipe_d(__m256d _1) {
1738 return (__m256d)__builtin_lasx_xvfrecipe_d((v4f64)_1);
1739}
1740
1741extern __inline1729extern __inline
1742 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m2561730 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
1743 __lasx_xvfrint_s(__m256 _1) {1731 __lasx_xvfrint_s(__m256 _1) {
...@@ -1762,18 +1750,6 @@ extern __inline...@@ -1762,18 +1750,6 @@ extern __inline
1762 return (__m256d)__builtin_lasx_xvfrsqrt_d((v4f64)_1);1750 return (__m256d)__builtin_lasx_xvfrsqrt_d((v4f64)_1);
1763}1751}
17641752
1765extern __inline
1766 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
1767 __lasx_xvfrsqrte_s(__m256 _1) {
1768 return (__m256)__builtin_lasx_xvfrsqrte_s((v8f32)_1);
1769}
1770
1771extern __inline
1772 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
1773 __lasx_xvfrsqrte_d(__m256d _1) {
1774 return (__m256d)__builtin_lasx_xvfrsqrte_d((v4f64)_1);
1775}
1776
1777extern __inline1753extern __inline
1778 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m2561754 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
1779 __lasx_xvflogb_s(__m256 _1) {1755 __lasx_xvflogb_s(__m256 _1) {
...@@ -2585,7 +2561,7 @@ extern __inline...@@ -2585,7 +2561,7 @@ extern __inline
2585extern __inline2561extern __inline
2586 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i2562 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i
2587 __lasx_xvorn_v(__m256i _1, __m256i _2) {2563 __lasx_xvorn_v(__m256i _1, __m256i _2) {
2588 return (__m256i)__builtin_lasx_xvorn_v((v32i8)_1, (v32i8)_2);2564 return (__m256i)__builtin_lasx_xvorn_v((v32u8)_1, (v32u8)_2);
2589}2565}
25902566
2591#define __lasx_xvldi(/*i13*/ _1) ((__m256i)__builtin_lasx_xvldi((_1)))2567#define __lasx_xvldi(/*i13*/ _1) ((__m256i)__builtin_lasx_xvldi((_1)))
...@@ -3866,6 +3842,32 @@ extern __inline...@@ -3866,6 +3842,32 @@ extern __inline
3866 return (__m256i)__builtin_lasx_xvfcmp_sun_s((v8f32)_1, (v8f32)_2);3842 return (__m256i)__builtin_lasx_xvfcmp_sun_s((v8f32)_1, (v8f32)_2);
3867}3843}
38683844
3845#if defined(__loongarch_frecipe)
3846extern __inline
3847 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
3848 __lasx_xvfrecipe_s(__m256 _1) {
3849 return (__m256)__builtin_lasx_xvfrecipe_s((v8f32)_1);
3850}
3851
3852extern __inline
3853 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
3854 __lasx_xvfrecipe_d(__m256d _1) {
3855 return (__m256d)__builtin_lasx_xvfrecipe_d((v4f64)_1);
3856}
3857
3858extern __inline
3859 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
3860 __lasx_xvfrsqrte_s(__m256 _1) {
3861 return (__m256)__builtin_lasx_xvfrsqrte_s((v8f32)_1);
3862}
3863
3864extern __inline
3865 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
3866 __lasx_xvfrsqrte_d(__m256d _1) {
3867 return (__m256d)__builtin_lasx_xvfrsqrte_d((v4f64)_1);
3868}
3869#endif
3870
3869#define __lasx_xvpickve_d_f(/*__m256d*/ _1, /*ui2*/ _2) \3871#define __lasx_xvpickve_d_f(/*__m256d*/ _1, /*ui2*/ _2) \
3870 ((__m256d)__builtin_lasx_xvpickve_d_f((v4f64)(_1), (_2)))3872 ((__m256d)__builtin_lasx_xvpickve_d_f((v4f64)(_1), (_2)))
38713873
lib/include/limits.h+7-4
...@@ -111,11 +111,14 @@...@@ -111,11 +111,14 @@
111#define ULLONG_MAX (__LONG_LONG_MAX__*2ULL+1ULL)111#define ULLONG_MAX (__LONG_LONG_MAX__*2ULL+1ULL)
112#endif112#endif
113113
114/* LONG_LONG_MIN/LONG_LONG_MAX/ULONG_LONG_MAX are a GNU extension. It's too bad114/* LONG_LONG_MIN/LONG_LONG_MAX/ULONG_LONG_MAX are a GNU extension. Android's
115 that we don't have something like #pragma poison that could be used to115 bionic also defines them. It's too bad that we don't have something like
116 deprecate a macro - the code should just use LLONG_MAX and friends.116 #pragma poison that could be used to deprecate a macro - the code should just
117 use LLONG_MAX and friends.
117 */118 */
118#if defined(__GNU_LIBRARY__) ? defined(__USE_GNU) : !defined(__STRICT_ANSI__)119#if (defined(__GNU_LIBRARY__) ? defined(__USE_GNU) \
120 : !defined(__STRICT_ANSI__)) || \
121 defined(__BIONIC__)
119122
120#undef LONG_LONG_MIN123#undef LONG_LONG_MIN
121#undef LONG_LONG_MAX124#undef LONG_LONG_MAX
lib/include/llvm_libc_wrappers/ctype.h+38
...@@ -51,6 +51,19 @@...@@ -51,6 +51,19 @@
51#pragma push_macro("toascii")51#pragma push_macro("toascii")
52#pragma push_macro("tolower")52#pragma push_macro("tolower")
53#pragma push_macro("toupper")53#pragma push_macro("toupper")
54#pragma push_macro("isalnum_l")
55#pragma push_macro("isalpha_l")
56#pragma push_macro("isascii_l")
57#pragma push_macro("isblank_l")
58#pragma push_macro("iscntrl_l")
59#pragma push_macro("isdigit_l")
60#pragma push_macro("isgraph_l")
61#pragma push_macro("islower_l")
62#pragma push_macro("isprint_l")
63#pragma push_macro("ispunct_l")
64#pragma push_macro("isspace_l")
65#pragma push_macro("isupper_l")
66#pragma push_macro("isxdigit_l")
5467
55#undef isalnum68#undef isalnum
56#undef isalpha69#undef isalpha
...@@ -68,6 +81,18 @@...@@ -68,6 +81,18 @@
68#undef toascii81#undef toascii
69#undef tolower82#undef tolower
70#undef toupper83#undef toupper
84#undef isalnum_l
85#undef isalpha_l
86#undef iscntrl_l
87#undef isdigit_l
88#undef islower_l
89#undef isgraph_l
90#undef isprint_l
91#undef ispunct_l
92#undef isspace_l
93#undef isupper_l
94#undef isblank_l
95#undef isxdigit_l
7196
72#pragma omp begin declare target97#pragma omp begin declare target
7398
...@@ -93,6 +118,19 @@...@@ -93,6 +118,19 @@
93#pragma pop_macro("toascii")118#pragma pop_macro("toascii")
94#pragma pop_macro("tolower")119#pragma pop_macro("tolower")
95#pragma pop_macro("toupper")120#pragma pop_macro("toupper")
121#pragma pop_macro("isalnum_l")
122#pragma pop_macro("isalpha_l")
123#pragma pop_macro("isascii_l")
124#pragma pop_macro("isblank_l")
125#pragma pop_macro("iscntrl_l")
126#pragma pop_macro("isdigit_l")
127#pragma pop_macro("isgraph_l")
128#pragma pop_macro("islower_l")
129#pragma pop_macro("isprint_l")
130#pragma pop_macro("ispunct_l")
131#pragma pop_macro("isspace_l")
132#pragma pop_macro("isupper_l")
133#pragma pop_macro("isxdigit_l")
96#endif134#endif
97135
98#undef __LIBC_ATTRS136#undef __LIBC_ATTRS
lib/include/llvm_libc_wrappers/stdlib.h+8
...@@ -34,8 +34,16 @@ _Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!");...@@ -34,8 +34,16 @@ _Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!");
34_Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!");34_Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!");
35_Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!");35_Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!");
3636
37#if defined(__GLIBC__) && __cplusplus >= 201703L
38#define at_quick_exit atexit
39#endif
40
37#include <llvm-libc-decls/stdlib.h>41#include <llvm-libc-decls/stdlib.h>
3842
43#if defined(__GLIBC__) && __cplusplus >= 201703L
44#undef at_quick_exit
45#endif
46
39#pragma omp end declare target47#pragma omp end declare target
4048
41#undef __LIBC_ATTRS49#undef __LIBC_ATTRS
lib/include/lsxintrin.h+27-25
...@@ -1776,18 +1776,6 @@ extern __inline...@@ -1776,18 +1776,6 @@ extern __inline
1776 return (__m128d)__builtin_lsx_vfrecip_d((v2f64)_1);1776 return (__m128d)__builtin_lsx_vfrecip_d((v2f64)_1);
1777}1777}
17781778
1779extern __inline
1780 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
1781 __lsx_vfrecipe_s(__m128 _1) {
1782 return (__m128)__builtin_lsx_vfrecipe_s((v4f32)_1);
1783}
1784
1785extern __inline
1786 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
1787 __lsx_vfrecipe_d(__m128d _1) {
1788 return (__m128d)__builtin_lsx_vfrecipe_d((v2f64)_1);
1789}
1790
1791extern __inline1779extern __inline
1792 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m1281780 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
1793 __lsx_vfrint_s(__m128 _1) {1781 __lsx_vfrint_s(__m128 _1) {
...@@ -1812,18 +1800,6 @@ extern __inline...@@ -1812,18 +1800,6 @@ extern __inline
1812 return (__m128d)__builtin_lsx_vfrsqrt_d((v2f64)_1);1800 return (__m128d)__builtin_lsx_vfrsqrt_d((v2f64)_1);
1813}1801}
18141802
1815extern __inline
1816 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
1817 __lsx_vfrsqrte_s(__m128 _1) {
1818 return (__m128)__builtin_lsx_vfrsqrte_s((v4f32)_1);
1819}
1820
1821extern __inline
1822 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
1823 __lsx_vfrsqrte_d(__m128d _1) {
1824 return (__m128d)__builtin_lsx_vfrsqrte_d((v2f64)_1);
1825}
1826
1827extern __inline1803extern __inline
1828 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m1281804 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
1829 __lsx_vflogb_s(__m128 _1) {1805 __lsx_vflogb_s(__m128 _1) {
...@@ -3425,7 +3401,7 @@ extern __inline...@@ -3425,7 +3401,7 @@ extern __inline
3425extern __inline3401extern __inline
3426 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128i3402 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128i
3427 __lsx_vorn_v(__m128i _1, __m128i _2) {3403 __lsx_vorn_v(__m128i _1, __m128i _2) {
3428 return (__m128i)__builtin_lsx_vorn_v((v16i8)_1, (v16i8)_2);3404 return (__m128i)__builtin_lsx_vorn_v((v16u8)_1, (v16u8)_2);
3429}3405}
34303406
3431#define __lsx_vldi(/*i13*/ _1) ((__m128i)__builtin_lsx_vldi((_1)))3407#define __lsx_vldi(/*i13*/ _1) ((__m128i)__builtin_lsx_vldi((_1)))
...@@ -3738,6 +3714,32 @@ extern __inline...@@ -3738,6 +3714,32 @@ extern __inline
3738 return (__m128i)__builtin_lsx_vfcmp_sun_s((v4f32)_1, (v4f32)_2);3714 return (__m128i)__builtin_lsx_vfcmp_sun_s((v4f32)_1, (v4f32)_2);
3739}3715}
37403716
3717#if defined(__loongarch_frecipe)
3718extern __inline
3719 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
3720 __lsx_vfrecipe_s(__m128 _1) {
3721 return (__m128)__builtin_lsx_vfrecipe_s((v4f32)_1);
3722}
3723
3724extern __inline
3725 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
3726 __lsx_vfrecipe_d(__m128d _1) {
3727 return (__m128d)__builtin_lsx_vfrecipe_d((v2f64)_1);
3728}
3729
3730extern __inline
3731 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
3732 __lsx_vfrsqrte_s(__m128 _1) {
3733 return (__m128)__builtin_lsx_vfrsqrte_s((v4f32)_1);
3734}
3735
3736extern __inline
3737 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
3738 __lsx_vfrsqrte_d(__m128d _1) {
3739 return (__m128d)__builtin_lsx_vfrsqrte_d((v2f64)_1);
3740}
3741#endif
3742
3741#define __lsx_vrepli_b(/*si10*/ _1) ((__m128i)__builtin_lsx_vrepli_b((_1)))3743#define __lsx_vrepli_b(/*si10*/ _1) ((__m128i)__builtin_lsx_vrepli_b((_1)))
37423744
3743#define __lsx_vrepli_d(/*si10*/ _1) ((__m128i)__builtin_lsx_vrepli_d((_1)))3745#define __lsx_vrepli_d(/*si10*/ _1) ((__m128i)__builtin_lsx_vrepli_d((_1)))
lib/include/lzcntintrin.h+10-7
...@@ -15,7 +15,13 @@...@@ -15,7 +15,13 @@
15#define __LZCNTINTRIN_H15#define __LZCNTINTRIN_H
1616
17/* Define the default attributes for the functions in this file. */17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("lzcnt")))18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("lzcnt"))) constexpr
21#else
22#define __DEFAULT_FN_ATTRS \
23 __attribute__((__always_inline__, __nodebug__, __target__("lzcnt")))
24#endif
1925
20#ifndef _MSC_VER26#ifndef _MSC_VER
21/// Counts the number of leading zero bits in the operand.27/// Counts the number of leading zero bits in the operand.
...@@ -43,8 +49,7 @@...@@ -43,8 +49,7 @@
43/// bits in the operand.49/// bits in the operand.
44/// \see _lzcnt_u3250/// \see _lzcnt_u32
45static __inline__ unsigned int __DEFAULT_FN_ATTRS51static __inline__ unsigned int __DEFAULT_FN_ATTRS
46__lzcnt32(unsigned int __X)52__lzcnt32(unsigned int __X) {
47{
48 return __builtin_ia32_lzcnt_u32(__X);53 return __builtin_ia32_lzcnt_u32(__X);
49}54}
5055
...@@ -60,8 +65,7 @@ __lzcnt32(unsigned int __X)...@@ -60,8 +65,7 @@ __lzcnt32(unsigned int __X)
60/// bits in the operand.65/// bits in the operand.
61/// \see __lzcnt3266/// \see __lzcnt32
62static __inline__ unsigned int __DEFAULT_FN_ATTRS67static __inline__ unsigned int __DEFAULT_FN_ATTRS
63_lzcnt_u32(unsigned int __X)68_lzcnt_u32(unsigned int __X) {
64{
65 return __builtin_ia32_lzcnt_u32(__X);69 return __builtin_ia32_lzcnt_u32(__X);
66}70}
6771
...@@ -93,8 +97,7 @@ _lzcnt_u32(unsigned int __X)...@@ -93,8 +97,7 @@ _lzcnt_u32(unsigned int __X)
93/// bits in the operand.97/// bits in the operand.
94/// \see __lzcnt6498/// \see __lzcnt64
95static __inline__ unsigned long long __DEFAULT_FN_ATTRS99static __inline__ unsigned long long __DEFAULT_FN_ATTRS
96_lzcnt_u64(unsigned long long __X)100_lzcnt_u64(unsigned long long __X) {
97{
98 return __builtin_ia32_lzcnt_u64(__X);101 return __builtin_ia32_lzcnt_u64(__X);
99}102}
100#endif103#endif
lib/include/mmintrin.h+209-155
...@@ -21,10 +21,45 @@ typedef int __v2si __attribute__((__vector_size__(8)));...@@ -21,10 +21,45 @@ typedef int __v2si __attribute__((__vector_size__(8)));
21typedef short __v4hi __attribute__((__vector_size__(8)));21typedef short __v4hi __attribute__((__vector_size__(8)));
22typedef char __v8qi __attribute__((__vector_size__(8)));22typedef char __v8qi __attribute__((__vector_size__(8)));
2323
24/* Unsigned types */
25typedef unsigned long long __v1du __attribute__ ((__vector_size__ (8)));
26typedef unsigned int __v2su __attribute__ ((__vector_size__ (8)));
27typedef unsigned short __v4hu __attribute__((__vector_size__(8)));
28typedef unsigned char __v8qu __attribute__((__vector_size__(8)));
29
30/* We need an explicitly signed variant for char. Note that this shouldn't
31 * appear in the interface though. */
32typedef signed char __v8qs __attribute__((__vector_size__(8)));
33
34/* SSE/SSE2 types */
35typedef long long __m128i __attribute__((__vector_size__(16), __aligned__(16)));
36typedef long long __v2di __attribute__ ((__vector_size__ (16)));
37typedef int __v4si __attribute__((__vector_size__(16)));
38typedef short __v8hi __attribute__((__vector_size__(16)));
39typedef char __v16qi __attribute__((__vector_size__(16)));
40
24/* Define the default attributes for the functions in this file. */41/* Define the default attributes for the functions in this file. */
25#define __DEFAULT_FN_ATTRS \42#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
26 __attribute__((__always_inline__, __nodebug__, __target__("mmx,no-evex512"), \43#define __DEFAULT_FN_ATTRS_SSE2 \
27 __min_vector_width__(64)))44 __attribute__((__always_inline__, __nodebug__, \
45 __target__("sse2,no-evex512"), __min_vector_width__(128)))
46#else
47#define __DEFAULT_FN_ATTRS_SSE2 \
48 __attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
49 __min_vector_width__(128)))
50#endif
51
52#if defined(__cplusplus) && (__cplusplus >= 201103L)
53#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2 constexpr
54#else
55#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2
56#endif
57
58#define __trunc64(x) \
59 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
60#define __anyext128(x) \
61 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
62 1, -1, -1)
2863
29/// Clears the MMX state by setting the state of the x87 stack registers64/// Clears the MMX state by setting the state of the x87 stack registers
30/// to empty.65/// to empty.
...@@ -50,10 +85,10 @@ _mm_empty(void) {...@@ -50,10 +85,10 @@ _mm_empty(void) {
50/// A 32-bit integer value.85/// A 32-bit integer value.
51/// \returns A 64-bit integer vector. The lower 32 bits contain the value of the86/// \returns A 64-bit integer vector. The lower 32 bits contain the value of the
52/// parameter. The upper 32 bits are set to 0.87/// parameter. The upper 32 bits are set to 0.
53static __inline__ __m64 __DEFAULT_FN_ATTRS88static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
54_mm_cvtsi32_si64(int __i)89_mm_cvtsi32_si64(int __i)
55{90{
56 return (__m64)__builtin_ia32_vec_init_v2si(__i, 0);91 return __extension__ (__m64)(__v2si){__i, 0};
57}92}
5893
59/// Returns the lower 32 bits of a 64-bit integer vector as a 32-bit94/// Returns the lower 32 bits of a 64-bit integer vector as a 32-bit
...@@ -67,10 +102,10 @@ _mm_cvtsi32_si64(int __i)...@@ -67,10 +102,10 @@ _mm_cvtsi32_si64(int __i)
67/// A 64-bit integer vector.102/// A 64-bit integer vector.
68/// \returns A 32-bit signed integer value containing the lower 32 bits of the103/// \returns A 32-bit signed integer value containing the lower 32 bits of the
69/// parameter.104/// parameter.
70static __inline__ int __DEFAULT_FN_ATTRS105static __inline__ int __DEFAULT_FN_ATTRS_SSE2
71_mm_cvtsi64_si32(__m64 __m)106_mm_cvtsi64_si32(__m64 __m)
72{107{
73 return __builtin_ia32_vec_ext_v2si((__v2si)__m, 0);108 return ((__v2si)__m)[0];
74}109}
75110
76/// Casts a 64-bit signed integer value into a 64-bit integer vector.111/// Casts a 64-bit signed integer value into a 64-bit integer vector.
...@@ -83,7 +118,7 @@ _mm_cvtsi64_si32(__m64 __m)...@@ -83,7 +118,7 @@ _mm_cvtsi64_si32(__m64 __m)
83/// A 64-bit signed integer.118/// A 64-bit signed integer.
84/// \returns A 64-bit integer vector containing the same bitwise pattern as the119/// \returns A 64-bit integer vector containing the same bitwise pattern as the
85/// parameter.120/// parameter.
86static __inline__ __m64 __DEFAULT_FN_ATTRS121static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
87_mm_cvtsi64_m64(long long __i)122_mm_cvtsi64_m64(long long __i)
88{123{
89 return (__m64)__i;124 return (__m64)__i;
...@@ -99,7 +134,7 @@ _mm_cvtsi64_m64(long long __i)...@@ -99,7 +134,7 @@ _mm_cvtsi64_m64(long long __i)
99/// A 64-bit integer vector.134/// A 64-bit integer vector.
100/// \returns A 64-bit signed integer containing the same bitwise pattern as the135/// \returns A 64-bit signed integer containing the same bitwise pattern as the
101/// parameter.136/// parameter.
102static __inline__ long long __DEFAULT_FN_ATTRS137static __inline__ long long __DEFAULT_FN_ATTRS_SSE2
103_mm_cvtm64_si64(__m64 __m)138_mm_cvtm64_si64(__m64 __m)
104{139{
105 return (long long)__m;140 return (long long)__m;
...@@ -124,10 +159,11 @@ _mm_cvtm64_si64(__m64 __m)...@@ -124,10 +159,11 @@ _mm_cvtm64_si64(__m64 __m)
124/// written to the upper 32 bits of the result.159/// written to the upper 32 bits of the result.
125/// \returns A 64-bit integer vector of [8 x i8] containing the converted160/// \returns A 64-bit integer vector of [8 x i8] containing the converted
126/// values.161/// values.
127static __inline__ __m64 __DEFAULT_FN_ATTRS162static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
128_mm_packs_pi16(__m64 __m1, __m64 __m2)163_mm_packs_pi16(__m64 __m1, __m64 __m2)
129{164{
130 return (__m64)__builtin_ia32_packsswb((__v4hi)__m1, (__v4hi)__m2);165 return __trunc64(__builtin_ia32_packsswb128(
166 (__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
131}167}
132168
133/// Converts, with saturation, 32-bit signed integers from both 64-bit integer169/// Converts, with saturation, 32-bit signed integers from both 64-bit integer
...@@ -149,10 +185,11 @@ _mm_packs_pi16(__m64 __m1, __m64 __m2)...@@ -149,10 +185,11 @@ _mm_packs_pi16(__m64 __m1, __m64 __m2)
149/// written to the upper 32 bits of the result.185/// written to the upper 32 bits of the result.
150/// \returns A 64-bit integer vector of [4 x i16] containing the converted186/// \returns A 64-bit integer vector of [4 x i16] containing the converted
151/// values.187/// values.
152static __inline__ __m64 __DEFAULT_FN_ATTRS188static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
153_mm_packs_pi32(__m64 __m1, __m64 __m2)189_mm_packs_pi32(__m64 __m1, __m64 __m2)
154{190{
155 return (__m64)__builtin_ia32_packssdw((__v2si)__m1, (__v2si)__m2);191 return __trunc64(__builtin_ia32_packssdw128(
192 (__v4si)__builtin_shufflevector(__m1, __m2, 0, 1), (__v4si){}));
156}193}
157194
158/// Converts, with saturation, 16-bit signed integers from both 64-bit integer195/// Converts, with saturation, 16-bit signed integers from both 64-bit integer
...@@ -174,10 +211,11 @@ _mm_packs_pi32(__m64 __m1, __m64 __m2)...@@ -174,10 +211,11 @@ _mm_packs_pi32(__m64 __m1, __m64 __m2)
174/// written to the upper 32 bits of the result.211/// written to the upper 32 bits of the result.
175/// \returns A 64-bit integer vector of [8 x i8] containing the converted212/// \returns A 64-bit integer vector of [8 x i8] containing the converted
176/// values.213/// values.
177static __inline__ __m64 __DEFAULT_FN_ATTRS214static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
178_mm_packs_pu16(__m64 __m1, __m64 __m2)215_mm_packs_pu16(__m64 __m1, __m64 __m2)
179{216{
180 return (__m64)__builtin_ia32_packuswb((__v4hi)__m1, (__v4hi)__m2);217 return __trunc64(__builtin_ia32_packuswb128(
218 (__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
181}219}
182220
183/// Unpacks the upper 32 bits from two 64-bit integer vectors of [8 x i8]221/// Unpacks the upper 32 bits from two 64-bit integer vectors of [8 x i8]
...@@ -201,10 +239,11 @@ _mm_packs_pu16(__m64 __m1, __m64 __m2)...@@ -201,10 +239,11 @@ _mm_packs_pu16(__m64 __m1, __m64 __m2)
201/// Bits [63:56] are written to bits [63:56] of the result.239/// Bits [63:56] are written to bits [63:56] of the result.
202/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved240/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
203/// values.241/// values.
204static __inline__ __m64 __DEFAULT_FN_ATTRS242static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
205_mm_unpackhi_pi8(__m64 __m1, __m64 __m2)243_mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
206{244{
207 return (__m64)__builtin_ia32_punpckhbw((__v8qi)__m1, (__v8qi)__m2);245 return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
246 4, 12, 5, 13, 6, 14, 7, 15);
208}247}
209248
210/// Unpacks the upper 32 bits from two 64-bit integer vectors of249/// Unpacks the upper 32 bits from two 64-bit integer vectors of
...@@ -224,10 +263,11 @@ _mm_unpackhi_pi8(__m64 __m1, __m64 __m2)...@@ -224,10 +263,11 @@ _mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
224/// Bits [63:48] are written to bits [63:48] of the result.263/// Bits [63:48] are written to bits [63:48] of the result.
225/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved264/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
226/// values.265/// values.
227static __inline__ __m64 __DEFAULT_FN_ATTRS266static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
228_mm_unpackhi_pi16(__m64 __m1, __m64 __m2)267_mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
229{268{
230 return (__m64)__builtin_ia32_punpckhwd((__v4hi)__m1, (__v4hi)__m2);269 return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
270 2, 6, 3, 7);
231}271}
232272
233/// Unpacks the upper 32 bits from two 64-bit integer vectors of273/// Unpacks the upper 32 bits from two 64-bit integer vectors of
...@@ -245,10 +285,10 @@ _mm_unpackhi_pi16(__m64 __m1, __m64 __m2)...@@ -245,10 +285,10 @@ _mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
245/// the upper 32 bits of the result.285/// the upper 32 bits of the result.
246/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved286/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
247/// values.287/// values.
248static __inline__ __m64 __DEFAULT_FN_ATTRS288static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
249_mm_unpackhi_pi32(__m64 __m1, __m64 __m2)289_mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
250{290{
251 return (__m64)__builtin_ia32_punpckhdq((__v2si)__m1, (__v2si)__m2);291 return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 3);
252}292}
253293
254/// Unpacks the lower 32 bits from two 64-bit integer vectors of [8 x i8]294/// Unpacks the lower 32 bits from two 64-bit integer vectors of [8 x i8]
...@@ -272,10 +312,11 @@ _mm_unpackhi_pi32(__m64 __m1, __m64 __m2)...@@ -272,10 +312,11 @@ _mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
272/// Bits [31:24] are written to bits [63:56] of the result.312/// Bits [31:24] are written to bits [63:56] of the result.
273/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved313/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
274/// values.314/// values.
275static __inline__ __m64 __DEFAULT_FN_ATTRS315static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
276_mm_unpacklo_pi8(__m64 __m1, __m64 __m2)316_mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
277{317{
278 return (__m64)__builtin_ia32_punpcklbw((__v8qi)__m1, (__v8qi)__m2);318 return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
319 0, 8, 1, 9, 2, 10, 3, 11);
279}320}
280321
281/// Unpacks the lower 32 bits from two 64-bit integer vectors of322/// Unpacks the lower 32 bits from two 64-bit integer vectors of
...@@ -295,10 +336,11 @@ _mm_unpacklo_pi8(__m64 __m1, __m64 __m2)...@@ -295,10 +336,11 @@ _mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
295/// Bits [31:16] are written to bits [63:48] of the result.336/// Bits [31:16] are written to bits [63:48] of the result.
296/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved337/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
297/// values.338/// values.
298static __inline__ __m64 __DEFAULT_FN_ATTRS339static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
299_mm_unpacklo_pi16(__m64 __m1, __m64 __m2)340_mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
300{341{
301 return (__m64)__builtin_ia32_punpcklwd((__v4hi)__m1, (__v4hi)__m2);342 return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
343 0, 4, 1, 5);
302}344}
303345
304/// Unpacks the lower 32 bits from two 64-bit integer vectors of346/// Unpacks the lower 32 bits from two 64-bit integer vectors of
...@@ -316,10 +358,10 @@ _mm_unpacklo_pi16(__m64 __m1, __m64 __m2)...@@ -316,10 +358,10 @@ _mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
316/// the upper 32 bits of the result.358/// the upper 32 bits of the result.
317/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved359/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
318/// values.360/// values.
319static __inline__ __m64 __DEFAULT_FN_ATTRS361static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
320_mm_unpacklo_pi32(__m64 __m1, __m64 __m2)362_mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
321{363{
322 return (__m64)__builtin_ia32_punpckldq((__v2si)__m1, (__v2si)__m2);364 return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2);
323}365}
324366
325/// Adds each 8-bit integer element of the first 64-bit integer vector367/// Adds each 8-bit integer element of the first 64-bit integer vector
...@@ -337,10 +379,10 @@ _mm_unpacklo_pi32(__m64 __m1, __m64 __m2)...@@ -337,10 +379,10 @@ _mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
337/// A 64-bit integer vector of [8 x i8].379/// A 64-bit integer vector of [8 x i8].
338/// \returns A 64-bit integer vector of [8 x i8] containing the sums of both380/// \returns A 64-bit integer vector of [8 x i8] containing the sums of both
339/// parameters.381/// parameters.
340static __inline__ __m64 __DEFAULT_FN_ATTRS382static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
341_mm_add_pi8(__m64 __m1, __m64 __m2)383_mm_add_pi8(__m64 __m1, __m64 __m2)
342{384{
343 return (__m64)__builtin_ia32_paddb((__v8qi)__m1, (__v8qi)__m2);385 return (__m64)(((__v8qu)__m1) + ((__v8qu)__m2));
344}386}
345387
346/// Adds each 16-bit integer element of the first 64-bit integer vector388/// Adds each 16-bit integer element of the first 64-bit integer vector
...@@ -358,10 +400,10 @@ _mm_add_pi8(__m64 __m1, __m64 __m2)...@@ -358,10 +400,10 @@ _mm_add_pi8(__m64 __m1, __m64 __m2)
358/// A 64-bit integer vector of [4 x i16].400/// A 64-bit integer vector of [4 x i16].
359/// \returns A 64-bit integer vector of [4 x i16] containing the sums of both401/// \returns A 64-bit integer vector of [4 x i16] containing the sums of both
360/// parameters.402/// parameters.
361static __inline__ __m64 __DEFAULT_FN_ATTRS403static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
362_mm_add_pi16(__m64 __m1, __m64 __m2)404_mm_add_pi16(__m64 __m1, __m64 __m2)
363{405{
364 return (__m64)__builtin_ia32_paddw((__v4hi)__m1, (__v4hi)__m2);406 return (__m64)(((__v4hu)__m1) + ((__v4hu)__m2));
365}407}
366408
367/// Adds each 32-bit integer element of the first 64-bit integer vector409/// Adds each 32-bit integer element of the first 64-bit integer vector
...@@ -379,10 +421,10 @@ _mm_add_pi16(__m64 __m1, __m64 __m2)...@@ -379,10 +421,10 @@ _mm_add_pi16(__m64 __m1, __m64 __m2)
379/// A 64-bit integer vector of [2 x i32].421/// A 64-bit integer vector of [2 x i32].
380/// \returns A 64-bit integer vector of [2 x i32] containing the sums of both422/// \returns A 64-bit integer vector of [2 x i32] containing the sums of both
381/// parameters.423/// parameters.
382static __inline__ __m64 __DEFAULT_FN_ATTRS424static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
383_mm_add_pi32(__m64 __m1, __m64 __m2)425_mm_add_pi32(__m64 __m1, __m64 __m2)
384{426{
385 return (__m64)__builtin_ia32_paddd((__v2si)__m1, (__v2si)__m2);427 return (__m64)(((__v2su)__m1) + ((__v2su)__m2));
386}428}
387429
388/// Adds, with saturation, each 8-bit signed integer element of the first430/// Adds, with saturation, each 8-bit signed integer element of the first
...@@ -403,10 +445,10 @@ _mm_add_pi32(__m64 __m1, __m64 __m2)...@@ -403,10 +445,10 @@ _mm_add_pi32(__m64 __m1, __m64 __m2)
403/// A 64-bit integer vector of [8 x i8].445/// A 64-bit integer vector of [8 x i8].
404/// \returns A 64-bit integer vector of [8 x i8] containing the saturated sums446/// \returns A 64-bit integer vector of [8 x i8] containing the saturated sums
405/// of both parameters.447/// of both parameters.
406static __inline__ __m64 __DEFAULT_FN_ATTRS448static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
407_mm_adds_pi8(__m64 __m1, __m64 __m2)449_mm_adds_pi8(__m64 __m1, __m64 __m2)
408{450{
409 return (__m64)__builtin_ia32_paddsb((__v8qi)__m1, (__v8qi)__m2);451 return (__m64)__builtin_elementwise_add_sat((__v8qs)__m1, (__v8qs)__m2);
410}452}
411453
412/// Adds, with saturation, each 16-bit signed integer element of the first454/// Adds, with saturation, each 16-bit signed integer element of the first
...@@ -427,10 +469,10 @@ _mm_adds_pi8(__m64 __m1, __m64 __m2)...@@ -427,10 +469,10 @@ _mm_adds_pi8(__m64 __m1, __m64 __m2)
427/// A 64-bit integer vector of [4 x i16].469/// A 64-bit integer vector of [4 x i16].
428/// \returns A 64-bit integer vector of [4 x i16] containing the saturated sums470/// \returns A 64-bit integer vector of [4 x i16] containing the saturated sums
429/// of both parameters.471/// of both parameters.
430static __inline__ __m64 __DEFAULT_FN_ATTRS472static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
431_mm_adds_pi16(__m64 __m1, __m64 __m2)473_mm_adds_pi16(__m64 __m1, __m64 __m2)
432{474{
433 return (__m64)__builtin_ia32_paddsw((__v4hi)__m1, (__v4hi)__m2);475 return (__m64)__builtin_elementwise_add_sat((__v4hi)__m1, (__v4hi)__m2);
434}476}
435477
436/// Adds, with saturation, each 8-bit unsigned integer element of the first478/// Adds, with saturation, each 8-bit unsigned integer element of the first
...@@ -450,10 +492,10 @@ _mm_adds_pi16(__m64 __m1, __m64 __m2)...@@ -450,10 +492,10 @@ _mm_adds_pi16(__m64 __m1, __m64 __m2)
450/// A 64-bit integer vector of [8 x i8].492/// A 64-bit integer vector of [8 x i8].
451/// \returns A 64-bit integer vector of [8 x i8] containing the saturated493/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
452/// unsigned sums of both parameters.494/// unsigned sums of both parameters.
453static __inline__ __m64 __DEFAULT_FN_ATTRS495static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
454_mm_adds_pu8(__m64 __m1, __m64 __m2)496_mm_adds_pu8(__m64 __m1, __m64 __m2)
455{497{
456 return (__m64)__builtin_ia32_paddusb((__v8qi)__m1, (__v8qi)__m2);498 return (__m64)__builtin_elementwise_add_sat((__v8qu)__m1, (__v8qu)__m2);
457}499}
458500
459/// Adds, with saturation, each 16-bit unsigned integer element of the first501/// Adds, with saturation, each 16-bit unsigned integer element of the first
...@@ -473,10 +515,10 @@ _mm_adds_pu8(__m64 __m1, __m64 __m2)...@@ -473,10 +515,10 @@ _mm_adds_pu8(__m64 __m1, __m64 __m2)
473/// A 64-bit integer vector of [4 x i16].515/// A 64-bit integer vector of [4 x i16].
474/// \returns A 64-bit integer vector of [4 x i16] containing the saturated516/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
475/// unsigned sums of both parameters.517/// unsigned sums of both parameters.
476static __inline__ __m64 __DEFAULT_FN_ATTRS518static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
477_mm_adds_pu16(__m64 __m1, __m64 __m2)519_mm_adds_pu16(__m64 __m1, __m64 __m2)
478{520{
479 return (__m64)__builtin_ia32_paddusw((__v4hi)__m1, (__v4hi)__m2);521 return (__m64)__builtin_elementwise_add_sat((__v4hu)__m1, (__v4hu)__m2);
480}522}
481523
482/// Subtracts each 8-bit integer element of the second 64-bit integer524/// Subtracts each 8-bit integer element of the second 64-bit integer
...@@ -494,10 +536,10 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2)...@@ -494,10 +536,10 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2)
494/// A 64-bit integer vector of [8 x i8] containing the subtrahends.536/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
495/// \returns A 64-bit integer vector of [8 x i8] containing the differences of537/// \returns A 64-bit integer vector of [8 x i8] containing the differences of
496/// both parameters.538/// both parameters.
497static __inline__ __m64 __DEFAULT_FN_ATTRS539static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
498_mm_sub_pi8(__m64 __m1, __m64 __m2)540_mm_sub_pi8(__m64 __m1, __m64 __m2)
499{541{
500 return (__m64)__builtin_ia32_psubb((__v8qi)__m1, (__v8qi)__m2);542 return (__m64)(((__v8qu)__m1) - ((__v8qu)__m2));
501}543}
502544
503/// Subtracts each 16-bit integer element of the second 64-bit integer545/// Subtracts each 16-bit integer element of the second 64-bit integer
...@@ -515,10 +557,10 @@ _mm_sub_pi8(__m64 __m1, __m64 __m2)...@@ -515,10 +557,10 @@ _mm_sub_pi8(__m64 __m1, __m64 __m2)
515/// A 64-bit integer vector of [4 x i16] containing the subtrahends.557/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
516/// \returns A 64-bit integer vector of [4 x i16] containing the differences of558/// \returns A 64-bit integer vector of [4 x i16] containing the differences of
517/// both parameters.559/// both parameters.
518static __inline__ __m64 __DEFAULT_FN_ATTRS560static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
519_mm_sub_pi16(__m64 __m1, __m64 __m2)561_mm_sub_pi16(__m64 __m1, __m64 __m2)
520{562{
521 return (__m64)__builtin_ia32_psubw((__v4hi)__m1, (__v4hi)__m2);563 return (__m64)(((__v4hu)__m1) - ((__v4hu)__m2));
522}564}
523565
524/// Subtracts each 32-bit integer element of the second 64-bit integer566/// Subtracts each 32-bit integer element of the second 64-bit integer
...@@ -536,10 +578,10 @@ _mm_sub_pi16(__m64 __m1, __m64 __m2)...@@ -536,10 +578,10 @@ _mm_sub_pi16(__m64 __m1, __m64 __m2)
536/// A 64-bit integer vector of [2 x i32] containing the subtrahends.578/// A 64-bit integer vector of [2 x i32] containing the subtrahends.
537/// \returns A 64-bit integer vector of [2 x i32] containing the differences of579/// \returns A 64-bit integer vector of [2 x i32] containing the differences of
538/// both parameters.580/// both parameters.
539static __inline__ __m64 __DEFAULT_FN_ATTRS581static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
540_mm_sub_pi32(__m64 __m1, __m64 __m2)582_mm_sub_pi32(__m64 __m1, __m64 __m2)
541{583{
542 return (__m64)__builtin_ia32_psubd((__v2si)__m1, (__v2si)__m2);584 return (__m64)(((__v2su)__m1) - ((__v2su)__m2));
543}585}
544586
545/// Subtracts, with saturation, each 8-bit signed integer element of the second587/// Subtracts, with saturation, each 8-bit signed integer element of the second
...@@ -560,10 +602,10 @@ _mm_sub_pi32(__m64 __m1, __m64 __m2)...@@ -560,10 +602,10 @@ _mm_sub_pi32(__m64 __m1, __m64 __m2)
560/// A 64-bit integer vector of [8 x i8] containing the subtrahends.602/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
561/// \returns A 64-bit integer vector of [8 x i8] containing the saturated603/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
562/// differences of both parameters.604/// differences of both parameters.
563static __inline__ __m64 __DEFAULT_FN_ATTRS605static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
564_mm_subs_pi8(__m64 __m1, __m64 __m2)606_mm_subs_pi8(__m64 __m1, __m64 __m2)
565{607{
566 return (__m64)__builtin_ia32_psubsb((__v8qi)__m1, (__v8qi)__m2);608 return (__m64)__builtin_elementwise_sub_sat((__v8qs)__m1, (__v8qs)__m2);
567}609}
568610
569/// Subtracts, with saturation, each 16-bit signed integer element of the611/// Subtracts, with saturation, each 16-bit signed integer element of the
...@@ -584,10 +626,10 @@ _mm_subs_pi8(__m64 __m1, __m64 __m2)...@@ -584,10 +626,10 @@ _mm_subs_pi8(__m64 __m1, __m64 __m2)
584/// A 64-bit integer vector of [4 x i16] containing the subtrahends.626/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
585/// \returns A 64-bit integer vector of [4 x i16] containing the saturated627/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
586/// differences of both parameters.628/// differences of both parameters.
587static __inline__ __m64 __DEFAULT_FN_ATTRS629static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
588_mm_subs_pi16(__m64 __m1, __m64 __m2)630_mm_subs_pi16(__m64 __m1, __m64 __m2)
589{631{
590 return (__m64)__builtin_ia32_psubsw((__v4hi)__m1, (__v4hi)__m2);632 return (__m64)__builtin_elementwise_sub_sat((__v4hi)__m1, (__v4hi)__m2);
591}633}
592634
593/// Subtracts each 8-bit unsigned integer element of the second 64-bit635/// Subtracts each 8-bit unsigned integer element of the second 64-bit
...@@ -608,10 +650,10 @@ _mm_subs_pi16(__m64 __m1, __m64 __m2)...@@ -608,10 +650,10 @@ _mm_subs_pi16(__m64 __m1, __m64 __m2)
608/// A 64-bit integer vector of [8 x i8] containing the subtrahends.650/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
609/// \returns A 64-bit integer vector of [8 x i8] containing the saturated651/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
610/// differences of both parameters.652/// differences of both parameters.
611static __inline__ __m64 __DEFAULT_FN_ATTRS653static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
612_mm_subs_pu8(__m64 __m1, __m64 __m2)654_mm_subs_pu8(__m64 __m1, __m64 __m2)
613{655{
614 return (__m64)__builtin_ia32_psubusb((__v8qi)__m1, (__v8qi)__m2);656 return (__m64)__builtin_elementwise_sub_sat((__v8qu)__m1, (__v8qu)__m2);
615}657}
616658
617/// Subtracts each 16-bit unsigned integer element of the second 64-bit659/// Subtracts each 16-bit unsigned integer element of the second 64-bit
...@@ -632,10 +674,10 @@ _mm_subs_pu8(__m64 __m1, __m64 __m2)...@@ -632,10 +674,10 @@ _mm_subs_pu8(__m64 __m1, __m64 __m2)
632/// A 64-bit integer vector of [4 x i16] containing the subtrahends.674/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
633/// \returns A 64-bit integer vector of [4 x i16] containing the saturated675/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
634/// differences of both parameters.676/// differences of both parameters.
635static __inline__ __m64 __DEFAULT_FN_ATTRS677static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
636_mm_subs_pu16(__m64 __m1, __m64 __m2)678_mm_subs_pu16(__m64 __m1, __m64 __m2)
637{679{
638 return (__m64)__builtin_ia32_psubusw((__v4hi)__m1, (__v4hi)__m2);680 return (__m64)__builtin_elementwise_sub_sat((__v4hu)__m1, (__v4hu)__m2);
639}681}
640682
641/// Multiplies each 16-bit signed integer element of the first 64-bit683/// Multiplies each 16-bit signed integer element of the first 64-bit
...@@ -659,10 +701,11 @@ _mm_subs_pu16(__m64 __m1, __m64 __m2)...@@ -659,10 +701,11 @@ _mm_subs_pu16(__m64 __m1, __m64 __m2)
659/// A 64-bit integer vector of [4 x i16].701/// A 64-bit integer vector of [4 x i16].
660/// \returns A 64-bit integer vector of [2 x i32] containing the sums of702/// \returns A 64-bit integer vector of [2 x i32] containing the sums of
661/// products of both parameters.703/// products of both parameters.
662static __inline__ __m64 __DEFAULT_FN_ATTRS704static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
663_mm_madd_pi16(__m64 __m1, __m64 __m2)705_mm_madd_pi16(__m64 __m1, __m64 __m2)
664{706{
665 return (__m64)__builtin_ia32_pmaddwd((__v4hi)__m1, (__v4hi)__m2);707 return __trunc64(__builtin_ia32_pmaddwd128((__v8hi)__anyext128(__m1),
708 (__v8hi)__anyext128(__m2)));
666}709}
667710
668/// Multiplies each 16-bit signed integer element of the first 64-bit711/// Multiplies each 16-bit signed integer element of the first 64-bit
...@@ -680,10 +723,11 @@ _mm_madd_pi16(__m64 __m1, __m64 __m2)...@@ -680,10 +723,11 @@ _mm_madd_pi16(__m64 __m1, __m64 __m2)
680/// A 64-bit integer vector of [4 x i16].723/// A 64-bit integer vector of [4 x i16].
681/// \returns A 64-bit integer vector of [4 x i16] containing the upper 16 bits724/// \returns A 64-bit integer vector of [4 x i16] containing the upper 16 bits
682/// of the products of both parameters.725/// of the products of both parameters.
683static __inline__ __m64 __DEFAULT_FN_ATTRS726static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
684_mm_mulhi_pi16(__m64 __m1, __m64 __m2)727_mm_mulhi_pi16(__m64 __m1, __m64 __m2)
685{728{
686 return (__m64)__builtin_ia32_pmulhw((__v4hi)__m1, (__v4hi)__m2);729 return __trunc64(__builtin_ia32_pmulhw128((__v8hi)__anyext128(__m1),
730 (__v8hi)__anyext128(__m2)));
687}731}
688732
689/// Multiplies each 16-bit signed integer element of the first 64-bit733/// Multiplies each 16-bit signed integer element of the first 64-bit
...@@ -701,10 +745,10 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2)...@@ -701,10 +745,10 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2)
701/// A 64-bit integer vector of [4 x i16].745/// A 64-bit integer vector of [4 x i16].
702/// \returns A 64-bit integer vector of [4 x i16] containing the lower 16 bits746/// \returns A 64-bit integer vector of [4 x i16] containing the lower 16 bits
703/// of the products of both parameters.747/// of the products of both parameters.
704static __inline__ __m64 __DEFAULT_FN_ATTRS748static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
705_mm_mullo_pi16(__m64 __m1, __m64 __m2)749_mm_mullo_pi16(__m64 __m1, __m64 __m2)
706{750{
707 return (__m64)__builtin_ia32_pmullw((__v4hi)__m1, (__v4hi)__m2);751 return (__m64)(((__v4hu)__m1) * ((__v4hu)__m2));
708}752}
709753
710/// Left-shifts each 16-bit signed integer element of the first754/// Left-shifts each 16-bit signed integer element of the first
...@@ -724,10 +768,11 @@ _mm_mullo_pi16(__m64 __m1, __m64 __m2)...@@ -724,10 +768,11 @@ _mm_mullo_pi16(__m64 __m1, __m64 __m2)
724/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted768/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted
725/// values. If \a __count is greater or equal to 16, the result is set to all769/// values. If \a __count is greater or equal to 16, the result is set to all
726/// 0.770/// 0.
727static __inline__ __m64 __DEFAULT_FN_ATTRS771static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
728_mm_sll_pi16(__m64 __m, __m64 __count)772_mm_sll_pi16(__m64 __m, __m64 __count)
729{773{
730 return (__m64)__builtin_ia32_psllw((__v4hi)__m, __count);774 return __trunc64(__builtin_ia32_psllw128((__v8hi)__anyext128(__m),
775 (__v8hi)__anyext128(__count)));
731}776}
732777
733/// Left-shifts each 16-bit signed integer element of a 64-bit integer778/// Left-shifts each 16-bit signed integer element of a 64-bit integer
...@@ -746,10 +791,11 @@ _mm_sll_pi16(__m64 __m, __m64 __count)...@@ -746,10 +791,11 @@ _mm_sll_pi16(__m64 __m, __m64 __count)
746/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted791/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted
747/// values. If \a __count is greater or equal to 16, the result is set to all792/// values. If \a __count is greater or equal to 16, the result is set to all
748/// 0.793/// 0.
749static __inline__ __m64 __DEFAULT_FN_ATTRS794static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
750_mm_slli_pi16(__m64 __m, int __count)795_mm_slli_pi16(__m64 __m, int __count)
751{796{
752 return (__m64)__builtin_ia32_psllwi((__v4hi)__m, __count);797 return __trunc64(__builtin_ia32_psllwi128((__v8hi)__anyext128(__m),
798 __count));
753}799}
754800
755/// Left-shifts each 32-bit signed integer element of the first801/// Left-shifts each 32-bit signed integer element of the first
...@@ -769,10 +815,11 @@ _mm_slli_pi16(__m64 __m, int __count)...@@ -769,10 +815,11 @@ _mm_slli_pi16(__m64 __m, int __count)
769/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted815/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted
770/// values. If \a __count is greater or equal to 32, the result is set to all816/// values. If \a __count is greater or equal to 32, the result is set to all
771/// 0.817/// 0.
772static __inline__ __m64 __DEFAULT_FN_ATTRS818static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
773_mm_sll_pi32(__m64 __m, __m64 __count)819_mm_sll_pi32(__m64 __m, __m64 __count)
774{820{
775 return (__m64)__builtin_ia32_pslld((__v2si)__m, __count);821 return __trunc64(__builtin_ia32_pslld128((__v4si)__anyext128(__m),
822 (__v4si)__anyext128(__count)));
776}823}
777824
778/// Left-shifts each 32-bit signed integer element of a 64-bit integer825/// Left-shifts each 32-bit signed integer element of a 64-bit integer
...@@ -791,10 +838,11 @@ _mm_sll_pi32(__m64 __m, __m64 __count)...@@ -791,10 +838,11 @@ _mm_sll_pi32(__m64 __m, __m64 __count)
791/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted838/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted
792/// values. If \a __count is greater or equal to 32, the result is set to all839/// values. If \a __count is greater or equal to 32, the result is set to all
793/// 0.840/// 0.
794static __inline__ __m64 __DEFAULT_FN_ATTRS841static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
795_mm_slli_pi32(__m64 __m, int __count)842_mm_slli_pi32(__m64 __m, int __count)
796{843{
797 return (__m64)__builtin_ia32_pslldi((__v2si)__m, __count);844 return __trunc64(__builtin_ia32_pslldi128((__v4si)__anyext128(__m),
845 __count));
798}846}
799847
800/// Left-shifts the first 64-bit integer parameter by the number of bits848/// Left-shifts the first 64-bit integer parameter by the number of bits
...@@ -811,10 +859,11 @@ _mm_slli_pi32(__m64 __m, int __count)...@@ -811,10 +859,11 @@ _mm_slli_pi32(__m64 __m, int __count)
811/// A 64-bit integer vector interpreted as a single 64-bit integer.859/// A 64-bit integer vector interpreted as a single 64-bit integer.
812/// \returns A 64-bit integer vector containing the left-shifted value. If860/// \returns A 64-bit integer vector containing the left-shifted value. If
813/// \a __count is greater or equal to 64, the result is set to 0.861/// \a __count is greater or equal to 64, the result is set to 0.
814static __inline__ __m64 __DEFAULT_FN_ATTRS862static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
815_mm_sll_si64(__m64 __m, __m64 __count)863_mm_sll_si64(__m64 __m, __m64 __count)
816{864{
817 return (__m64)__builtin_ia32_psllq((__v1di)__m, __count);865 return __trunc64(__builtin_ia32_psllq128((__v2di)__anyext128(__m),
866 (__v2di)__anyext128(__count)));
818}867}
819868
820/// Left-shifts the first parameter, which is a 64-bit integer, by the869/// Left-shifts the first parameter, which is a 64-bit integer, by the
...@@ -831,10 +880,11 @@ _mm_sll_si64(__m64 __m, __m64 __count)...@@ -831,10 +880,11 @@ _mm_sll_si64(__m64 __m, __m64 __count)
831/// A 32-bit integer value.880/// A 32-bit integer value.
832/// \returns A 64-bit integer vector containing the left-shifted value. If881/// \returns A 64-bit integer vector containing the left-shifted value. If
833/// \a __count is greater or equal to 64, the result is set to 0.882/// \a __count is greater or equal to 64, the result is set to 0.
834static __inline__ __m64 __DEFAULT_FN_ATTRS883static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
835_mm_slli_si64(__m64 __m, int __count)884_mm_slli_si64(__m64 __m, int __count)
836{885{
837 return (__m64)__builtin_ia32_psllqi((__v1di)__m, __count);886 return __trunc64(__builtin_ia32_psllqi128((__v2di)__anyext128(__m),
887 __count));
838}888}
839889
840/// Right-shifts each 16-bit integer element of the first parameter,890/// Right-shifts each 16-bit integer element of the first parameter,
...@@ -855,10 +905,11 @@ _mm_slli_si64(__m64 __m, int __count)...@@ -855,10 +905,11 @@ _mm_slli_si64(__m64 __m, int __count)
855/// A 64-bit integer vector interpreted as a single 64-bit integer.905/// A 64-bit integer vector interpreted as a single 64-bit integer.
856/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted906/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
857/// values.907/// values.
858static __inline__ __m64 __DEFAULT_FN_ATTRS908static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
859_mm_sra_pi16(__m64 __m, __m64 __count)909_mm_sra_pi16(__m64 __m, __m64 __count)
860{910{
861 return (__m64)__builtin_ia32_psraw((__v4hi)__m, __count);911 return __trunc64(__builtin_ia32_psraw128((__v8hi)__anyext128(__m),
912 (__v8hi)__anyext128(__count)));
862}913}
863914
864/// Right-shifts each 16-bit integer element of a 64-bit integer vector915/// Right-shifts each 16-bit integer element of a 64-bit integer vector
...@@ -878,10 +929,11 @@ _mm_sra_pi16(__m64 __m, __m64 __count)...@@ -878,10 +929,11 @@ _mm_sra_pi16(__m64 __m, __m64 __count)
878/// A 32-bit integer value.929/// A 32-bit integer value.
879/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted930/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
880/// values.931/// values.
881static __inline__ __m64 __DEFAULT_FN_ATTRS932static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
882_mm_srai_pi16(__m64 __m, int __count)933_mm_srai_pi16(__m64 __m, int __count)
883{934{
884 return (__m64)__builtin_ia32_psrawi((__v4hi)__m, __count);935 return __trunc64(__builtin_ia32_psrawi128((__v8hi)__anyext128(__m),
936 __count));
885}937}
886938
887/// Right-shifts each 32-bit integer element of the first parameter,939/// Right-shifts each 32-bit integer element of the first parameter,
...@@ -902,10 +954,11 @@ _mm_srai_pi16(__m64 __m, int __count)...@@ -902,10 +954,11 @@ _mm_srai_pi16(__m64 __m, int __count)
902/// A 64-bit integer vector interpreted as a single 64-bit integer.954/// A 64-bit integer vector interpreted as a single 64-bit integer.
903/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted955/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
904/// values.956/// values.
905static __inline__ __m64 __DEFAULT_FN_ATTRS957static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
906_mm_sra_pi32(__m64 __m, __m64 __count)958_mm_sra_pi32(__m64 __m, __m64 __count)
907{959{
908 return (__m64)__builtin_ia32_psrad((__v2si)__m, __count);960 return __trunc64(__builtin_ia32_psrad128((__v4si)__anyext128(__m),
961 (__v4si)__anyext128(__count)));
909}962}
910963
911/// Right-shifts each 32-bit integer element of a 64-bit integer vector964/// Right-shifts each 32-bit integer element of a 64-bit integer vector
...@@ -925,10 +978,11 @@ _mm_sra_pi32(__m64 __m, __m64 __count)...@@ -925,10 +978,11 @@ _mm_sra_pi32(__m64 __m, __m64 __count)
925/// A 32-bit integer value.978/// A 32-bit integer value.
926/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted979/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
927/// values.980/// values.
928static __inline__ __m64 __DEFAULT_FN_ATTRS981static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
929_mm_srai_pi32(__m64 __m, int __count)982_mm_srai_pi32(__m64 __m, int __count)
930{983{
931 return (__m64)__builtin_ia32_psradi((__v2si)__m, __count);984 return __trunc64(__builtin_ia32_psradi128((__v4si)__anyext128(__m),
985 __count));
932}986}
933987
934/// Right-shifts each 16-bit integer element of the first parameter,988/// Right-shifts each 16-bit integer element of the first parameter,
...@@ -948,10 +1002,11 @@ _mm_srai_pi32(__m64 __m, int __count)...@@ -948,10 +1002,11 @@ _mm_srai_pi32(__m64 __m, int __count)
948/// A 64-bit integer vector interpreted as a single 64-bit integer.1002/// A 64-bit integer vector interpreted as a single 64-bit integer.
949/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted1003/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
950/// values.1004/// values.
951static __inline__ __m64 __DEFAULT_FN_ATTRS1005static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
952_mm_srl_pi16(__m64 __m, __m64 __count)1006_mm_srl_pi16(__m64 __m, __m64 __count)
953{1007{
954 return (__m64)__builtin_ia32_psrlw((__v4hi)__m, __count);1008 return __trunc64(__builtin_ia32_psrlw128((__v8hi)__anyext128(__m),
1009 (__v8hi)__anyext128(__count)));
955}1010}
9561011
957/// Right-shifts each 16-bit integer element of a 64-bit integer vector1012/// Right-shifts each 16-bit integer element of a 64-bit integer vector
...@@ -970,10 +1025,11 @@ _mm_srl_pi16(__m64 __m, __m64 __count)...@@ -970,10 +1025,11 @@ _mm_srl_pi16(__m64 __m, __m64 __count)
970/// A 32-bit integer value.1025/// A 32-bit integer value.
971/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted1026/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
972/// values.1027/// values.
973static __inline__ __m64 __DEFAULT_FN_ATTRS1028static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
974_mm_srli_pi16(__m64 __m, int __count)1029_mm_srli_pi16(__m64 __m, int __count)
975{1030{
976 return (__m64)__builtin_ia32_psrlwi((__v4hi)__m, __count);1031 return __trunc64(__builtin_ia32_psrlwi128((__v8hi)__anyext128(__m),
1032 __count));
977}1033}
9781034
979/// Right-shifts each 32-bit integer element of the first parameter,1035/// Right-shifts each 32-bit integer element of the first parameter,
...@@ -993,10 +1049,11 @@ _mm_srli_pi16(__m64 __m, int __count)...@@ -993,10 +1049,11 @@ _mm_srli_pi16(__m64 __m, int __count)
993/// A 64-bit integer vector interpreted as a single 64-bit integer.1049/// A 64-bit integer vector interpreted as a single 64-bit integer.
994/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted1050/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
995/// values.1051/// values.
996static __inline__ __m64 __DEFAULT_FN_ATTRS1052static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
997_mm_srl_pi32(__m64 __m, __m64 __count)1053_mm_srl_pi32(__m64 __m, __m64 __count)
998{1054{
999 return (__m64)__builtin_ia32_psrld((__v2si)__m, __count);1055 return __trunc64(__builtin_ia32_psrld128((__v4si)__anyext128(__m),
1056 (__v4si)__anyext128(__count)));
1000}1057}
10011058
1002/// Right-shifts each 32-bit integer element of a 64-bit integer vector1059/// Right-shifts each 32-bit integer element of a 64-bit integer vector
...@@ -1015,10 +1072,11 @@ _mm_srl_pi32(__m64 __m, __m64 __count)...@@ -1015,10 +1072,11 @@ _mm_srl_pi32(__m64 __m, __m64 __count)
1015/// A 32-bit integer value.1072/// A 32-bit integer value.
1016/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted1073/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
1017/// values.1074/// values.
1018static __inline__ __m64 __DEFAULT_FN_ATTRS1075static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1019_mm_srli_pi32(__m64 __m, int __count)1076_mm_srli_pi32(__m64 __m, int __count)
1020{1077{
1021 return (__m64)__builtin_ia32_psrldi((__v2si)__m, __count);1078 return __trunc64(__builtin_ia32_psrldi128((__v4si)__anyext128(__m),
1079 __count));
1022}1080}
10231081
1024/// Right-shifts the first 64-bit integer parameter by the number of bits1082/// Right-shifts the first 64-bit integer parameter by the number of bits
...@@ -1035,10 +1093,11 @@ _mm_srli_pi32(__m64 __m, int __count)...@@ -1035,10 +1093,11 @@ _mm_srli_pi32(__m64 __m, int __count)
1035/// \param __count1093/// \param __count
1036/// A 64-bit integer vector interpreted as a single 64-bit integer.1094/// A 64-bit integer vector interpreted as a single 64-bit integer.
1037/// \returns A 64-bit integer vector containing the right-shifted value.1095/// \returns A 64-bit integer vector containing the right-shifted value.
1038static __inline__ __m64 __DEFAULT_FN_ATTRS1096static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1039_mm_srl_si64(__m64 __m, __m64 __count)1097_mm_srl_si64(__m64 __m, __m64 __count)
1040{1098{
1041 return (__m64)__builtin_ia32_psrlq((__v1di)__m, __count);1099 return __trunc64(__builtin_ia32_psrlq128((__v2di)__anyext128(__m),
1100 (__v2di)__anyext128(__count)));
1042}1101}
10431102
1044/// Right-shifts the first parameter, which is a 64-bit integer, by the1103/// Right-shifts the first parameter, which is a 64-bit integer, by the
...@@ -1056,10 +1115,11 @@ _mm_srl_si64(__m64 __m, __m64 __count)...@@ -1056,10 +1115,11 @@ _mm_srl_si64(__m64 __m, __m64 __count)
1056/// \param __count1115/// \param __count
1057/// A 32-bit integer value.1116/// A 32-bit integer value.
1058/// \returns A 64-bit integer vector containing the right-shifted value.1117/// \returns A 64-bit integer vector containing the right-shifted value.
1059static __inline__ __m64 __DEFAULT_FN_ATTRS1118static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1060_mm_srli_si64(__m64 __m, int __count)1119_mm_srli_si64(__m64 __m, int __count)
1061{1120{
1062 return (__m64)__builtin_ia32_psrlqi((__v1di)__m, __count);1121 return __trunc64(__builtin_ia32_psrlqi128((__v2di)__anyext128(__m),
1122 __count));
1063}1123}
10641124
1065/// Performs a bitwise AND of two 64-bit integer vectors.1125/// Performs a bitwise AND of two 64-bit integer vectors.
...@@ -1074,10 +1134,10 @@ _mm_srli_si64(__m64 __m, int __count)...@@ -1074,10 +1134,10 @@ _mm_srli_si64(__m64 __m, int __count)
1074/// A 64-bit integer vector.1134/// A 64-bit integer vector.
1075/// \returns A 64-bit integer vector containing the bitwise AND of both1135/// \returns A 64-bit integer vector containing the bitwise AND of both
1076/// parameters.1136/// parameters.
1077static __inline__ __m64 __DEFAULT_FN_ATTRS1137static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1078_mm_and_si64(__m64 __m1, __m64 __m2)1138_mm_and_si64(__m64 __m1, __m64 __m2)
1079{1139{
1080 return __builtin_ia32_pand((__v1di)__m1, (__v1di)__m2);1140 return (__m64)(((__v1du)__m1) & ((__v1du)__m2));
1081}1141}
10821142
1083/// Performs a bitwise NOT of the first 64-bit integer vector, and then1143/// Performs a bitwise NOT of the first 64-bit integer vector, and then
...@@ -1095,10 +1155,10 @@ _mm_and_si64(__m64 __m1, __m64 __m2)...@@ -1095,10 +1155,10 @@ _mm_and_si64(__m64 __m1, __m64 __m2)
1095/// A 64-bit integer vector.1155/// A 64-bit integer vector.
1096/// \returns A 64-bit integer vector containing the bitwise AND of the second1156/// \returns A 64-bit integer vector containing the bitwise AND of the second
1097/// parameter and the one's complement of the first parameter.1157/// parameter and the one's complement of the first parameter.
1098static __inline__ __m64 __DEFAULT_FN_ATTRS1158static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1099_mm_andnot_si64(__m64 __m1, __m64 __m2)1159_mm_andnot_si64(__m64 __m1, __m64 __m2)
1100{1160{
1101 return __builtin_ia32_pandn((__v1di)__m1, (__v1di)__m2);1161 return (__m64)(~((__v1du)__m1) & ((__v1du)__m2));
1102}1162}
11031163
1104/// Performs a bitwise OR of two 64-bit integer vectors.1164/// Performs a bitwise OR of two 64-bit integer vectors.
...@@ -1113,10 +1173,10 @@ _mm_andnot_si64(__m64 __m1, __m64 __m2)...@@ -1113,10 +1173,10 @@ _mm_andnot_si64(__m64 __m1, __m64 __m2)
1113/// A 64-bit integer vector.1173/// A 64-bit integer vector.
1114/// \returns A 64-bit integer vector containing the bitwise OR of both1174/// \returns A 64-bit integer vector containing the bitwise OR of both
1115/// parameters.1175/// parameters.
1116static __inline__ __m64 __DEFAULT_FN_ATTRS1176static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1117_mm_or_si64(__m64 __m1, __m64 __m2)1177_mm_or_si64(__m64 __m1, __m64 __m2)
1118{1178{
1119 return __builtin_ia32_por((__v1di)__m1, (__v1di)__m2);1179 return (__m64)(((__v1du)__m1) | ((__v1du)__m2));
1120}1180}
11211181
1122/// Performs a bitwise exclusive OR of two 64-bit integer vectors.1182/// Performs a bitwise exclusive OR of two 64-bit integer vectors.
...@@ -1131,10 +1191,10 @@ _mm_or_si64(__m64 __m1, __m64 __m2)...@@ -1131,10 +1191,10 @@ _mm_or_si64(__m64 __m1, __m64 __m2)
1131/// A 64-bit integer vector.1191/// A 64-bit integer vector.
1132/// \returns A 64-bit integer vector containing the bitwise exclusive OR of both1192/// \returns A 64-bit integer vector containing the bitwise exclusive OR of both
1133/// parameters.1193/// parameters.
1134static __inline__ __m64 __DEFAULT_FN_ATTRS1194static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1135_mm_xor_si64(__m64 __m1, __m64 __m2)1195_mm_xor_si64(__m64 __m1, __m64 __m2)
1136{1196{
1137 return __builtin_ia32_pxor((__v1di)__m1, (__v1di)__m2);1197 return (__m64)(((__v1du)__m1) ^ ((__v1du)__m2));
1138}1198}
11391199
1140/// Compares the 8-bit integer elements of two 64-bit integer vectors of1200/// Compares the 8-bit integer elements of two 64-bit integer vectors of
...@@ -1153,10 +1213,10 @@ _mm_xor_si64(__m64 __m1, __m64 __m2)...@@ -1153,10 +1213,10 @@ _mm_xor_si64(__m64 __m1, __m64 __m2)
1153/// A 64-bit integer vector of [8 x i8].1213/// A 64-bit integer vector of [8 x i8].
1154/// \returns A 64-bit integer vector of [8 x i8] containing the comparison1214/// \returns A 64-bit integer vector of [8 x i8] containing the comparison
1155/// results.1215/// results.
1156static __inline__ __m64 __DEFAULT_FN_ATTRS1216static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1157_mm_cmpeq_pi8(__m64 __m1, __m64 __m2)1217_mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
1158{1218{
1159 return (__m64)__builtin_ia32_pcmpeqb((__v8qi)__m1, (__v8qi)__m2);1219 return (__m64)(((__v8qi)__m1) == ((__v8qi)__m2));
1160}1220}
11611221
1162/// Compares the 16-bit integer elements of two 64-bit integer vectors of1222/// Compares the 16-bit integer elements of two 64-bit integer vectors of
...@@ -1175,10 +1235,10 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)...@@ -1175,10 +1235,10 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
1175/// A 64-bit integer vector of [4 x i16].1235/// A 64-bit integer vector of [4 x i16].
1176/// \returns A 64-bit integer vector of [4 x i16] containing the comparison1236/// \returns A 64-bit integer vector of [4 x i16] containing the comparison
1177/// results.1237/// results.
1178static __inline__ __m64 __DEFAULT_FN_ATTRS1238static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1179_mm_cmpeq_pi16(__m64 __m1, __m64 __m2)1239_mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
1180{1240{
1181 return (__m64)__builtin_ia32_pcmpeqw((__v4hi)__m1, (__v4hi)__m2);1241 return (__m64)(((__v4hi)__m1) == ((__v4hi)__m2));
1182}1242}
11831243
1184/// Compares the 32-bit integer elements of two 64-bit integer vectors of1244/// Compares the 32-bit integer elements of two 64-bit integer vectors of
...@@ -1197,10 +1257,10 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)...@@ -1197,10 +1257,10 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
1197/// A 64-bit integer vector of [2 x i32].1257/// A 64-bit integer vector of [2 x i32].
1198/// \returns A 64-bit integer vector of [2 x i32] containing the comparison1258/// \returns A 64-bit integer vector of [2 x i32] containing the comparison
1199/// results.1259/// results.
1200static __inline__ __m64 __DEFAULT_FN_ATTRS1260static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1201_mm_cmpeq_pi32(__m64 __m1, __m64 __m2)1261_mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
1202{1262{
1203 return (__m64)__builtin_ia32_pcmpeqd((__v2si)__m1, (__v2si)__m2);1263 return (__m64)(((__v2si)__m1) == ((__v2si)__m2));
1204}1264}
12051265
1206/// Compares the 8-bit integer elements of two 64-bit integer vectors of1266/// Compares the 8-bit integer elements of two 64-bit integer vectors of
...@@ -1219,10 +1279,12 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)...@@ -1219,10 +1279,12 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
1219/// A 64-bit integer vector of [8 x i8].1279/// A 64-bit integer vector of [8 x i8].
1220/// \returns A 64-bit integer vector of [8 x i8] containing the comparison1280/// \returns A 64-bit integer vector of [8 x i8] containing the comparison
1221/// results.1281/// results.
1222static __inline__ __m64 __DEFAULT_FN_ATTRS1282static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1223_mm_cmpgt_pi8(__m64 __m1, __m64 __m2)1283_mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
1224{1284{
1225 return (__m64)__builtin_ia32_pcmpgtb((__v8qi)__m1, (__v8qi)__m2);1285 /* This function always performs a signed comparison, but __v8qi is a char
1286 which may be signed or unsigned, so use __v8qs. */
1287 return (__m64)((__v8qs)__m1 > (__v8qs)__m2);
1226}1288}
12271289
1228/// Compares the 16-bit integer elements of two 64-bit integer vectors of1290/// Compares the 16-bit integer elements of two 64-bit integer vectors of
...@@ -1241,10 +1303,10 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)...@@ -1241,10 +1303,10 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
1241/// A 64-bit integer vector of [4 x i16].1303/// A 64-bit integer vector of [4 x i16].
1242/// \returns A 64-bit integer vector of [4 x i16] containing the comparison1304/// \returns A 64-bit integer vector of [4 x i16] containing the comparison
1243/// results.1305/// results.
1244static __inline__ __m64 __DEFAULT_FN_ATTRS1306static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1245_mm_cmpgt_pi16(__m64 __m1, __m64 __m2)1307_mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
1246{1308{
1247 return (__m64)__builtin_ia32_pcmpgtw((__v4hi)__m1, (__v4hi)__m2);1309 return (__m64)((__v4hi)__m1 > (__v4hi)__m2);
1248}1310}
12491311
1250/// Compares the 32-bit integer elements of two 64-bit integer vectors of1312/// Compares the 32-bit integer elements of two 64-bit integer vectors of
...@@ -1263,10 +1325,10 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)...@@ -1263,10 +1325,10 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
1263/// A 64-bit integer vector of [2 x i32].1325/// A 64-bit integer vector of [2 x i32].
1264/// \returns A 64-bit integer vector of [2 x i32] containing the comparison1326/// \returns A 64-bit integer vector of [2 x i32] containing the comparison
1265/// results.1327/// results.
1266static __inline__ __m64 __DEFAULT_FN_ATTRS1328static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1267_mm_cmpgt_pi32(__m64 __m1, __m64 __m2)1329_mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
1268{1330{
1269 return (__m64)__builtin_ia32_pcmpgtd((__v2si)__m1, (__v2si)__m2);1331 return (__m64)((__v2si)__m1 > (__v2si)__m2);
1270}1332}
12711333
1272/// Constructs a 64-bit integer vector initialized to zero.1334/// Constructs a 64-bit integer vector initialized to zero.
...@@ -1276,10 +1338,9 @@ _mm_cmpgt_pi32(__m64 __m1, __m64 __m2)...@@ -1276,10 +1338,9 @@ _mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
1276/// This intrinsic corresponds to the <c> PXOR </c> instruction.1338/// This intrinsic corresponds to the <c> PXOR </c> instruction.
1277///1339///
1278/// \returns An initialized 64-bit integer vector with all elements set to zero.1340/// \returns An initialized 64-bit integer vector with all elements set to zero.
1279static __inline__ __m64 __DEFAULT_FN_ATTRS1341static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1280_mm_setzero_si64(void)1342_mm_setzero_si64(void) {
1281{1343 return __extension__(__m64){0LL};
1282 return __extension__ (__m64){ 0LL };
1283}1344}
12841345
1285/// Constructs a 64-bit integer vector initialized with the specified1346/// Constructs a 64-bit integer vector initialized with the specified
...@@ -1297,10 +1358,9 @@ _mm_setzero_si64(void)...@@ -1297,10 +1358,9 @@ _mm_setzero_si64(void)
1297/// A 32-bit integer value used to initialize the lower 32 bits of the1358/// A 32-bit integer value used to initialize the lower 32 bits of the
1298/// result.1359/// result.
1299/// \returns An initialized 64-bit integer vector.1360/// \returns An initialized 64-bit integer vector.
1300static __inline__ __m64 __DEFAULT_FN_ATTRS1361static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1301_mm_set_pi32(int __i1, int __i0)1362_mm_set_pi32(int __i1, int __i0) {
1302{1363 return __extension__(__m64)(__v2si){__i0, __i1};
1303 return (__m64)__builtin_ia32_vec_init_v2si(__i0, __i1);
1304}1364}
13051365
1306/// Constructs a 64-bit integer vector initialized with the specified1366/// Constructs a 64-bit integer vector initialized with the specified
...@@ -1320,10 +1380,9 @@ _mm_set_pi32(int __i1, int __i0)...@@ -1320,10 +1380,9 @@ _mm_set_pi32(int __i1, int __i0)
1320/// \param __s01380/// \param __s0
1321/// A 16-bit integer value used to initialize bits [15:0] of the result.1381/// A 16-bit integer value used to initialize bits [15:0] of the result.
1322/// \returns An initialized 64-bit integer vector.1382/// \returns An initialized 64-bit integer vector.
1323static __inline__ __m64 __DEFAULT_FN_ATTRS1383static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1324_mm_set_pi16(short __s3, short __s2, short __s1, short __s0)1384_mm_set_pi16(short __s3, short __s2, short __s1, short __s0) {
1325{1385 return __extension__(__m64)(__v4hi){__s0, __s1, __s2, __s3};
1326 return (__m64)__builtin_ia32_vec_init_v4hi(__s0, __s1, __s2, __s3);
1327}1386}
13281387
1329/// Constructs a 64-bit integer vector initialized with the specified1388/// Constructs a 64-bit integer vector initialized with the specified
...@@ -1351,12 +1410,11 @@ _mm_set_pi16(short __s3, short __s2, short __s1, short __s0)...@@ -1351,12 +1410,11 @@ _mm_set_pi16(short __s3, short __s2, short __s1, short __s0)
1351/// \param __b01410/// \param __b0
1352/// An 8-bit integer value used to initialize bits [7:0] of the result.1411/// An 8-bit integer value used to initialize bits [7:0] of the result.
1353/// \returns An initialized 64-bit integer vector.1412/// \returns An initialized 64-bit integer vector.
1354static __inline__ __m64 __DEFAULT_FN_ATTRS1413static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1355_mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,1414_mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
1356 char __b1, char __b0)1415 char __b1, char __b0) {
1357{1416 return __extension__(__m64)(__v8qi){__b0, __b1, __b2, __b3,
1358 return (__m64)__builtin_ia32_vec_init_v8qi(__b0, __b1, __b2, __b3,1417 __b4, __b5, __b6, __b7};
1359 __b4, __b5, __b6, __b7);
1360}1418}
13611419
1362/// Constructs a 64-bit integer vector of [2 x i32], with each of the1420/// Constructs a 64-bit integer vector of [2 x i32], with each of the
...@@ -1372,10 +1430,9 @@ _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,...@@ -1372,10 +1430,9 @@ _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
1372/// A 32-bit integer value used to initialize each vector element of the1430/// A 32-bit integer value used to initialize each vector element of the
1373/// result.1431/// result.
1374/// \returns An initialized 64-bit integer vector of [2 x i32].1432/// \returns An initialized 64-bit integer vector of [2 x i32].
1375static __inline__ __m64 __DEFAULT_FN_ATTRS1433static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1376_mm_set1_pi32(int __i)1434_mm_set1_pi32(int __i) {
1377{1435 return _mm_set_pi32(__i, __i);
1378 return _mm_set_pi32(__i, __i);
1379}1436}
13801437
1381/// Constructs a 64-bit integer vector of [4 x i16], with each of the1438/// Constructs a 64-bit integer vector of [4 x i16], with each of the
...@@ -1391,10 +1448,9 @@ _mm_set1_pi32(int __i)...@@ -1391,10 +1448,9 @@ _mm_set1_pi32(int __i)
1391/// A 16-bit integer value used to initialize each vector element of the1448/// A 16-bit integer value used to initialize each vector element of the
1392/// result.1449/// result.
1393/// \returns An initialized 64-bit integer vector of [4 x i16].1450/// \returns An initialized 64-bit integer vector of [4 x i16].
1394static __inline__ __m64 __DEFAULT_FN_ATTRS1451static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1395_mm_set1_pi16(short __w)1452_mm_set1_pi16(short __w) {
1396{1453 return _mm_set_pi16(__w, __w, __w, __w);
1397 return _mm_set_pi16(__w, __w, __w, __w);
1398}1454}
13991455
1400/// Constructs a 64-bit integer vector of [8 x i8], with each of the1456/// Constructs a 64-bit integer vector of [8 x i8], with each of the
...@@ -1409,10 +1465,9 @@ _mm_set1_pi16(short __w)...@@ -1409,10 +1465,9 @@ _mm_set1_pi16(short __w)
1409/// An 8-bit integer value used to initialize each vector element of the1465/// An 8-bit integer value used to initialize each vector element of the
1410/// result.1466/// result.
1411/// \returns An initialized 64-bit integer vector of [8 x i8].1467/// \returns An initialized 64-bit integer vector of [8 x i8].
1412static __inline__ __m64 __DEFAULT_FN_ATTRS1468static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1413_mm_set1_pi8(char __b)1469_mm_set1_pi8(char __b) {
1414{1470 return _mm_set_pi8(__b, __b, __b, __b, __b, __b, __b, __b);
1415 return _mm_set_pi8(__b, __b, __b, __b, __b, __b, __b, __b);
1416}1471}
14171472
1418/// Constructs a 64-bit integer vector, initialized in reverse order with1473/// Constructs a 64-bit integer vector, initialized in reverse order with
...@@ -1430,10 +1485,9 @@ _mm_set1_pi8(char __b)...@@ -1430,10 +1485,9 @@ _mm_set1_pi8(char __b)
1430/// A 32-bit integer value used to initialize the upper 32 bits of the1485/// A 32-bit integer value used to initialize the upper 32 bits of the
1431/// result.1486/// result.
1432/// \returns An initialized 64-bit integer vector.1487/// \returns An initialized 64-bit integer vector.
1433static __inline__ __m64 __DEFAULT_FN_ATTRS1488static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1434_mm_setr_pi32(int __i0, int __i1)1489_mm_setr_pi32(int __i0, int __i1) {
1435{1490 return _mm_set_pi32(__i1, __i0);
1436 return _mm_set_pi32(__i1, __i0);
1437}1491}
14381492
1439/// Constructs a 64-bit integer vector, initialized in reverse order with1493/// Constructs a 64-bit integer vector, initialized in reverse order with
...@@ -1453,10 +1507,9 @@ _mm_setr_pi32(int __i0, int __i1)...@@ -1453,10 +1507,9 @@ _mm_setr_pi32(int __i0, int __i1)
1453/// \param __w31507/// \param __w3
1454/// A 16-bit integer value used to initialize bits [63:48] of the result.1508/// A 16-bit integer value used to initialize bits [63:48] of the result.
1455/// \returns An initialized 64-bit integer vector.1509/// \returns An initialized 64-bit integer vector.
1456static __inline__ __m64 __DEFAULT_FN_ATTRS1510static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1457_mm_setr_pi16(short __w0, short __w1, short __w2, short __w3)1511_mm_setr_pi16(short __w0, short __w1, short __w2, short __w3) {
1458{1512 return _mm_set_pi16(__w3, __w2, __w1, __w0);
1459 return _mm_set_pi16(__w3, __w2, __w1, __w0);
1460}1513}
14611514
1462/// Constructs a 64-bit integer vector, initialized in reverse order with1515/// Constructs a 64-bit integer vector, initialized in reverse order with
...@@ -1484,14 +1537,15 @@ _mm_setr_pi16(short __w0, short __w1, short __w2, short __w3)...@@ -1484,14 +1537,15 @@ _mm_setr_pi16(short __w0, short __w1, short __w2, short __w3)
1484/// \param __b71537/// \param __b7
1485/// An 8-bit integer value used to initialize bits [63:56] of the result.1538/// An 8-bit integer value used to initialize bits [63:56] of the result.
1486/// \returns An initialized 64-bit integer vector.1539/// \returns An initialized 64-bit integer vector.
1487static __inline__ __m64 __DEFAULT_FN_ATTRS1540static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1488_mm_setr_pi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,1541_mm_setr_pi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
1489 char __b6, char __b7)1542 char __b6, char __b7) {
1490{1543 return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
1491 return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
1492}1544}
14931545
1494#undef __DEFAULT_FN_ATTRS1546#undef __anyext128
1547#undef __trunc64
1548#undef __DEFAULT_FN_ATTRS_SSE2
14951549
1496/* Aliases for compatibility. */1550/* Aliases for compatibility. */
1497#define _m_empty _mm_empty1551#define _m_empty _mm_empty
lib/include/module.modulemap+2-4
...@@ -66,6 +66,8 @@ module _Builtin_intrinsics [system] [extern_c] {...@@ -66,6 +66,8 @@ module _Builtin_intrinsics [system] [extern_c] {
66 textual header "__wmmintrin_aes.h"66 textual header "__wmmintrin_aes.h"
67 textual header "__wmmintrin_pclmul.h"67 textual header "__wmmintrin_pclmul.h"
6868
69 textual header "mm3dnow.h"
70
69 explicit module mm_malloc {71 explicit module mm_malloc {
70 requires !freestanding72 requires !freestanding
71 header "mm_malloc.h"73 header "mm_malloc.h"
...@@ -122,10 +124,6 @@ module _Builtin_intrinsics [system] [extern_c] {...@@ -122,10 +124,6 @@ module _Builtin_intrinsics [system] [extern_c] {
122 header "popcntintrin.h"124 header "popcntintrin.h"
123 }125 }
124126
125 explicit module mm3dnow {
126 header "mm3dnow.h"
127 }
128
129 explicit module aes_pclmul {127 explicit module aes_pclmul {
130 header "wmmintrin.h"128 header "wmmintrin.h"
131 export aes129 export aes
lib/include/movrs_avx10_2_512intrin.h created+98
...@@ -0,0 +1,98 @@
1/*===----- movrs_avx10_2_512intrin.h - AVX10.2-512-MOVRS intrinsics --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <movrs_avx10_2_512intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __MOVRS_AVX10_2_512INTRIN_H
15#define __MOVRS_AVX10_2_512INTRIN_H
16#ifdef __x86_64__
17
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS512 \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("movrs, avx10.2-512"), __min_vector_width__(512)))
22
23static __inline__ __m512i __DEFAULT_FN_ATTRS512
24_mm512_loadrs_epi8(void const *__A) {
25 return (__m512i)__builtin_ia32_vmovrsb512((const __v64qi *)(__A));
26}
27
28static __inline__ __m512i __DEFAULT_FN_ATTRS512
29_mm512_mask_loadrs_epi8(__m512i __W, __mmask64 __U, void const *__A) {
30 return (__m512i)__builtin_ia32_selectb_512(
31 (__mmask64)__U, (__v64qi)_mm512_loadrs_epi8(__A), (__v64qi)__W);
32}
33
34static __inline__ __m512i __DEFAULT_FN_ATTRS512
35_mm512_maskz_loadrs_epi8(__mmask64 __U, void const *__A) {
36 return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
37 (__v64qi)_mm512_loadrs_epi8(__A),
38 (__v64qi)_mm512_setzero_si512());
39}
40
41static __inline__ __m512i __DEFAULT_FN_ATTRS512
42_mm512_loadrs_epi32(void const *__A) {
43 return (__m512i)__builtin_ia32_vmovrsd512((const __v16si *)(__A));
44}
45
46static __inline__ __m512i __DEFAULT_FN_ATTRS512
47_mm512_mask_loadrs_epi32(__m512i __W, __mmask16 __U, void const *__A) {
48 return (__m512i)__builtin_ia32_selectd_512(
49 (__mmask16)__U, (__v16si)_mm512_loadrs_epi32(__A), (__v16si)__W);
50}
51
52static __inline__ __m512i __DEFAULT_FN_ATTRS512
53_mm512_maskz_loadrs_epi32(__mmask16 __U, void const *__A) {
54 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
55 (__v16si)_mm512_loadrs_epi32(__A),
56 (__v16si)_mm512_setzero_si512());
57}
58
59static __inline__ __m512i __DEFAULT_FN_ATTRS512
60_mm512_loadrs_epi64(void const *__A) {
61 return (__m512i)__builtin_ia32_vmovrsq512((const __v8di *)(__A));
62}
63
64static __inline__ __m512i __DEFAULT_FN_ATTRS512
65_mm512_mask_loadrs_epi64(__m512i __W, __mmask8 __U, void const *__A) {
66 return (__m512i)__builtin_ia32_selectq_512(
67 (__mmask8)__U, (__v8di)_mm512_loadrs_epi64(__A), (__v8di)__W);
68}
69
70static __inline__ __m512i __DEFAULT_FN_ATTRS512
71_mm512_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
72 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
73 (__v8di)_mm512_loadrs_epi64(__A),
74 (__v8di)_mm512_setzero_si512());
75}
76
77static __inline__ __m512i __DEFAULT_FN_ATTRS512
78_mm512_loadrs_epi16(void const *__A) {
79 return (__m512i)__builtin_ia32_vmovrsw512((const __v32hi *)(__A));
80}
81
82static __inline__ __m512i __DEFAULT_FN_ATTRS512
83_mm512_mask_loadrs_epi16(__m512i __W, __mmask32 __U, void const *__A) {
84 return (__m512i)__builtin_ia32_selectw_512(
85 (__mmask32)__U, (__v32hi)_mm512_loadrs_epi16(__A), (__v32hi)__W);
86}
87
88static __inline__ __m512i __DEFAULT_FN_ATTRS512
89_mm512_maskz_loadrs_epi16(__mmask32 __U, void const *__A) {
90 return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
91 (__v32hi)_mm512_loadrs_epi16(__A),
92 (__v32hi)_mm512_setzero_si512());
93}
94
95#undef __DEFAULT_FN_ATTRS512
96
97#endif /* __x86_64__ */
98#endif /* __MOVRS_AVX10_2_512INTRIN_H */
lib/include/movrs_avx10_2intrin.h created+174
...@@ -0,0 +1,174 @@
1/*===--------- movrs_avx10_2intrin.h - AVX10.2-MOVRS intrinsics ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <movrs_avx10_2intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __MOVRS_AVX10_2INTRIN_H
15#define __MOVRS_AVX10_2INTRIN_H
16#ifdef __x86_64__
17
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS128 \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("movrs,avx10.2-256"), __min_vector_width__(128)))
22#define __DEFAULT_FN_ATTRS256 \
23 __attribute__((__always_inline__, __nodebug__, \
24 __target__("movrs,avx10.2-256"), __min_vector_width__(256)))
25
26static __inline__ __m128i __DEFAULT_FN_ATTRS128
27_mm_loadrs_epi8(void const *__A) {
28 return (__m128i)__builtin_ia32_vmovrsb128((const __v16qi *)(__A));
29}
30
31static __inline__ __m128i __DEFAULT_FN_ATTRS128
32_mm_mask_loadrs_epi8(__m128i __W, __mmask16 __U, void const *__A) {
33 return (__m128i)__builtin_ia32_selectb_128(
34 (__mmask16)__U, (__v16qi)_mm_loadrs_epi8(__A), (__v16qi)__W);
35}
36
37static __inline__ __m128i __DEFAULT_FN_ATTRS128
38_mm_maskz_loadrs_epi8(__mmask16 __U, void const *__A) {
39 return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
40 (__v16qi)_mm_loadrs_epi8(__A),
41 (__v16qi)_mm_setzero_si128());
42}
43
44static __inline__ __m256i __DEFAULT_FN_ATTRS256
45_mm256_loadrs_epi8(void const *__A) {
46 return (__m256i)__builtin_ia32_vmovrsb256((const __v32qi *)(__A));
47}
48
49static __inline__ __m256i __DEFAULT_FN_ATTRS256
50_mm256_mask_loadrs_epi8(__m256i __W, __mmask32 __U, void const *__A) {
51 return (__m256i)__builtin_ia32_selectb_256(
52 (__mmask32)__U, (__v32qi)_mm256_loadrs_epi8(__A), (__v32qi)__W);
53}
54
55static __inline__ __m256i __DEFAULT_FN_ATTRS256
56_mm256_maskz_loadrs_epi8(__mmask32 __U, void const *__A) {
57 return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
58 (__v32qi)_mm256_loadrs_epi8(__A),
59 (__v32qi)_mm256_setzero_si256());
60}
61
62static __inline__ __m128i __DEFAULT_FN_ATTRS128
63_mm_loadrs_epi32(void const *__A) {
64 return (__m128i)__builtin_ia32_vmovrsd128((const __v4si *)(__A));
65}
66
67static __inline__ __m128i __DEFAULT_FN_ATTRS128
68_mm_mask_loadrs_epi32(__m128i __W, __mmask8 __U, void const *__A) {
69 return (__m128i)__builtin_ia32_selectd_128(
70 (__mmask8)__U, (__v4si)_mm_loadrs_epi32(__A), (__v4si)__W);
71}
72
73static __inline__ __m128i __DEFAULT_FN_ATTRS128
74_mm_maskz_loadrs_epi32(__mmask8 __U, void const *__A) {
75 return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
76 (__v4si)_mm_loadrs_epi32(__A),
77 (__v4si)_mm_setzero_si128());
78}
79
80static __inline__ __m256i __DEFAULT_FN_ATTRS256
81_mm256_loadrs_epi32(void const *__A) {
82 return (__m256i)__builtin_ia32_vmovrsd256((const __v8si *)(__A));
83}
84
85static __inline__ __m256i __DEFAULT_FN_ATTRS256
86_mm256_mask_loadrs_epi32(__m256i __W, __mmask8 __U, void const *__A) {
87 return (__m256i)__builtin_ia32_selectd_256(
88 (__mmask8)__U, (__v8si)_mm256_loadrs_epi32(__A), (__v8si)__W);
89}
90
91static __inline__ __m256i __DEFAULT_FN_ATTRS256
92_mm256_maskz_loadrs_epi32(__mmask8 __U, void const *__A) {
93 return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
94 (__v8si)_mm256_loadrs_epi32(__A),
95 (__v8si)_mm256_setzero_si256());
96}
97
98static __inline__ __m128i __DEFAULT_FN_ATTRS128
99_mm_loadrs_epi64(void const *__A) {
100 return (__m128i)__builtin_ia32_vmovrsq128((const __v2di *)(__A));
101}
102
103static __inline__ __m128i __DEFAULT_FN_ATTRS128
104_mm_mask_loadrs_epi64(__m128i __W, __mmask8 __U, void const *__A) {
105 return (__m128i)__builtin_ia32_selectq_128(
106 (__mmask8)__U, (__v2di)_mm_loadrs_epi64(__A), (__v2di)__W);
107}
108
109static __inline__ __m128i __DEFAULT_FN_ATTRS128
110_mm_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
111 return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
112 (__v2di)_mm_loadrs_epi64(__A),
113 (__v2di)_mm_setzero_si128());
114}
115
116static __inline__ __m256i __DEFAULT_FN_ATTRS256
117_mm256_loadrs_epi64(void const *__A) {
118 return (__m256i)__builtin_ia32_vmovrsq256((const __v4di *)(__A));
119}
120
121static __inline__ __m256i __DEFAULT_FN_ATTRS256
122_mm256_mask_loadrs_epi64(__m256i __W, __mmask8 __U, void const *__A) {
123 return (__m256i)__builtin_ia32_selectq_256(
124 (__mmask8)__U, (__v4di)_mm256_loadrs_epi64(__A), (__v4di)__W);
125}
126
127static __inline__ __m256i __DEFAULT_FN_ATTRS256
128_mm256_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
129 return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
130 (__v4di)_mm256_loadrs_epi64(__A),
131 (__v4di)_mm256_setzero_si256());
132}
133
134static __inline__ __m128i __DEFAULT_FN_ATTRS128
135_mm_loadrs_epi16(void const *__A) {
136 return (__m128i)__builtin_ia32_vmovrsw128((const __v8hi *)(__A));
137}
138
139static __inline__ __m128i __DEFAULT_FN_ATTRS128
140_mm_mask_loadrs_epi16(__m128i __W, __mmask8 __U, void const *__A) {
141 return (__m128i)__builtin_ia32_selectw_128(
142 (__mmask8)__U, (__v8hi)_mm_loadrs_epi16(__A), (__v8hi)__W);
143}
144
145static __inline__ __m128i __DEFAULT_FN_ATTRS128
146_mm_maskz_loadrs_epi16(__mmask8 __U, void const *__A) {
147 return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
148 (__v8hi)_mm_loadrs_epi16(__A),
149 (__v8hi)_mm_setzero_si128());
150}
151
152static __inline__ __m256i __DEFAULT_FN_ATTRS256
153_mm256_loadrs_epi16(void const *__A) {
154 return (__m256i)__builtin_ia32_vmovrsw256((const __v16hi *)(__A));
155}
156
157static __inline__ __m256i __DEFAULT_FN_ATTRS256
158_mm256_mask_loadrs_epi16(__m256i __W, __mmask16 __U, void const *__A) {
159 return (__m256i)__builtin_ia32_selectw_256(
160 (__mmask16)__U, (__v16hi)_mm256_loadrs_epi16(__A), (__v16hi)__W);
161}
162
163static __inline__ __m256i __DEFAULT_FN_ATTRS256
164_mm256_maskz_loadrs_epi16(__mmask16 __U, void const *__A) {
165 return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
166 (__v16hi)_mm256_loadrs_epi16(__A),
167 (__v16hi)_mm256_setzero_si256());
168}
169
170#undef __DEFAULT_FN_ATTRS128
171#undef __DEFAULT_FN_ATTRS256
172
173#endif /* __x86_64__ */
174#endif /* __MOVRS_AVX10_2INTRIN_H */
lib/include/movrsintrin.h created+59
...@@ -0,0 +1,59 @@
1/*===---------------- movrsintrin.h - MOVRS intrinsics ----------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===----------------------------------------------------------------------===*/
8
9#ifndef __IMMINTRIN_H
10#error "Never use <movrsintrin.h> directly; include <immintrin.h> instead."
11#endif // __IMMINTRIN_H
12
13#ifndef __MOVRSINTRIN_H
14#define __MOVRSINTRIN_H
15
16#define __DEFAULT_FN_ATTRS \
17 __attribute__((__always_inline__, __nodebug__, __target__("movrs")))
18
19#ifdef __x86_64__
20static __inline__ char __DEFAULT_FN_ATTRS _movrs_i8(const void *__A) {
21 return (char)__builtin_ia32_movrsqi((const void *)__A);
22}
23
24static __inline__ short __DEFAULT_FN_ATTRS _movrs_i16(const void *__A) {
25 return (short)__builtin_ia32_movrshi((const void *)__A);
26}
27
28static __inline__ int __DEFAULT_FN_ATTRS _movrs_i32(const void *__A) {
29 return (int)__builtin_ia32_movrssi((const void *)__A);
30}
31
32static __inline__ long long __DEFAULT_FN_ATTRS _movrs_i64(const void *__A) {
33 return (long long)__builtin_ia32_movrsdi((const void *)__A);
34}
35#endif // __x86_64__
36
37// Loads a memory sequence containing the specified memory address into
38/// the L3 data cache. Data will be shared (read/written) to by requesting
39/// core and other cores.
40///
41/// Note that the effect of this intrinsic is dependent on the processor
42/// implementation.
43///
44/// \headerfile <x86intrin.h>
45///
46/// This intrinsic corresponds to the \c PREFETCHRS instruction.
47///
48/// \param __P
49/// A pointer specifying the memory address to be prefetched.
50static __inline__ void __DEFAULT_FN_ATTRS
51_m_prefetchrs(volatile const void *__P) {
52#pragma clang diagnostic push
53#pragma clang diagnostic ignored "-Wcast-qual"
54 __builtin_ia32_prefetchrs((const void *)__P);
55#pragma clang diagnostic pop
56}
57
58#undef __DEFAULT_FN_ATTRS
59#endif // __MOVRSINTRIN_H
\ No newline at end of file
lib/include/openmp_wrappers/__clang_openmp_device_functions.h+4-5
...@@ -10,17 +10,15 @@...@@ -10,17 +10,15 @@
10#ifndef __CLANG_OPENMP_DEVICE_FUNCTIONS_H__10#ifndef __CLANG_OPENMP_DEVICE_FUNCTIONS_H__
11#define __CLANG_OPENMP_DEVICE_FUNCTIONS_H__11#define __CLANG_OPENMP_DEVICE_FUNCTIONS_H__
1212
13#ifndef _OPENMP
14#error "This file is for OpenMP compilation only."
15#endif
16
17#ifdef __cplusplus13#ifdef __cplusplus
18extern "C" {14extern "C" {
19#endif15#endif
2016
17#ifdef __NVPTX__
21#pragma omp begin declare variant match( \18#pragma omp begin declare variant match( \
22 device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any)})19 device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any)})
2320
21#pragma push_macro("__CUDA__")
24#define __CUDA__22#define __CUDA__
25#define __OPENMP_NVPTX__23#define __OPENMP_NVPTX__
2624
...@@ -31,9 +29,10 @@ extern "C" {...@@ -31,9 +29,10 @@ extern "C" {
31#include <__clang_cuda_device_functions.h>29#include <__clang_cuda_device_functions.h>
3230
33#undef __OPENMP_NVPTX__31#undef __OPENMP_NVPTX__
34#undef __CUDA__32#pragma pop_macro("__CUDA__")
3533
36#pragma omp end declare variant34#pragma omp end declare variant
35#endif
3736
38#ifdef __AMDGCN__37#ifdef __AMDGCN__
39#pragma omp begin declare variant match(device = {arch(amdgcn)})38#pragma omp begin declare variant match(device = {arch(amdgcn)})
lib/include/openmp_wrappers/complex_cmath.h+7-2
...@@ -64,8 +64,13 @@ template <class _Tp> __DEVICE__ _Tp norm(const std::complex<_Tp> &__c) {...@@ -64,8 +64,13 @@ template <class _Tp> __DEVICE__ _Tp norm(const std::complex<_Tp> &__c) {
64}64}
6565
66// conj66// conj
6767#ifdef _GLIBCXX20_CONSTEXPR
68template <class _Tp> std::complex<_Tp> conj(const std::complex<_Tp> &__c) {68#define CXX20_CONSTEXPR_DEVICE __DEVICE__
69#else
70#define CXX20_CONSTEXPR_DEVICE
71#endif
72template <class _Tp>
73CXX20_CONSTEXPR_DEVICE std::complex<_Tp> conj(const std::complex<_Tp> &__c) {
69 return std::complex<_Tp>(__c.real(), -__c.imag());74 return std::complex<_Tp>(__c.real(), -__c.imag());
70}75}
7176
lib/include/pmmintrin.h+16-3
...@@ -17,9 +17,21 @@...@@ -17,9 +17,21 @@
17#include <emmintrin.h>17#include <emmintrin.h>
1818
19/* Define the default attributes for the functions in this file. */19/* Define the default attributes for the functions in this file. */
20#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
20#define __DEFAULT_FN_ATTRS \21#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \22 __attribute__((__always_inline__, __nodebug__, \
22 __target__("sse3,no-evex512"), __min_vector_width__(128)))23 __target__("sse3,no-evex512"), __min_vector_width__(128)))
24#else
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("sse3"), \
27 __min_vector_width__(128)))
28#endif
29
30#if defined(__cplusplus) && (__cplusplus >= 201103L)
31#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
32#else
33#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
34#endif
2335
24/// Loads data from an unaligned memory location to elements in a 128-bit36/// Loads data from an unaligned memory location to elements in a 128-bit
25/// vector.37/// vector.
...@@ -122,7 +134,7 @@ _mm_hsub_ps(__m128 __a, __m128 __b)...@@ -122,7 +134,7 @@ _mm_hsub_ps(__m128 __a, __m128 __b)
122/// destination.134/// destination.
123/// \returns A 128-bit vector of [4 x float] containing the moved and duplicated135/// \returns A 128-bit vector of [4 x float] containing the moved and duplicated
124/// values.136/// values.
125static __inline__ __m128 __DEFAULT_FN_ATTRS137static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
126_mm_movehdup_ps(__m128 __a)138_mm_movehdup_ps(__m128 __a)
127{139{
128 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 1, 1, 3, 3);140 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 1, 1, 3, 3);
...@@ -143,7 +155,7 @@ _mm_movehdup_ps(__m128 __a)...@@ -143,7 +155,7 @@ _mm_movehdup_ps(__m128 __a)
143/// destination.155/// destination.
144/// \returns A 128-bit vector of [4 x float] containing the moved and duplicated156/// \returns A 128-bit vector of [4 x float] containing the moved and duplicated
145/// values.157/// values.
146static __inline__ __m128 __DEFAULT_FN_ATTRS158static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
147_mm_moveldup_ps(__m128 __a)159_mm_moveldup_ps(__m128 __a)
148{160{
149 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 0, 2, 2);161 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 0, 2, 2);
...@@ -244,7 +256,7 @@ _mm_hsub_pd(__m128d __a, __m128d __b)...@@ -244,7 +256,7 @@ _mm_hsub_pd(__m128d __a, __m128d __b)
244/// [127:64] and [63:0] of the destination.256/// [127:64] and [63:0] of the destination.
245/// \returns A 128-bit vector of [2 x double] containing the moved and257/// \returns A 128-bit vector of [2 x double] containing the moved and
246/// duplicated values.258/// duplicated values.
247static __inline__ __m128d __DEFAULT_FN_ATTRS259static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
248_mm_movedup_pd(__m128d __a)260_mm_movedup_pd(__m128d __a)
249{261{
250 return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);262 return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);
...@@ -297,5 +309,6 @@ _mm_mwait(unsigned __extensions, unsigned __hints)...@@ -297,5 +309,6 @@ _mm_mwait(unsigned __extensions, unsigned __hints)
297}309}
298310
299#undef __DEFAULT_FN_ATTRS311#undef __DEFAULT_FN_ATTRS
312#undef __DEFAULT_FN_ATTRS_CONSTEXPR
300313
301#endif /* __PMMINTRIN_H */314#endif /* __PMMINTRIN_H */
lib/include/popcntintrin.h+7-7
...@@ -11,12 +11,13 @@...@@ -11,12 +11,13 @@
11#define __POPCNTINTRIN_H11#define __POPCNTINTRIN_H
1212
13/* Define the default attributes for the functions in this file. */13/* Define the default attributes for the functions in this file. */
14#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("popcnt")))
15
16#if defined(__cplusplus) && (__cplusplus >= 201103L)14#if defined(__cplusplus) && (__cplusplus >= 201103L)
17#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr15#define __DEFAULT_FN_ATTRS \
16 __attribute__((__always_inline__, __nodebug__, \
17 __target__("popcnt"))) constexpr
18#else18#else
19#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("popcnt")))
20#endif21#endif
2122
22/// Counts the number of bits in the source operand having a value of 1.23/// Counts the number of bits in the source operand having a value of 1.
...@@ -29,7 +30,7 @@...@@ -29,7 +30,7 @@
29/// An unsigned 32-bit integer operand.30/// An unsigned 32-bit integer operand.
30/// \returns A 32-bit integer containing the number of bits with value 1 in the31/// \returns A 32-bit integer containing the number of bits with value 1 in the
31/// source operand.32/// source operand.
32static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR33static __inline__ int __DEFAULT_FN_ATTRS
33_mm_popcnt_u32(unsigned int __A)34_mm_popcnt_u32(unsigned int __A)
34{35{
35 return __builtin_popcount(__A);36 return __builtin_popcount(__A);
...@@ -46,7 +47,7 @@ _mm_popcnt_u32(unsigned int __A)...@@ -46,7 +47,7 @@ _mm_popcnt_u32(unsigned int __A)
46/// An unsigned 64-bit integer operand.47/// An unsigned 64-bit integer operand.
47/// \returns A 64-bit integer containing the number of bits with value 1 in the48/// \returns A 64-bit integer containing the number of bits with value 1 in the
48/// source operand.49/// source operand.
49static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR50static __inline__ long long __DEFAULT_FN_ATTRS
50_mm_popcnt_u64(unsigned long long __A)51_mm_popcnt_u64(unsigned long long __A)
51{52{
52 return __builtin_popcountll(__A);53 return __builtin_popcountll(__A);
...@@ -54,6 +55,5 @@ _mm_popcnt_u64(unsigned long long __A)...@@ -54,6 +55,5 @@ _mm_popcnt_u64(unsigned long long __A)
54#endif /* __x86_64__ */55#endif /* __x86_64__ */
5556
56#undef __DEFAULT_FN_ATTRS57#undef __DEFAULT_FN_ATTRS
57#undef __DEFAULT_FN_ATTRS_CONSTEXPR
5858
59#endif /* __POPCNTINTRIN_H */59#endif /* __POPCNTINTRIN_H */
lib/include/ptrauth.h+6
...@@ -42,6 +42,9 @@ typedef enum {...@@ -42,6 +42,9 @@ typedef enum {
42 The extra data is always 0. */42 The extra data is always 0. */
43 ptrauth_key_cxx_vtable_pointer = ptrauth_key_process_independent_data,43 ptrauth_key_cxx_vtable_pointer = ptrauth_key_process_independent_data,
4444
45 /* The key used to sign pointers in ELF .init_array/.fini_array. */
46 ptrauth_key_init_fini_pointer = ptrauth_key_process_independent_code,
47
45 /* Other pointers signed under the ABI use private ABI rules. */48 /* Other pointers signed under the ABI use private ABI rules. */
4649
47} ptrauth_key;50} ptrauth_key;
...@@ -253,6 +256,9 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;...@@ -253,6 +256,9 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
253 [[clang::ptrauth_vtable_pointer(key, address_discrimination, \256 [[clang::ptrauth_vtable_pointer(key, address_discrimination, \
254 extra_discrimination)]]257 extra_discrimination)]]
255258
259/* The value is ptrauth_string_discriminator("init_fini") */
260#define __ptrauth_init_fini_discriminator 0xd9d4
261
256#else262#else
257263
258#define ptrauth_strip(__value, __key) \264#define ptrauth_strip(__value, __key) \
lib/include/riscv_corev_alu.h created+128
...@@ -0,0 +1,128 @@
1/*===---- riscv_corev_alu.h - CORE-V ALU intrinsics ------------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __RISCV_COREV_ALU_H
11#define __RISCV_COREV_ALU_H
12
13#include <stdint.h>
14
15#if defined(__cplusplus)
16extern "C" {
17#endif
18
19#if defined(__riscv_xcvalu)
20
21#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
22
23static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_abs(long a) {
24 return __builtin_abs(a);
25}
26
27static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_slet(long a, long b) {
28 return __builtin_riscv_cv_alu_slet(a, b);
29}
30
31static __inline__ long __DEFAULT_FN_ATTRS
32__riscv_cv_alu_sletu(unsigned long a, unsigned long b) {
33 return __builtin_riscv_cv_alu_sletu(a, b);
34}
35
36static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_min(long a, long b) {
37 return __builtin_elementwise_min(a, b);
38}
39
40static __inline__ unsigned long __DEFAULT_FN_ATTRS
41__riscv_cv_alu_minu(unsigned long a, unsigned long b) {
42 return __builtin_elementwise_min(a, b);
43}
44
45static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_max(long a, long b) {
46 return __builtin_elementwise_max(a, b);
47}
48
49static __inline__ unsigned long __DEFAULT_FN_ATTRS
50__riscv_cv_alu_maxu(unsigned long a, unsigned long b) {
51 return __builtin_elementwise_max(a, b);
52}
53
54static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_exths(int16_t a) {
55 return __builtin_riscv_cv_alu_exths(a);
56}
57
58static __inline__ unsigned long __DEFAULT_FN_ATTRS
59__riscv_cv_alu_exthz(uint16_t a) {
60 return __builtin_riscv_cv_alu_exthz(a);
61}
62
63static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_extbs(int8_t a) {
64 return __builtin_riscv_cv_alu_extbs(a);
65}
66
67static __inline__ unsigned long __DEFAULT_FN_ATTRS
68__riscv_cv_alu_extbz(uint8_t a) {
69 return __builtin_riscv_cv_alu_extbz(a);
70}
71
72static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_clip(long a,
73 unsigned long b) {
74 return __builtin_riscv_cv_alu_clip(a, b);
75}
76
77static __inline__ unsigned long __DEFAULT_FN_ATTRS
78__riscv_cv_alu_clipu(unsigned long a, unsigned long b) {
79 return __builtin_riscv_cv_alu_clipu(a, b);
80}
81
82static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_addN(long a, long b,
83 uint8_t shft) {
84 return __builtin_riscv_cv_alu_addN(a, b, shft);
85}
86
87static __inline__ unsigned long __DEFAULT_FN_ATTRS
88__riscv_cv_alu_adduN(unsigned long a, unsigned long b, uint8_t shft) {
89 return __builtin_riscv_cv_alu_adduN(a, b, shft);
90}
91
92static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_addRN(long a, long b,
93 uint8_t shft) {
94 return __builtin_riscv_cv_alu_addRN(a, b, shft);
95}
96
97static __inline__ unsigned long __DEFAULT_FN_ATTRS
98__riscv_cv_alu_adduRN(unsigned long a, unsigned long b, uint8_t shft) {
99 return __builtin_riscv_cv_alu_adduRN(a, b, shft);
100}
101
102static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_subN(long a, long b,
103 uint8_t shft) {
104 return __builtin_riscv_cv_alu_subN(a, b, shft);
105}
106
107static __inline__ unsigned long __DEFAULT_FN_ATTRS
108__riscv_cv_alu_subuN(unsigned long a, unsigned long b, uint8_t shft) {
109 return __builtin_riscv_cv_alu_subuN(a, b, shft);
110}
111
112static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_subRN(long a, long b,
113 uint8_t shft) {
114 return __builtin_riscv_cv_alu_subRN(a, b, shft);
115}
116
117static __inline__ unsigned long __DEFAULT_FN_ATTRS
118__riscv_cv_alu_subuRN(unsigned long a, unsigned long b, uint8_t shft) {
119 return __builtin_riscv_cv_alu_subuRN(a, b, shft);
120}
121
122#endif // defined(__riscv_xcvalu)
123
124#if defined(__cplusplus)
125}
126#endif
127
128#endif // define __RISCV_COREV_ALU_H
lib/include/riscv_vector.h-1
...@@ -419,7 +419,6 @@ typedef __rvv_bfloat16m2x4_t vbfloat16m2x4_t;...@@ -419,7 +419,6 @@ typedef __rvv_bfloat16m2x4_t vbfloat16m2x4_t;
419typedef __rvv_bfloat16m4_t vbfloat16m4_t;419typedef __rvv_bfloat16m4_t vbfloat16m4_t;
420typedef __rvv_bfloat16m4x2_t vbfloat16m4x2_t;420typedef __rvv_bfloat16m4x2_t vbfloat16m4x2_t;
421typedef __rvv_bfloat16m8_t vbfloat16m8_t;421typedef __rvv_bfloat16m8_t vbfloat16m8_t;
422#define __riscv_v_intrinsic_overloading 1
423422
424#ifdef __cplusplus423#ifdef __cplusplus
425}424}
lib/include/sm4evexintrin.h created+32
...@@ -0,0 +1,32 @@
1/*===--------------- sm4evexintrin.h - SM4 EVEX intrinsics -----------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <sm4evexintrin.h> directly; include <immintrin.h> instead."
11#endif // __IMMINTRIN_H
12
13#ifndef __SM4EVEXINTRIN_H
14#define __SM4EVEXINTRIN_H
15
16#define __DEFAULT_FN_ATTRS512 \
17 __attribute__((__always_inline__, __nodebug__, \
18 __target__("sm4,avx10.2-512"), __min_vector_width__(512)))
19
20static __inline__ __m512i __DEFAULT_FN_ATTRS512
21_mm512_sm4key4_epi32(__m512i __A, __m512i __B) {
22 return (__m512i)__builtin_ia32_vsm4key4512((__v16su)__A, (__v16su)__B);
23}
24
25static __inline__ __m512i __DEFAULT_FN_ATTRS512
26_mm512_sm4rnds4_epi32(__m512i __A, __m512i __B) {
27 return (__m512i)__builtin_ia32_vsm4rnds4512((__v16su)__A, (__v16su)__B);
28}
29
30#undef __DEFAULT_FN_ATTRS512
31
32#endif // __SM4EVEXINTRIN_H
lib/include/smmintrin.h+6
...@@ -17,9 +17,15 @@...@@ -17,9 +17,15 @@
17#include <tmmintrin.h>17#include <tmmintrin.h>
1818
19/* Define the default attributes for the functions in this file. */19/* Define the default attributes for the functions in this file. */
20#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
20#define __DEFAULT_FN_ATTRS \21#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \22 __attribute__((__always_inline__, __nodebug__, \
22 __target__("sse4.1,no-evex512"), __min_vector_width__(128)))23 __target__("sse4.1,no-evex512"), __min_vector_width__(128)))
24#else
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("sse4.1"), \
27 __min_vector_width__(128)))
28#endif
2329
24/* SSE4 Rounding macros. */30/* SSE4 Rounding macros. */
25#define _MM_FROUND_TO_NEAREST_INT 0x0031#define _MM_FROUND_TO_NEAREST_INT 0x00
lib/include/stdalign.h-5
...@@ -10,10 +10,6 @@...@@ -10,10 +10,6 @@
10#ifndef __STDALIGN_H10#ifndef __STDALIGN_H
11#define __STDALIGN_H11#define __STDALIGN_H
1212
13#if defined(__MVS__) && __has_include_next(<stdalign.h>)
14#include_next <stdalign.h>
15#else
16
17#if defined(__cplusplus) || \13#if defined(__cplusplus) || \
18 (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202311L)14 (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202311L)
19#ifndef __cplusplus15#ifndef __cplusplus
...@@ -25,5 +21,4 @@...@@ -25,5 +21,4 @@
25#define __alignof_is_defined 121#define __alignof_is_defined 1
26#endif /* __STDC_VERSION__ */22#endif /* __STDC_VERSION__ */
2723
28#endif /* __MVS__ */
29#endif /* __STDALIGN_H */24#endif /* __STDALIGN_H */
lib/include/tbmintrin.h+25-37
...@@ -15,63 +15,60 @@...@@ -15,63 +15,60 @@
15#define __TBMINTRIN_H15#define __TBMINTRIN_H
1616
17/* Define the default attributes for the functions in this file. */17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("tbm")))18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("tbm"))) constexpr
21#else
22#define __DEFAULT_FN_ATTRS \
23 __attribute__((__always_inline__, __nodebug__, __target__("tbm")))
24#endif
1925
20#define __bextri_u32(a, b) \26#define __bextri_u32(a, b) \
21 ((unsigned int)__builtin_ia32_bextri_u32((unsigned int)(a), \27 ((unsigned int)__builtin_ia32_bextri_u32((unsigned int)(a), \
22 (unsigned int)(b)))28 (unsigned int)(b)))
2329
24static __inline__ unsigned int __DEFAULT_FN_ATTRS30static __inline__ unsigned int __DEFAULT_FN_ATTRS
25__blcfill_u32(unsigned int __a)31__blcfill_u32(unsigned int __a) {
26{
27 return __a & (__a + 1);32 return __a & (__a + 1);
28}33}
2934
30static __inline__ unsigned int __DEFAULT_FN_ATTRS35static __inline__ unsigned int __DEFAULT_FN_ATTRS
31__blci_u32(unsigned int __a)36__blci_u32(unsigned int __a) {
32{
33 return __a | ~(__a + 1);37 return __a | ~(__a + 1);
34}38}
3539
36static __inline__ unsigned int __DEFAULT_FN_ATTRS40static __inline__ unsigned int __DEFAULT_FN_ATTRS
37__blcic_u32(unsigned int __a)41__blcic_u32(unsigned int __a) {
38{
39 return ~__a & (__a + 1);42 return ~__a & (__a + 1);
40}43}
4144
42static __inline__ unsigned int __DEFAULT_FN_ATTRS45static __inline__ unsigned int __DEFAULT_FN_ATTRS
43__blcmsk_u32(unsigned int __a)46__blcmsk_u32(unsigned int __a) {
44{
45 return __a ^ (__a + 1);47 return __a ^ (__a + 1);
46}48}
4749
48static __inline__ unsigned int __DEFAULT_FN_ATTRS50static __inline__ unsigned int __DEFAULT_FN_ATTRS
49__blcs_u32(unsigned int __a)51__blcs_u32(unsigned int __a) {
50{
51 return __a | (__a + 1);52 return __a | (__a + 1);
52}53}
5354
54static __inline__ unsigned int __DEFAULT_FN_ATTRS55static __inline__ unsigned int __DEFAULT_FN_ATTRS
55__blsfill_u32(unsigned int __a)56__blsfill_u32(unsigned int __a) {
56{
57 return __a | (__a - 1);57 return __a | (__a - 1);
58}58}
5959
60static __inline__ unsigned int __DEFAULT_FN_ATTRS60static __inline__ unsigned int __DEFAULT_FN_ATTRS
61__blsic_u32(unsigned int __a)61__blsic_u32(unsigned int __a) {
62{
63 return ~__a | (__a - 1);62 return ~__a | (__a - 1);
64}63}
6564
66static __inline__ unsigned int __DEFAULT_FN_ATTRS65static __inline__ unsigned int __DEFAULT_FN_ATTRS
67__t1mskc_u32(unsigned int __a)66__t1mskc_u32(unsigned int __a) {
68{
69 return ~__a | (__a + 1);67 return ~__a | (__a + 1);
70}68}
7169
72static __inline__ unsigned int __DEFAULT_FN_ATTRS70static __inline__ unsigned int __DEFAULT_FN_ATTRS
73__tzmsk_u32(unsigned int __a)71__tzmsk_u32(unsigned int __a) {
74{
75 return ~__a & (__a - 1);72 return ~__a & (__a - 1);
76}73}
7774
...@@ -81,56 +78,47 @@ __tzmsk_u32(unsigned int __a)...@@ -81,56 +78,47 @@ __tzmsk_u32(unsigned int __a)
81 (unsigned long long)(b)))78 (unsigned long long)(b)))
8279
83static __inline__ unsigned long long __DEFAULT_FN_ATTRS80static __inline__ unsigned long long __DEFAULT_FN_ATTRS
84__blcfill_u64(unsigned long long __a)81__blcfill_u64(unsigned long long __a) {
85{
86 return __a & (__a + 1);82 return __a & (__a + 1);
87}83}
8884
89static __inline__ unsigned long long __DEFAULT_FN_ATTRS85static __inline__ unsigned long long __DEFAULT_FN_ATTRS
90__blci_u64(unsigned long long __a)86__blci_u64(unsigned long long __a) {
91{
92 return __a | ~(__a + 1);87 return __a | ~(__a + 1);
93}88}
9489
95static __inline__ unsigned long long __DEFAULT_FN_ATTRS90static __inline__ unsigned long long __DEFAULT_FN_ATTRS
96__blcic_u64(unsigned long long __a)91__blcic_u64(unsigned long long __a) {
97{
98 return ~__a & (__a + 1);92 return ~__a & (__a + 1);
99}93}
10094
101static __inline__ unsigned long long __DEFAULT_FN_ATTRS95static __inline__ unsigned long long __DEFAULT_FN_ATTRS
102__blcmsk_u64(unsigned long long __a)96__blcmsk_u64(unsigned long long __a) {
103{
104 return __a ^ (__a + 1);97 return __a ^ (__a + 1);
105}98}
10699
107static __inline__ unsigned long long __DEFAULT_FN_ATTRS100static __inline__ unsigned long long __DEFAULT_FN_ATTRS
108__blcs_u64(unsigned long long __a)101__blcs_u64(unsigned long long __a) {
109{
110 return __a | (__a + 1);102 return __a | (__a + 1);
111}103}
112104
113static __inline__ unsigned long long __DEFAULT_FN_ATTRS105static __inline__ unsigned long long __DEFAULT_FN_ATTRS
114__blsfill_u64(unsigned long long __a)106__blsfill_u64(unsigned long long __a) {
115{
116 return __a | (__a - 1);107 return __a | (__a - 1);
117}108}
118109
119static __inline__ unsigned long long __DEFAULT_FN_ATTRS110static __inline__ unsigned long long __DEFAULT_FN_ATTRS
120__blsic_u64(unsigned long long __a)111__blsic_u64(unsigned long long __a) {
121{
122 return ~__a | (__a - 1);112 return ~__a | (__a - 1);
123}113}
124114
125static __inline__ unsigned long long __DEFAULT_FN_ATTRS115static __inline__ unsigned long long __DEFAULT_FN_ATTRS
126__t1mskc_u64(unsigned long long __a)116__t1mskc_u64(unsigned long long __a) {
127{
128 return ~__a | (__a + 1);117 return ~__a | (__a + 1);
129}118}
130119
131static __inline__ unsigned long long __DEFAULT_FN_ATTRS120static __inline__ unsigned long long __DEFAULT_FN_ATTRS
132__tzmsk_u64(unsigned long long __a)121__tzmsk_u64(unsigned long long __a) {
133{
134 return ~__a & (__a - 1);122 return ~__a & (__a - 1);
135}123}
136#endif124#endif
lib/include/tmmintrin.h+64-38
...@@ -17,13 +17,21 @@...@@ -17,13 +17,21 @@
17#include <pmmintrin.h>17#include <pmmintrin.h>
1818
19/* Define the default attributes for the functions in this file. */19/* Define the default attributes for the functions in this file. */
20#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
20#define __DEFAULT_FN_ATTRS \21#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \22 __attribute__((__always_inline__, __nodebug__, \
22 __target__("ssse3,no-evex512"), __min_vector_width__(64)))23 __target__("ssse3,no-evex512"), __min_vector_width__(128)))
23#define __DEFAULT_FN_ATTRS_MMX \24#else
24 __attribute__((__always_inline__, __nodebug__, \25#define __DEFAULT_FN_ATTRS \
25 __target__("mmx,ssse3,no-evex512"), \26 __attribute__((__always_inline__, __nodebug__, __target__("ssse3"), \
26 __min_vector_width__(64)))27 __min_vector_width__(128)))
28#endif
29
30#define __trunc64(x) \
31 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
32#define __anyext128(x) \
33 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
34 1, -1, -1)
2735
28/// Computes the absolute value of each of the packed 8-bit signed36/// Computes the absolute value of each of the packed 8-bit signed
29/// integers in the source operand and stores the 8-bit unsigned integer37/// integers in the source operand and stores the 8-bit unsigned integer
...@@ -37,10 +45,10 @@...@@ -37,10 +45,10 @@
37/// A 64-bit vector of [8 x i8].45/// A 64-bit vector of [8 x i8].
38/// \returns A 64-bit integer vector containing the absolute values of the46/// \returns A 64-bit integer vector containing the absolute values of the
39/// elements in the operand.47/// elements in the operand.
40static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX48static __inline__ __m64 __DEFAULT_FN_ATTRS
41_mm_abs_pi8(__m64 __a)49_mm_abs_pi8(__m64 __a)
42{50{
43 return (__m64)__builtin_ia32_pabsb((__v8qi)__a);51 return (__m64)__builtin_elementwise_abs((__v8qs)__a);
44}52}
4553
46/// Computes the absolute value of each of the packed 8-bit signed54/// Computes the absolute value of each of the packed 8-bit signed
...@@ -73,10 +81,10 @@ _mm_abs_epi8(__m128i __a)...@@ -73,10 +81,10 @@ _mm_abs_epi8(__m128i __a)
73/// A 64-bit vector of [4 x i16].81/// A 64-bit vector of [4 x i16].
74/// \returns A 64-bit integer vector containing the absolute values of the82/// \returns A 64-bit integer vector containing the absolute values of the
75/// elements in the operand.83/// elements in the operand.
76static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX84static __inline__ __m64 __DEFAULT_FN_ATTRS
77_mm_abs_pi16(__m64 __a)85_mm_abs_pi16(__m64 __a)
78{86{
79 return (__m64)__builtin_ia32_pabsw((__v4hi)__a);87 return (__m64)__builtin_elementwise_abs((__v4hi)__a);
80}88}
8189
82/// Computes the absolute value of each of the packed 16-bit signed90/// Computes the absolute value of each of the packed 16-bit signed
...@@ -109,10 +117,10 @@ _mm_abs_epi16(__m128i __a)...@@ -109,10 +117,10 @@ _mm_abs_epi16(__m128i __a)
109/// A 64-bit vector of [2 x i32].117/// A 64-bit vector of [2 x i32].
110/// \returns A 64-bit integer vector containing the absolute values of the118/// \returns A 64-bit integer vector containing the absolute values of the
111/// elements in the operand.119/// elements in the operand.
112static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX120static __inline__ __m64 __DEFAULT_FN_ATTRS
113_mm_abs_pi32(__m64 __a)121_mm_abs_pi32(__m64 __a)
114{122{
115 return (__m64)__builtin_ia32_pabsd((__v2si)__a);123 return (__m64)__builtin_elementwise_abs((__v2si)__a);
116}124}
117125
118/// Computes the absolute value of each of the packed 32-bit signed126/// Computes the absolute value of each of the packed 32-bit signed
...@@ -177,7 +185,10 @@ _mm_abs_epi32(__m128i __a)...@@ -177,7 +185,10 @@ _mm_abs_epi32(__m128i __a)
177/// \returns A 64-bit integer vector containing the concatenated right-shifted185/// \returns A 64-bit integer vector containing the concatenated right-shifted
178/// value.186/// value.
179#define _mm_alignr_pi8(a, b, n) \187#define _mm_alignr_pi8(a, b, n) \
180 ((__m64)__builtin_ia32_palignr((__v8qi)(__m64)(a), (__v8qi)(__m64)(b), (n)))188 ((__m64)__builtin_shufflevector( \
189 __builtin_ia32_psrldqi128_byteshift( \
190 __builtin_shufflevector((__v1di)(a), (__v1di)(b), 1, 0), \
191 (n)), __extension__ (__v2di){}, 0))
181192
182/// Horizontally adds the adjacent pairs of values contained in 2 packed193/// Horizontally adds the adjacent pairs of values contained in 2 packed
183/// 128-bit vectors of [8 x i16].194/// 128-bit vectors of [8 x i16].
...@@ -242,10 +253,11 @@ _mm_hadd_epi32(__m128i __a, __m128i __b)...@@ -242,10 +253,11 @@ _mm_hadd_epi32(__m128i __a, __m128i __b)
242/// destination.253/// destination.
243/// \returns A 64-bit vector of [4 x i16] containing the horizontal sums of both254/// \returns A 64-bit vector of [4 x i16] containing the horizontal sums of both
244/// operands.255/// operands.
245static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX256static __inline__ __m64 __DEFAULT_FN_ATTRS
246_mm_hadd_pi16(__m64 __a, __m64 __b)257_mm_hadd_pi16(__m64 __a, __m64 __b)
247{258{
248 return (__m64)__builtin_ia32_phaddw((__v4hi)__a, (__v4hi)__b);259 return __trunc64(__builtin_ia32_phaddw128(
260 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
249}261}
250262
251/// Horizontally adds the adjacent pairs of values contained in 2 packed263/// Horizontally adds the adjacent pairs of values contained in 2 packed
...@@ -265,10 +277,11 @@ _mm_hadd_pi16(__m64 __a, __m64 __b)...@@ -265,10 +277,11 @@ _mm_hadd_pi16(__m64 __a, __m64 __b)
265/// destination.277/// destination.
266/// \returns A 64-bit vector of [2 x i32] containing the horizontal sums of both278/// \returns A 64-bit vector of [2 x i32] containing the horizontal sums of both
267/// operands.279/// operands.
268static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX280static __inline__ __m64 __DEFAULT_FN_ATTRS
269_mm_hadd_pi32(__m64 __a, __m64 __b)281_mm_hadd_pi32(__m64 __a, __m64 __b)
270{282{
271 return (__m64)__builtin_ia32_phaddd((__v2si)__a, (__v2si)__b);283 return __trunc64(__builtin_ia32_phaddd128(
284 (__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
272}285}
273286
274/// Horizontally adds, with saturation, the adjacent pairs of values contained287/// Horizontally adds, with saturation, the adjacent pairs of values contained
...@@ -317,10 +330,11 @@ _mm_hadds_epi16(__m128i __a, __m128i __b)...@@ -317,10 +330,11 @@ _mm_hadds_epi16(__m128i __a, __m128i __b)
317/// destination.330/// destination.
318/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated331/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated
319/// sums of both operands.332/// sums of both operands.
320static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX333static __inline__ __m64 __DEFAULT_FN_ATTRS
321_mm_hadds_pi16(__m64 __a, __m64 __b)334_mm_hadds_pi16(__m64 __a, __m64 __b)
322{335{
323 return (__m64)__builtin_ia32_phaddsw((__v4hi)__a, (__v4hi)__b);336 return __trunc64(__builtin_ia32_phaddsw128(
337 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
324}338}
325339
326/// Horizontally subtracts the adjacent pairs of values contained in 2340/// Horizontally subtracts the adjacent pairs of values contained in 2
...@@ -386,10 +400,11 @@ _mm_hsub_epi32(__m128i __a, __m128i __b)...@@ -386,10 +400,11 @@ _mm_hsub_epi32(__m128i __a, __m128i __b)
386/// the destination.400/// the destination.
387/// \returns A 64-bit vector of [4 x i16] containing the horizontal differences401/// \returns A 64-bit vector of [4 x i16] containing the horizontal differences
388/// of both operands.402/// of both operands.
389static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX403static __inline__ __m64 __DEFAULT_FN_ATTRS
390_mm_hsub_pi16(__m64 __a, __m64 __b)404_mm_hsub_pi16(__m64 __a, __m64 __b)
391{405{
392 return (__m64)__builtin_ia32_phsubw((__v4hi)__a, (__v4hi)__b);406 return __trunc64(__builtin_ia32_phsubw128(
407 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
393}408}
394409
395/// Horizontally subtracts the adjacent pairs of values contained in 2410/// Horizontally subtracts the adjacent pairs of values contained in 2
...@@ -409,10 +424,11 @@ _mm_hsub_pi16(__m64 __a, __m64 __b)...@@ -409,10 +424,11 @@ _mm_hsub_pi16(__m64 __a, __m64 __b)
409/// the destination.424/// the destination.
410/// \returns A 64-bit vector of [2 x i32] containing the horizontal differences425/// \returns A 64-bit vector of [2 x i32] containing the horizontal differences
411/// of both operands.426/// of both operands.
412static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX427static __inline__ __m64 __DEFAULT_FN_ATTRS
413_mm_hsub_pi32(__m64 __a, __m64 __b)428_mm_hsub_pi32(__m64 __a, __m64 __b)
414{429{
415 return (__m64)__builtin_ia32_phsubd((__v2si)__a, (__v2si)__b);430 return __trunc64(__builtin_ia32_phsubd128(
431 (__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
416}432}
417433
418/// Horizontally subtracts, with saturation, the adjacent pairs of values434/// Horizontally subtracts, with saturation, the adjacent pairs of values
...@@ -461,10 +477,11 @@ _mm_hsubs_epi16(__m128i __a, __m128i __b)...@@ -461,10 +477,11 @@ _mm_hsubs_epi16(__m128i __a, __m128i __b)
461/// the destination.477/// the destination.
462/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated478/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated
463/// differences of both operands.479/// differences of both operands.
464static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX480static __inline__ __m64 __DEFAULT_FN_ATTRS
465_mm_hsubs_pi16(__m64 __a, __m64 __b)481_mm_hsubs_pi16(__m64 __a, __m64 __b)
466{482{
467 return (__m64)__builtin_ia32_phsubsw((__v4hi)__a, (__v4hi)__b);483 return __trunc64(__builtin_ia32_phsubsw128(
484 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
468}485}
469486
470/// Multiplies corresponding pairs of packed 8-bit unsigned integer487/// Multiplies corresponding pairs of packed 8-bit unsigned integer
...@@ -525,10 +542,11 @@ _mm_maddubs_epi16(__m128i __a, __m128i __b)...@@ -525,10 +542,11 @@ _mm_maddubs_epi16(__m128i __a, __m128i __b)
525/// \a R1 := (\a __a2 * \a __b2) + (\a __a3 * \a __b3) \n542/// \a R1 := (\a __a2 * \a __b2) + (\a __a3 * \a __b3) \n
526/// \a R2 := (\a __a4 * \a __b4) + (\a __a5 * \a __b5) \n543/// \a R2 := (\a __a4 * \a __b4) + (\a __a5 * \a __b5) \n
527/// \a R3 := (\a __a6 * \a __b6) + (\a __a7 * \a __b7)544/// \a R3 := (\a __a6 * \a __b6) + (\a __a7 * \a __b7)
528static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX545static __inline__ __m64 __DEFAULT_FN_ATTRS
529_mm_maddubs_pi16(__m64 __a, __m64 __b)546_mm_maddubs_pi16(__m64 __a, __m64 __b)
530{547{
531 return (__m64)__builtin_ia32_pmaddubsw((__v8qi)__a, (__v8qi)__b);548 return __trunc64(__builtin_ia32_pmaddubsw128((__v16qi)__anyext128(__a),
549 (__v16qi)__anyext128(__b)));
532}550}
533551
534/// Multiplies packed 16-bit signed integer values, truncates the 32-bit552/// Multiplies packed 16-bit signed integer values, truncates the 32-bit
...@@ -565,10 +583,11 @@ _mm_mulhrs_epi16(__m128i __a, __m128i __b)...@@ -565,10 +583,11 @@ _mm_mulhrs_epi16(__m128i __a, __m128i __b)
565/// A 64-bit vector of [4 x i16] containing one of the source operands.583/// A 64-bit vector of [4 x i16] containing one of the source operands.
566/// \returns A 64-bit vector of [4 x i16] containing the rounded and scaled584/// \returns A 64-bit vector of [4 x i16] containing the rounded and scaled
567/// products of both operands.585/// products of both operands.
568static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX586static __inline__ __m64 __DEFAULT_FN_ATTRS
569_mm_mulhrs_pi16(__m64 __a, __m64 __b)587_mm_mulhrs_pi16(__m64 __a, __m64 __b)
570{588{
571 return (__m64)__builtin_ia32_pmulhrsw((__v4hi)__a, (__v4hi)__b);589 return __trunc64(__builtin_ia32_pmulhrsw128((__v8hi)__anyext128(__a),
590 (__v8hi)__anyext128(__b)));
572}591}
573592
574/// Copies the 8-bit integers from a 128-bit integer vector to the593/// Copies the 8-bit integers from a 128-bit integer vector to the
...@@ -614,12 +633,15 @@ _mm_shuffle_epi8(__m128i __a, __m128i __b)...@@ -614,12 +633,15 @@ _mm_shuffle_epi8(__m128i __a, __m128i __b)
614/// 1: Clear the corresponding byte in the destination. \n633/// 1: Clear the corresponding byte in the destination. \n
615/// 0: Copy the selected source byte to the corresponding byte in the634/// 0: Copy the selected source byte to the corresponding byte in the
616/// destination. \n635/// destination. \n
617/// Bits [3:0] select the source byte to be copied.636/// Bits [2:0] select the source byte to be copied.
618/// \returns A 64-bit integer vector containing the copied or cleared values.637/// \returns A 64-bit integer vector containing the copied or cleared values.
619static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX638static __inline__ __m64 __DEFAULT_FN_ATTRS
620_mm_shuffle_pi8(__m64 __a, __m64 __b)639_mm_shuffle_pi8(__m64 __a, __m64 __b)
621{640{
622 return (__m64)__builtin_ia32_pshufb((__v8qi)__a, (__v8qi)__b);641 return __trunc64(__builtin_ia32_pshufb128(
642 (__v16qi)__builtin_shufflevector(
643 (__v2si)(__a), __extension__ (__v2si){}, 0, 1, 0, 1),
644 (__v16qi)__anyext128(__b)));
623}645}
624646
625/// For each 8-bit integer in the first source operand, perform one of647/// For each 8-bit integer in the first source operand, perform one of
...@@ -720,10 +742,11 @@ _mm_sign_epi32(__m128i __a, __m128i __b)...@@ -720,10 +742,11 @@ _mm_sign_epi32(__m128i __a, __m128i __b)
720/// A 64-bit integer vector containing control bytes corresponding to742/// A 64-bit integer vector containing control bytes corresponding to
721/// positions in the destination.743/// positions in the destination.
722/// \returns A 64-bit integer vector containing the resultant values.744/// \returns A 64-bit integer vector containing the resultant values.
723static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX745static __inline__ __m64 __DEFAULT_FN_ATTRS
724_mm_sign_pi8(__m64 __a, __m64 __b)746_mm_sign_pi8(__m64 __a, __m64 __b)
725{747{
726 return (__m64)__builtin_ia32_psignb((__v8qi)__a, (__v8qi)__b);748 return __trunc64(__builtin_ia32_psignb128((__v16qi)__anyext128(__a),
749 (__v16qi)__anyext128(__b)));
727}750}
728751
729/// For each 16-bit integer in the first source operand, perform one of752/// For each 16-bit integer in the first source operand, perform one of
...@@ -746,10 +769,11 @@ _mm_sign_pi8(__m64 __a, __m64 __b)...@@ -746,10 +769,11 @@ _mm_sign_pi8(__m64 __a, __m64 __b)
746/// A 64-bit integer vector containing control words corresponding to769/// A 64-bit integer vector containing control words corresponding to
747/// positions in the destination.770/// positions in the destination.
748/// \returns A 64-bit integer vector containing the resultant values.771/// \returns A 64-bit integer vector containing the resultant values.
749static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX772static __inline__ __m64 __DEFAULT_FN_ATTRS
750_mm_sign_pi16(__m64 __a, __m64 __b)773_mm_sign_pi16(__m64 __a, __m64 __b)
751{774{
752 return (__m64)__builtin_ia32_psignw((__v4hi)__a, (__v4hi)__b);775 return __trunc64(__builtin_ia32_psignw128((__v8hi)__anyext128(__a),
776 (__v8hi)__anyext128(__b)));
753}777}
754778
755/// For each 32-bit integer in the first source operand, perform one of779/// For each 32-bit integer in the first source operand, perform one of
...@@ -772,13 +796,15 @@ _mm_sign_pi16(__m64 __a, __m64 __b)...@@ -772,13 +796,15 @@ _mm_sign_pi16(__m64 __a, __m64 __b)
772/// A 64-bit integer vector containing two control doublewords corresponding796/// A 64-bit integer vector containing two control doublewords corresponding
773/// to positions in the destination.797/// to positions in the destination.
774/// \returns A 64-bit integer vector containing the resultant values.798/// \returns A 64-bit integer vector containing the resultant values.
775static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX799static __inline__ __m64 __DEFAULT_FN_ATTRS
776_mm_sign_pi32(__m64 __a, __m64 __b)800_mm_sign_pi32(__m64 __a, __m64 __b)
777{801{
778 return (__m64)__builtin_ia32_psignd((__v2si)__a, (__v2si)__b);802 return __trunc64(__builtin_ia32_psignd128((__v4si)__anyext128(__a),
803 (__v4si)__anyext128(__b)));
779}804}
780805
806#undef __anyext128
807#undef __trunc64
781#undef __DEFAULT_FN_ATTRS808#undef __DEFAULT_FN_ATTRS
782#undef __DEFAULT_FN_ATTRS_MMX
783809
784#endif /* __TMMINTRIN_H */810#endif /* __TMMINTRIN_H */
lib/include/vecintrin.h+1778-60
...@@ -468,6 +468,27 @@ vec_perm(__vector __bool long long __a, __vector __bool long long __b,...@@ -468,6 +468,27 @@ vec_perm(__vector __bool long long __a, __vector __bool long long __b,
468 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);468 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
469}469}
470470
471static inline __ATTRS_o_ai __vector signed __int128
472vec_perm(__vector signed __int128 __a, __vector signed __int128 __b,
473 __vector unsigned char __c) {
474 return (__vector signed __int128)__builtin_s390_vperm(
475 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
476}
477
478static inline __ATTRS_o_ai __vector unsigned __int128
479vec_perm(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
480 __vector unsigned char __c) {
481 return (__vector unsigned __int128)__builtin_s390_vperm(
482 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
483}
484
485static inline __ATTRS_o_ai __vector __bool __int128
486vec_perm(__vector __bool __int128 __a, __vector __bool __int128 __b,
487 __vector unsigned char __c) {
488 return (__vector __bool __int128)__builtin_s390_vperm(
489 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
490}
491
471#if __ARCH__ >= 12492#if __ARCH__ >= 12
472static inline __ATTRS_o_ai __vector float493static inline __ATTRS_o_ai __vector float
473vec_perm(__vector float __a, __vector float __b,494vec_perm(__vector float __a, __vector float __b,
...@@ -514,9 +535,19 @@ vec_permi(__vector double __a, __vector double __b, int __c)...@@ -514,9 +535,19 @@ vec_permi(__vector double __a, __vector double __b, int __c)
514 (__vector unsigned long long)(Y), \535 (__vector unsigned long long)(Y), \
515 (((Z) & 2) << 1) | ((Z) & 1)))536 (((Z) & 2) << 1) | ((Z) & 1)))
516537
538/*-- vec_bperm --------------------------------------------------------------*/
539
540#if __ARCH__ >= 12
541static inline __ATTRS_ai __vector unsigned long long
542vec_bperm(__vector unsigned __int128 __a, __vector unsigned char __b) {
543 return __builtin_s390_vbperm((__vector unsigned char)__a, __b);
544}
545#endif
546
517/*-- vec_bperm_u128 ---------------------------------------------------------*/547/*-- vec_bperm_u128 ---------------------------------------------------------*/
518548
519#if __ARCH__ >= 12549#if __ARCH__ >= 12
550// This prototype is deprecated.
520static inline __ATTRS_ai __vector unsigned long long551static inline __ATTRS_ai __vector unsigned long long
521vec_bperm_u128(__vector unsigned char __a, __vector unsigned char __b) {552vec_bperm_u128(__vector unsigned char __a, __vector unsigned char __b) {
522 return __builtin_s390_vbperm(__a, __b);553 return __builtin_s390_vbperm(__a, __b);
...@@ -558,6 +589,18 @@ vec_revb(__vector unsigned long long __vec) {...@@ -558,6 +589,18 @@ vec_revb(__vector unsigned long long __vec) {
558 return __builtin_s390_vlbrg(__vec);589 return __builtin_s390_vlbrg(__vec);
559}590}
560591
592static inline __ATTRS_o_ai __vector signed __int128
593vec_revb(__vector signed __int128 __vec) {
594 return (__vector signed __int128)
595 __builtin_s390_vlbrq((unsigned __int128)__vec);
596}
597
598static inline __ATTRS_o_ai __vector unsigned __int128
599vec_revb(__vector unsigned __int128 __vec) {
600 return (__vector unsigned __int128)
601 __builtin_s390_vlbrq((unsigned __int128)__vec);
602}
603
561#if __ARCH__ >= 12604#if __ARCH__ >= 12
562static inline __ATTRS_o_ai __vector float605static inline __ATTRS_o_ai __vector float
563vec_revb(__vector float __vec) {606vec_revb(__vector float __vec) {
...@@ -820,6 +863,46 @@ vec_sel(__vector unsigned long long __a, __vector unsigned long long __b,...@@ -820,6 +863,46 @@ vec_sel(__vector unsigned long long __a, __vector unsigned long long __b,
820 (~(__vector unsigned long long)__c & __a));863 (~(__vector unsigned long long)__c & __a));
821}864}
822865
866static inline __ATTRS_o_ai __vector signed __int128
867vec_sel(__vector signed __int128 __a, __vector signed __int128 __b,
868 __vector unsigned __int128 __c) {
869 return (((__vector signed __int128)__c & __b) |
870 (~(__vector signed __int128)__c & __a));
871}
872
873static inline __ATTRS_o_ai __vector signed __int128
874vec_sel(__vector signed __int128 __a, __vector signed __int128 __b,
875 __vector __bool __int128 __c) {
876 return (((__vector signed __int128)__c & __b) |
877 (~(__vector signed __int128)__c & __a));
878}
879
880static inline __ATTRS_o_ai __vector __bool __int128
881vec_sel(__vector __bool __int128 __a, __vector __bool __int128 __b,
882 __vector unsigned __int128 __c) {
883 return (((__vector __bool __int128)__c & __b) |
884 (~(__vector __bool __int128)__c & __a));
885}
886
887static inline __ATTRS_o_ai __vector __bool __int128
888vec_sel(__vector __bool __int128 __a, __vector __bool __int128 __b,
889 __vector __bool __int128 __c) {
890 return (__c & __b) | (~__c & __a);
891}
892
893static inline __ATTRS_o_ai __vector unsigned __int128
894vec_sel(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
895 __vector unsigned __int128 __c) {
896 return (__c & __b) | (~__c & __a);
897}
898
899static inline __ATTRS_o_ai __vector unsigned __int128
900vec_sel(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
901 __vector __bool __int128 __c) {
902 return (((__vector unsigned __int128)__c & __b) |
903 (~(__vector unsigned __int128)__c & __a));
904}
905
823#if __ARCH__ >= 12906#if __ARCH__ >= 12
824static inline __ATTRS_o_ai __vector float907static inline __ATTRS_o_ai __vector float
825vec_sel(__vector float __a, __vector float __b, __vector unsigned int __c) {908vec_sel(__vector float __a, __vector float __b, __vector unsigned int __c) {
...@@ -1078,6 +1161,22 @@ vec_xl(long __offset, const unsigned long long *__ptr) {...@@ -1078,6 +1161,22 @@ vec_xl(long __offset, const unsigned long long *__ptr) {
1078 return V;1161 return V;
1079}1162}
10801163
1164static inline __ATTRS_o_ai __vector signed __int128
1165vec_xl(long __offset, const signed __int128 *__ptr) {
1166 __vector signed __int128 V;
1167 __builtin_memcpy(&V, ((const char *)__ptr + __offset),
1168 sizeof(__vector signed __int128));
1169 return V;
1170}
1171
1172static inline __ATTRS_o_ai __vector unsigned __int128
1173vec_xl(long __offset, const unsigned __int128 *__ptr) {
1174 __vector unsigned __int128 V;
1175 __builtin_memcpy(&V, ((const char *)__ptr + __offset),
1176 sizeof(__vector unsigned __int128));
1177 return V;
1178}
1179
1081#if __ARCH__ >= 121180#if __ARCH__ >= 12
1082static inline __ATTRS_o_ai __vector float1181static inline __ATTRS_o_ai __vector float
1083vec_xl(long __offset, const float *__ptr) {1182vec_xl(long __offset, const float *__ptr) {
...@@ -1294,6 +1393,22 @@ vec_xst(__vector unsigned long long __vec, long __offset,...@@ -1294,6 +1393,22 @@ vec_xst(__vector unsigned long long __vec, long __offset,
1294 sizeof(__vector unsigned long long));1393 sizeof(__vector unsigned long long));
1295}1394}
12961395
1396static inline __ATTRS_o_ai void
1397vec_xst(__vector signed __int128 __vec, long __offset,
1398 signed __int128 *__ptr) {
1399 __vector signed __int128 V = __vec;
1400 __builtin_memcpy(((char *)__ptr + __offset), &V,
1401 sizeof(__vector signed __int128));
1402}
1403
1404static inline __ATTRS_o_ai void
1405vec_xst(__vector unsigned __int128 __vec, long __offset,
1406 unsigned __int128 *__ptr) {
1407 __vector unsigned __int128 V = __vec;
1408 __builtin_memcpy(((char *)__ptr + __offset), &V,
1409 sizeof(__vector unsigned __int128));
1410}
1411
1297#if __ARCH__ >= 121412#if __ARCH__ >= 12
1298static inline __ATTRS_o_ai void1413static inline __ATTRS_o_ai void
1299vec_xst(__vector float __vec, long __offset, float *__ptr) {1414vec_xst(__vector float __vec, long __offset, float *__ptr) {
...@@ -1465,6 +1580,14 @@ extern __ATTRS_o __vector unsigned long long...@@ -1465,6 +1580,14 @@ extern __ATTRS_o __vector unsigned long long
1465vec_load_bndry(const unsigned long long *__ptr, unsigned short __len)1580vec_load_bndry(const unsigned long long *__ptr, unsigned short __len)
1466 __constant_pow2_range(__len, 64, 4096);1581 __constant_pow2_range(__len, 64, 4096);
14671582
1583extern __ATTRS_o __vector signed __int128
1584vec_load_bndry(const signed __int128 *__ptr, unsigned short __len)
1585 __constant_pow2_range(__len, 64, 4096);
1586
1587extern __ATTRS_o __vector unsigned __int128
1588vec_load_bndry(const unsigned __int128 *__ptr, unsigned short __len)
1589 __constant_pow2_range(__len, 64, 4096);
1590
1468#if __ARCH__ >= 121591#if __ARCH__ >= 12
1469extern __ATTRS_o __vector float1592extern __ATTRS_o __vector float
1470vec_load_bndry(const float *__ptr, unsigned short __len)1593vec_load_bndry(const float *__ptr, unsigned short __len)
...@@ -1496,43 +1619,51 @@ vec_load_len(const unsigned char *__ptr, unsigned int __len) {...@@ -1496,43 +1619,51 @@ vec_load_len(const unsigned char *__ptr, unsigned int __len) {
1496 return (__vector unsigned char)__builtin_s390_vll(__len, __ptr);1619 return (__vector unsigned char)__builtin_s390_vll(__len, __ptr);
1497}1620}
14981621
1622// This prototype is deprecated.
1499static inline __ATTRS_o_ai __vector signed short1623static inline __ATTRS_o_ai __vector signed short
1500vec_load_len(const signed short *__ptr, unsigned int __len) {1624vec_load_len(const signed short *__ptr, unsigned int __len) {
1501 return (__vector signed short)__builtin_s390_vll(__len, __ptr);1625 return (__vector signed short)__builtin_s390_vll(__len, __ptr);
1502}1626}
15031627
1628// This prototype is deprecated.
1504static inline __ATTRS_o_ai __vector unsigned short1629static inline __ATTRS_o_ai __vector unsigned short
1505vec_load_len(const unsigned short *__ptr, unsigned int __len) {1630vec_load_len(const unsigned short *__ptr, unsigned int __len) {
1506 return (__vector unsigned short)__builtin_s390_vll(__len, __ptr);1631 return (__vector unsigned short)__builtin_s390_vll(__len, __ptr);
1507}1632}
15081633
1634// This prototype is deprecated.
1509static inline __ATTRS_o_ai __vector signed int1635static inline __ATTRS_o_ai __vector signed int
1510vec_load_len(const signed int *__ptr, unsigned int __len) {1636vec_load_len(const signed int *__ptr, unsigned int __len) {
1511 return (__vector signed int)__builtin_s390_vll(__len, __ptr);1637 return (__vector signed int)__builtin_s390_vll(__len, __ptr);
1512}1638}
15131639
1640// This prototype is deprecated.
1514static inline __ATTRS_o_ai __vector unsigned int1641static inline __ATTRS_o_ai __vector unsigned int
1515vec_load_len(const unsigned int *__ptr, unsigned int __len) {1642vec_load_len(const unsigned int *__ptr, unsigned int __len) {
1516 return (__vector unsigned int)__builtin_s390_vll(__len, __ptr);1643 return (__vector unsigned int)__builtin_s390_vll(__len, __ptr);
1517}1644}
15181645
1646// This prototype is deprecated.
1519static inline __ATTRS_o_ai __vector signed long long1647static inline __ATTRS_o_ai __vector signed long long
1520vec_load_len(const signed long long *__ptr, unsigned int __len) {1648vec_load_len(const signed long long *__ptr, unsigned int __len) {
1521 return (__vector signed long long)__builtin_s390_vll(__len, __ptr);1649 return (__vector signed long long)__builtin_s390_vll(__len, __ptr);
1522}1650}
15231651
1652// This prototype is deprecated.
1524static inline __ATTRS_o_ai __vector unsigned long long1653static inline __ATTRS_o_ai __vector unsigned long long
1525vec_load_len(const unsigned long long *__ptr, unsigned int __len) {1654vec_load_len(const unsigned long long *__ptr, unsigned int __len) {
1526 return (__vector unsigned long long)__builtin_s390_vll(__len, __ptr);1655 return (__vector unsigned long long)__builtin_s390_vll(__len, __ptr);
1527}1656}
15281657
1529#if __ARCH__ >= 121658#if __ARCH__ >= 12
1659// This prototype is deprecated.
1530static inline __ATTRS_o_ai __vector float1660static inline __ATTRS_o_ai __vector float
1531vec_load_len(const float *__ptr, unsigned int __len) {1661vec_load_len(const float *__ptr, unsigned int __len) {
1532 return (__vector float)__builtin_s390_vll(__len, __ptr);1662 return (__vector float)__builtin_s390_vll(__len, __ptr);
1533}1663}
1534#endif1664#endif
15351665
1666// This prototype is deprecated.
1536static inline __ATTRS_o_ai __vector double1667static inline __ATTRS_o_ai __vector double
1537vec_load_len(const double *__ptr, unsigned int __len) {1668vec_load_len(const double *__ptr, unsigned int __len) {
1538 return (__vector double)__builtin_s390_vll(__len, __ptr);1669 return (__vector double)__builtin_s390_vll(__len, __ptr);
...@@ -1541,7 +1672,12 @@ vec_load_len(const double *__ptr, unsigned int __len) {...@@ -1541,7 +1672,12 @@ vec_load_len(const double *__ptr, unsigned int __len) {
1541/*-- vec_load_len_r ---------------------------------------------------------*/1672/*-- vec_load_len_r ---------------------------------------------------------*/
15421673
1543#if __ARCH__ >= 121674#if __ARCH__ >= 12
1544static inline __ATTRS_ai __vector unsigned char1675static inline __ATTRS_o_ai __vector signed char
1676vec_load_len_r(const signed char *__ptr, unsigned int __len) {
1677 return (__vector signed char)__builtin_s390_vlrlr(__len, __ptr);
1678}
1679
1680static inline __ATTRS_o_ai __vector unsigned char
1545vec_load_len_r(const unsigned char *__ptr, unsigned int __len) {1681vec_load_len_r(const unsigned char *__ptr, unsigned int __len) {
1546 return (__vector unsigned char)__builtin_s390_vlrlr(__len, __ptr);1682 return (__vector unsigned char)__builtin_s390_vlrlr(__len, __ptr);
1547}1683}
...@@ -1561,36 +1697,42 @@ vec_store_len(__vector unsigned char __vec, unsigned char *__ptr,...@@ -1561,36 +1697,42 @@ vec_store_len(__vector unsigned char __vec, unsigned char *__ptr,
1561 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);1697 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
1562}1698}
15631699
1700// This prototype is deprecated.
1564static inline __ATTRS_o_ai void1701static inline __ATTRS_o_ai void
1565vec_store_len(__vector signed short __vec, signed short *__ptr,1702vec_store_len(__vector signed short __vec, signed short *__ptr,
1566 unsigned int __len) {1703 unsigned int __len) {
1567 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);1704 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
1568}1705}
15691706
1707// This prototype is deprecated.
1570static inline __ATTRS_o_ai void1708static inline __ATTRS_o_ai void
1571vec_store_len(__vector unsigned short __vec, unsigned short *__ptr,1709vec_store_len(__vector unsigned short __vec, unsigned short *__ptr,
1572 unsigned int __len) {1710 unsigned int __len) {
1573 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);1711 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
1574}1712}
15751713
1714// This prototype is deprecated.
1576static inline __ATTRS_o_ai void1715static inline __ATTRS_o_ai void
1577vec_store_len(__vector signed int __vec, signed int *__ptr,1716vec_store_len(__vector signed int __vec, signed int *__ptr,
1578 unsigned int __len) {1717 unsigned int __len) {
1579 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);1718 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
1580}1719}
15811720
1721// This prototype is deprecated.
1582static inline __ATTRS_o_ai void1722static inline __ATTRS_o_ai void
1583vec_store_len(__vector unsigned int __vec, unsigned int *__ptr,1723vec_store_len(__vector unsigned int __vec, unsigned int *__ptr,
1584 unsigned int __len) {1724 unsigned int __len) {
1585 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);1725 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
1586}1726}
15871727
1728// This prototype is deprecated.
1588static inline __ATTRS_o_ai void1729static inline __ATTRS_o_ai void
1589vec_store_len(__vector signed long long __vec, signed long long *__ptr,1730vec_store_len(__vector signed long long __vec, signed long long *__ptr,
1590 unsigned int __len) {1731 unsigned int __len) {
1591 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);1732 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
1592}1733}
15931734
1735// This prototype is deprecated.
1594static inline __ATTRS_o_ai void1736static inline __ATTRS_o_ai void
1595vec_store_len(__vector unsigned long long __vec, unsigned long long *__ptr,1737vec_store_len(__vector unsigned long long __vec, unsigned long long *__ptr,
1596 unsigned int __len) {1738 unsigned int __len) {
...@@ -1598,6 +1740,7 @@ vec_store_len(__vector unsigned long long __vec, unsigned long long *__ptr,...@@ -1598,6 +1740,7 @@ vec_store_len(__vector unsigned long long __vec, unsigned long long *__ptr,
1598}1740}
15991741
1600#if __ARCH__ >= 121742#if __ARCH__ >= 12
1743// This prototype is deprecated.
1601static inline __ATTRS_o_ai void1744static inline __ATTRS_o_ai void
1602vec_store_len(__vector float __vec, float *__ptr,1745vec_store_len(__vector float __vec, float *__ptr,
1603 unsigned int __len) {1746 unsigned int __len) {
...@@ -1605,6 +1748,7 @@ vec_store_len(__vector float __vec, float *__ptr,...@@ -1605,6 +1748,7 @@ vec_store_len(__vector float __vec, float *__ptr,
1605}1748}
1606#endif1749#endif
16071750
1751// This prototype is deprecated.
1608static inline __ATTRS_o_ai void1752static inline __ATTRS_o_ai void
1609vec_store_len(__vector double __vec, double *__ptr,1753vec_store_len(__vector double __vec, double *__ptr,
1610 unsigned int __len) {1754 unsigned int __len) {
...@@ -1614,7 +1758,13 @@ vec_store_len(__vector double __vec, double *__ptr,...@@ -1614,7 +1758,13 @@ vec_store_len(__vector double __vec, double *__ptr,
1614/*-- vec_store_len_r --------------------------------------------------------*/1758/*-- vec_store_len_r --------------------------------------------------------*/
16151759
1616#if __ARCH__ >= 121760#if __ARCH__ >= 12
1617static inline __ATTRS_ai void1761static inline __ATTRS_o_ai void
1762vec_store_len_r(__vector signed char __vec, signed char *__ptr,
1763 unsigned int __len) {
1764 __builtin_s390_vstrlr(__vec, __len, __ptr);
1765}
1766
1767static inline __ATTRS_o_ai void
1618vec_store_len_r(__vector unsigned char __vec, unsigned char *__ptr,1768vec_store_len_r(__vector unsigned char __vec, unsigned char *__ptr,
1619 unsigned int __len) {1769 unsigned int __len) {
1620 __builtin_s390_vstrlr((__vector signed char)__vec, __len, __ptr);1770 __builtin_s390_vstrlr((__vector signed char)__vec, __len, __ptr);
...@@ -1711,6 +1861,35 @@ vec_genmasks_64(unsigned char __first, unsigned char __last)...@@ -1711,6 +1861,35 @@ vec_genmasks_64(unsigned char __first, unsigned char __last)
1711 return (__vector unsigned long long)__value;1861 return (__vector unsigned long long)__value;
1712}1862}
17131863
1864/*-- vec_gen_element_masks_* ------------------------------------------------*/
1865
1866#if __ARCH__ >= 15
1867static inline __ATTRS_ai __vector unsigned char
1868vec_gen_element_masks_8(__vector unsigned short __mask) {
1869 return __builtin_s390_vgemb(__mask);
1870}
1871
1872static inline __ATTRS_ai __vector unsigned short
1873vec_gen_element_masks_16(__vector unsigned char __mask) {
1874 return __builtin_s390_vgemh(__mask);
1875}
1876
1877static inline __ATTRS_ai __vector unsigned int
1878vec_gen_element_masks_32(__vector unsigned char __mask) {
1879 return __builtin_s390_vgemf(__mask);
1880}
1881
1882static inline __ATTRS_ai __vector unsigned long long
1883vec_gen_element_masks_64(__vector unsigned char __mask) {
1884 return __builtin_s390_vgemg(__mask);
1885}
1886
1887static inline __ATTRS_ai __vector unsigned __int128
1888vec_gen_element_masks_128(__vector unsigned char __mask) {
1889 return (__vector unsigned __int128)__builtin_s390_vgemq(__mask);
1890}
1891#endif
1892
1714/*-- vec_splat --------------------------------------------------------------*/1893/*-- vec_splat --------------------------------------------------------------*/
17151894
1716static inline __ATTRS_o_ai __vector signed char1895static inline __ATTRS_o_ai __vector signed char
...@@ -1894,6 +2073,16 @@ vec_splats(unsigned long long __scalar) {...@@ -1894,6 +2073,16 @@ vec_splats(unsigned long long __scalar) {
1894 return (__vector unsigned long long)__scalar;2073 return (__vector unsigned long long)__scalar;
1895}2074}
18962075
2076static inline __ATTRS_o_ai __vector signed __int128
2077vec_splats(signed __int128 __scalar) {
2078 return (__vector signed __int128)__scalar;
2079}
2080
2081static inline __ATTRS_o_ai __vector unsigned __int128
2082vec_splats(unsigned __int128 __scalar) {
2083 return (__vector unsigned __int128)__scalar;
2084}
2085
1897#if __ARCH__ >= 122086#if __ARCH__ >= 12
1898static inline __ATTRS_o_ai __vector float2087static inline __ATTRS_o_ai __vector float
1899vec_splats(float __scalar) {2088vec_splats(float __scalar) {
...@@ -2166,6 +2355,27 @@ vec_pack(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -2166,6 +2355,27 @@ vec_pack(__vector unsigned long long __a, __vector unsigned long long __b) {
2166 return (__vector unsigned int)(__ac[1], __ac[3], __bc[1], __bc[3]);2355 return (__vector unsigned int)(__ac[1], __ac[3], __bc[1], __bc[3]);
2167}2356}
21682357
2358static inline __ATTRS_o_ai __vector signed long long
2359vec_pack(__vector signed __int128 __a, __vector signed __int128 __b) {
2360 __vector signed long long __ac = (__vector signed long long)__a;
2361 __vector signed long long __bc = (__vector signed long long)__b;
2362 return (__vector signed long long)(__ac[1], __bc[1]);
2363}
2364
2365static inline __ATTRS_o_ai __vector __bool long long
2366vec_pack(__vector __bool __int128 __a, __vector __bool __int128 __b) {
2367 __vector __bool long long __ac = (__vector __bool long long)__a;
2368 __vector __bool long long __bc = (__vector __bool long long)__b;
2369 return (__vector __bool long long)(__ac[1], __bc[1]);
2370}
2371
2372static inline __ATTRS_o_ai __vector unsigned long long
2373vec_pack(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2374 __vector unsigned long long __ac = (__vector unsigned long long)__a;
2375 __vector unsigned long long __bc = (__vector unsigned long long)__b;
2376 return (__vector unsigned long long)(__ac[1], __bc[1]);
2377}
2378
2169/*-- vec_packs --------------------------------------------------------------*/2379/*-- vec_packs --------------------------------------------------------------*/
21702380
2171static inline __ATTRS_o_ai __vector signed char2381static inline __ATTRS_o_ai __vector signed char
...@@ -2344,6 +2554,24 @@ vec_unpackh(__vector unsigned int __a) {...@@ -2344,6 +2554,24 @@ vec_unpackh(__vector unsigned int __a) {
2344 return __builtin_s390_vuplhf(__a);2554 return __builtin_s390_vuplhf(__a);
2345}2555}
23462556
2557#if __ARCH__ >= 15
2558static inline __ATTRS_o_ai __vector signed __int128
2559vec_unpackh(__vector signed long long __a) {
2560 return (__vector signed __int128)__builtin_s390_vuphg(__a);
2561}
2562
2563static inline __ATTRS_o_ai __vector __bool __int128
2564vec_unpackh(__vector __bool long long __a) {
2565 return ((__vector __bool __int128)
2566 __builtin_s390_vuphg((__vector signed long long)__a));
2567}
2568
2569static inline __ATTRS_o_ai __vector unsigned __int128
2570vec_unpackh(__vector unsigned long long __a) {
2571 return (__vector unsigned __int128)__builtin_s390_vuplhg(__a);
2572}
2573#endif
2574
2347/*-- vec_unpackl ------------------------------------------------------------*/2575/*-- vec_unpackl ------------------------------------------------------------*/
23482576
2349static inline __ATTRS_o_ai __vector signed short2577static inline __ATTRS_o_ai __vector signed short
...@@ -2394,6 +2622,24 @@ vec_unpackl(__vector unsigned int __a) {...@@ -2394,6 +2622,24 @@ vec_unpackl(__vector unsigned int __a) {
2394 return __builtin_s390_vupllf(__a);2622 return __builtin_s390_vupllf(__a);
2395}2623}
23962624
2625#if __ARCH__ >= 15
2626static inline __ATTRS_o_ai __vector signed __int128
2627vec_unpackl(__vector signed long long __a) {
2628 return (__vector signed __int128)__builtin_s390_vuplg(__a);
2629}
2630
2631static inline __ATTRS_o_ai __vector __bool __int128
2632vec_unpackl(__vector __bool long long __a) {
2633 return ((__vector __bool __int128)
2634 __builtin_s390_vuplg((__vector signed long long)__a));
2635}
2636
2637static inline __ATTRS_o_ai __vector unsigned __int128
2638vec_unpackl(__vector unsigned long long __a) {
2639 return (__vector unsigned __int128)__builtin_s390_vupllg(__a);
2640}
2641#endif
2642
2397/*-- vec_cmpeq --------------------------------------------------------------*/2643/*-- vec_cmpeq --------------------------------------------------------------*/
23982644
2399static inline __ATTRS_o_ai __vector __bool char2645static inline __ATTRS_o_ai __vector __bool char
...@@ -2456,6 +2702,21 @@ vec_cmpeq(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -2456,6 +2702,21 @@ vec_cmpeq(__vector unsigned long long __a, __vector unsigned long long __b) {
2456 return (__vector __bool long long)(__a == __b);2702 return (__vector __bool long long)(__a == __b);
2457}2703}
24582704
2705static inline __ATTRS_o_ai __vector __bool __int128
2706vec_cmpeq(__vector __bool __int128 __a, __vector __bool __int128 __b) {
2707 return (__vector __bool __int128)(__a == __b);
2708}
2709
2710static inline __ATTRS_o_ai __vector __bool __int128
2711vec_cmpeq(__vector signed __int128 __a, __vector signed __int128 __b) {
2712 return (__vector __bool __int128)(__a == __b);
2713}
2714
2715static inline __ATTRS_o_ai __vector __bool __int128
2716vec_cmpeq(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2717 return (__vector __bool __int128)(__a == __b);
2718}
2719
2459#if __ARCH__ >= 122720#if __ARCH__ >= 12
2460static inline __ATTRS_o_ai __vector __bool int2721static inline __ATTRS_o_ai __vector __bool int
2461vec_cmpeq(__vector float __a, __vector float __b) {2722vec_cmpeq(__vector float __a, __vector float __b) {
...@@ -2510,6 +2771,16 @@ vec_cmpge(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -2510,6 +2771,16 @@ vec_cmpge(__vector unsigned long long __a, __vector unsigned long long __b) {
2510 return (__vector __bool long long)(__a >= __b);2771 return (__vector __bool long long)(__a >= __b);
2511}2772}
25122773
2774static inline __ATTRS_o_ai __vector __bool __int128
2775vec_cmpge(__vector signed __int128 __a, __vector signed __int128 __b) {
2776 return (__vector __bool __int128)(__a >= __b);
2777}
2778
2779static inline __ATTRS_o_ai __vector __bool __int128
2780vec_cmpge(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2781 return (__vector __bool __int128)(__a >= __b);
2782}
2783
2513#if __ARCH__ >= 122784#if __ARCH__ >= 12
2514static inline __ATTRS_o_ai __vector __bool int2785static inline __ATTRS_o_ai __vector __bool int
2515vec_cmpge(__vector float __a, __vector float __b) {2786vec_cmpge(__vector float __a, __vector float __b) {
...@@ -2564,6 +2835,16 @@ vec_cmpgt(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -2564,6 +2835,16 @@ vec_cmpgt(__vector unsigned long long __a, __vector unsigned long long __b) {
2564 return (__vector __bool long long)(__a > __b);2835 return (__vector __bool long long)(__a > __b);
2565}2836}
25662837
2838static inline __ATTRS_o_ai __vector __bool __int128
2839vec_cmpgt(__vector signed __int128 __a, __vector signed __int128 __b) {
2840 return (__vector __bool __int128)(__a > __b);
2841}
2842
2843static inline __ATTRS_o_ai __vector __bool __int128
2844vec_cmpgt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2845 return (__vector __bool __int128)(__a > __b);
2846}
2847
2567#if __ARCH__ >= 122848#if __ARCH__ >= 12
2568static inline __ATTRS_o_ai __vector __bool int2849static inline __ATTRS_o_ai __vector __bool int
2569vec_cmpgt(__vector float __a, __vector float __b) {2850vec_cmpgt(__vector float __a, __vector float __b) {
...@@ -2618,6 +2899,16 @@ vec_cmple(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -2618,6 +2899,16 @@ vec_cmple(__vector unsigned long long __a, __vector unsigned long long __b) {
2618 return (__vector __bool long long)(__a <= __b);2899 return (__vector __bool long long)(__a <= __b);
2619}2900}
26202901
2902static inline __ATTRS_o_ai __vector __bool __int128
2903vec_cmple(__vector signed __int128 __a, __vector signed __int128 __b) {
2904 return (__vector __bool __int128)(__a <= __b);
2905}
2906
2907static inline __ATTRS_o_ai __vector __bool __int128
2908vec_cmple(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2909 return (__vector __bool __int128)(__a <= __b);
2910}
2911
2621#if __ARCH__ >= 122912#if __ARCH__ >= 12
2622static inline __ATTRS_o_ai __vector __bool int2913static inline __ATTRS_o_ai __vector __bool int
2623vec_cmple(__vector float __a, __vector float __b) {2914vec_cmple(__vector float __a, __vector float __b) {
...@@ -2672,6 +2963,16 @@ vec_cmplt(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -2672,6 +2963,16 @@ vec_cmplt(__vector unsigned long long __a, __vector unsigned long long __b) {
2672 return (__vector __bool long long)(__a < __b);2963 return (__vector __bool long long)(__a < __b);
2673}2964}
26742965
2966static inline __ATTRS_o_ai __vector __bool __int128
2967vec_cmplt(__vector signed __int128 __a, __vector signed __int128 __b) {
2968 return (__vector __bool __int128)(__a < __b);
2969}
2970
2971static inline __ATTRS_o_ai __vector __bool __int128
2972vec_cmplt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2973 return (__vector __bool __int128)(__a < __b);
2974}
2975
2675#if __ARCH__ >= 122976#if __ARCH__ >= 12
2676static inline __ATTRS_o_ai __vector __bool int2977static inline __ATTRS_o_ai __vector __bool int
2677vec_cmplt(__vector float __a, __vector float __b) {2978vec_cmplt(__vector float __a, __vector float __b) {
...@@ -2914,6 +3215,29 @@ vec_all_eq(__vector __bool long long __a, __vector __bool long long __b) {...@@ -2914,6 +3215,29 @@ vec_all_eq(__vector __bool long long __a, __vector __bool long long __b) {
2914 return __cc == 0;3215 return __cc == 0;
2915}3216}
29163217
3218#if __ARCH__ >= 15
3219static inline __ATTRS_o_ai int
3220vec_all_eq(__vector signed __int128 __a, __vector signed __int128 __b) {
3221 int __cc;
3222 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3223 return __cc == 0;
3224}
3225
3226static inline __ATTRS_o_ai int
3227vec_all_eq(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
3228 int __cc;
3229 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3230 return __cc == 0;
3231}
3232
3233static inline __ATTRS_o_ai int
3234vec_all_eq(__vector __bool __int128 __a, __vector __bool __int128 __b) {
3235 int __cc;
3236 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3237 return __cc == 0;
3238}
3239#endif
3240
2917#if __ARCH__ >= 123241#if __ARCH__ >= 12
2918static inline __ATTRS_o_ai int3242static inline __ATTRS_o_ai int
2919vec_all_eq(__vector float __a, __vector float __b) {3243vec_all_eq(__vector float __a, __vector float __b) {
...@@ -3161,6 +3485,29 @@ vec_all_ne(__vector __bool long long __a, __vector __bool long long __b) {...@@ -3161,6 +3485,29 @@ vec_all_ne(__vector __bool long long __a, __vector __bool long long __b) {
3161 return __cc == 3;3485 return __cc == 3;
3162}3486}
31633487
3488#if __ARCH__ >= 15
3489static inline __ATTRS_o_ai int
3490vec_all_ne(__vector signed __int128 __a, __vector signed __int128 __b) {
3491 int __cc;
3492 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3493 return __cc == 3;
3494}
3495
3496static inline __ATTRS_o_ai int
3497vec_all_ne(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
3498 int __cc;
3499 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3500 return __cc == 3;
3501}
3502
3503static inline __ATTRS_o_ai int
3504vec_all_ne(__vector __bool __int128 __a, __vector __bool __int128 __b) {
3505 int __cc;
3506 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3507 return __cc == 3;
3508}
3509#endif
3510
3164#if __ARCH__ >= 123511#if __ARCH__ >= 12
3165static inline __ATTRS_o_ai int3512static inline __ATTRS_o_ai int
3166vec_all_ne(__vector float __a, __vector float __b) {3513vec_all_ne(__vector float __a, __vector float __b) {
...@@ -3399,6 +3746,22 @@ vec_all_ge(__vector __bool long long __a, __vector __bool long long __b) {...@@ -3399,6 +3746,22 @@ vec_all_ge(__vector __bool long long __a, __vector __bool long long __b) {
3399 return __cc == 3;3746 return __cc == 3;
3400}3747}
34013748
3749#if __ARCH__ >= 15
3750static inline __ATTRS_o_ai int
3751vec_all_ge(__vector signed __int128 __a, __vector signed __int128 __b) {
3752 int __cc;
3753 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
3754 return __cc == 3;
3755}
3756
3757static inline __ATTRS_o_ai int
3758vec_all_ge(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
3759 int __cc;
3760 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
3761 return __cc == 3;
3762}
3763#endif
3764
3402#if __ARCH__ >= 123765#if __ARCH__ >= 12
3403static inline __ATTRS_o_ai int3766static inline __ATTRS_o_ai int
3404vec_all_ge(__vector float __a, __vector float __b) {3767vec_all_ge(__vector float __a, __vector float __b) {
...@@ -3637,6 +4000,22 @@ vec_all_gt(__vector __bool long long __a, __vector __bool long long __b) {...@@ -3637,6 +4000,22 @@ vec_all_gt(__vector __bool long long __a, __vector __bool long long __b) {
3637 return __cc == 0;4000 return __cc == 0;
3638}4001}
36394002
4003#if __ARCH__ >= 15
4004static inline __ATTRS_o_ai int
4005vec_all_gt(__vector signed __int128 __a, __vector signed __int128 __b) {
4006 int __cc;
4007 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
4008 return __cc == 0;
4009}
4010
4011static inline __ATTRS_o_ai int
4012vec_all_gt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4013 int __cc;
4014 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4015 return __cc == 0;
4016}
4017#endif
4018
3640#if __ARCH__ >= 124019#if __ARCH__ >= 12
3641static inline __ATTRS_o_ai int4020static inline __ATTRS_o_ai int
3642vec_all_gt(__vector float __a, __vector float __b) {4021vec_all_gt(__vector float __a, __vector float __b) {
...@@ -3875,6 +4254,22 @@ vec_all_le(__vector __bool long long __a, __vector __bool long long __b) {...@@ -3875,6 +4254,22 @@ vec_all_le(__vector __bool long long __a, __vector __bool long long __b) {
3875 return __cc == 3;4254 return __cc == 3;
3876}4255}
38774256
4257#if __ARCH__ >= 15
4258static inline __ATTRS_o_ai int
4259vec_all_le(__vector signed __int128 __a, __vector signed __int128 __b) {
4260 int __cc;
4261 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
4262 return __cc == 3;
4263}
4264
4265static inline __ATTRS_o_ai int
4266vec_all_le(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4267 int __cc;
4268 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4269 return __cc == 3;
4270}
4271#endif
4272
3878#if __ARCH__ >= 124273#if __ARCH__ >= 12
3879static inline __ATTRS_o_ai int4274static inline __ATTRS_o_ai int
3880vec_all_le(__vector float __a, __vector float __b) {4275vec_all_le(__vector float __a, __vector float __b) {
...@@ -4113,6 +4508,22 @@ vec_all_lt(__vector __bool long long __a, __vector __bool long long __b) {...@@ -4113,6 +4508,22 @@ vec_all_lt(__vector __bool long long __a, __vector __bool long long __b) {
4113 return __cc == 0;4508 return __cc == 0;
4114}4509}
41154510
4511#if __ARCH__ >= 15
4512static inline __ATTRS_o_ai int
4513vec_all_lt(__vector signed __int128 __a, __vector signed __int128 __b) {
4514 int __cc;
4515 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
4516 return __cc == 0;
4517}
4518
4519static inline __ATTRS_o_ai int
4520vec_all_lt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4521 int __cc;
4522 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
4523 return __cc == 0;
4524}
4525#endif
4526
4116#if __ARCH__ >= 124527#if __ARCH__ >= 12
4117static inline __ATTRS_o_ai int4528static inline __ATTRS_o_ai int
4118vec_all_lt(__vector float __a, __vector float __b) {4529vec_all_lt(__vector float __a, __vector float __b) {
...@@ -4467,6 +4878,29 @@ vec_any_eq(__vector __bool long long __a, __vector __bool long long __b) {...@@ -4467,6 +4878,29 @@ vec_any_eq(__vector __bool long long __a, __vector __bool long long __b) {
4467 return __cc <= 1;4878 return __cc <= 1;
4468}4879}
44694880
4881#if __ARCH__ >= 15
4882static inline __ATTRS_o_ai int
4883vec_any_eq(__vector signed __int128 __a, __vector signed __int128 __b) {
4884 int __cc;
4885 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4886 return __cc <= 1;
4887}
4888
4889static inline __ATTRS_o_ai int
4890vec_any_eq(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4891 int __cc;
4892 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4893 return __cc <= 1;
4894}
4895
4896static inline __ATTRS_o_ai int
4897vec_any_eq(__vector __bool __int128 __a, __vector __bool __int128 __b) {
4898 int __cc;
4899 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4900 return __cc <= 1;
4901}
4902#endif
4903
4470#if __ARCH__ >= 124904#if __ARCH__ >= 12
4471static inline __ATTRS_o_ai int4905static inline __ATTRS_o_ai int
4472vec_any_eq(__vector float __a, __vector float __b) {4906vec_any_eq(__vector float __a, __vector float __b) {
...@@ -4713,28 +5147,51 @@ vec_any_ne(__vector __bool long long __a, __vector __bool long long __b) {...@@ -4713,28 +5147,51 @@ vec_any_ne(__vector __bool long long __a, __vector __bool long long __b) {
4713 return __cc != 0;5147 return __cc != 0;
4714}5148}
47155149
4716#if __ARCH__ >= 125150#if __ARCH__ >= 15
4717static inline __ATTRS_o_ai int5151static inline __ATTRS_o_ai int
4718vec_any_ne(__vector float __a, __vector float __b) {5152vec_any_ne(__vector signed __int128 __a, __vector signed __int128 __b) {
4719 int __cc;5153 int __cc;
4720 __builtin_s390_vfcesbs(__a, __b, &__cc);5154 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4721 return __cc != 0;5155 return __cc != 0;
4722}5156}
4723#endif
47245157
4725static inline __ATTRS_o_ai int5158static inline __ATTRS_o_ai int
4726vec_any_ne(__vector double __a, __vector double __b) {5159vec_any_ne(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4727 int __cc;5160 int __cc;
4728 __builtin_s390_vfcedbs(__a, __b, &__cc);5161 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4729 return __cc != 0;5162 return __cc != 0;
4730}5163}
47315164
4732/*-- vec_any_ge -------------------------------------------------------------*/
4733
4734static inline __ATTRS_o_ai int5165static inline __ATTRS_o_ai int
4735vec_any_ge(__vector signed char __a, __vector signed char __b) {5166vec_any_ne(__vector __bool __int128 __a, __vector __bool __int128 __b) {
4736 int __cc;5167 int __cc;
4737 __builtin_s390_vchbs(__b, __a, &__cc);5168 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
5169 return __cc != 0;
5170}
5171#endif
5172
5173#if __ARCH__ >= 12
5174static inline __ATTRS_o_ai int
5175vec_any_ne(__vector float __a, __vector float __b) {
5176 int __cc;
5177 __builtin_s390_vfcesbs(__a, __b, &__cc);
5178 return __cc != 0;
5179}
5180#endif
5181
5182static inline __ATTRS_o_ai int
5183vec_any_ne(__vector double __a, __vector double __b) {
5184 int __cc;
5185 __builtin_s390_vfcedbs(__a, __b, &__cc);
5186 return __cc != 0;
5187}
5188
5189/*-- vec_any_ge -------------------------------------------------------------*/
5190
5191static inline __ATTRS_o_ai int
5192vec_any_ge(__vector signed char __a, __vector signed char __b) {
5193 int __cc;
5194 __builtin_s390_vchbs(__b, __a, &__cc);
4738 return __cc != 0;5195 return __cc != 0;
4739}5196}
47405197
...@@ -4951,6 +5408,22 @@ vec_any_ge(__vector __bool long long __a, __vector __bool long long __b) {...@@ -4951,6 +5408,22 @@ vec_any_ge(__vector __bool long long __a, __vector __bool long long __b) {
4951 return __cc != 0;5408 return __cc != 0;
4952}5409}
49535410
5411#if __ARCH__ >= 15
5412static inline __ATTRS_o_ai int
5413vec_any_ge(__vector signed __int128 __a, __vector signed __int128 __b) {
5414 int __cc;
5415 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
5416 return __cc != 0;
5417}
5418
5419static inline __ATTRS_o_ai int
5420vec_any_ge(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
5421 int __cc;
5422 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
5423 return __cc != 0;
5424}
5425#endif
5426
4954#if __ARCH__ >= 125427#if __ARCH__ >= 12
4955static inline __ATTRS_o_ai int5428static inline __ATTRS_o_ai int
4956vec_any_ge(__vector float __a, __vector float __b) {5429vec_any_ge(__vector float __a, __vector float __b) {
...@@ -5189,6 +5662,22 @@ vec_any_gt(__vector __bool long long __a, __vector __bool long long __b) {...@@ -5189,6 +5662,22 @@ vec_any_gt(__vector __bool long long __a, __vector __bool long long __b) {
5189 return __cc <= 1;5662 return __cc <= 1;
5190}5663}
51915664
5665#if __ARCH__ >= 15
5666static inline __ATTRS_o_ai int
5667vec_any_gt(__vector signed __int128 __a, __vector signed __int128 __b) {
5668 int __cc;
5669 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
5670 return __cc <= 1;
5671}
5672
5673static inline __ATTRS_o_ai int
5674vec_any_gt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
5675 int __cc;
5676 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
5677 return __cc <= 1;
5678}
5679#endif
5680
5192#if __ARCH__ >= 125681#if __ARCH__ >= 12
5193static inline __ATTRS_o_ai int5682static inline __ATTRS_o_ai int
5194vec_any_gt(__vector float __a, __vector float __b) {5683vec_any_gt(__vector float __a, __vector float __b) {
...@@ -5427,6 +5916,22 @@ vec_any_le(__vector __bool long long __a, __vector __bool long long __b) {...@@ -5427,6 +5916,22 @@ vec_any_le(__vector __bool long long __a, __vector __bool long long __b) {
5427 return __cc != 0;5916 return __cc != 0;
5428}5917}
54295918
5919#if __ARCH__ >= 15
5920static inline __ATTRS_o_ai int
5921vec_any_le(__vector signed __int128 __a, __vector signed __int128 __b) {
5922 int __cc;
5923 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
5924 return __cc != 0;
5925}
5926
5927static inline __ATTRS_o_ai int
5928vec_any_le(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
5929 int __cc;
5930 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
5931 return __cc != 0;
5932}
5933#endif
5934
5430#if __ARCH__ >= 125935#if __ARCH__ >= 12
5431static inline __ATTRS_o_ai int5936static inline __ATTRS_o_ai int
5432vec_any_le(__vector float __a, __vector float __b) {5937vec_any_le(__vector float __a, __vector float __b) {
...@@ -5665,6 +6170,22 @@ vec_any_lt(__vector __bool long long __a, __vector __bool long long __b) {...@@ -5665,6 +6170,22 @@ vec_any_lt(__vector __bool long long __a, __vector __bool long long __b) {
5665 return __cc <= 1;6170 return __cc <= 1;
5666}6171}
56676172
6173#if __ARCH__ >= 15
6174static inline __ATTRS_o_ai int
6175vec_any_lt(__vector signed __int128 __a, __vector signed __int128 __b) {
6176 int __cc;
6177 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
6178 return __cc <= 1;
6179}
6180
6181static inline __ATTRS_o_ai int
6182vec_any_lt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6183 int __cc;
6184 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
6185 return __cc <= 1;
6186}
6187#endif
6188
5668#if __ARCH__ >= 126189#if __ARCH__ >= 12
5669static inline __ATTRS_o_ai int6190static inline __ATTRS_o_ai int
5670vec_any_lt(__vector float __a, __vector float __b) {6191vec_any_lt(__vector float __a, __vector float __b) {
...@@ -5753,40 +6274,419 @@ vec_any_nlt(__vector double __a, __vector double __b) {...@@ -5753,40 +6274,419 @@ vec_any_nlt(__vector double __a, __vector double __b) {
5753 return __cc != 0;6274 return __cc != 0;
5754}6275}
57556276
5756/*-- vec_any_nan ------------------------------------------------------------*/6277/*-- vec_any_nan ------------------------------------------------------------*/
6278
6279#if __ARCH__ >= 12
6280static inline __ATTRS_o_ai int
6281vec_any_nan(__vector float __a) {
6282 int __cc;
6283 __builtin_s390_vftcisb(__a, 15, &__cc);
6284 return __cc != 3;
6285}
6286#endif
6287
6288static inline __ATTRS_o_ai int
6289vec_any_nan(__vector double __a) {
6290 int __cc;
6291 __builtin_s390_vftcidb(__a, 15, &__cc);
6292 return __cc != 3;
6293}
6294
6295/*-- vec_any_numeric --------------------------------------------------------*/
6296
6297#if __ARCH__ >= 12
6298static inline __ATTRS_o_ai int
6299vec_any_numeric(__vector float __a) {
6300 int __cc;
6301 __builtin_s390_vftcisb(__a, 15, &__cc);
6302 return __cc != 0;
6303}
6304#endif
6305
6306static inline __ATTRS_o_ai int
6307vec_any_numeric(__vector double __a) {
6308 int __cc;
6309 __builtin_s390_vftcidb(__a, 15, &__cc);
6310 return __cc != 0;
6311}
6312
6313/*-- vec_blend --------------------------------------------------------------*/
6314
6315#if __ARCH__ >= 15
6316static inline __ATTRS_o_ai __vector signed char
6317vec_blend(__vector signed char __a, __vector signed char __b,
6318 __vector signed char __c) {
6319 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed char)0));
6320}
6321
6322static inline __ATTRS_o_ai __vector __bool char
6323vec_blend(__vector __bool char __a, __vector __bool char __b,
6324 __vector signed char __c) {
6325 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed char)0));
6326}
6327
6328static inline __ATTRS_o_ai __vector unsigned char
6329vec_blend(__vector unsigned char __a, __vector unsigned char __b,
6330 __vector signed char __c) {
6331 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed char)0));
6332}
6333
6334static inline __ATTRS_o_ai __vector signed short
6335vec_blend(__vector signed short __a, __vector signed short __b,
6336 __vector signed short __c) {
6337 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed short)0));
6338}
6339
6340static inline __ATTRS_o_ai __vector __bool short
6341vec_blend(__vector __bool short __a, __vector __bool short __b,
6342 __vector signed short __c) {
6343 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed short)0));
6344}
6345
6346static inline __ATTRS_o_ai __vector unsigned short
6347vec_blend(__vector unsigned short __a, __vector unsigned short __b,
6348 __vector signed short __c) {
6349 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed short)0));
6350}
6351
6352static inline __ATTRS_o_ai __vector signed int
6353vec_blend(__vector signed int __a, __vector signed int __b,
6354 __vector signed int __c) {
6355 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6356}
6357
6358static inline __ATTRS_o_ai __vector __bool int
6359vec_blend(__vector __bool int __a, __vector __bool int __b,
6360 __vector signed int __c) {
6361 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6362}
6363
6364static inline __ATTRS_o_ai __vector unsigned int
6365vec_blend(__vector unsigned int __a, __vector unsigned int __b,
6366 __vector signed int __c) {
6367 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6368}
6369
6370static inline __ATTRS_o_ai __vector signed long long
6371vec_blend(__vector signed long long __a, __vector signed long long __b,
6372 __vector signed long long __c) {
6373 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6374}
6375
6376static inline __ATTRS_o_ai __vector __bool long long
6377vec_blend(__vector __bool long long __a, __vector __bool long long __b,
6378 __vector signed long long __c) {
6379 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6380}
6381
6382static inline __ATTRS_o_ai __vector unsigned long long
6383vec_blend(__vector unsigned long long __a, __vector unsigned long long __b,
6384 __vector signed long long __c) {
6385 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6386}
6387
6388static inline __ATTRS_o_ai __vector signed __int128
6389vec_blend(__vector signed __int128 __a, __vector signed __int128 __b,
6390 __vector signed __int128 __c) {
6391 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed __int128)0));
6392}
6393
6394static inline __ATTRS_o_ai __vector __bool __int128
6395vec_blend(__vector __bool __int128 __a, __vector __bool __int128 __b,
6396 __vector signed __int128 __c) {
6397 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed __int128)0));
6398}
6399
6400static inline __ATTRS_o_ai __vector unsigned __int128
6401vec_blend(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
6402 __vector signed __int128 __c) {
6403 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed __int128)0));
6404}
6405
6406static inline __ATTRS_o_ai __vector float
6407vec_blend(__vector float __a, __vector float __b,
6408 __vector signed int __c) {
6409 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6410}
6411
6412static inline __ATTRS_o_ai __vector double
6413vec_blend(__vector double __a, __vector double __b,
6414 __vector signed long long __c) {
6415 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6416}
6417#endif
6418
6419/*-- vec_and ---------------------------------------------------------------*/
6420
6421static inline __ATTRS_o_ai __vector __bool char
6422vec_and(__vector __bool char __a, __vector __bool char __b) {
6423 return __a & __b;
6424}
6425
6426static inline __ATTRS_o_ai __vector signed char
6427vec_and(__vector signed char __a, __vector signed char __b) {
6428 return __a & __b;
6429}
6430
6431static inline __ATTRS_o_ai __vector unsigned char
6432vec_and(__vector unsigned char __a, __vector unsigned char __b) {
6433 return __a & __b;
6434}
6435
6436static inline __ATTRS_o_ai __vector __bool short
6437vec_and(__vector __bool short __a, __vector __bool short __b) {
6438 return __a & __b;
6439}
6440
6441static inline __ATTRS_o_ai __vector signed short
6442vec_and(__vector signed short __a, __vector signed short __b) {
6443 return __a & __b;
6444}
6445
6446static inline __ATTRS_o_ai __vector unsigned short
6447vec_and(__vector unsigned short __a, __vector unsigned short __b) {
6448 return __a & __b;
6449}
6450
6451static inline __ATTRS_o_ai __vector __bool int
6452vec_and(__vector __bool int __a, __vector __bool int __b) {
6453 return __a & __b;
6454}
6455
6456static inline __ATTRS_o_ai __vector signed int
6457vec_and(__vector signed int __a, __vector signed int __b) {
6458 return __a & __b;
6459}
6460
6461static inline __ATTRS_o_ai __vector unsigned int
6462vec_and(__vector unsigned int __a, __vector unsigned int __b) {
6463 return __a & __b;
6464}
6465
6466static inline __ATTRS_o_ai __vector __bool long long
6467vec_and(__vector __bool long long __a, __vector __bool long long __b) {
6468 return __a & __b;
6469}
6470
6471static inline __ATTRS_o_ai __vector signed long long
6472vec_and(__vector signed long long __a, __vector signed long long __b) {
6473 return __a & __b;
6474}
6475
6476static inline __ATTRS_o_ai __vector unsigned long long
6477vec_and(__vector unsigned long long __a, __vector unsigned long long __b) {
6478 return __a & __b;
6479}
6480
6481static inline __ATTRS_o_ai __vector __bool __int128
6482vec_and(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6483 return __a & __b;
6484}
6485
6486static inline __ATTRS_o_ai __vector signed __int128
6487vec_and(__vector signed __int128 __a, __vector signed __int128 __b) {
6488 return __a & __b;
6489}
6490
6491static inline __ATTRS_o_ai __vector unsigned __int128
6492vec_and(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6493 return __a & __b;
6494}
6495
6496#if __ARCH__ >= 12
6497static inline __ATTRS_o_ai __vector float
6498vec_and(__vector float __a, __vector float __b) {
6499 return (__vector float)((__vector unsigned int)__a &
6500 (__vector unsigned int)__b);
6501}
6502#endif
6503
6504static inline __ATTRS_o_ai __vector double
6505vec_and(__vector double __a, __vector double __b) {
6506 return (__vector double)((__vector unsigned long long)__a &
6507 (__vector unsigned long long)__b);
6508}
6509
6510/*-- vec_or ----------------------------------------------------------------*/
6511
6512static inline __ATTRS_o_ai __vector __bool char
6513vec_or(__vector __bool char __a, __vector __bool char __b) {
6514 return __a | __b;
6515}
6516
6517static inline __ATTRS_o_ai __vector signed char
6518vec_or(__vector signed char __a, __vector signed char __b) {
6519 return __a | __b;
6520}
6521
6522static inline __ATTRS_o_ai __vector unsigned char
6523vec_or(__vector unsigned char __a, __vector unsigned char __b) {
6524 return __a | __b;
6525}
6526
6527static inline __ATTRS_o_ai __vector __bool short
6528vec_or(__vector __bool short __a, __vector __bool short __b) {
6529 return __a | __b;
6530}
6531
6532static inline __ATTRS_o_ai __vector signed short
6533vec_or(__vector signed short __a, __vector signed short __b) {
6534 return __a | __b;
6535}
6536
6537static inline __ATTRS_o_ai __vector unsigned short
6538vec_or(__vector unsigned short __a, __vector unsigned short __b) {
6539 return __a | __b;
6540}
6541
6542static inline __ATTRS_o_ai __vector __bool int
6543vec_or(__vector __bool int __a, __vector __bool int __b) {
6544 return __a | __b;
6545}
6546
6547static inline __ATTRS_o_ai __vector signed int
6548vec_or(__vector signed int __a, __vector signed int __b) {
6549 return __a | __b;
6550}
6551
6552static inline __ATTRS_o_ai __vector unsigned int
6553vec_or(__vector unsigned int __a, __vector unsigned int __b) {
6554 return __a | __b;
6555}
6556
6557static inline __ATTRS_o_ai __vector __bool long long
6558vec_or(__vector __bool long long __a, __vector __bool long long __b) {
6559 return __a | __b;
6560}
6561
6562static inline __ATTRS_o_ai __vector signed long long
6563vec_or(__vector signed long long __a, __vector signed long long __b) {
6564 return __a | __b;
6565}
6566
6567static inline __ATTRS_o_ai __vector unsigned long long
6568vec_or(__vector unsigned long long __a, __vector unsigned long long __b) {
6569 return __a | __b;
6570}
6571
6572static inline __ATTRS_o_ai __vector __bool __int128
6573vec_or(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6574 return __a | __b;
6575}
6576
6577static inline __ATTRS_o_ai __vector signed __int128
6578vec_or(__vector signed __int128 __a, __vector signed __int128 __b) {
6579 return __a | __b;
6580}
6581
6582static inline __ATTRS_o_ai __vector unsigned __int128
6583vec_or(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6584 return __a | __b;
6585}
6586
6587#if __ARCH__ >= 12
6588static inline __ATTRS_o_ai __vector float
6589vec_or(__vector float __a, __vector float __b) {
6590 return (__vector float)((__vector unsigned int)__a |
6591 (__vector unsigned int)__b);
6592}
6593#endif
6594
6595static inline __ATTRS_o_ai __vector double
6596vec_or(__vector double __a, __vector double __b) {
6597 return (__vector double)((__vector unsigned long long)__a |
6598 (__vector unsigned long long)__b);
6599}
6600
6601/*-- vec_xor ----------------------------------------------------------------*/
6602
6603static inline __ATTRS_o_ai __vector __bool char
6604vec_xor(__vector __bool char __a, __vector __bool char __b) {
6605 return __a ^ __b;
6606}
6607
6608static inline __ATTRS_o_ai __vector signed char
6609vec_xor(__vector signed char __a, __vector signed char __b) {
6610 return __a ^ __b;
6611}
6612
6613static inline __ATTRS_o_ai __vector unsigned char
6614vec_xor(__vector unsigned char __a, __vector unsigned char __b) {
6615 return __a ^ __b;
6616}
6617
6618static inline __ATTRS_o_ai __vector __bool short
6619vec_xor(__vector __bool short __a, __vector __bool short __b) {
6620 return __a ^ __b;
6621}
6622
6623static inline __ATTRS_o_ai __vector signed short
6624vec_xor(__vector signed short __a, __vector signed short __b) {
6625 return __a ^ __b;
6626}
6627
6628static inline __ATTRS_o_ai __vector unsigned short
6629vec_xor(__vector unsigned short __a, __vector unsigned short __b) {
6630 return __a ^ __b;
6631}
6632
6633static inline __ATTRS_o_ai __vector __bool int
6634vec_xor(__vector __bool int __a, __vector __bool int __b) {
6635 return __a ^ __b;
6636}
6637
6638static inline __ATTRS_o_ai __vector signed int
6639vec_xor(__vector signed int __a, __vector signed int __b) {
6640 return __a ^ __b;
6641}
6642
6643static inline __ATTRS_o_ai __vector unsigned int
6644vec_xor(__vector unsigned int __a, __vector unsigned int __b) {
6645 return __a ^ __b;
6646}
6647
6648static inline __ATTRS_o_ai __vector __bool long long
6649vec_xor(__vector __bool long long __a, __vector __bool long long __b) {
6650 return __a ^ __b;
6651}
6652
6653static inline __ATTRS_o_ai __vector signed long long
6654vec_xor(__vector signed long long __a, __vector signed long long __b) {
6655 return __a ^ __b;
6656}
6657
6658static inline __ATTRS_o_ai __vector unsigned long long
6659vec_xor(__vector unsigned long long __a, __vector unsigned long long __b) {
6660 return __a ^ __b;
6661}
57576662
5758#if __ARCH__ >= 126663static inline __ATTRS_o_ai __vector __bool __int128
5759static inline __ATTRS_o_ai int6664vec_xor(__vector __bool __int128 __a, __vector __bool __int128 __b) {
5760vec_any_nan(__vector float __a) {6665 return __a ^ __b;
5761 int __cc;
5762 __builtin_s390_vftcisb(__a, 15, &__cc);
5763 return __cc != 3;
5764}6666}
5765#endif
57666667
5767static inline __ATTRS_o_ai int6668static inline __ATTRS_o_ai __vector signed __int128
5768vec_any_nan(__vector double __a) {6669vec_xor(__vector signed __int128 __a, __vector signed __int128 __b) {
5769 int __cc;6670 return __a ^ __b;
5770 __builtin_s390_vftcidb(__a, 15, &__cc);
5771 return __cc != 3;
5772}6671}
57736672
5774/*-- vec_any_numeric --------------------------------------------------------*/6673static inline __ATTRS_o_ai __vector unsigned __int128
6674vec_xor(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6675 return __a ^ __b;
6676}
57756677
5776#if __ARCH__ >= 126678#if __ARCH__ >= 12
5777static inline __ATTRS_o_ai int6679static inline __ATTRS_o_ai __vector float
5778vec_any_numeric(__vector float __a) {6680vec_xor(__vector float __a, __vector float __b) {
5779 int __cc;6681 return (__vector float)((__vector unsigned int)__a ^
5780 __builtin_s390_vftcisb(__a, 15, &__cc);6682 (__vector unsigned int)__b);
5781 return __cc != 0;
5782}6683}
5783#endif6684#endif
57846685
5785static inline __ATTRS_o_ai int6686static inline __ATTRS_o_ai __vector double
5786vec_any_numeric(__vector double __a) {6687vec_xor(__vector double __a, __vector double __b) {
5787 int __cc;6688 return (__vector double)((__vector unsigned long long)__a ^
5788 __builtin_s390_vftcidb(__a, 15, &__cc);6689 (__vector unsigned long long)__b);
5789 return __cc != 0;
5790}6690}
57916691
5792/*-- vec_andc ---------------------------------------------------------------*/6692/*-- vec_andc ---------------------------------------------------------------*/
...@@ -5947,6 +6847,21 @@ vec_andc(__vector unsigned long long __a, __vector __bool long long __b) {...@@ -5947,6 +6847,21 @@ vec_andc(__vector unsigned long long __a, __vector __bool long long __b) {
5947 return __a & ~__b;6847 return __a & ~__b;
5948}6848}
59496849
6850static inline __ATTRS_o_ai __vector __bool __int128
6851vec_andc(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6852 return __a & ~__b;
6853}
6854
6855static inline __ATTRS_o_ai __vector signed __int128
6856vec_andc(__vector signed __int128 __a, __vector signed __int128 __b) {
6857 return __a & ~__b;
6858}
6859
6860static inline __ATTRS_o_ai __vector unsigned __int128
6861vec_andc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6862 return __a & ~__b;
6863}
6864
5950#if __ARCH__ >= 126865#if __ARCH__ >= 12
5951static inline __ATTRS_o_ai __vector float6866static inline __ATTRS_o_ai __vector float
5952vec_andc(__vector float __a, __vector float __b) {6867vec_andc(__vector float __a, __vector float __b) {
...@@ -6133,6 +7048,21 @@ vec_nor(__vector unsigned long long __a, __vector __bool long long __b) {...@@ -6133,6 +7048,21 @@ vec_nor(__vector unsigned long long __a, __vector __bool long long __b) {
6133 return ~(__a | __b);7048 return ~(__a | __b);
6134}7049}
61357050
7051static inline __ATTRS_o_ai __vector __bool __int128
7052vec_nor(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7053 return ~(__a | __b);
7054}
7055
7056static inline __ATTRS_o_ai __vector signed __int128
7057vec_nor(__vector signed __int128 __a, __vector signed __int128 __b) {
7058 return ~(__a | __b);
7059}
7060
7061static inline __ATTRS_o_ai __vector unsigned __int128
7062vec_nor(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7063 return ~(__a | __b);
7064}
7065
6136#if __ARCH__ >= 127066#if __ARCH__ >= 12
6137static inline __ATTRS_o_ai __vector float7067static inline __ATTRS_o_ai __vector float
6138vec_nor(__vector float __a, __vector float __b) {7068vec_nor(__vector float __a, __vector float __b) {
...@@ -6224,6 +7154,21 @@ vec_orc(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -6224,6 +7154,21 @@ vec_orc(__vector unsigned long long __a, __vector unsigned long long __b) {
6224 return __a | ~__b;7154 return __a | ~__b;
6225}7155}
62267156
7157static inline __ATTRS_o_ai __vector __bool __int128
7158vec_orc(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7159 return __a | ~__b;
7160}
7161
7162static inline __ATTRS_o_ai __vector signed __int128
7163vec_orc(__vector signed __int128 __a, __vector signed __int128 __b) {
7164 return __a | ~__b;
7165}
7166
7167static inline __ATTRS_o_ai __vector unsigned __int128
7168vec_orc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7169 return __a | ~__b;
7170}
7171
6227static inline __ATTRS_o_ai __vector float7172static inline __ATTRS_o_ai __vector float
6228vec_orc(__vector float __a, __vector float __b) {7173vec_orc(__vector float __a, __vector float __b) {
6229 return (__vector float)((__vector unsigned int)__a |7174 return (__vector float)((__vector unsigned int)__a |
...@@ -6300,6 +7245,21 @@ vec_nand(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -6300,6 +7245,21 @@ vec_nand(__vector unsigned long long __a, __vector unsigned long long __b) {
6300 return ~(__a & __b);7245 return ~(__a & __b);
6301}7246}
63027247
7248static inline __ATTRS_o_ai __vector __bool __int128
7249vec_nand(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7250 return ~(__a & __b);
7251}
7252
7253static inline __ATTRS_o_ai __vector signed __int128
7254vec_nand(__vector signed __int128 __a, __vector signed __int128 __b) {
7255 return ~(__a & __b);
7256}
7257
7258static inline __ATTRS_o_ai __vector unsigned __int128
7259vec_nand(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7260 return ~(__a & __b);
7261}
7262
6303static inline __ATTRS_o_ai __vector float7263static inline __ATTRS_o_ai __vector float
6304vec_nand(__vector float __a, __vector float __b) {7264vec_nand(__vector float __a, __vector float __b) {
6305 return (__vector float)~((__vector unsigned int)__a &7265 return (__vector float)~((__vector unsigned int)__a &
...@@ -6376,6 +7336,21 @@ vec_eqv(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -6376,6 +7336,21 @@ vec_eqv(__vector unsigned long long __a, __vector unsigned long long __b) {
6376 return ~(__a ^ __b);7336 return ~(__a ^ __b);
6377}7337}
63787338
7339static inline __ATTRS_o_ai __vector __bool __int128
7340vec_eqv(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7341 return ~(__a ^ __b);
7342}
7343
7344static inline __ATTRS_o_ai __vector signed __int128
7345vec_eqv(__vector signed __int128 __a, __vector signed __int128 __b) {
7346 return ~(__a ^ __b);
7347}
7348
7349static inline __ATTRS_o_ai __vector unsigned __int128
7350vec_eqv(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7351 return ~(__a ^ __b);
7352}
7353
6379static inline __ATTRS_o_ai __vector float7354static inline __ATTRS_o_ai __vector float
6380vec_eqv(__vector float __a, __vector float __b) {7355vec_eqv(__vector float __a, __vector float __b) {
6381 return (__vector float)~((__vector unsigned int)__a ^7356 return (__vector float)~((__vector unsigned int)__a ^
...@@ -6389,6 +7364,91 @@ vec_eqv(__vector double __a, __vector double __b) {...@@ -6389,6 +7364,91 @@ vec_eqv(__vector double __a, __vector double __b) {
6389}7364}
6390#endif7365#endif
63917366
7367/*-- vec_evaluate -----------------------------------------------------------*/
7368
7369#if __ARCH__ >= 15
7370extern __ATTRS_o __vector signed char
7371vec_evaluate(__vector signed char __a, __vector signed char __b,
7372 __vector signed char __c, unsigned char __d)
7373 __constant(__d);
7374
7375extern __ATTRS_o __vector unsigned char
7376vec_evaluate(__vector unsigned char __a, __vector unsigned char __b,
7377 __vector unsigned char __c, unsigned char __d)
7378 __constant(__d);
7379
7380extern __ATTRS_o __vector __bool char
7381vec_evaluate(__vector __bool char __a, __vector __bool char __b,
7382 __vector __bool char __c, unsigned char __d)
7383 __constant(__d);
7384
7385extern __ATTRS_o __vector signed short
7386vec_evaluate(__vector signed short __a, __vector signed short __b,
7387 __vector signed short __c, unsigned char __d)
7388 __constant(__d);
7389
7390extern __ATTRS_o __vector unsigned short
7391vec_evaluate(__vector unsigned short __a, __vector unsigned short __b,
7392 __vector unsigned short __c, unsigned char __d)
7393 __constant(__d);
7394
7395extern __ATTRS_o __vector __bool short
7396vec_evaluate(__vector __bool short __a, __vector __bool short __b,
7397 __vector __bool short __c, unsigned char __d)
7398 __constant(__d);
7399
7400extern __ATTRS_o __vector signed int
7401vec_evaluate(__vector signed int __a, __vector signed int __b,
7402 __vector signed int __c, unsigned char __d)
7403 __constant(__d);
7404
7405extern __ATTRS_o __vector unsigned int
7406vec_evaluate(__vector unsigned int __a, __vector unsigned int __b,
7407 __vector unsigned int __c, unsigned char __d)
7408 __constant(__d);
7409
7410extern __ATTRS_o __vector __bool int
7411vec_evaluate(__vector __bool int __a, __vector __bool int __b,
7412 __vector __bool int __c, unsigned char __d)
7413 __constant(__d);
7414
7415extern __ATTRS_o __vector signed long long
7416vec_evaluate(__vector signed long long __a, __vector signed long long __b,
7417 __vector signed long long __c, unsigned char __d)
7418 __constant(__d);
7419
7420extern __ATTRS_o __vector unsigned long long
7421vec_evaluate(__vector unsigned long long __a, __vector unsigned long long __b,
7422 __vector unsigned long long __c, unsigned char __d)
7423 __constant(__d);
7424
7425extern __ATTRS_o __vector __bool long long
7426vec_evaluate(__vector __bool long long __a, __vector __bool long long __b,
7427 __vector __bool long long __c, unsigned char __d)
7428 __constant(__d);
7429
7430extern __ATTRS_o __vector signed __int128
7431vec_evaluate(__vector signed __int128 __a, __vector signed __int128 __b,
7432 __vector signed __int128 __c, unsigned char __d)
7433 __constant(__d);
7434
7435extern __ATTRS_o __vector unsigned __int128
7436vec_evaluate(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
7437 __vector unsigned __int128 __c, unsigned char __d)
7438 __constant(__d);
7439
7440extern __ATTRS_o __vector __bool __int128
7441vec_evaluate(__vector __bool __int128 __a, __vector __bool __int128 __b,
7442 __vector __bool __int128 __c, unsigned char __d)
7443 __constant(__d);
7444
7445#define vec_evaluate(A, B, C, D) \
7446 ((__typeof__((vec_evaluate)((A), (B), (C), (D)))) \
7447 __builtin_s390_veval((__vector unsigned char)(A), \
7448 (__vector unsigned char)(B), \
7449 (__vector unsigned char)(C), (D)))
7450#endif
7451
6392/*-- vec_cntlz --------------------------------------------------------------*/7452/*-- vec_cntlz --------------------------------------------------------------*/
63937453
6394static inline __ATTRS_o_ai __vector unsigned char7454static inline __ATTRS_o_ai __vector unsigned char
...@@ -6431,6 +7491,20 @@ vec_cntlz(__vector unsigned long long __a) {...@@ -6431,6 +7491,20 @@ vec_cntlz(__vector unsigned long long __a) {
6431 return __builtin_s390_vclzg(__a);7491 return __builtin_s390_vclzg(__a);
6432}7492}
64337493
7494#if __ARCH__ >= 15
7495static inline __ATTRS_o_ai __vector unsigned __int128
7496vec_cntlz(__vector signed __int128 __a) {
7497 return (__vector unsigned __int128)
7498 __builtin_s390_vclzq((unsigned __int128)__a);
7499}
7500
7501static inline __ATTRS_o_ai __vector unsigned __int128
7502vec_cntlz(__vector unsigned __int128 __a) {
7503 return (__vector unsigned __int128)
7504 __builtin_s390_vclzq((unsigned __int128)__a);
7505}
7506#endif
7507
6434/*-- vec_cnttz --------------------------------------------------------------*/7508/*-- vec_cnttz --------------------------------------------------------------*/
64357509
6436static inline __ATTRS_o_ai __vector unsigned char7510static inline __ATTRS_o_ai __vector unsigned char
...@@ -6473,46 +7547,60 @@ vec_cnttz(__vector unsigned long long __a) {...@@ -6473,46 +7547,60 @@ vec_cnttz(__vector unsigned long long __a) {
6473 return __builtin_s390_vctzg(__a);7547 return __builtin_s390_vctzg(__a);
6474}7548}
64757549
7550#if __ARCH__ >= 15
7551static inline __ATTRS_o_ai __vector unsigned __int128
7552vec_cnttz(__vector signed __int128 __a) {
7553 return (__vector unsigned __int128)
7554 __builtin_s390_vctzq((unsigned __int128)__a);
7555}
7556
7557static inline __ATTRS_o_ai __vector unsigned __int128
7558vec_cnttz(__vector unsigned __int128 __a) {
7559 return (__vector unsigned __int128)
7560 __builtin_s390_vctzq((unsigned __int128)__a);
7561}
7562#endif
7563
6476/*-- vec_popcnt -------------------------------------------------------------*/7564/*-- vec_popcnt -------------------------------------------------------------*/
64777565
6478static inline __ATTRS_o_ai __vector unsigned char7566static inline __ATTRS_o_ai __vector unsigned char
6479vec_popcnt(__vector signed char __a) {7567vec_popcnt(__vector signed char __a) {
6480 return __builtin_s390_vpopctb((__vector unsigned char)__a);7568 return __builtin_elementwise_popcount((__vector unsigned char)__a);
6481}7569}
64827570
6483static inline __ATTRS_o_ai __vector unsigned char7571static inline __ATTRS_o_ai __vector unsigned char
6484vec_popcnt(__vector unsigned char __a) {7572vec_popcnt(__vector unsigned char __a) {
6485 return __builtin_s390_vpopctb(__a);7573 return __builtin_elementwise_popcount(__a);
6486}7574}
64877575
6488static inline __ATTRS_o_ai __vector unsigned short7576static inline __ATTRS_o_ai __vector unsigned short
6489vec_popcnt(__vector signed short __a) {7577vec_popcnt(__vector signed short __a) {
6490 return __builtin_s390_vpopcth((__vector unsigned short)__a);7578 return __builtin_elementwise_popcount((__vector unsigned short)__a);
6491}7579}
64927580
6493static inline __ATTRS_o_ai __vector unsigned short7581static inline __ATTRS_o_ai __vector unsigned short
6494vec_popcnt(__vector unsigned short __a) {7582vec_popcnt(__vector unsigned short __a) {
6495 return __builtin_s390_vpopcth(__a);7583 return __builtin_elementwise_popcount(__a);
6496}7584}
64977585
6498static inline __ATTRS_o_ai __vector unsigned int7586static inline __ATTRS_o_ai __vector unsigned int
6499vec_popcnt(__vector signed int __a) {7587vec_popcnt(__vector signed int __a) {
6500 return __builtin_s390_vpopctf((__vector unsigned int)__a);7588 return __builtin_elementwise_popcount((__vector unsigned int)__a);
6501}7589}
65027590
6503static inline __ATTRS_o_ai __vector unsigned int7591static inline __ATTRS_o_ai __vector unsigned int
6504vec_popcnt(__vector unsigned int __a) {7592vec_popcnt(__vector unsigned int __a) {
6505 return __builtin_s390_vpopctf(__a);7593 return __builtin_elementwise_popcount(__a);
6506}7594}
65077595
6508static inline __ATTRS_o_ai __vector unsigned long long7596static inline __ATTRS_o_ai __vector unsigned long long
6509vec_popcnt(__vector signed long long __a) {7597vec_popcnt(__vector signed long long __a) {
6510 return __builtin_s390_vpopctg((__vector unsigned long long)__a);7598 return __builtin_elementwise_popcount((__vector unsigned long long)__a);
6511}7599}
65127600
6513static inline __ATTRS_o_ai __vector unsigned long long7601static inline __ATTRS_o_ai __vector unsigned long long
6514vec_popcnt(__vector unsigned long long __a) {7602vec_popcnt(__vector unsigned long long __a) {
6515 return __builtin_s390_vpopctg(__a);7603 return __builtin_elementwise_popcount(__a);
6516}7604}
65177605
6518/*-- vec_rl -----------------------------------------------------------------*/7606/*-- vec_rl -----------------------------------------------------------------*/
...@@ -6904,8 +7992,21 @@ vec_sll(__vector unsigned long long __a, __vector unsigned int __b) {...@@ -6904,8 +7992,21 @@ vec_sll(__vector unsigned long long __a, __vector unsigned int __b) {
6904 (__vector unsigned char)__a, (__vector unsigned char)__b);7992 (__vector unsigned char)__a, (__vector unsigned char)__b);
6905}7993}
69067994
7995static inline __ATTRS_o_ai __vector signed __int128
7996vec_sll(__vector signed __int128 __a, __vector unsigned char __b) {
7997 return (__vector signed __int128)__builtin_s390_vsl(
7998 (__vector unsigned char)__a, __b);
7999}
8000
8001static inline __ATTRS_o_ai __vector unsigned __int128
8002vec_sll(__vector unsigned __int128 __a, __vector unsigned char __b) {
8003 return (__vector unsigned __int128)__builtin_s390_vsl(
8004 (__vector unsigned char)__a, __b);
8005}
8006
6907/*-- vec_slb ----------------------------------------------------------------*/8007/*-- vec_slb ----------------------------------------------------------------*/
69088008
8009// This prototype is deprecated.
6909static inline __ATTRS_o_ai __vector signed char8010static inline __ATTRS_o_ai __vector signed char
6910vec_slb(__vector signed char __a, __vector signed char __b) {8011vec_slb(__vector signed char __a, __vector signed char __b) {
6911 return (__vector signed char)__builtin_s390_vslb(8012 return (__vector signed char)__builtin_s390_vslb(
...@@ -6918,6 +8019,7 @@ vec_slb(__vector signed char __a, __vector unsigned char __b) {...@@ -6918,6 +8019,7 @@ vec_slb(__vector signed char __a, __vector unsigned char __b) {
6918 (__vector unsigned char)__a, __b);8019 (__vector unsigned char)__a, __b);
6919}8020}
69208021
8022// This prototype is deprecated.
6921static inline __ATTRS_o_ai __vector unsigned char8023static inline __ATTRS_o_ai __vector unsigned char
6922vec_slb(__vector unsigned char __a, __vector signed char __b) {8024vec_slb(__vector unsigned char __a, __vector signed char __b) {
6923 return __builtin_s390_vslb(__a, (__vector unsigned char)__b);8025 return __builtin_s390_vslb(__a, (__vector unsigned char)__b);
...@@ -6928,110 +8030,187 @@ vec_slb(__vector unsigned char __a, __vector unsigned char __b) {...@@ -6928,110 +8030,187 @@ vec_slb(__vector unsigned char __a, __vector unsigned char __b) {
6928 return __builtin_s390_vslb(__a, __b);8030 return __builtin_s390_vslb(__a, __b);
6929}8031}
69308032
8033// This prototype is deprecated.
6931static inline __ATTRS_o_ai __vector signed short8034static inline __ATTRS_o_ai __vector signed short
6932vec_slb(__vector signed short __a, __vector signed short __b) {8035vec_slb(__vector signed short __a, __vector signed short __b) {
6933 return (__vector signed short)__builtin_s390_vslb(8036 return (__vector signed short)__builtin_s390_vslb(
6934 (__vector unsigned char)__a, (__vector unsigned char)__b);8037 (__vector unsigned char)__a, (__vector unsigned char)__b);
6935}8038}
69368039
8040// This prototype is deprecated.
6937static inline __ATTRS_o_ai __vector signed short8041static inline __ATTRS_o_ai __vector signed short
6938vec_slb(__vector signed short __a, __vector unsigned short __b) {8042vec_slb(__vector signed short __a, __vector unsigned short __b) {
6939 return (__vector signed short)__builtin_s390_vslb(8043 return (__vector signed short)__builtin_s390_vslb(
6940 (__vector unsigned char)__a, (__vector unsigned char)__b);8044 (__vector unsigned char)__a, (__vector unsigned char)__b);
6941}8045}
69428046
8047static inline __ATTRS_o_ai __vector signed short
8048vec_slb(__vector signed short __a, __vector unsigned char __b) {
8049 return (__vector signed short)__builtin_s390_vslb(
8050 (__vector unsigned char)__a, __b);
8051}
8052
8053// This prototype is deprecated.
6943static inline __ATTRS_o_ai __vector unsigned short8054static inline __ATTRS_o_ai __vector unsigned short
6944vec_slb(__vector unsigned short __a, __vector signed short __b) {8055vec_slb(__vector unsigned short __a, __vector signed short __b) {
6945 return (__vector unsigned short)__builtin_s390_vslb(8056 return (__vector unsigned short)__builtin_s390_vslb(
6946 (__vector unsigned char)__a, (__vector unsigned char)__b);8057 (__vector unsigned char)__a, (__vector unsigned char)__b);
6947}8058}
69488059
8060// This prototype is deprecated.
6949static inline __ATTRS_o_ai __vector unsigned short8061static inline __ATTRS_o_ai __vector unsigned short
6950vec_slb(__vector unsigned short __a, __vector unsigned short __b) {8062vec_slb(__vector unsigned short __a, __vector unsigned short __b) {
6951 return (__vector unsigned short)__builtin_s390_vslb(8063 return (__vector unsigned short)__builtin_s390_vslb(
6952 (__vector unsigned char)__a, (__vector unsigned char)__b);8064 (__vector unsigned char)__a, (__vector unsigned char)__b);
6953}8065}
69548066
8067static inline __ATTRS_o_ai __vector unsigned short
8068vec_slb(__vector unsigned short __a, __vector unsigned char __b) {
8069 return (__vector unsigned short)__builtin_s390_vslb(
8070 (__vector unsigned char)__a, __b);
8071}
8072
8073// This prototype is deprecated.
6955static inline __ATTRS_o_ai __vector signed int8074static inline __ATTRS_o_ai __vector signed int
6956vec_slb(__vector signed int __a, __vector signed int __b) {8075vec_slb(__vector signed int __a, __vector signed int __b) {
6957 return (__vector signed int)__builtin_s390_vslb(8076 return (__vector signed int)__builtin_s390_vslb(
6958 (__vector unsigned char)__a, (__vector unsigned char)__b);8077 (__vector unsigned char)__a, (__vector unsigned char)__b);
6959}8078}
69608079
8080// This prototype is deprecated.
6961static inline __ATTRS_o_ai __vector signed int8081static inline __ATTRS_o_ai __vector signed int
6962vec_slb(__vector signed int __a, __vector unsigned int __b) {8082vec_slb(__vector signed int __a, __vector unsigned int __b) {
6963 return (__vector signed int)__builtin_s390_vslb(8083 return (__vector signed int)__builtin_s390_vslb(
6964 (__vector unsigned char)__a, (__vector unsigned char)__b);8084 (__vector unsigned char)__a, (__vector unsigned char)__b);
6965}8085}
69668086
8087static inline __ATTRS_o_ai __vector signed int
8088vec_slb(__vector signed int __a, __vector unsigned char __b) {
8089 return (__vector signed int)__builtin_s390_vslb(
8090 (__vector unsigned char)__a, __b);
8091}
8092
8093// This prototype is deprecated.
6967static inline __ATTRS_o_ai __vector unsigned int8094static inline __ATTRS_o_ai __vector unsigned int
6968vec_slb(__vector unsigned int __a, __vector signed int __b) {8095vec_slb(__vector unsigned int __a, __vector signed int __b) {
6969 return (__vector unsigned int)__builtin_s390_vslb(8096 return (__vector unsigned int)__builtin_s390_vslb(
6970 (__vector unsigned char)__a, (__vector unsigned char)__b);8097 (__vector unsigned char)__a, (__vector unsigned char)__b);
6971}8098}
69728099
8100// This prototype is deprecated.
6973static inline __ATTRS_o_ai __vector unsigned int8101static inline __ATTRS_o_ai __vector unsigned int
6974vec_slb(__vector unsigned int __a, __vector unsigned int __b) {8102vec_slb(__vector unsigned int __a, __vector unsigned int __b) {
6975 return (__vector unsigned int)__builtin_s390_vslb(8103 return (__vector unsigned int)__builtin_s390_vslb(
6976 (__vector unsigned char)__a, (__vector unsigned char)__b);8104 (__vector unsigned char)__a, (__vector unsigned char)__b);
6977}8105}
69788106
8107static inline __ATTRS_o_ai __vector unsigned int
8108vec_slb(__vector unsigned int __a, __vector unsigned char __b) {
8109 return (__vector unsigned int)__builtin_s390_vslb(
8110 (__vector unsigned char)__a, __b);
8111}
8112
8113// This prototype is deprecated.
6979static inline __ATTRS_o_ai __vector signed long long8114static inline __ATTRS_o_ai __vector signed long long
6980vec_slb(__vector signed long long __a, __vector signed long long __b) {8115vec_slb(__vector signed long long __a, __vector signed long long __b) {
6981 return (__vector signed long long)__builtin_s390_vslb(8116 return (__vector signed long long)__builtin_s390_vslb(
6982 (__vector unsigned char)__a, (__vector unsigned char)__b);8117 (__vector unsigned char)__a, (__vector unsigned char)__b);
6983}8118}
69848119
8120// This prototype is deprecated.
6985static inline __ATTRS_o_ai __vector signed long long8121static inline __ATTRS_o_ai __vector signed long long
6986vec_slb(__vector signed long long __a, __vector unsigned long long __b) {8122vec_slb(__vector signed long long __a, __vector unsigned long long __b) {
6987 return (__vector signed long long)__builtin_s390_vslb(8123 return (__vector signed long long)__builtin_s390_vslb(
6988 (__vector unsigned char)__a, (__vector unsigned char)__b);8124 (__vector unsigned char)__a, (__vector unsigned char)__b);
6989}8125}
69908126
8127static inline __ATTRS_o_ai __vector signed long long
8128vec_slb(__vector signed long long __a, __vector unsigned char __b) {
8129 return (__vector signed long long)__builtin_s390_vslb(
8130 (__vector unsigned char)__a, __b);
8131}
8132
8133// This prototype is deprecated.
6991static inline __ATTRS_o_ai __vector unsigned long long8134static inline __ATTRS_o_ai __vector unsigned long long
6992vec_slb(__vector unsigned long long __a, __vector signed long long __b) {8135vec_slb(__vector unsigned long long __a, __vector signed long long __b) {
6993 return (__vector unsigned long long)__builtin_s390_vslb(8136 return (__vector unsigned long long)__builtin_s390_vslb(
6994 (__vector unsigned char)__a, (__vector unsigned char)__b);8137 (__vector unsigned char)__a, (__vector unsigned char)__b);
6995}8138}
69968139
8140// This prototype is deprecated.
6997static inline __ATTRS_o_ai __vector unsigned long long8141static inline __ATTRS_o_ai __vector unsigned long long
6998vec_slb(__vector unsigned long long __a, __vector unsigned long long __b) {8142vec_slb(__vector unsigned long long __a, __vector unsigned long long __b) {
6999 return (__vector unsigned long long)__builtin_s390_vslb(8143 return (__vector unsigned long long)__builtin_s390_vslb(
7000 (__vector unsigned char)__a, (__vector unsigned char)__b);8144 (__vector unsigned char)__a, (__vector unsigned char)__b);
7001}8145}
70028146
8147static inline __ATTRS_o_ai __vector unsigned long long
8148vec_slb(__vector unsigned long long __a, __vector unsigned char __b) {
8149 return (__vector unsigned long long)__builtin_s390_vslb(
8150 (__vector unsigned char)__a, __b);
8151}
8152
8153static inline __ATTRS_o_ai __vector signed __int128
8154vec_slb(__vector signed __int128 __a, __vector unsigned char __b) {
8155 return (__vector signed __int128)__builtin_s390_vslb(
8156 (__vector unsigned char)__a, __b);
8157}
8158
8159static inline __ATTRS_o_ai __vector unsigned __int128
8160vec_slb(__vector unsigned __int128 __a, __vector unsigned char __b) {
8161 return (__vector unsigned __int128)__builtin_s390_vslb(
8162 (__vector unsigned char)__a, __b);
8163}
8164
7003#if __ARCH__ >= 128165#if __ARCH__ >= 12
8166// This prototype is deprecated.
7004static inline __ATTRS_o_ai __vector float8167static inline __ATTRS_o_ai __vector float
7005vec_slb(__vector float __a, __vector signed int __b) {8168vec_slb(__vector float __a, __vector signed int __b) {
7006 return (__vector float)__builtin_s390_vslb(8169 return (__vector float)__builtin_s390_vslb(
7007 (__vector unsigned char)__a, (__vector unsigned char)__b);8170 (__vector unsigned char)__a, (__vector unsigned char)__b);
7008}8171}
70098172
8173// This prototype is deprecated.
7010static inline __ATTRS_o_ai __vector float8174static inline __ATTRS_o_ai __vector float
7011vec_slb(__vector float __a, __vector unsigned int __b) {8175vec_slb(__vector float __a, __vector unsigned int __b) {
7012 return (__vector float)__builtin_s390_vslb(8176 return (__vector float)__builtin_s390_vslb(
7013 (__vector unsigned char)__a, (__vector unsigned char)__b);8177 (__vector unsigned char)__a, (__vector unsigned char)__b);
7014}8178}
8179
8180static inline __ATTRS_o_ai __vector float
8181vec_slb(__vector float __a, __vector unsigned char __b) {
8182 return (__vector float)__builtin_s390_vslb(
8183 (__vector unsigned char)__a, __b);
8184}
7015#endif8185#endif
70168186
8187// This prototype is deprecated.
7017static inline __ATTRS_o_ai __vector double8188static inline __ATTRS_o_ai __vector double
7018vec_slb(__vector double __a, __vector signed long long __b) {8189vec_slb(__vector double __a, __vector signed long long __b) {
7019 return (__vector double)__builtin_s390_vslb(8190 return (__vector double)__builtin_s390_vslb(
7020 (__vector unsigned char)__a, (__vector unsigned char)__b);8191 (__vector unsigned char)__a, (__vector unsigned char)__b);
7021}8192}
70228193
8194// This prototype is deprecated.
7023static inline __ATTRS_o_ai __vector double8195static inline __ATTRS_o_ai __vector double
7024vec_slb(__vector double __a, __vector unsigned long long __b) {8196vec_slb(__vector double __a, __vector unsigned long long __b) {
7025 return (__vector double)__builtin_s390_vslb(8197 return (__vector double)__builtin_s390_vslb(
7026 (__vector unsigned char)__a, (__vector unsigned char)__b);8198 (__vector unsigned char)__a, (__vector unsigned char)__b);
7027}8199}
70288200
8201static inline __ATTRS_o_ai __vector double
8202vec_slb(__vector double __a, __vector unsigned char __b) {
8203 return (__vector double)__builtin_s390_vslb(
8204 (__vector unsigned char)__a, __b);
8205}
8206
7029/*-- vec_sld ----------------------------------------------------------------*/8207/*-- vec_sld ----------------------------------------------------------------*/
70308208
7031extern __ATTRS_o __vector signed char8209extern __ATTRS_o __vector signed char
7032vec_sld(__vector signed char __a, __vector signed char __b, int __c)8210vec_sld(__vector signed char __a, __vector signed char __b, int __c)
7033 __constant_range(__c, 0, 15);8211 __constant_range(__c, 0, 15);
70348212
8213// This prototype is deprecated.
7035extern __ATTRS_o __vector __bool char8214extern __ATTRS_o __vector __bool char
7036vec_sld(__vector __bool char __a, __vector __bool char __b, int __c)8215vec_sld(__vector __bool char __a, __vector __bool char __b, int __c)
7037 __constant_range(__c, 0, 15);8216 __constant_range(__c, 0, 15);
...@@ -7044,6 +8223,7 @@ extern __ATTRS_o __vector signed short...@@ -7044,6 +8223,7 @@ extern __ATTRS_o __vector signed short
7044vec_sld(__vector signed short __a, __vector signed short __b, int __c)8223vec_sld(__vector signed short __a, __vector signed short __b, int __c)
7045 __constant_range(__c, 0, 15);8224 __constant_range(__c, 0, 15);
70468225
8226// This prototype is deprecated.
7047extern __ATTRS_o __vector __bool short8227extern __ATTRS_o __vector __bool short
7048vec_sld(__vector __bool short __a, __vector __bool short __b, int __c)8228vec_sld(__vector __bool short __a, __vector __bool short __b, int __c)
7049 __constant_range(__c, 0, 15);8229 __constant_range(__c, 0, 15);
...@@ -7056,6 +8236,7 @@ extern __ATTRS_o __vector signed int...@@ -7056,6 +8236,7 @@ extern __ATTRS_o __vector signed int
7056vec_sld(__vector signed int __a, __vector signed int __b, int __c)8236vec_sld(__vector signed int __a, __vector signed int __b, int __c)
7057 __constant_range(__c, 0, 15);8237 __constant_range(__c, 0, 15);
70588238
8239// This prototype is deprecated.
7059extern __ATTRS_o __vector __bool int8240extern __ATTRS_o __vector __bool int
7060vec_sld(__vector __bool int __a, __vector __bool int __b, int __c)8241vec_sld(__vector __bool int __a, __vector __bool int __b, int __c)
7061 __constant_range(__c, 0, 15);8242 __constant_range(__c, 0, 15);
...@@ -7068,6 +8249,7 @@ extern __ATTRS_o __vector signed long long...@@ -7068,6 +8249,7 @@ extern __ATTRS_o __vector signed long long
7068vec_sld(__vector signed long long __a, __vector signed long long __b, int __c)8249vec_sld(__vector signed long long __a, __vector signed long long __b, int __c)
7069 __constant_range(__c, 0, 15);8250 __constant_range(__c, 0, 15);
70708251
8252// This prototype is deprecated.
7071extern __ATTRS_o __vector __bool long long8253extern __ATTRS_o __vector __bool long long
7072vec_sld(__vector __bool long long __a, __vector __bool long long __b, int __c)8254vec_sld(__vector __bool long long __a, __vector __bool long long __b, int __c)
7073 __constant_range(__c, 0, 15);8255 __constant_range(__c, 0, 15);
...@@ -7077,6 +8259,15 @@ vec_sld(__vector unsigned long long __a, __vector unsigned long long __b,...@@ -7077,6 +8259,15 @@ vec_sld(__vector unsigned long long __a, __vector unsigned long long __b,
7077 int __c)8259 int __c)
7078 __constant_range(__c, 0, 15);8260 __constant_range(__c, 0, 15);
70798261
8262extern __ATTRS_o __vector signed __int128
8263vec_sld(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
8264 __constant_range(__c, 0, 15);
8265
8266extern __ATTRS_o __vector unsigned __int128
8267vec_sld(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
8268 int __c)
8269 __constant_range(__c, 0, 15);
8270
7080#if __ARCH__ >= 128271#if __ARCH__ >= 12
7081extern __ATTRS_o __vector float8272extern __ATTRS_o __vector float
7082vec_sld(__vector float __a, __vector float __b, int __c)8273vec_sld(__vector float __a, __vector float __b, int __c)
...@@ -7126,6 +8317,15 @@ vec_sldw(__vector unsigned long long __a, __vector unsigned long long __b,...@@ -7126,6 +8317,15 @@ vec_sldw(__vector unsigned long long __a, __vector unsigned long long __b,
7126 int __c)8317 int __c)
7127 __constant_range(__c, 0, 3);8318 __constant_range(__c, 0, 3);
71288319
8320extern __ATTRS_o __vector signed __int128
8321vec_sldw(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
8322 __constant_range(__c, 0, 3);
8323
8324extern __ATTRS_o __vector unsigned __int128
8325vec_sldw(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
8326 int __c)
8327 __constant_range(__c, 0, 3);
8328
7129// This prototype is deprecated.8329// This prototype is deprecated.
7130extern __ATTRS_o __vector double8330extern __ATTRS_o __vector double
7131vec_sldw(__vector double __a, __vector double __b, int __c)8331vec_sldw(__vector double __a, __vector double __b, int __c)
...@@ -7172,6 +8372,15 @@ vec_sldb(__vector unsigned long long __a, __vector unsigned long long __b,...@@ -7172,6 +8372,15 @@ vec_sldb(__vector unsigned long long __a, __vector unsigned long long __b,
7172 int __c)8372 int __c)
7173 __constant_range(__c, 0, 7);8373 __constant_range(__c, 0, 7);
71748374
8375extern __ATTRS_o __vector signed __int128
8376vec_sldb(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
8377 __constant_range(__c, 0, 7);
8378
8379extern __ATTRS_o __vector unsigned __int128
8380vec_sldb(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
8381 int __c)
8382 __constant_range(__c, 0, 7);
8383
7175extern __ATTRS_o __vector float8384extern __ATTRS_o __vector float
7176vec_sldb(__vector float __a, __vector float __b, int __c)8385vec_sldb(__vector float __a, __vector float __b, int __c)
7177 __constant_range(__c, 0, 7);8386 __constant_range(__c, 0, 7);
...@@ -7429,8 +8638,21 @@ vec_sral(__vector unsigned long long __a, __vector unsigned int __b) {...@@ -7429,8 +8638,21 @@ vec_sral(__vector unsigned long long __a, __vector unsigned int __b) {
7429 (__vector unsigned char)__a, (__vector unsigned char)__b);8638 (__vector unsigned char)__a, (__vector unsigned char)__b);
7430}8639}
74318640
8641static inline __ATTRS_o_ai __vector signed __int128
8642vec_sral(__vector signed __int128 __a, __vector unsigned char __b) {
8643 return (__vector signed __int128)__builtin_s390_vsra(
8644 (__vector unsigned char)__a, __b);
8645}
8646
8647static inline __ATTRS_o_ai __vector unsigned __int128
8648vec_sral(__vector unsigned __int128 __a, __vector unsigned char __b) {
8649 return (__vector unsigned __int128)__builtin_s390_vsra(
8650 (__vector unsigned char)__a, __b);
8651}
8652
7432/*-- vec_srab ---------------------------------------------------------------*/8653/*-- vec_srab ---------------------------------------------------------------*/
74338654
8655// This prototype is deprecated.
7434static inline __ATTRS_o_ai __vector signed char8656static inline __ATTRS_o_ai __vector signed char
7435vec_srab(__vector signed char __a, __vector signed char __b) {8657vec_srab(__vector signed char __a, __vector signed char __b) {
7436 return (__vector signed char)__builtin_s390_vsrab(8658 return (__vector signed char)__builtin_s390_vsrab(
...@@ -7443,6 +8665,7 @@ vec_srab(__vector signed char __a, __vector unsigned char __b) {...@@ -7443,6 +8665,7 @@ vec_srab(__vector signed char __a, __vector unsigned char __b) {
7443 (__vector unsigned char)__a, __b);8665 (__vector unsigned char)__a, __b);
7444}8666}
74458667
8668// This prototype is deprecated.
7446static inline __ATTRS_o_ai __vector unsigned char8669static inline __ATTRS_o_ai __vector unsigned char
7447vec_srab(__vector unsigned char __a, __vector signed char __b) {8670vec_srab(__vector unsigned char __a, __vector signed char __b) {
7448 return __builtin_s390_vsrab(__a, (__vector unsigned char)__b);8671 return __builtin_s390_vsrab(__a, (__vector unsigned char)__b);
...@@ -7453,104 +8676,180 @@ vec_srab(__vector unsigned char __a, __vector unsigned char __b) {...@@ -7453,104 +8676,180 @@ vec_srab(__vector unsigned char __a, __vector unsigned char __b) {
7453 return __builtin_s390_vsrab(__a, __b);8676 return __builtin_s390_vsrab(__a, __b);
7454}8677}
74558678
8679// This prototype is deprecated.
7456static inline __ATTRS_o_ai __vector signed short8680static inline __ATTRS_o_ai __vector signed short
7457vec_srab(__vector signed short __a, __vector signed short __b) {8681vec_srab(__vector signed short __a, __vector signed short __b) {
7458 return (__vector signed short)__builtin_s390_vsrab(8682 return (__vector signed short)__builtin_s390_vsrab(
7459 (__vector unsigned char)__a, (__vector unsigned char)__b);8683 (__vector unsigned char)__a, (__vector unsigned char)__b);
7460}8684}
74618685
8686// This prototype is deprecated.
7462static inline __ATTRS_o_ai __vector signed short8687static inline __ATTRS_o_ai __vector signed short
7463vec_srab(__vector signed short __a, __vector unsigned short __b) {8688vec_srab(__vector signed short __a, __vector unsigned short __b) {
7464 return (__vector signed short)__builtin_s390_vsrab(8689 return (__vector signed short)__builtin_s390_vsrab(
7465 (__vector unsigned char)__a, (__vector unsigned char)__b);8690 (__vector unsigned char)__a, (__vector unsigned char)__b);
7466}8691}
74678692
8693static inline __ATTRS_o_ai __vector signed short
8694vec_srab(__vector signed short __a, __vector unsigned char __b) {
8695 return (__vector signed short)__builtin_s390_vsrab(
8696 (__vector unsigned char)__a, __b);
8697}
8698
8699// This prototype is deprecated.
7468static inline __ATTRS_o_ai __vector unsigned short8700static inline __ATTRS_o_ai __vector unsigned short
7469vec_srab(__vector unsigned short __a, __vector signed short __b) {8701vec_srab(__vector unsigned short __a, __vector signed short __b) {
7470 return (__vector unsigned short)__builtin_s390_vsrab(8702 return (__vector unsigned short)__builtin_s390_vsrab(
7471 (__vector unsigned char)__a, (__vector unsigned char)__b);8703 (__vector unsigned char)__a, (__vector unsigned char)__b);
7472}8704}
74738705
8706// This prototype is deprecated.
7474static inline __ATTRS_o_ai __vector unsigned short8707static inline __ATTRS_o_ai __vector unsigned short
7475vec_srab(__vector unsigned short __a, __vector unsigned short __b) {8708vec_srab(__vector unsigned short __a, __vector unsigned short __b) {
7476 return (__vector unsigned short)__builtin_s390_vsrab(8709 return (__vector unsigned short)__builtin_s390_vsrab(
7477 (__vector unsigned char)__a, (__vector unsigned char)__b);8710 (__vector unsigned char)__a, (__vector unsigned char)__b);
7478}8711}
74798712
8713static inline __ATTRS_o_ai __vector unsigned short
8714vec_srab(__vector unsigned short __a, __vector unsigned char __b) {
8715 return (__vector unsigned short)__builtin_s390_vsrab(
8716 (__vector unsigned char)__a, __b);
8717}
8718
8719// This prototype is deprecated.
7480static inline __ATTRS_o_ai __vector signed int8720static inline __ATTRS_o_ai __vector signed int
7481vec_srab(__vector signed int __a, __vector signed int __b) {8721vec_srab(__vector signed int __a, __vector signed int __b) {
7482 return (__vector signed int)__builtin_s390_vsrab(8722 return (__vector signed int)__builtin_s390_vsrab(
7483 (__vector unsigned char)__a, (__vector unsigned char)__b);8723 (__vector unsigned char)__a, (__vector unsigned char)__b);
7484}8724}
74858725
8726// This prototype is deprecated.
7486static inline __ATTRS_o_ai __vector signed int8727static inline __ATTRS_o_ai __vector signed int
7487vec_srab(__vector signed int __a, __vector unsigned int __b) {8728vec_srab(__vector signed int __a, __vector unsigned int __b) {
7488 return (__vector signed int)__builtin_s390_vsrab(8729 return (__vector signed int)__builtin_s390_vsrab(
7489 (__vector unsigned char)__a, (__vector unsigned char)__b);8730 (__vector unsigned char)__a, (__vector unsigned char)__b);
7490}8731}
74918732
8733static inline __ATTRS_o_ai __vector signed int
8734vec_srab(__vector signed int __a, __vector unsigned char __b) {
8735 return (__vector signed int)__builtin_s390_vsrab(
8736 (__vector unsigned char)__a, __b);
8737}
8738
8739// This prototype is deprecated.
7492static inline __ATTRS_o_ai __vector unsigned int8740static inline __ATTRS_o_ai __vector unsigned int
7493vec_srab(__vector unsigned int __a, __vector signed int __b) {8741vec_srab(__vector unsigned int __a, __vector signed int __b) {
7494 return (__vector unsigned int)__builtin_s390_vsrab(8742 return (__vector unsigned int)__builtin_s390_vsrab(
7495 (__vector unsigned char)__a, (__vector unsigned char)__b);8743 (__vector unsigned char)__a, (__vector unsigned char)__b);
7496}8744}
74978745
8746// This prototype is deprecated.
7498static inline __ATTRS_o_ai __vector unsigned int8747static inline __ATTRS_o_ai __vector unsigned int
7499vec_srab(__vector unsigned int __a, __vector unsigned int __b) {8748vec_srab(__vector unsigned int __a, __vector unsigned int __b) {
7500 return (__vector unsigned int)__builtin_s390_vsrab(8749 return (__vector unsigned int)__builtin_s390_vsrab(
7501 (__vector unsigned char)__a, (__vector unsigned char)__b);8750 (__vector unsigned char)__a, (__vector unsigned char)__b);
7502}8751}
75038752
8753static inline __ATTRS_o_ai __vector unsigned int
8754vec_srab(__vector unsigned int __a, __vector unsigned char __b) {
8755 return (__vector unsigned int)__builtin_s390_vsrab(
8756 (__vector unsigned char)__a, __b);
8757}
8758
8759// This prototype is deprecated.
7504static inline __ATTRS_o_ai __vector signed long long8760static inline __ATTRS_o_ai __vector signed long long
7505vec_srab(__vector signed long long __a, __vector signed long long __b) {8761vec_srab(__vector signed long long __a, __vector signed long long __b) {
7506 return (__vector signed long long)__builtin_s390_vsrab(8762 return (__vector signed long long)__builtin_s390_vsrab(
7507 (__vector unsigned char)__a, (__vector unsigned char)__b);8763 (__vector unsigned char)__a, (__vector unsigned char)__b);
7508}8764}
75098765
8766// This prototype is deprecated.
7510static inline __ATTRS_o_ai __vector signed long long8767static inline __ATTRS_o_ai __vector signed long long
7511vec_srab(__vector signed long long __a, __vector unsigned long long __b) {8768vec_srab(__vector signed long long __a, __vector unsigned long long __b) {
7512 return (__vector signed long long)__builtin_s390_vsrab(8769 return (__vector signed long long)__builtin_s390_vsrab(
7513 (__vector unsigned char)__a, (__vector unsigned char)__b);8770 (__vector unsigned char)__a, (__vector unsigned char)__b);
7514}8771}
75158772
8773static inline __ATTRS_o_ai __vector signed long long
8774vec_srab(__vector signed long long __a, __vector unsigned char __b) {
8775 return (__vector signed long long)__builtin_s390_vsrab(
8776 (__vector unsigned char)__a, __b);
8777}
8778
8779// This prototype is deprecated.
7516static inline __ATTRS_o_ai __vector unsigned long long8780static inline __ATTRS_o_ai __vector unsigned long long
7517vec_srab(__vector unsigned long long __a, __vector signed long long __b) {8781vec_srab(__vector unsigned long long __a, __vector signed long long __b) {
7518 return (__vector unsigned long long)__builtin_s390_vsrab(8782 return (__vector unsigned long long)__builtin_s390_vsrab(
7519 (__vector unsigned char)__a, (__vector unsigned char)__b);8783 (__vector unsigned char)__a, (__vector unsigned char)__b);
7520}8784}
75218785
7522static inline __ATTRS_o_ai __vector unsigned long long8786// This prototype is deprecated.
7523vec_srab(__vector unsigned long long __a, __vector unsigned long long __b) {8787static inline __ATTRS_o_ai __vector unsigned long long
7524 return (__vector unsigned long long)__builtin_s390_vsrab(8788vec_srab(__vector unsigned long long __a, __vector unsigned long long __b) {
7525 (__vector unsigned char)__a, (__vector unsigned char)__b);8789 return (__vector unsigned long long)__builtin_s390_vsrab(
8790 (__vector unsigned char)__a, (__vector unsigned char)__b);
8791}
8792
8793static inline __ATTRS_o_ai __vector unsigned long long
8794vec_srab(__vector unsigned long long __a, __vector unsigned char __b) {
8795 return (__vector unsigned long long)__builtin_s390_vsrab(
8796 (__vector unsigned char)__a, __b);
8797}
8798
8799static inline __ATTRS_o_ai __vector signed __int128
8800vec_srab(__vector signed __int128 __a, __vector unsigned char __b) {
8801 return (__vector signed __int128)__builtin_s390_vsrab(
8802 (__vector unsigned char)__a, __b);
8803}
8804
8805static inline __ATTRS_o_ai __vector unsigned __int128
8806vec_srab(__vector unsigned __int128 __a, __vector unsigned char __b) {
8807 return (__vector unsigned __int128)__builtin_s390_vsrab(
8808 (__vector unsigned char)__a, __b);
7526}8809}
75278810
7528#if __ARCH__ >= 128811#if __ARCH__ >= 12
8812// This prototype is deprecated.
7529static inline __ATTRS_o_ai __vector float8813static inline __ATTRS_o_ai __vector float
7530vec_srab(__vector float __a, __vector signed int __b) {8814vec_srab(__vector float __a, __vector signed int __b) {
7531 return (__vector float)__builtin_s390_vsrab(8815 return (__vector float)__builtin_s390_vsrab(
7532 (__vector unsigned char)__a, (__vector unsigned char)__b);8816 (__vector unsigned char)__a, (__vector unsigned char)__b);
7533}8817}
75348818
8819// This prototype is deprecated.
7535static inline __ATTRS_o_ai __vector float8820static inline __ATTRS_o_ai __vector float
7536vec_srab(__vector float __a, __vector unsigned int __b) {8821vec_srab(__vector float __a, __vector unsigned int __b) {
7537 return (__vector float)__builtin_s390_vsrab(8822 return (__vector float)__builtin_s390_vsrab(
7538 (__vector unsigned char)__a, (__vector unsigned char)__b);8823 (__vector unsigned char)__a, (__vector unsigned char)__b);
7539}8824}
8825
8826static inline __ATTRS_o_ai __vector float
8827vec_srab(__vector float __a, __vector unsigned char __b) {
8828 return (__vector float)__builtin_s390_vsrab(
8829 (__vector unsigned char)__a, __b);
8830}
7540#endif8831#endif
75418832
8833// This prototype is deprecated.
7542static inline __ATTRS_o_ai __vector double8834static inline __ATTRS_o_ai __vector double
7543vec_srab(__vector double __a, __vector signed long long __b) {8835vec_srab(__vector double __a, __vector signed long long __b) {
7544 return (__vector double)__builtin_s390_vsrab(8836 return (__vector double)__builtin_s390_vsrab(
7545 (__vector unsigned char)__a, (__vector unsigned char)__b);8837 (__vector unsigned char)__a, (__vector unsigned char)__b);
7546}8838}
75478839
8840// This prototype is deprecated.
7548static inline __ATTRS_o_ai __vector double8841static inline __ATTRS_o_ai __vector double
7549vec_srab(__vector double __a, __vector unsigned long long __b) {8842vec_srab(__vector double __a, __vector unsigned long long __b) {
7550 return (__vector double)__builtin_s390_vsrab(8843 return (__vector double)__builtin_s390_vsrab(
7551 (__vector unsigned char)__a, (__vector unsigned char)__b);8844 (__vector unsigned char)__a, (__vector unsigned char)__b);
7552}8845}
75538846
8847static inline __ATTRS_o_ai __vector double
8848vec_srab(__vector double __a, __vector unsigned char __b) {
8849 return (__vector double)__builtin_s390_vsrab(
8850 (__vector unsigned char)__a, __b);
8851}
8852
7554/*-- vec_srl ----------------------------------------------------------------*/8853/*-- vec_srl ----------------------------------------------------------------*/
75558854
7556static inline __ATTRS_o_ai __vector signed char8855static inline __ATTRS_o_ai __vector signed char
...@@ -7794,8 +9093,21 @@ vec_srl(__vector unsigned long long __a, __vector unsigned int __b) {...@@ -7794,8 +9093,21 @@ vec_srl(__vector unsigned long long __a, __vector unsigned int __b) {
7794 (__vector unsigned char)__a, (__vector unsigned char)__b);9093 (__vector unsigned char)__a, (__vector unsigned char)__b);
7795}9094}
77969095
9096static inline __ATTRS_o_ai __vector signed __int128
9097vec_srl(__vector signed __int128 __a, __vector unsigned char __b) {
9098 return (__vector signed __int128)__builtin_s390_vsrl(
9099 (__vector unsigned char)__a, __b);
9100}
9101
9102static inline __ATTRS_o_ai __vector unsigned __int128
9103vec_srl(__vector unsigned __int128 __a, __vector unsigned char __b) {
9104 return (__vector unsigned __int128)__builtin_s390_vsrl(
9105 (__vector unsigned char)__a, __b);
9106}
9107
7797/*-- vec_srb ----------------------------------------------------------------*/9108/*-- vec_srb ----------------------------------------------------------------*/
77989109
9110// This prototype is deprecated.
7799static inline __ATTRS_o_ai __vector signed char9111static inline __ATTRS_o_ai __vector signed char
7800vec_srb(__vector signed char __a, __vector signed char __b) {9112vec_srb(__vector signed char __a, __vector signed char __b) {
7801 return (__vector signed char)__builtin_s390_vsrlb(9113 return (__vector signed char)__builtin_s390_vsrlb(
...@@ -7808,6 +9120,7 @@ vec_srb(__vector signed char __a, __vector unsigned char __b) {...@@ -7808,6 +9120,7 @@ vec_srb(__vector signed char __a, __vector unsigned char __b) {
7808 (__vector unsigned char)__a, __b);9120 (__vector unsigned char)__a, __b);
7809}9121}
78109122
9123// This prototype is deprecated.
7811static inline __ATTRS_o_ai __vector unsigned char9124static inline __ATTRS_o_ai __vector unsigned char
7812vec_srb(__vector unsigned char __a, __vector signed char __b) {9125vec_srb(__vector unsigned char __a, __vector signed char __b) {
7813 return __builtin_s390_vsrlb(__a, (__vector unsigned char)__b);9126 return __builtin_s390_vsrlb(__a, (__vector unsigned char)__b);
...@@ -7818,104 +9131,180 @@ vec_srb(__vector unsigned char __a, __vector unsigned char __b) {...@@ -7818,104 +9131,180 @@ vec_srb(__vector unsigned char __a, __vector unsigned char __b) {
7818 return __builtin_s390_vsrlb(__a, __b);9131 return __builtin_s390_vsrlb(__a, __b);
7819}9132}
78209133
9134// This prototype is deprecated.
7821static inline __ATTRS_o_ai __vector signed short9135static inline __ATTRS_o_ai __vector signed short
7822vec_srb(__vector signed short __a, __vector signed short __b) {9136vec_srb(__vector signed short __a, __vector signed short __b) {
7823 return (__vector signed short)__builtin_s390_vsrlb(9137 return (__vector signed short)__builtin_s390_vsrlb(
7824 (__vector unsigned char)__a, (__vector unsigned char)__b);9138 (__vector unsigned char)__a, (__vector unsigned char)__b);
7825}9139}
78269140
9141// This prototype is deprecated.
7827static inline __ATTRS_o_ai __vector signed short9142static inline __ATTRS_o_ai __vector signed short
7828vec_srb(__vector signed short __a, __vector unsigned short __b) {9143vec_srb(__vector signed short __a, __vector unsigned short __b) {
7829 return (__vector signed short)__builtin_s390_vsrlb(9144 return (__vector signed short)__builtin_s390_vsrlb(
7830 (__vector unsigned char)__a, (__vector unsigned char)__b);9145 (__vector unsigned char)__a, (__vector unsigned char)__b);
7831}9146}
78329147
9148static inline __ATTRS_o_ai __vector signed short
9149vec_srb(__vector signed short __a, __vector unsigned char __b) {
9150 return (__vector signed short)__builtin_s390_vsrlb(
9151 (__vector unsigned char)__a, __b);
9152}
9153
9154// This prototype is deprecated.
7833static inline __ATTRS_o_ai __vector unsigned short9155static inline __ATTRS_o_ai __vector unsigned short
7834vec_srb(__vector unsigned short __a, __vector signed short __b) {9156vec_srb(__vector unsigned short __a, __vector signed short __b) {
7835 return (__vector unsigned short)__builtin_s390_vsrlb(9157 return (__vector unsigned short)__builtin_s390_vsrlb(
7836 (__vector unsigned char)__a, (__vector unsigned char)__b);9158 (__vector unsigned char)__a, (__vector unsigned char)__b);
7837}9159}
78389160
9161// This prototype is deprecated.
7839static inline __ATTRS_o_ai __vector unsigned short9162static inline __ATTRS_o_ai __vector unsigned short
7840vec_srb(__vector unsigned short __a, __vector unsigned short __b) {9163vec_srb(__vector unsigned short __a, __vector unsigned short __b) {
7841 return (__vector unsigned short)__builtin_s390_vsrlb(9164 return (__vector unsigned short)__builtin_s390_vsrlb(
7842 (__vector unsigned char)__a, (__vector unsigned char)__b);9165 (__vector unsigned char)__a, (__vector unsigned char)__b);
7843}9166}
78449167
9168static inline __ATTRS_o_ai __vector unsigned short
9169vec_srb(__vector unsigned short __a, __vector unsigned char __b) {
9170 return (__vector unsigned short)__builtin_s390_vsrlb(
9171 (__vector unsigned char)__a, __b);
9172}
9173
9174// This prototype is deprecated.
7845static inline __ATTRS_o_ai __vector signed int9175static inline __ATTRS_o_ai __vector signed int
7846vec_srb(__vector signed int __a, __vector signed int __b) {9176vec_srb(__vector signed int __a, __vector signed int __b) {
7847 return (__vector signed int)__builtin_s390_vsrlb(9177 return (__vector signed int)__builtin_s390_vsrlb(
7848 (__vector unsigned char)__a, (__vector unsigned char)__b);9178 (__vector unsigned char)__a, (__vector unsigned char)__b);
7849}9179}
78509180
9181// This prototype is deprecated.
7851static inline __ATTRS_o_ai __vector signed int9182static inline __ATTRS_o_ai __vector signed int
7852vec_srb(__vector signed int __a, __vector unsigned int __b) {9183vec_srb(__vector signed int __a, __vector unsigned int __b) {
7853 return (__vector signed int)__builtin_s390_vsrlb(9184 return (__vector signed int)__builtin_s390_vsrlb(
7854 (__vector unsigned char)__a, (__vector unsigned char)__b);9185 (__vector unsigned char)__a, (__vector unsigned char)__b);
7855}9186}
78569187
9188static inline __ATTRS_o_ai __vector signed int
9189vec_srb(__vector signed int __a, __vector unsigned char __b) {
9190 return (__vector signed int)__builtin_s390_vsrlb(
9191 (__vector unsigned char)__a, __b);
9192}
9193
9194// This prototype is deprecated.
7857static inline __ATTRS_o_ai __vector unsigned int9195static inline __ATTRS_o_ai __vector unsigned int
7858vec_srb(__vector unsigned int __a, __vector signed int __b) {9196vec_srb(__vector unsigned int __a, __vector signed int __b) {
7859 return (__vector unsigned int)__builtin_s390_vsrlb(9197 return (__vector unsigned int)__builtin_s390_vsrlb(
7860 (__vector unsigned char)__a, (__vector unsigned char)__b);9198 (__vector unsigned char)__a, (__vector unsigned char)__b);
7861}9199}
78629200
9201// This prototype is deprecated.
7863static inline __ATTRS_o_ai __vector unsigned int9202static inline __ATTRS_o_ai __vector unsigned int
7864vec_srb(__vector unsigned int __a, __vector unsigned int __b) {9203vec_srb(__vector unsigned int __a, __vector unsigned int __b) {
7865 return (__vector unsigned int)__builtin_s390_vsrlb(9204 return (__vector unsigned int)__builtin_s390_vsrlb(
7866 (__vector unsigned char)__a, (__vector unsigned char)__b);9205 (__vector unsigned char)__a, (__vector unsigned char)__b);
7867}9206}
78689207
9208static inline __ATTRS_o_ai __vector unsigned int
9209vec_srb(__vector unsigned int __a, __vector unsigned char __b) {
9210 return (__vector unsigned int)__builtin_s390_vsrlb(
9211 (__vector unsigned char)__a, __b);
9212}
9213
9214// This prototype is deprecated.
7869static inline __ATTRS_o_ai __vector signed long long9215static inline __ATTRS_o_ai __vector signed long long
7870vec_srb(__vector signed long long __a, __vector signed long long __b) {9216vec_srb(__vector signed long long __a, __vector signed long long __b) {
7871 return (__vector signed long long)__builtin_s390_vsrlb(9217 return (__vector signed long long)__builtin_s390_vsrlb(
7872 (__vector unsigned char)__a, (__vector unsigned char)__b);9218 (__vector unsigned char)__a, (__vector unsigned char)__b);
7873}9219}
78749220
9221// This prototype is deprecated.
7875static inline __ATTRS_o_ai __vector signed long long9222static inline __ATTRS_o_ai __vector signed long long
7876vec_srb(__vector signed long long __a, __vector unsigned long long __b) {9223vec_srb(__vector signed long long __a, __vector unsigned long long __b) {
7877 return (__vector signed long long)__builtin_s390_vsrlb(9224 return (__vector signed long long)__builtin_s390_vsrlb(
7878 (__vector unsigned char)__a, (__vector unsigned char)__b);9225 (__vector unsigned char)__a, (__vector unsigned char)__b);
7879}9226}
78809227
9228static inline __ATTRS_o_ai __vector signed long long
9229vec_srb(__vector signed long long __a, __vector unsigned char __b) {
9230 return (__vector signed long long)__builtin_s390_vsrlb(
9231 (__vector unsigned char)__a, __b);
9232}
9233
9234// This prototype is deprecated.
7881static inline __ATTRS_o_ai __vector unsigned long long9235static inline __ATTRS_o_ai __vector unsigned long long
7882vec_srb(__vector unsigned long long __a, __vector signed long long __b) {9236vec_srb(__vector unsigned long long __a, __vector signed long long __b) {
7883 return (__vector unsigned long long)__builtin_s390_vsrlb(9237 return (__vector unsigned long long)__builtin_s390_vsrlb(
7884 (__vector unsigned char)__a, (__vector unsigned char)__b);9238 (__vector unsigned char)__a, (__vector unsigned char)__b);
7885}9239}
78869240
9241// This prototype is deprecated.
7887static inline __ATTRS_o_ai __vector unsigned long long9242static inline __ATTRS_o_ai __vector unsigned long long
7888vec_srb(__vector unsigned long long __a, __vector unsigned long long __b) {9243vec_srb(__vector unsigned long long __a, __vector unsigned long long __b) {
7889 return (__vector unsigned long long)__builtin_s390_vsrlb(9244 return (__vector unsigned long long)__builtin_s390_vsrlb(
7890 (__vector unsigned char)__a, (__vector unsigned char)__b);9245 (__vector unsigned char)__a, (__vector unsigned char)__b);
7891}9246}
78929247
9248static inline __ATTRS_o_ai __vector unsigned long long
9249vec_srb(__vector unsigned long long __a, __vector unsigned char __b) {
9250 return (__vector unsigned long long)__builtin_s390_vsrlb(
9251 (__vector unsigned char)__a, __b);
9252}
9253
9254static inline __ATTRS_o_ai __vector signed __int128
9255vec_srb(__vector signed __int128 __a, __vector unsigned char __b) {
9256 return (__vector signed __int128)__builtin_s390_vsrlb(
9257 (__vector unsigned char)__a, __b);
9258}
9259
9260static inline __ATTRS_o_ai __vector unsigned __int128
9261vec_srb(__vector unsigned __int128 __a, __vector unsigned char __b) {
9262 return (__vector unsigned __int128)__builtin_s390_vsrlb(
9263 (__vector unsigned char)__a, __b);
9264}
9265
7893#if __ARCH__ >= 129266#if __ARCH__ >= 12
9267// This prototype is deprecated.
7894static inline __ATTRS_o_ai __vector float9268static inline __ATTRS_o_ai __vector float
7895vec_srb(__vector float __a, __vector signed int __b) {9269vec_srb(__vector float __a, __vector signed int __b) {
7896 return (__vector float)__builtin_s390_vsrlb(9270 return (__vector float)__builtin_s390_vsrlb(
7897 (__vector unsigned char)__a, (__vector unsigned char)__b);9271 (__vector unsigned char)__a, (__vector unsigned char)__b);
7898}9272}
78999273
9274// This prototype is deprecated.
7900static inline __ATTRS_o_ai __vector float9275static inline __ATTRS_o_ai __vector float
7901vec_srb(__vector float __a, __vector unsigned int __b) {9276vec_srb(__vector float __a, __vector unsigned int __b) {
7902 return (__vector float)__builtin_s390_vsrlb(9277 return (__vector float)__builtin_s390_vsrlb(
7903 (__vector unsigned char)__a, (__vector unsigned char)__b);9278 (__vector unsigned char)__a, (__vector unsigned char)__b);
7904}9279}
9280
9281static inline __ATTRS_o_ai __vector float
9282vec_srb(__vector float __a, __vector unsigned char __b) {
9283 return (__vector float)__builtin_s390_vsrlb(
9284 (__vector unsigned char)__a, __b);
9285}
7905#endif9286#endif
79069287
9288// This prototype is deprecated.
7907static inline __ATTRS_o_ai __vector double9289static inline __ATTRS_o_ai __vector double
7908vec_srb(__vector double __a, __vector signed long long __b) {9290vec_srb(__vector double __a, __vector signed long long __b) {
7909 return (__vector double)__builtin_s390_vsrlb(9291 return (__vector double)__builtin_s390_vsrlb(
7910 (__vector unsigned char)__a, (__vector unsigned char)__b);9292 (__vector unsigned char)__a, (__vector unsigned char)__b);
7911}9293}
79129294
9295// This prototype is deprecated.
7913static inline __ATTRS_o_ai __vector double9296static inline __ATTRS_o_ai __vector double
7914vec_srb(__vector double __a, __vector unsigned long long __b) {9297vec_srb(__vector double __a, __vector unsigned long long __b) {
7915 return (__vector double)__builtin_s390_vsrlb(9298 return (__vector double)__builtin_s390_vsrlb(
7916 (__vector unsigned char)__a, (__vector unsigned char)__b);9299 (__vector unsigned char)__a, (__vector unsigned char)__b);
7917}9300}
79189301
9302static inline __ATTRS_o_ai __vector double
9303vec_srb(__vector double __a, __vector unsigned char __b) {
9304 return (__vector double)__builtin_s390_vsrlb(
9305 (__vector unsigned char)__a, __b);
9306}
9307
7919/*-- vec_srdb ---------------------------------------------------------------*/9308/*-- vec_srdb ---------------------------------------------------------------*/
79209309
7921#if __ARCH__ >= 139310#if __ARCH__ >= 13
...@@ -7953,6 +9342,15 @@ vec_srdb(__vector unsigned long long __a, __vector unsigned long long __b,...@@ -7953,6 +9342,15 @@ vec_srdb(__vector unsigned long long __a, __vector unsigned long long __b,
7953 int __c)9342 int __c)
7954 __constant_range(__c, 0, 7);9343 __constant_range(__c, 0, 7);
79559344
9345extern __ATTRS_o __vector signed __int128
9346vec_srdb(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
9347 __constant_range(__c, 0, 7);
9348
9349extern __ATTRS_o __vector unsigned __int128
9350vec_srdb(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
9351 int __c)
9352 __constant_range(__c, 0, 7);
9353
7956extern __ATTRS_o __vector float9354extern __ATTRS_o __vector float
7957vec_srdb(__vector float __a, __vector float __b, int __c)9355vec_srdb(__vector float __a, __vector float __b, int __c)
7958 __constant_range(__c, 0, 7);9356 __constant_range(__c, 0, 7);
...@@ -7989,6 +9387,11 @@ vec_abs(__vector signed long long __a) {...@@ -7989,6 +9387,11 @@ vec_abs(__vector signed long long __a) {
7989 return vec_sel(__a, -__a, vec_cmplt(__a, (__vector signed long long)0));9387 return vec_sel(__a, -__a, vec_cmplt(__a, (__vector signed long long)0));
7990}9388}
79919389
9390static inline __ATTRS_o_ai __vector signed __int128
9391vec_abs(__vector signed __int128 __a) {
9392 return vec_sel(__a, -__a, vec_cmplt(__a, (__vector signed __int128)0));
9393}
9394
7992#if __ARCH__ >= 129395#if __ARCH__ >= 12
7993static inline __ATTRS_o_ai __vector float9396static inline __ATTRS_o_ai __vector float
7994vec_abs(__vector float __a) {9397vec_abs(__vector float __a) {
...@@ -8169,6 +9572,16 @@ vec_max(__vector __bool long long __a, __vector unsigned long long __b) {...@@ -8169,6 +9572,16 @@ vec_max(__vector __bool long long __a, __vector unsigned long long __b) {
8169 return vec_sel(__b, __ac, vec_cmpgt(__ac, __b));9572 return vec_sel(__b, __ac, vec_cmpgt(__ac, __b));
8170}9573}
81719574
9575static inline __ATTRS_o_ai __vector signed __int128
9576vec_max(__vector signed __int128 __a, __vector signed __int128 __b) {
9577 return vec_sel(__b, __a, vec_cmpgt(__a, __b));
9578}
9579
9580static inline __ATTRS_o_ai __vector unsigned __int128
9581vec_max(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9582 return vec_sel(__b, __a, vec_cmpgt(__a, __b));
9583}
9584
8172#if __ARCH__ >= 129585#if __ARCH__ >= 12
8173static inline __ATTRS_o_ai __vector float9586static inline __ATTRS_o_ai __vector float
8174vec_max(__vector float __a, __vector float __b) {9587vec_max(__vector float __a, __vector float __b) {
...@@ -8339,6 +9752,16 @@ vec_min(__vector __bool long long __a, __vector unsigned long long __b) {...@@ -8339,6 +9752,16 @@ vec_min(__vector __bool long long __a, __vector unsigned long long __b) {
8339 return vec_sel(__ac, __b, vec_cmpgt(__ac, __b));9752 return vec_sel(__ac, __b, vec_cmpgt(__ac, __b));
8340}9753}
83419754
9755static inline __ATTRS_o_ai __vector signed __int128
9756vec_min(__vector signed __int128 __a, __vector signed __int128 __b) {
9757 return vec_sel(__a, __b, vec_cmpgt(__a, __b));
9758}
9759
9760static inline __ATTRS_o_ai __vector unsigned __int128
9761vec_min(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9762 return vec_sel(__a, __b, vec_cmpgt(__a, __b));
9763}
9764
8342#if __ARCH__ >= 129765#if __ARCH__ >= 12
8343static inline __ATTRS_o_ai __vector float9766static inline __ATTRS_o_ai __vector float
8344vec_min(__vector float __a, __vector float __b) {9767vec_min(__vector float __a, __vector float __b) {
...@@ -8357,9 +9780,11 @@ vec_min(__vector double __a, __vector double __b) {...@@ -8357,9 +9780,11 @@ vec_min(__vector double __a, __vector double __b) {
83579780
8358/*-- vec_add_u128 -----------------------------------------------------------*/9781/*-- vec_add_u128 -----------------------------------------------------------*/
83599782
9783// This prototype is deprecated.
8360static inline __ATTRS_ai __vector unsigned char9784static inline __ATTRS_ai __vector unsigned char
8361vec_add_u128(__vector unsigned char __a, __vector unsigned char __b) {9785vec_add_u128(__vector unsigned char __a, __vector unsigned char __b) {
8362 return (__vector unsigned char)((__int128)__a + (__int128)__b);9786 return (__vector unsigned char)(__vector unsigned __int128)
9787 ((__int128)__a + (__int128)__b);
8363}9788}
83649789
8365/*-- vec_addc ---------------------------------------------------------------*/9790/*-- vec_addc ---------------------------------------------------------------*/
...@@ -8384,30 +9809,59 @@ vec_addc(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -8384,30 +9809,59 @@ vec_addc(__vector unsigned long long __a, __vector unsigned long long __b) {
8384 return __builtin_s390_vaccg(__a, __b);9809 return __builtin_s390_vaccg(__a, __b);
8385}9810}
83869811
9812static inline __ATTRS_o_ai __vector unsigned __int128
9813vec_addc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9814 return (__vector unsigned __int128)
9815 __builtin_s390_vaccq((unsigned __int128)__a, (unsigned __int128)__b);
9816}
9817
8387/*-- vec_addc_u128 ----------------------------------------------------------*/9818/*-- vec_addc_u128 ----------------------------------------------------------*/
83889819
9820// This prototype is deprecated.
8389static inline __ATTRS_ai __vector unsigned char9821static inline __ATTRS_ai __vector unsigned char
8390vec_addc_u128(__vector unsigned char __a, __vector unsigned char __b) {9822vec_addc_u128(__vector unsigned char __a, __vector unsigned char __b) {
8391 return (__vector unsigned char)9823 return (__vector unsigned char)(__vector unsigned __int128)
8392 __builtin_s390_vaccq((unsigned __int128)__a, (unsigned __int128)__b);9824 __builtin_s390_vaccq((unsigned __int128)__a, (unsigned __int128)__b);
8393}9825}
83949826
9827/*-- vec_adde ---------------------------------------------------------------*/
9828
9829static inline __ATTRS_ai __vector unsigned __int128
9830vec_adde(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
9831 __vector unsigned __int128 __c) {
9832 return (__vector unsigned __int128)
9833 __builtin_s390_vacq((unsigned __int128)__a, (unsigned __int128)__b,
9834 (unsigned __int128)__c);
9835}
9836
8395/*-- vec_adde_u128 ----------------------------------------------------------*/9837/*-- vec_adde_u128 ----------------------------------------------------------*/
83969838
9839// This prototype is deprecated.
8397static inline __ATTRS_ai __vector unsigned char9840static inline __ATTRS_ai __vector unsigned char
8398vec_adde_u128(__vector unsigned char __a, __vector unsigned char __b,9841vec_adde_u128(__vector unsigned char __a, __vector unsigned char __b,
8399 __vector unsigned char __c) {9842 __vector unsigned char __c) {
8400 return (__vector unsigned char)9843 return (__vector unsigned char)(__vector unsigned __int128)
8401 __builtin_s390_vacq((unsigned __int128)__a, (unsigned __int128)__b,9844 __builtin_s390_vacq((unsigned __int128)__a, (unsigned __int128)__b,
8402 (unsigned __int128)__c);9845 (unsigned __int128)__c);
8403}9846}
84049847
9848/*-- vec_addec --------------------------------------------------------------*/
9849
9850static inline __ATTRS_ai __vector unsigned __int128
9851vec_addec(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
9852 __vector unsigned __int128 __c) {
9853 return (__vector unsigned __int128)
9854 __builtin_s390_vacccq((unsigned __int128)__a, (unsigned __int128)__b,
9855 (unsigned __int128)__c);
9856}
9857
8405/*-- vec_addec_u128 ---------------------------------------------------------*/9858/*-- vec_addec_u128 ---------------------------------------------------------*/
84069859
9860// This prototype is deprecated.
8407static inline __ATTRS_ai __vector unsigned char9861static inline __ATTRS_ai __vector unsigned char
8408vec_addec_u128(__vector unsigned char __a, __vector unsigned char __b,9862vec_addec_u128(__vector unsigned char __a, __vector unsigned char __b,
8409 __vector unsigned char __c) {9863 __vector unsigned char __c) {
8410 return (__vector unsigned char)9864 return (__vector unsigned char)(__vector unsigned __int128)
8411 __builtin_s390_vacccq((unsigned __int128)__a, (unsigned __int128)__b,9865 __builtin_s390_vacccq((unsigned __int128)__a, (unsigned __int128)__b,
8412 (unsigned __int128)__c);9866 (unsigned __int128)__c);
8413}9867}
...@@ -8434,6 +9888,14 @@ vec_avg(__vector signed long long __a, __vector signed long long __b) {...@@ -8434,6 +9888,14 @@ vec_avg(__vector signed long long __a, __vector signed long long __b) {
8434 return __builtin_s390_vavgg(__a, __b);9888 return __builtin_s390_vavgg(__a, __b);
8435}9889}
84369890
9891#if __ARCH__ >= 15
9892static inline __ATTRS_o_ai __vector signed __int128
9893vec_avg(__vector signed __int128 __a, __vector signed __int128 __b) {
9894 return (__vector signed __int128)
9895 __builtin_s390_vavgq((signed __int128)__a, (signed __int128)__b);
9896}
9897#endif
9898
8437static inline __ATTRS_o_ai __vector unsigned char9899static inline __ATTRS_o_ai __vector unsigned char
8438vec_avg(__vector unsigned char __a, __vector unsigned char __b) {9900vec_avg(__vector unsigned char __a, __vector unsigned char __b) {
8439 return __builtin_s390_vavglb(__a, __b);9901 return __builtin_s390_vavglb(__a, __b);
...@@ -8454,6 +9916,14 @@ vec_avg(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -8454,6 +9916,14 @@ vec_avg(__vector unsigned long long __a, __vector unsigned long long __b) {
8454 return __builtin_s390_vavglg(__a, __b);9916 return __builtin_s390_vavglg(__a, __b);
8455}9917}
84569918
9919#if __ARCH__ >= 15
9920static inline __ATTRS_o_ai __vector unsigned __int128
9921vec_avg(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9922 return (__vector unsigned __int128)
9923 __builtin_s390_vavglq((unsigned __int128)__a, (unsigned __int128)__b);
9924}
9925#endif
9926
8457/*-- vec_checksum -----------------------------------------------------------*/9927/*-- vec_checksum -----------------------------------------------------------*/
84589928
8459static inline __ATTRS_ai __vector unsigned int9929static inline __ATTRS_ai __vector unsigned int
...@@ -8478,12 +9948,19 @@ vec_gfmsum(__vector unsigned int __a, __vector unsigned int __b) {...@@ -8478,12 +9948,19 @@ vec_gfmsum(__vector unsigned int __a, __vector unsigned int __b) {
8478 return __builtin_s390_vgfmf(__a, __b);9948 return __builtin_s390_vgfmf(__a, __b);
8479}9949}
84809950
9951static inline __ATTRS_o_ai __vector unsigned __int128
9952vec_gfmsum(__vector unsigned long long __a, __vector unsigned long long __b) {
9953 return (__vector unsigned __int128)__builtin_s390_vgfmg(__a, __b);
9954}
9955
8481/*-- vec_gfmsum_128 ---------------------------------------------------------*/9956/*-- vec_gfmsum_128 ---------------------------------------------------------*/
84829957
9958// This prototype is deprecated.
8483static inline __ATTRS_o_ai __vector unsigned char9959static inline __ATTRS_o_ai __vector unsigned char
8484vec_gfmsum_128(__vector unsigned long long __a,9960vec_gfmsum_128(__vector unsigned long long __a,
8485 __vector unsigned long long __b) {9961 __vector unsigned long long __b) {
8486 return (__vector unsigned char)__builtin_s390_vgfmg(__a, __b);9962 return (__vector unsigned char)(__vector unsigned __int128)
9963 __builtin_s390_vgfmg(__a, __b);
8487}9964}
84889965
8489/*-- vec_gfmsum_accum -------------------------------------------------------*/9966/*-- vec_gfmsum_accum -------------------------------------------------------*/
...@@ -8506,13 +9983,21 @@ vec_gfmsum_accum(__vector unsigned int __a, __vector unsigned int __b,...@@ -8506,13 +9983,21 @@ vec_gfmsum_accum(__vector unsigned int __a, __vector unsigned int __b,
8506 return __builtin_s390_vgfmaf(__a, __b, __c);9983 return __builtin_s390_vgfmaf(__a, __b, __c);
8507}9984}
85089985
9986static inline __ATTRS_o_ai __vector unsigned __int128
9987vec_gfmsum_accum(__vector unsigned long long __a, __vector unsigned long long __b,
9988 __vector unsigned __int128 __c) {
9989 return (__vector unsigned __int128)
9990 __builtin_s390_vgfmag(__a, __b, (unsigned __int128)__c);
9991}
9992
8509/*-- vec_gfmsum_accum_128 ---------------------------------------------------*/9993/*-- vec_gfmsum_accum_128 ---------------------------------------------------*/
85109994
9995// This prototype is deprecated.
8511static inline __ATTRS_o_ai __vector unsigned char9996static inline __ATTRS_o_ai __vector unsigned char
8512vec_gfmsum_accum_128(__vector unsigned long long __a,9997vec_gfmsum_accum_128(__vector unsigned long long __a,
8513 __vector unsigned long long __b,9998 __vector unsigned long long __b,
8514 __vector unsigned char __c) {9999 __vector unsigned char __c) {
8515 return (__vector unsigned char)10000 return (__vector unsigned char)(__vector unsigned __int128)
8516 __builtin_s390_vgfmag(__a, __b, (unsigned __int128)__c);10001 __builtin_s390_vgfmag(__a, __b, (unsigned __int128)__c);
8517}10002}
851810003
...@@ -8590,6 +10075,56 @@ vec_mladd(__vector unsigned int __a, __vector unsigned int __b,...@@ -8590,6 +10075,56 @@ vec_mladd(__vector unsigned int __a, __vector unsigned int __b,
8590 return __a * __b + __c;10075 return __a * __b + __c;
8591}10076}
859210077
10078#if __ARCH__ >= 15
10079static inline __ATTRS_o_ai __vector signed long long
10080vec_mladd(__vector signed long long __a, __vector signed long long __b,
10081 __vector signed long long __c) {
10082 return __a * __b + __c;
10083}
10084
10085static inline __ATTRS_o_ai __vector signed long long
10086vec_mladd(__vector unsigned long long __a, __vector signed long long __b,
10087 __vector signed long long __c) {
10088 return (__vector signed long long)__a * __b + __c;
10089}
10090
10091static inline __ATTRS_o_ai __vector signed long long
10092vec_mladd(__vector signed long long __a, __vector unsigned long long __b,
10093 __vector unsigned long long __c) {
10094 return __a * (__vector signed long long)__b + (__vector signed long long)__c;
10095}
10096
10097static inline __ATTRS_o_ai __vector unsigned long long
10098vec_mladd(__vector unsigned long long __a, __vector unsigned long long __b,
10099 __vector unsigned long long __c) {
10100 return __a * __b + __c;
10101}
10102
10103static inline __ATTRS_o_ai __vector signed __int128
10104vec_mladd(__vector signed __int128 __a, __vector signed __int128 __b,
10105 __vector signed __int128 __c) {
10106 return __a * __b + __c;
10107}
10108
10109static inline __ATTRS_o_ai __vector signed __int128
10110vec_mladd(__vector unsigned __int128 __a, __vector signed __int128 __b,
10111 __vector signed __int128 __c) {
10112 return (__vector signed __int128)__a * __b + __c;
10113}
10114
10115static inline __ATTRS_o_ai __vector signed __int128
10116vec_mladd(__vector signed __int128 __a, __vector unsigned __int128 __b,
10117 __vector unsigned __int128 __c) {
10118 return __a * (__vector signed __int128)__b + (__vector signed __int128)__c;
10119}
10120
10121static inline __ATTRS_o_ai __vector unsigned __int128
10122vec_mladd(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10123 __vector unsigned __int128 __c) {
10124 return __a * __b + __c;
10125}
10126#endif
10127
8593/*-- vec_mhadd --------------------------------------------------------------*/10128/*-- vec_mhadd --------------------------------------------------------------*/
859410129
8595static inline __ATTRS_o_ai __vector signed char10130static inline __ATTRS_o_ai __vector signed char
...@@ -8628,6 +10163,34 @@ vec_mhadd(__vector unsigned int __a, __vector unsigned int __b,...@@ -8628,6 +10163,34 @@ vec_mhadd(__vector unsigned int __a, __vector unsigned int __b,
8628 return __builtin_s390_vmalhf(__a, __b, __c);10163 return __builtin_s390_vmalhf(__a, __b, __c);
8629}10164}
863010165
10166#if __ARCH__ >= 15
10167static inline __ATTRS_o_ai __vector signed long long
10168vec_mhadd(__vector signed long long __a, __vector signed long long __b,
10169 __vector signed long long __c) {
10170 return __builtin_s390_vmahg(__a, __b, __c);
10171}
10172
10173static inline __ATTRS_o_ai __vector unsigned long long
10174vec_mhadd(__vector unsigned long long __a, __vector unsigned long long __b,
10175 __vector unsigned long long __c) {
10176 return __builtin_s390_vmalhg(__a, __b, __c);
10177}
10178
10179static inline __ATTRS_o_ai __vector signed __int128
10180vec_mhadd(__vector signed __int128 __a, __vector signed __int128 __b,
10181 __vector signed __int128 __c) {
10182 return (__vector signed __int128)
10183 __builtin_s390_vmahq((signed __int128)__a, (signed __int128)__b, (signed __int128)__c);
10184}
10185
10186static inline __ATTRS_o_ai __vector unsigned __int128
10187vec_mhadd(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10188 __vector unsigned __int128 __c) {
10189 return (__vector unsigned __int128)
10190 __builtin_s390_vmalhq((unsigned __int128)__a, (unsigned __int128)__b, (unsigned __int128)__c);
10191}
10192#endif
10193
8631/*-- vec_meadd --------------------------------------------------------------*/10194/*-- vec_meadd --------------------------------------------------------------*/
863210195
8633static inline __ATTRS_o_ai __vector signed short10196static inline __ATTRS_o_ai __vector signed short
...@@ -8666,6 +10229,22 @@ vec_meadd(__vector unsigned int __a, __vector unsigned int __b,...@@ -8666,6 +10229,22 @@ vec_meadd(__vector unsigned int __a, __vector unsigned int __b,
8666 return __builtin_s390_vmalef(__a, __b, __c);10229 return __builtin_s390_vmalef(__a, __b, __c);
8667}10230}
866810231
10232#if __ARCH__ >= 15
10233static inline __ATTRS_o_ai __vector signed __int128
10234vec_meadd(__vector signed long long __a, __vector signed long long __b,
10235 __vector signed __int128 __c) {
10236 return (__vector signed __int128)
10237 __builtin_s390_vmaeg(__a, __b, (signed __int128)__c);
10238}
10239
10240static inline __ATTRS_o_ai __vector unsigned __int128
10241vec_meadd(__vector unsigned long long __a, __vector unsigned long long __b,
10242 __vector unsigned __int128 __c) {
10243 return (__vector unsigned __int128)
10244 __builtin_s390_vmaleg(__a, __b, (unsigned __int128)__c);
10245}
10246#endif
10247
8669/*-- vec_moadd --------------------------------------------------------------*/10248/*-- vec_moadd --------------------------------------------------------------*/
867010249
8671static inline __ATTRS_o_ai __vector signed short10250static inline __ATTRS_o_ai __vector signed short
...@@ -8704,6 +10283,22 @@ vec_moadd(__vector unsigned int __a, __vector unsigned int __b,...@@ -8704,6 +10283,22 @@ vec_moadd(__vector unsigned int __a, __vector unsigned int __b,
8704 return __builtin_s390_vmalof(__a, __b, __c);10283 return __builtin_s390_vmalof(__a, __b, __c);
8705}10284}
870610285
10286#if __ARCH__ >= 15
10287static inline __ATTRS_o_ai __vector signed __int128
10288vec_moadd(__vector signed long long __a, __vector signed long long __b,
10289 __vector signed __int128 __c) {
10290 return (__vector signed __int128)
10291 __builtin_s390_vmaog(__a, __b, (signed __int128)__c);
10292}
10293
10294static inline __ATTRS_o_ai __vector unsigned __int128
10295vec_moadd(__vector unsigned long long __a, __vector unsigned long long __b,
10296 __vector unsigned __int128 __c) {
10297 return (__vector unsigned __int128)
10298 __builtin_s390_vmalog(__a, __b, (unsigned __int128)__c);
10299}
10300#endif
10301
8707/*-- vec_mulh ---------------------------------------------------------------*/10302/*-- vec_mulh ---------------------------------------------------------------*/
870810303
8709static inline __ATTRS_o_ai __vector signed char10304static inline __ATTRS_o_ai __vector signed char
...@@ -8736,6 +10331,30 @@ vec_mulh(__vector unsigned int __a, __vector unsigned int __b) {...@@ -8736,6 +10331,30 @@ vec_mulh(__vector unsigned int __a, __vector unsigned int __b) {
8736 return __builtin_s390_vmlhf(__a, __b);10331 return __builtin_s390_vmlhf(__a, __b);
8737}10332}
873810333
10334#if __ARCH__ >= 15
10335static inline __ATTRS_o_ai __vector signed long long
10336vec_mulh(__vector signed long long __a, __vector signed long long __b) {
10337 return __builtin_s390_vmhg(__a, __b);
10338}
10339
10340static inline __ATTRS_o_ai __vector unsigned long long
10341vec_mulh(__vector unsigned long long __a, __vector unsigned long long __b) {
10342 return __builtin_s390_vmlhg(__a, __b);
10343}
10344
10345static inline __ATTRS_o_ai __vector signed __int128
10346vec_mulh(__vector signed __int128 __a, __vector signed __int128 __b) {
10347 return (__vector signed __int128)
10348 __builtin_s390_vmhq((signed __int128)__a, (signed __int128)__b);
10349}
10350
10351static inline __ATTRS_o_ai __vector unsigned __int128
10352vec_mulh(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
10353 return (__vector unsigned __int128)
10354 __builtin_s390_vmlhq((unsigned __int128)__a, (unsigned __int128)__b);
10355}
10356#endif
10357
8739/*-- vec_mule ---------------------------------------------------------------*/10358/*-- vec_mule ---------------------------------------------------------------*/
874010359
8741static inline __ATTRS_o_ai __vector signed short10360static inline __ATTRS_o_ai __vector signed short
...@@ -8768,6 +10387,18 @@ vec_mule(__vector unsigned int __a, __vector unsigned int __b) {...@@ -8768,6 +10387,18 @@ vec_mule(__vector unsigned int __a, __vector unsigned int __b) {
8768 return __builtin_s390_vmlef(__a, __b);10387 return __builtin_s390_vmlef(__a, __b);
8769}10388}
877010389
10390#if __ARCH__ >= 15
10391static inline __ATTRS_o_ai __vector signed __int128
10392vec_mule(__vector signed long long __a, __vector signed long long __b) {
10393 return (__vector signed __int128)__builtin_s390_vmeg(__a, __b);
10394}
10395
10396static inline __ATTRS_o_ai __vector unsigned __int128
10397vec_mule(__vector unsigned long long __a, __vector unsigned long long __b) {
10398 return (__vector unsigned __int128)__builtin_s390_vmleg(__a, __b);
10399}
10400#endif
10401
8771/*-- vec_mulo ---------------------------------------------------------------*/10402/*-- vec_mulo ---------------------------------------------------------------*/
877210403
8773static inline __ATTRS_o_ai __vector signed short10404static inline __ATTRS_o_ai __vector signed short
...@@ -8800,9 +10431,35 @@ vec_mulo(__vector unsigned int __a, __vector unsigned int __b) {...@@ -8800,9 +10431,35 @@ vec_mulo(__vector unsigned int __a, __vector unsigned int __b) {
8800 return __builtin_s390_vmlof(__a, __b);10431 return __builtin_s390_vmlof(__a, __b);
8801}10432}
880210433
10434#if __ARCH__ >= 15
10435static inline __ATTRS_o_ai __vector signed __int128
10436vec_mulo(__vector signed long long __a, __vector signed long long __b) {
10437 return (__vector signed __int128)__builtin_s390_vmog(__a, __b);
10438}
10439
10440static inline __ATTRS_o_ai __vector unsigned __int128
10441vec_mulo(__vector unsigned long long __a, __vector unsigned long long __b) {
10442 return (__vector unsigned __int128)__builtin_s390_vmlog(__a, __b);
10443}
10444#endif
10445
10446/*-- vec_msum ---------------------------------------------------------------*/
10447
10448#if __ARCH__ >= 12
10449extern __ATTRS_o __vector unsigned __int128
10450vec_msum(__vector unsigned long long __a, __vector unsigned long long __b,
10451 __vector unsigned __int128 __c, int __d)
10452 __constant_range(__d, 0, 15);
10453
10454#define vec_msum(X, Y, Z, W) \
10455 ((__typeof__((vec_msum)((X), (Y), (Z), (W)))) \
10456 __builtin_s390_vmslg((X), (Y), (unsigned __int128)(Z), (W)))
10457#endif
10458
8803/*-- vec_msum_u128 ----------------------------------------------------------*/10459/*-- vec_msum_u128 ----------------------------------------------------------*/
880410460
8805#if __ARCH__ >= 1210461#if __ARCH__ >= 12
10462// This prototype is deprecated.
8806extern __ATTRS_o __vector unsigned char10463extern __ATTRS_o __vector unsigned char
8807vec_msum_u128(__vector unsigned long long __a, __vector unsigned long long __b,10464vec_msum_u128(__vector unsigned long long __a, __vector unsigned long long __b,
8808 __vector unsigned char __c, int __d)10465 __vector unsigned char __c, int __d)
...@@ -8810,14 +10467,17 @@ vec_msum_u128(__vector unsigned long long __a, __vector unsigned long long __b,...@@ -8810,14 +10467,17 @@ vec_msum_u128(__vector unsigned long long __a, __vector unsigned long long __b,
881010467
8811#define vec_msum_u128(X, Y, Z, W) \10468#define vec_msum_u128(X, Y, Z, W) \
8812 ((__typeof__((vec_msum_u128)((X), (Y), (Z), (W)))) \10469 ((__typeof__((vec_msum_u128)((X), (Y), (Z), (W)))) \
10470 (__vector unsigned __int128) \
8813 __builtin_s390_vmslg((X), (Y), (unsigned __int128)(Z), (W)))10471 __builtin_s390_vmslg((X), (Y), (unsigned __int128)(Z), (W)))
8814#endif10472#endif
881510473
8816/*-- vec_sub_u128 -----------------------------------------------------------*/10474/*-- vec_sub_u128 -----------------------------------------------------------*/
881710475
10476// This prototype is deprecated.
8818static inline __ATTRS_ai __vector unsigned char10477static inline __ATTRS_ai __vector unsigned char
8819vec_sub_u128(__vector unsigned char __a, __vector unsigned char __b) {10478vec_sub_u128(__vector unsigned char __a, __vector unsigned char __b) {
8820 return (__vector unsigned char)((__int128)__a - (__int128)__b);10479 return (__vector unsigned char)(__vector unsigned __int128)
10480 ((__int128)__a - (__int128)__b);
8821}10481}
882210482
8823/*-- vec_subc ---------------------------------------------------------------*/10483/*-- vec_subc ---------------------------------------------------------------*/
...@@ -8842,30 +10502,59 @@ vec_subc(__vector unsigned long long __a, __vector unsigned long long __b) {...@@ -8842,30 +10502,59 @@ vec_subc(__vector unsigned long long __a, __vector unsigned long long __b) {
8842 return __builtin_s390_vscbig(__a, __b);10502 return __builtin_s390_vscbig(__a, __b);
8843}10503}
884410504
10505static inline __ATTRS_o_ai __vector unsigned __int128
10506vec_subc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
10507 return (__vector unsigned __int128)
10508 __builtin_s390_vscbiq((unsigned __int128)__a, (unsigned __int128)__b);
10509}
10510
8845/*-- vec_subc_u128 ----------------------------------------------------------*/10511/*-- vec_subc_u128 ----------------------------------------------------------*/
884610512
10513// This prototype is deprecated.
8847static inline __ATTRS_ai __vector unsigned char10514static inline __ATTRS_ai __vector unsigned char
8848vec_subc_u128(__vector unsigned char __a, __vector unsigned char __b) {10515vec_subc_u128(__vector unsigned char __a, __vector unsigned char __b) {
8849 return (__vector unsigned char)10516 return (__vector unsigned char)(__vector unsigned __int128)
8850 __builtin_s390_vscbiq((unsigned __int128)__a, (unsigned __int128)__b);10517 __builtin_s390_vscbiq((unsigned __int128)__a, (unsigned __int128)__b);
8851}10518}
885210519
10520/*-- vec_sube ---------------------------------------------------------------*/
10521
10522static inline __ATTRS_ai __vector unsigned __int128
10523vec_sube(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10524 __vector unsigned __int128 __c) {
10525 return (__vector unsigned __int128)
10526 __builtin_s390_vsbiq((unsigned __int128)__a, (unsigned __int128)__b,
10527 (unsigned __int128)__c);
10528}
10529
8853/*-- vec_sube_u128 ----------------------------------------------------------*/10530/*-- vec_sube_u128 ----------------------------------------------------------*/
885410531
10532// This prototype is deprecated.
8855static inline __ATTRS_ai __vector unsigned char10533static inline __ATTRS_ai __vector unsigned char
8856vec_sube_u128(__vector unsigned char __a, __vector unsigned char __b,10534vec_sube_u128(__vector unsigned char __a, __vector unsigned char __b,
8857 __vector unsigned char __c) {10535 __vector unsigned char __c) {
8858 return (__vector unsigned char)10536 return (__vector unsigned char)(__vector unsigned __int128)
8859 __builtin_s390_vsbiq((unsigned __int128)__a, (unsigned __int128)__b,10537 __builtin_s390_vsbiq((unsigned __int128)__a, (unsigned __int128)__b,
8860 (unsigned __int128)__c);10538 (unsigned __int128)__c);
8861}10539}
886210540
10541/*-- vec_subec --------------------------------------------------------------*/
10542
10543static inline __ATTRS_ai __vector unsigned __int128
10544vec_subec(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10545 __vector unsigned __int128 __c) {
10546 return (__vector unsigned __int128)
10547 __builtin_s390_vsbcbiq((unsigned __int128)__a, (unsigned __int128)__b,
10548 (unsigned __int128)__c);
10549}
10550
8863/*-- vec_subec_u128 ---------------------------------------------------------*/10551/*-- vec_subec_u128 ---------------------------------------------------------*/
886410552
10553// This prototype is deprecated.
8865static inline __ATTRS_ai __vector unsigned char10554static inline __ATTRS_ai __vector unsigned char
8866vec_subec_u128(__vector unsigned char __a, __vector unsigned char __b,10555vec_subec_u128(__vector unsigned char __a, __vector unsigned char __b,
8867 __vector unsigned char __c) {10556 __vector unsigned char __c) {
8868 return (__vector unsigned char)10557 return (__vector unsigned char)(__vector unsigned __int128)
8869 __builtin_s390_vsbcbiq((unsigned __int128)__a, (unsigned __int128)__b,10558 __builtin_s390_vsbcbiq((unsigned __int128)__a, (unsigned __int128)__b,
8870 (unsigned __int128)__c);10559 (unsigned __int128)__c);
8871}10560}
...@@ -8882,16 +10571,32 @@ vec_sum2(__vector unsigned int __a, __vector unsigned int __b) {...@@ -8882,16 +10571,32 @@ vec_sum2(__vector unsigned int __a, __vector unsigned int __b) {
8882 return __builtin_s390_vsumgf(__a, __b);10571 return __builtin_s390_vsumgf(__a, __b);
8883}10572}
888410573
10574/*-- vec_sum ----------------------------------------------------------------*/
10575
10576static inline __ATTRS_o_ai __vector unsigned __int128
10577vec_sum(__vector unsigned int __a, __vector unsigned int __b) {
10578 return (__vector unsigned __int128)__builtin_s390_vsumqf(__a, __b);
10579}
10580
10581static inline __ATTRS_o_ai __vector unsigned __int128
10582vec_sum(__vector unsigned long long __a, __vector unsigned long long __b) {
10583 return (__vector unsigned __int128)__builtin_s390_vsumqg(__a, __b);
10584}
10585
8885/*-- vec_sum_u128 -----------------------------------------------------------*/10586/*-- vec_sum_u128 -----------------------------------------------------------*/
888610587
10588// This prototype is deprecated.
8887static inline __ATTRS_o_ai __vector unsigned char10589static inline __ATTRS_o_ai __vector unsigned char
8888vec_sum_u128(__vector unsigned int __a, __vector unsigned int __b) {10590vec_sum_u128(__vector unsigned int __a, __vector unsigned int __b) {
8889 return (__vector unsigned char)__builtin_s390_vsumqf(__a, __b);10591 return (__vector unsigned char)(__vector unsigned __int128)
10592 __builtin_s390_vsumqf(__a, __b);
8890}10593}
889110594
10595// This prototype is deprecated.
8892static inline __ATTRS_o_ai __vector unsigned char10596static inline __ATTRS_o_ai __vector unsigned char
8893vec_sum_u128(__vector unsigned long long __a, __vector unsigned long long __b) {10597vec_sum_u128(__vector unsigned long long __a, __vector unsigned long long __b) {
8894 return (__vector unsigned char)__builtin_s390_vsumqg(__a, __b);10598 return (__vector unsigned char)(__vector unsigned __int128)
10599 __builtin_s390_vsumqg(__a, __b);
8895}10600}
889610601
8897/*-- vec_sum4 ---------------------------------------------------------------*/10602/*-- vec_sum4 ---------------------------------------------------------------*/
...@@ -8956,6 +10661,19 @@ vec_test_mask(__vector unsigned long long __a,...@@ -8956,6 +10661,19 @@ vec_test_mask(__vector unsigned long long __a,
8956 (__vector unsigned char)__b);10661 (__vector unsigned char)__b);
8957}10662}
895810663
10664static inline __ATTRS_o_ai int
10665vec_test_mask(__vector signed __int128 __a, __vector unsigned __int128 __b) {
10666 return __builtin_s390_vtm((__vector unsigned char)__a,
10667 (__vector unsigned char)__b);
10668}
10669
10670static inline __ATTRS_o_ai int
10671vec_test_mask(__vector unsigned __int128 __a,
10672 __vector unsigned __int128 __b) {
10673 return __builtin_s390_vtm((__vector unsigned char)__a,
10674 (__vector unsigned char)__b);
10675}
10676
8959#if __ARCH__ >= 1210677#if __ARCH__ >= 12
8960static inline __ATTRS_o_ai int10678static inline __ATTRS_o_ai int
8961vec_test_mask(__vector float __a, __vector unsigned int __b) {10679vec_test_mask(__vector float __a, __vector unsigned int __b) {
lib/include/wasm_simd128.h+167-21
...@@ -33,6 +33,7 @@ typedef unsigned long long __u64x2...@@ -33,6 +33,7 @@ typedef unsigned long long __u64x2
33 __attribute__((__vector_size__(16), __aligned__(16)));33 __attribute__((__vector_size__(16), __aligned__(16)));
34typedef float __f32x4 __attribute__((__vector_size__(16), __aligned__(16)));34typedef float __f32x4 __attribute__((__vector_size__(16), __aligned__(16)));
35typedef double __f64x2 __attribute__((__vector_size__(16), __aligned__(16)));35typedef double __f64x2 __attribute__((__vector_size__(16), __aligned__(16)));
36typedef __fp16 __f16x8 __attribute__((__vector_size__(16), __aligned__(16)));
3637
37typedef signed char __i8x8 __attribute__((__vector_size__(8), __aligned__(8)));38typedef signed char __i8x8 __attribute__((__vector_size__(8), __aligned__(8)));
38typedef unsigned char __u8x839typedef unsigned char __u8x8
...@@ -956,7 +957,7 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i8x16_bitmask(v128_t __a) {...@@ -956,7 +957,7 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i8x16_bitmask(v128_t __a) {
956}957}
957958
958static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) {959static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) {
959 return (v128_t)__builtin_wasm_popcnt_i8x16((__i8x16)__a);960 return (v128_t)__builtin_elementwise_popcount((__i8x16)__a);
960}961}
961962
962static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,963static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,
...@@ -981,12 +982,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,...@@ -981,12 +982,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,
981982
982static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add_sat(v128_t __a,983static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add_sat(v128_t __a,
983 v128_t __b) {984 v128_t __b) {
984 return (v128_t)__builtin_wasm_add_sat_s_i8x16((__i8x16)__a, (__i8x16)__b);985 return (v128_t)__builtin_elementwise_add_sat((__i8x16)__a, (__i8x16)__b);
985}986}
986987
987static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_add_sat(v128_t __a,988static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_add_sat(v128_t __a,
988 v128_t __b) {989 v128_t __b) {
989 return (v128_t)__builtin_wasm_add_sat_u_i8x16((__u8x16)__a, (__u8x16)__b);990 return (v128_t)__builtin_elementwise_add_sat((__u8x16)__a, (__u8x16)__b);
990}991}
991992
992static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,993static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
...@@ -996,32 +997,32 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,...@@ -996,32 +997,32 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
996997
997static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub_sat(v128_t __a,998static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub_sat(v128_t __a,
998 v128_t __b) {999 v128_t __b) {
999 return (v128_t)__builtin_wasm_sub_sat_s_i8x16((__i8x16)__a, (__i8x16)__b);1000 return (v128_t)__builtin_elementwise_sub_sat((__i8x16)__a, (__i8x16)__b);
1000}1001}
10011002
1002static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_sub_sat(v128_t __a,1003static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_sub_sat(v128_t __a,
1003 v128_t __b) {1004 v128_t __b) {
1004 return (v128_t)__builtin_wasm_sub_sat_u_i8x16((__u8x16)__a, (__u8x16)__b);1005 return (v128_t)__builtin_elementwise_sub_sat((__u8x16)__a, (__u8x16)__b);
1005}1006}
10061007
1007static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,1008static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,
1008 v128_t __b) {1009 v128_t __b) {
1009 return (v128_t)__builtin_wasm_min_s_i8x16((__i8x16)__a, (__i8x16)__b);1010 return (v128_t)__builtin_elementwise_min((__i8x16)__a, (__i8x16)__b);
1010}1011}
10111012
1012static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_min(v128_t __a,1013static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_min(v128_t __a,
1013 v128_t __b) {1014 v128_t __b) {
1014 return (v128_t)__builtin_wasm_min_u_i8x16((__u8x16)__a, (__u8x16)__b);1015 return (v128_t)__builtin_elementwise_min((__u8x16)__a, (__u8x16)__b);
1015}1016}
10161017
1017static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_max(v128_t __a,1018static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_max(v128_t __a,
1018 v128_t __b) {1019 v128_t __b) {
1019 return (v128_t)__builtin_wasm_max_s_i8x16((__i8x16)__a, (__i8x16)__b);1020 return (v128_t)__builtin_elementwise_max((__i8x16)__a, (__i8x16)__b);
1020}1021}
10211022
1022static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_max(v128_t __a,1023static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_max(v128_t __a,
1023 v128_t __b) {1024 v128_t __b) {
1024 return (v128_t)__builtin_wasm_max_u_i8x16((__u8x16)__a, (__u8x16)__b);1025 return (v128_t)__builtin_elementwise_max((__u8x16)__a, (__u8x16)__b);
1025}1026}
10261027
1027static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_avgr(v128_t __a,1028static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_avgr(v128_t __a,
...@@ -1067,12 +1068,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,...@@ -1067,12 +1068,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,
10671068
1068static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add_sat(v128_t __a,1069static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add_sat(v128_t __a,
1069 v128_t __b) {1070 v128_t __b) {
1070 return (v128_t)__builtin_wasm_add_sat_s_i16x8((__i16x8)__a, (__i16x8)__b);1071 return (v128_t)__builtin_elementwise_add_sat((__i16x8)__a, (__i16x8)__b);
1071}1072}
10721073
1073static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_add_sat(v128_t __a,1074static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_add_sat(v128_t __a,
1074 v128_t __b) {1075 v128_t __b) {
1075 return (v128_t)__builtin_wasm_add_sat_u_i16x8((__u16x8)__a, (__u16x8)__b);1076 return (v128_t)__builtin_elementwise_add_sat((__u16x8)__a, (__u16x8)__b);
1076}1077}
10771078
1078static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,1079static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
...@@ -1082,12 +1083,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,...@@ -1082,12 +1083,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
10821083
1083static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub_sat(v128_t __a,1084static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub_sat(v128_t __a,
1084 v128_t __b) {1085 v128_t __b) {
1085 return (v128_t)__builtin_wasm_sub_sat_s_i16x8((__i16x8)__a, (__i16x8)__b);1086 return (v128_t)__builtin_elementwise_sub_sat((__i16x8)__a, (__i16x8)__b);
1086}1087}
10871088
1088static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_sub_sat(v128_t __a,1089static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_sub_sat(v128_t __a,
1089 v128_t __b) {1090 v128_t __b) {
1090 return (v128_t)__builtin_wasm_sub_sat_u_i16x8((__u16x8)__a, (__u16x8)__b);1091 return (v128_t)__builtin_elementwise_sub_sat((__u16x8)__a, (__u16x8)__b);
1091}1092}
10921093
1093static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,1094static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
...@@ -1097,22 +1098,22 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,...@@ -1097,22 +1098,22 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
10971098
1098static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_min(v128_t __a,1099static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_min(v128_t __a,
1099 v128_t __b) {1100 v128_t __b) {
1100 return (v128_t)__builtin_wasm_min_s_i16x8((__i16x8)__a, (__i16x8)__b);1101 return (v128_t)__builtin_elementwise_min((__i16x8)__a, (__i16x8)__b);
1101}1102}
11021103
1103static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_min(v128_t __a,1104static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_min(v128_t __a,
1104 v128_t __b) {1105 v128_t __b) {
1105 return (v128_t)__builtin_wasm_min_u_i16x8((__u16x8)__a, (__u16x8)__b);1106 return (v128_t)__builtin_elementwise_min((__u16x8)__a, (__u16x8)__b);
1106}1107}
11071108
1108static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_max(v128_t __a,1109static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_max(v128_t __a,
1109 v128_t __b) {1110 v128_t __b) {
1110 return (v128_t)__builtin_wasm_max_s_i16x8((__i16x8)__a, (__i16x8)__b);1111 return (v128_t)__builtin_elementwise_max((__i16x8)__a, (__i16x8)__b);
1111}1112}
11121113
1113static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_max(v128_t __a,1114static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_max(v128_t __a,
1114 v128_t __b) {1115 v128_t __b) {
1115 return (v128_t)__builtin_wasm_max_u_i16x8((__u16x8)__a, (__u16x8)__b);1116 return (v128_t)__builtin_elementwise_max((__u16x8)__a, (__u16x8)__b);
1116}1117}
11171118
1118static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_avgr(v128_t __a,1119static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_avgr(v128_t __a,
...@@ -1168,22 +1169,22 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_mul(v128_t __a,...@@ -1168,22 +1169,22 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_mul(v128_t __a,
11681169
1169static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_min(v128_t __a,1170static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_min(v128_t __a,
1170 v128_t __b) {1171 v128_t __b) {
1171 return (v128_t)__builtin_wasm_min_s_i32x4((__i32x4)__a, (__i32x4)__b);1172 return (v128_t)__builtin_elementwise_min((__i32x4)__a, (__i32x4)__b);
1172}1173}
11731174
1174static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_min(v128_t __a,1175static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_min(v128_t __a,
1175 v128_t __b) {1176 v128_t __b) {
1176 return (v128_t)__builtin_wasm_min_u_i32x4((__u32x4)__a, (__u32x4)__b);1177 return (v128_t)__builtin_elementwise_min((__u32x4)__a, (__u32x4)__b);
1177}1178}
11781179
1179static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_max(v128_t __a,1180static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_max(v128_t __a,
1180 v128_t __b) {1181 v128_t __b) {
1181 return (v128_t)__builtin_wasm_max_s_i32x4((__i32x4)__a, (__i32x4)__b);1182 return (v128_t)__builtin_elementwise_max((__i32x4)__a, (__i32x4)__b);
1182}1183}
11831184
1184static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_max(v128_t __a,1185static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_max(v128_t __a,
1185 v128_t __b) {1186 v128_t __b) {
1186 return (v128_t)__builtin_wasm_max_u_i32x4((__u32x4)__a, (__u32x4)__b);1187 return (v128_t)__builtin_elementwise_max((__u32x4)__a, (__u32x4)__b);
1187}1188}
11881189
1189static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_dot_i16x8(v128_t __a,1190static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_dot_i16x8(v128_t __a,
...@@ -1878,6 +1879,151 @@ wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v128_t __a, v128_t __b, v128_t __c) {...@@ -1878,6 +1879,151 @@ wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v128_t __a, v128_t __b, v128_t __c) {
1878 (__i8x16)__a, (__i8x16)__b, (__i32x4)__c);1879 (__i8x16)__a, (__i8x16)__b, (__i32x4)__c);
1879}1880}
18801881
1882// FP16 intrinsics
1883#define __FP16_FN_ATTRS \
1884 __attribute__((__always_inline__, __nodebug__, __target__("fp16"), \
1885 __min_vector_width__(128)))
1886
1887static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_splat(float __a) {
1888 return (v128_t)__builtin_wasm_splat_f16x8(__a);
1889}
1890
1891#ifdef __wasm_fp16__
1892// TODO Replace the following macros with regular C functions and use normal
1893// target-independent vector code like the other replace/extract instructions.
1894
1895#define wasm_f16x8_extract_lane(__a, __i) \
1896 (__builtin_wasm_extract_lane_f16x8((__f16x8)(__a), __i))
1897
1898#define wasm_f16x8_replace_lane(__a, __i, __b) \
1899 ((v128_t)__builtin_wasm_replace_lane_f16x8((__f16x8)(__a), __i, __b))
1900
1901#endif
1902
1903static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_abs(v128_t __a) {
1904 return (v128_t)__builtin_wasm_abs_f16x8((__f16x8)__a);
1905}
1906
1907static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_neg(v128_t __a) {
1908 return (v128_t)(-(__f16x8)__a);
1909}
1910
1911static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_sqrt(v128_t __a) {
1912 return (v128_t)__builtin_wasm_sqrt_f16x8((__f16x8)__a);
1913}
1914
1915static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_ceil(v128_t __a) {
1916 return (v128_t)__builtin_wasm_ceil_f16x8((__f16x8)__a);
1917}
1918
1919static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_floor(v128_t __a) {
1920 return (v128_t)__builtin_wasm_floor_f16x8((__f16x8)__a);
1921}
1922
1923static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_trunc(v128_t __a) {
1924 return (v128_t)__builtin_wasm_trunc_f16x8((__f16x8)__a);
1925}
1926
1927static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_nearest(v128_t __a) {
1928 return (v128_t)__builtin_wasm_nearest_f16x8((__f16x8)__a);
1929}
1930
1931static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_eq(v128_t __a, v128_t __b) {
1932 return (v128_t)((__f16x8)__a == (__f16x8)__b);
1933}
1934
1935static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_ne(v128_t __a, v128_t __b) {
1936 return (v128_t)((__f16x8)__a != (__f16x8)__b);
1937}
1938
1939static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_lt(v128_t __a, v128_t __b) {
1940 return (v128_t)((__f16x8)__a < (__f16x8)__b);
1941}
1942
1943static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_gt(v128_t __a, v128_t __b) {
1944 return (v128_t)((__f16x8)__a > (__f16x8)__b);
1945}
1946
1947static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_le(v128_t __a, v128_t __b) {
1948 return (v128_t)((__f16x8)__a <= (__f16x8)__b);
1949}
1950
1951static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_ge(v128_t __a, v128_t __b) {
1952 return (v128_t)((__f16x8)__a >= (__f16x8)__b);
1953}
1954
1955static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_add(v128_t __a,
1956 v128_t __b) {
1957 return (v128_t)((__f16x8)__a + (__f16x8)__b);
1958}
1959
1960static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_sub(v128_t __a,
1961 v128_t __b) {
1962 return (v128_t)((__f16x8)__a - (__f16x8)__b);
1963}
1964
1965static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_mul(v128_t __a,
1966 v128_t __b) {
1967 return (v128_t)((__f16x8)__a * (__f16x8)__b);
1968}
1969
1970static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_div(v128_t __a,
1971 v128_t __b) {
1972 return (v128_t)((__f16x8)__a / (__f16x8)__b);
1973}
1974
1975static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_min(v128_t __a,
1976 v128_t __b) {
1977 return (v128_t)__builtin_wasm_min_f16x8((__f16x8)__a, (__f16x8)__b);
1978}
1979
1980static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_max(v128_t __a,
1981 v128_t __b) {
1982 return (v128_t)__builtin_wasm_max_f16x8((__f16x8)__a, (__f16x8)__b);
1983}
1984
1985static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_pmin(v128_t __a,
1986 v128_t __b) {
1987 return (v128_t)__builtin_wasm_pmin_f16x8((__f16x8)__a, (__f16x8)__b);
1988}
1989
1990static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_pmax(v128_t __a,
1991 v128_t __b) {
1992 return (v128_t)__builtin_wasm_pmax_f16x8((__f16x8)__a, (__f16x8)__b);
1993}
1994
1995static __inline__ v128_t __FP16_FN_ATTRS
1996wasm_i16x8_trunc_sat_f16x8(v128_t __a) {
1997 return (v128_t)__builtin_wasm_trunc_saturate_s_i16x8_f16x8((__f16x8)__a);
1998}
1999
2000static __inline__ v128_t __FP16_FN_ATTRS
2001wasm_u16x8_trunc_sat_f16x8(v128_t __a) {
2002 return (v128_t)__builtin_wasm_trunc_saturate_u_i16x8_f16x8((__f16x8)__a);
2003}
2004
2005static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_convert_i16x8(v128_t __a) {
2006 return (v128_t) __builtin_convertvector((__i16x8)__a, __f16x8);
2007}
2008
2009static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_convert_u16x8(v128_t __a) {
2010 return (v128_t) __builtin_convertvector((__u16x8)__a, __f16x8);
2011}
2012
2013static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_relaxed_madd(v128_t __a,
2014 v128_t __b,
2015 v128_t __c) {
2016 return (v128_t)__builtin_wasm_relaxed_madd_f16x8((__f16x8)__a, (__f16x8)__b,
2017 (__f16x8)__c);
2018}
2019
2020static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_relaxed_nmadd(v128_t __a,
2021 v128_t __b,
2022 v128_t __c) {
2023 return (v128_t)__builtin_wasm_relaxed_nmadd_f16x8((__f16x8)__a, (__f16x8)__b,
2024 (__f16x8)__c);
2025}
2026
1881// Deprecated intrinsics2027// Deprecated intrinsics
18822028
1883static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle")2029static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle")
lib/include/xmmintrin.h+170-177
...@@ -32,12 +32,41 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));...@@ -32,12 +32,41 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));
32#endif32#endif
3333
34/* Define the default attributes for the functions in this file. */34/* Define the default attributes for the functions in this file. */
35#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
35#define __DEFAULT_FN_ATTRS \36#define __DEFAULT_FN_ATTRS \
36 __attribute__((__always_inline__, __nodebug__, __target__("sse,no-evex512"), \37 __attribute__((__always_inline__, __nodebug__, __target__("sse,no-evex512"), \
37 __min_vector_width__(128)))38 __min_vector_width__(128)))
38#define __DEFAULT_FN_ATTRS_MMX \39#define __DEFAULT_FN_ATTRS_SSE2 \
39 __attribute__((__always_inline__, __nodebug__, \40 __attribute__((__always_inline__, __nodebug__, \
40 __target__("mmx,sse,no-evex512"), __min_vector_width__(64)))41 __target__("sse2,no-evex512"), __min_vector_width__(128)))
42#else
43#define __DEFAULT_FN_ATTRS \
44 __attribute__((__always_inline__, __nodebug__, __target__("sse"), \
45 __min_vector_width__(128)))
46#define __DEFAULT_FN_ATTRS_SSE2 \
47 __attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
48 __min_vector_width__(128)))
49#endif
50
51#if defined(__cplusplus) && (__cplusplus >= 201103L)
52#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
53#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2 constexpr
54#else
55#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
56#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2
57#endif
58
59#define __trunc64(x) \
60 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
61#define __zext128(x) \
62 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
63 1, 2, 3)
64#define __anyext128(x) \
65 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
66 1, -1, -1)
67#define __zeroupper64(x) \
68 (__m128i) __builtin_shufflevector((__v4si)(x), __extension__(__v4si){}, 0, \
69 1, 4, 5)
4170
42/// Adds the 32-bit float values in the low-order bits of the operands.71/// Adds the 32-bit float values in the low-order bits of the operands.
43///72///
...@@ -54,9 +83,8 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));...@@ -54,9 +83,8 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));
54/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the sum83/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the sum
55/// of the lower 32 bits of both operands. The upper 96 bits are copied from84/// of the lower 32 bits of both operands. The upper 96 bits are copied from
56/// the upper 96 bits of the first source operand.85/// the upper 96 bits of the first source operand.
57static __inline__ __m128 __DEFAULT_FN_ATTRS86static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
58_mm_add_ss(__m128 __a, __m128 __b)87_mm_add_ss(__m128 __a, __m128 __b) {
59{
60 __a[0] += __b[0];88 __a[0] += __b[0];
61 return __a;89 return __a;
62}90}
...@@ -74,9 +102,8 @@ _mm_add_ss(__m128 __a, __m128 __b)...@@ -74,9 +102,8 @@ _mm_add_ss(__m128 __a, __m128 __b)
74/// A 128-bit vector of [4 x float] containing one of the source operands.102/// A 128-bit vector of [4 x float] containing one of the source operands.
75/// \returns A 128-bit vector of [4 x float] containing the sums of both103/// \returns A 128-bit vector of [4 x float] containing the sums of both
76/// operands.104/// operands.
77static __inline__ __m128 __DEFAULT_FN_ATTRS105static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
78_mm_add_ps(__m128 __a, __m128 __b)106_mm_add_ps(__m128 __a, __m128 __b) {
79{
80 return (__m128)((__v4sf)__a + (__v4sf)__b);107 return (__m128)((__v4sf)__a + (__v4sf)__b);
81}108}
82109
...@@ -96,9 +123,8 @@ _mm_add_ps(__m128 __a, __m128 __b)...@@ -96,9 +123,8 @@ _mm_add_ps(__m128 __a, __m128 __b)
96/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the123/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
97/// difference of the lower 32 bits of both operands. The upper 96 bits are124/// difference of the lower 32 bits of both operands. The upper 96 bits are
98/// copied from the upper 96 bits of the first source operand.125/// copied from the upper 96 bits of the first source operand.
99static __inline__ __m128 __DEFAULT_FN_ATTRS126static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
100_mm_sub_ss(__m128 __a, __m128 __b)127_mm_sub_ss(__m128 __a, __m128 __b) {
101{
102 __a[0] -= __b[0];128 __a[0] -= __b[0];
103 return __a;129 return __a;
104}130}
...@@ -117,9 +143,8 @@ _mm_sub_ss(__m128 __a, __m128 __b)...@@ -117,9 +143,8 @@ _mm_sub_ss(__m128 __a, __m128 __b)
117/// A 128-bit vector of [4 x float] containing the subtrahend.143/// A 128-bit vector of [4 x float] containing the subtrahend.
118/// \returns A 128-bit vector of [4 x float] containing the differences between144/// \returns A 128-bit vector of [4 x float] containing the differences between
119/// both operands.145/// both operands.
120static __inline__ __m128 __DEFAULT_FN_ATTRS146static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
121_mm_sub_ps(__m128 __a, __m128 __b)147_mm_sub_ps(__m128 __a, __m128 __b) {
122{
123 return (__m128)((__v4sf)__a - (__v4sf)__b);148 return (__m128)((__v4sf)__a - (__v4sf)__b);
124}149}
125150
...@@ -139,9 +164,8 @@ _mm_sub_ps(__m128 __a, __m128 __b)...@@ -139,9 +164,8 @@ _mm_sub_ps(__m128 __a, __m128 __b)
139/// \returns A 128-bit vector of [4 x float] containing the product of the lower164/// \returns A 128-bit vector of [4 x float] containing the product of the lower
140/// 32 bits of both operands. The upper 96 bits are copied from the upper 96165/// 32 bits of both operands. The upper 96 bits are copied from the upper 96
141/// bits of the first source operand.166/// bits of the first source operand.
142static __inline__ __m128 __DEFAULT_FN_ATTRS167static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
143_mm_mul_ss(__m128 __a, __m128 __b)168_mm_mul_ss(__m128 __a, __m128 __b) {
144{
145 __a[0] *= __b[0];169 __a[0] *= __b[0];
146 return __a;170 return __a;
147}171}
...@@ -159,9 +183,8 @@ _mm_mul_ss(__m128 __a, __m128 __b)...@@ -159,9 +183,8 @@ _mm_mul_ss(__m128 __a, __m128 __b)
159/// A 128-bit vector of [4 x float] containing one of the source operands.183/// A 128-bit vector of [4 x float] containing one of the source operands.
160/// \returns A 128-bit vector of [4 x float] containing the products of both184/// \returns A 128-bit vector of [4 x float] containing the products of both
161/// operands.185/// operands.
162static __inline__ __m128 __DEFAULT_FN_ATTRS186static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
163_mm_mul_ps(__m128 __a, __m128 __b)187_mm_mul_ps(__m128 __a, __m128 __b) {
164{
165 return (__m128)((__v4sf)__a * (__v4sf)__b);188 return (__m128)((__v4sf)__a * (__v4sf)__b);
166}189}
167190
...@@ -181,9 +204,8 @@ _mm_mul_ps(__m128 __a, __m128 __b)...@@ -181,9 +204,8 @@ _mm_mul_ps(__m128 __a, __m128 __b)
181/// \returns A 128-bit vector of [4 x float] containing the quotients of the204/// \returns A 128-bit vector of [4 x float] containing the quotients of the
182/// lower 32 bits of both operands. The upper 96 bits are copied from the205/// lower 32 bits of both operands. The upper 96 bits are copied from the
183/// upper 96 bits of the first source operand.206/// upper 96 bits of the first source operand.
184static __inline__ __m128 __DEFAULT_FN_ATTRS207static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
185_mm_div_ss(__m128 __a, __m128 __b)208_mm_div_ss(__m128 __a, __m128 __b) {
186{
187 __a[0] /= __b[0];209 __a[0] /= __b[0];
188 return __a;210 return __a;
189}211}
...@@ -200,9 +222,8 @@ _mm_div_ss(__m128 __a, __m128 __b)...@@ -200,9 +222,8 @@ _mm_div_ss(__m128 __a, __m128 __b)
200/// A 128-bit vector of [4 x float] containing the divisor.222/// A 128-bit vector of [4 x float] containing the divisor.
201/// \returns A 128-bit vector of [4 x float] containing the quotients of both223/// \returns A 128-bit vector of [4 x float] containing the quotients of both
202/// operands.224/// operands.
203static __inline__ __m128 __DEFAULT_FN_ATTRS225static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
204_mm_div_ps(__m128 __a, __m128 __b)226_mm_div_ps(__m128 __a, __m128 __b) {
205{
206 return (__m128)((__v4sf)__a / (__v4sf)__b);227 return (__m128)((__v4sf)__a / (__v4sf)__b);
207}228}
208229
...@@ -416,9 +437,8 @@ _mm_max_ps(__m128 __a, __m128 __b)...@@ -416,9 +437,8 @@ _mm_max_ps(__m128 __a, __m128 __b)
416/// A 128-bit vector containing one of the source operands.437/// A 128-bit vector containing one of the source operands.
417/// \returns A 128-bit vector of [4 x float] containing the bitwise AND of the438/// \returns A 128-bit vector of [4 x float] containing the bitwise AND of the
418/// values between both operands.439/// values between both operands.
419static __inline__ __m128 __DEFAULT_FN_ATTRS440static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
420_mm_and_ps(__m128 __a, __m128 __b)441_mm_and_ps(__m128 __a, __m128 __b) {
421{
422 return (__m128)((__v4su)__a & (__v4su)__b);442 return (__m128)((__v4su)__a & (__v4su)__b);
423}443}
424444
...@@ -438,9 +458,8 @@ _mm_and_ps(__m128 __a, __m128 __b)...@@ -438,9 +458,8 @@ _mm_and_ps(__m128 __a, __m128 __b)
438/// \returns A 128-bit vector of [4 x float] containing the bitwise AND of the458/// \returns A 128-bit vector of [4 x float] containing the bitwise AND of the
439/// one's complement of the first operand and the values in the second459/// one's complement of the first operand and the values in the second
440/// operand.460/// operand.
441static __inline__ __m128 __DEFAULT_FN_ATTRS461static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
442_mm_andnot_ps(__m128 __a, __m128 __b)462_mm_andnot_ps(__m128 __a, __m128 __b) {
443{
444 return (__m128)(~(__v4su)__a & (__v4su)__b);463 return (__m128)(~(__v4su)__a & (__v4su)__b);
445}464}
446465
...@@ -456,9 +475,8 @@ _mm_andnot_ps(__m128 __a, __m128 __b)...@@ -456,9 +475,8 @@ _mm_andnot_ps(__m128 __a, __m128 __b)
456/// A 128-bit vector of [4 x float] containing one of the source operands.475/// A 128-bit vector of [4 x float] containing one of the source operands.
457/// \returns A 128-bit vector of [4 x float] containing the bitwise OR of the476/// \returns A 128-bit vector of [4 x float] containing the bitwise OR of the
458/// values between both operands.477/// values between both operands.
459static __inline__ __m128 __DEFAULT_FN_ATTRS478static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
460_mm_or_ps(__m128 __a, __m128 __b)479_mm_or_ps(__m128 __a, __m128 __b) {
461{
462 return (__m128)((__v4su)__a | (__v4su)__b);480 return (__m128)((__v4su)__a | (__v4su)__b);
463}481}
464482
...@@ -475,9 +493,8 @@ _mm_or_ps(__m128 __a, __m128 __b)...@@ -475,9 +493,8 @@ _mm_or_ps(__m128 __a, __m128 __b)
475/// A 128-bit vector of [4 x float] containing one of the source operands.493/// A 128-bit vector of [4 x float] containing one of the source operands.
476/// \returns A 128-bit vector of [4 x float] containing the bitwise exclusive OR494/// \returns A 128-bit vector of [4 x float] containing the bitwise exclusive OR
477/// of the values between both operands.495/// of the values between both operands.
478static __inline__ __m128 __DEFAULT_FN_ATTRS496static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
479_mm_xor_ps(__m128 __a, __m128 __b)497_mm_xor_ps(__m128 __a, __m128 __b) {
480{
481 return (__m128)((__v4su)__a ^ (__v4su)__b);498 return (__m128)((__v4su)__a ^ (__v4su)__b);
482}499}
483500
...@@ -1448,10 +1465,10 @@ _mm_cvtss_si64(__m128 __a)...@@ -1448,10 +1465,10 @@ _mm_cvtss_si64(__m128 __a)
1448/// \param __a1465/// \param __a
1449/// A 128-bit vector of [4 x float].1466/// A 128-bit vector of [4 x float].
1450/// \returns A 64-bit integer vector containing the converted values.1467/// \returns A 64-bit integer vector containing the converted values.
1451static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX1468static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1452_mm_cvtps_pi32(__m128 __a)1469_mm_cvtps_pi32(__m128 __a)
1453{1470{
1454 return (__m64)__builtin_ia32_cvtps2pi((__v4sf)__a);1471 return __trunc64(__builtin_ia32_cvtps2dq((__v4sf)__zeroupper64(__a)));
1455}1472}
14561473
1457/// Converts two low-order float values in a 128-bit vector of1474/// Converts two low-order float values in a 128-bit vector of
...@@ -1468,7 +1485,7 @@ _mm_cvtps_pi32(__m128 __a)...@@ -1468,7 +1485,7 @@ _mm_cvtps_pi32(__m128 __a)
1468/// \param __a1485/// \param __a
1469/// A 128-bit vector of [4 x float].1486/// A 128-bit vector of [4 x float].
1470/// \returns A 64-bit integer vector containing the converted values.1487/// \returns A 64-bit integer vector containing the converted values.
1471static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX1488static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1472_mm_cvt_ps2pi(__m128 __a)1489_mm_cvt_ps2pi(__m128 __a)
1473{1490{
1474 return _mm_cvtps_pi32(__a);1491 return _mm_cvtps_pi32(__a);
...@@ -1558,10 +1575,10 @@ _mm_cvttss_si64(__m128 __a)...@@ -1558,10 +1575,10 @@ _mm_cvttss_si64(__m128 __a)
1558/// \param __a1575/// \param __a
1559/// A 128-bit vector of [4 x float].1576/// A 128-bit vector of [4 x float].
1560/// \returns A 64-bit integer vector containing the converted values.1577/// \returns A 64-bit integer vector containing the converted values.
1561static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX1578static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1562_mm_cvttps_pi32(__m128 __a)1579_mm_cvttps_pi32(__m128 __a)
1563{1580{
1564 return (__m64)__builtin_ia32_cvttps2pi((__v4sf)__a);1581 return __trunc64(__builtin_ia32_cvttps2dq((__v4sf)__zeroupper64(__a)));
1565}1582}
15661583
1567/// Converts the lower (first) two elements of a 128-bit vector of [4 x float]1584/// Converts the lower (first) two elements of a 128-bit vector of [4 x float]
...@@ -1579,7 +1596,7 @@ _mm_cvttps_pi32(__m128 __a)...@@ -1579,7 +1596,7 @@ _mm_cvttps_pi32(__m128 __a)
1579/// \param __a1596/// \param __a
1580/// A 128-bit vector of [4 x float].1597/// A 128-bit vector of [4 x float].
1581/// \returns A 64-bit integer vector containing the converted values.1598/// \returns A 64-bit integer vector containing the converted values.
1582static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX1599static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1583_mm_cvtt_ps2pi(__m128 __a)1600_mm_cvtt_ps2pi(__m128 __a)
1584{1601{
1585 return _mm_cvttps_pi32(__a);1602 return _mm_cvttps_pi32(__a);
...@@ -1601,9 +1618,8 @@ _mm_cvtt_ps2pi(__m128 __a)...@@ -1601,9 +1618,8 @@ _mm_cvtt_ps2pi(__m128 __a)
1601/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the1618/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
1602/// converted value of the second operand. The upper 96 bits are copied from1619/// converted value of the second operand. The upper 96 bits are copied from
1603/// the upper 96 bits of the first operand.1620/// the upper 96 bits of the first operand.
1604static __inline__ __m128 __DEFAULT_FN_ATTRS1621static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvtsi32_ss(__m128 __a,
1605_mm_cvtsi32_ss(__m128 __a, int __b)1622 int __b) {
1606{
1607 __a[0] = __b;1623 __a[0] = __b;
1608 return __a;1624 return __a;
1609}1625}
...@@ -1624,9 +1640,8 @@ _mm_cvtsi32_ss(__m128 __a, int __b)...@@ -1624,9 +1640,8 @@ _mm_cvtsi32_ss(__m128 __a, int __b)
1624/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the1640/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
1625/// converted value of the second operand. The upper 96 bits are copied from1641/// converted value of the second operand. The upper 96 bits are copied from
1626/// the upper 96 bits of the first operand.1642/// the upper 96 bits of the first operand.
1627static __inline__ __m128 __DEFAULT_FN_ATTRS1643static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvt_si2ss(__m128 __a,
1628_mm_cvt_si2ss(__m128 __a, int __b)1644 int __b) {
1629{
1630 return _mm_cvtsi32_ss(__a, __b);1645 return _mm_cvtsi32_ss(__a, __b);
1631}1646}
16321647
...@@ -1648,9 +1663,8 @@ _mm_cvt_si2ss(__m128 __a, int __b)...@@ -1648,9 +1663,8 @@ _mm_cvt_si2ss(__m128 __a, int __b)
1648/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the1663/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
1649/// converted value of the second operand. The upper 96 bits are copied from1664/// converted value of the second operand. The upper 96 bits are copied from
1650/// the upper 96 bits of the first operand.1665/// the upper 96 bits of the first operand.
1651static __inline__ __m128 __DEFAULT_FN_ATTRS1666static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1652_mm_cvtsi64_ss(__m128 __a, long long __b)1667_mm_cvtsi64_ss(__m128 __a, long long __b) {
1653{
1654 __a[0] = __b;1668 __a[0] = __b;
1655 return __a;1669 return __a;
1656}1670}
...@@ -1674,10 +1688,13 @@ _mm_cvtsi64_ss(__m128 __a, long long __b)...@@ -1674,10 +1688,13 @@ _mm_cvtsi64_ss(__m128 __a, long long __b)
1674/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the1688/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
1675/// converted value of the second operand. The upper 64 bits are copied from1689/// converted value of the second operand. The upper 64 bits are copied from
1676/// the upper 64 bits of the first operand.1690/// the upper 64 bits of the first operand.
1677static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX1691static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
1678_mm_cvtpi32_ps(__m128 __a, __m64 __b)1692_mm_cvtpi32_ps(__m128 __a, __m64 __b)
1679{1693{
1680 return __builtin_ia32_cvtpi2ps((__v4sf)__a, (__v2si)__b);1694 return (__m128)__builtin_shufflevector(
1695 (__v4sf)__a,
1696 __builtin_convertvector((__v4si)__zext128(__b), __v4sf),
1697 4, 5, 2, 3);
1681}1698}
16821699
1683/// Converts two elements of a 64-bit vector of [2 x i32] into two1700/// Converts two elements of a 64-bit vector of [2 x i32] into two
...@@ -1697,7 +1714,7 @@ _mm_cvtpi32_ps(__m128 __a, __m64 __b)...@@ -1697,7 +1714,7 @@ _mm_cvtpi32_ps(__m128 __a, __m64 __b)
1697/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the1714/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
1698/// converted value from the second operand. The upper 64 bits are copied1715/// converted value from the second operand. The upper 64 bits are copied
1699/// from the upper 64 bits of the first operand.1716/// from the upper 64 bits of the first operand.
1700static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX1717static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
1701_mm_cvt_pi2ps(__m128 __a, __m64 __b)1718_mm_cvt_pi2ps(__m128 __a, __m64 __b)
1702{1719{
1703 return _mm_cvtpi32_ps(__a, __b);1720 return _mm_cvtpi32_ps(__a, __b);
...@@ -1714,9 +1731,8 @@ _mm_cvt_pi2ps(__m128 __a, __m64 __b)...@@ -1714,9 +1731,8 @@ _mm_cvt_pi2ps(__m128 __a, __m64 __b)
1714/// A 128-bit vector of [4 x float]. The lower 32 bits of this operand are1731/// A 128-bit vector of [4 x float]. The lower 32 bits of this operand are
1715/// used in the extraction.1732/// used in the extraction.
1716/// \returns A 32-bit float containing the extracted value.1733/// \returns A 32-bit float containing the extracted value.
1717static __inline__ float __DEFAULT_FN_ATTRS1734static __inline__ float __DEFAULT_FN_ATTRS_CONSTEXPR
1718_mm_cvtss_f32(__m128 __a)1735_mm_cvtss_f32(__m128 __a) {
1719{
1720 return __a[0];1736 return __a[0];
1721}1737}
17221738
...@@ -1907,9 +1923,8 @@ _mm_undefined_ps(void)...@@ -1907,9 +1923,8 @@ _mm_undefined_ps(void)
1907/// \returns An initialized 128-bit floating-point vector of [4 x float]. The1923/// \returns An initialized 128-bit floating-point vector of [4 x float]. The
1908/// lower 32 bits contain the value provided in the source operand. The1924/// lower 32 bits contain the value provided in the source operand. The
1909/// upper 96 bits are set to zero.1925/// upper 96 bits are set to zero.
1910static __inline__ __m128 __DEFAULT_FN_ATTRS1926static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1911_mm_set_ss(float __w)1927_mm_set_ss(float __w) {
1912{
1913 return __extension__ (__m128){ __w, 0.0f, 0.0f, 0.0f };1928 return __extension__ (__m128){ __w, 0.0f, 0.0f, 0.0f };
1914}1929}
19151930
...@@ -1925,9 +1940,8 @@ _mm_set_ss(float __w)...@@ -1925,9 +1940,8 @@ _mm_set_ss(float __w)
1925/// A single-precision floating-point value used to initialize each vector1940/// A single-precision floating-point value used to initialize each vector
1926/// element of the result.1941/// element of the result.
1927/// \returns An initialized 128-bit floating-point vector of [4 x float].1942/// \returns An initialized 128-bit floating-point vector of [4 x float].
1928static __inline__ __m128 __DEFAULT_FN_ATTRS1943static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1929_mm_set1_ps(float __w)1944_mm_set1_ps(float __w) {
1930{
1931 return __extension__ (__m128){ __w, __w, __w, __w };1945 return __extension__ (__m128){ __w, __w, __w, __w };
1932}1946}
19331947
...@@ -1944,9 +1958,8 @@ _mm_set1_ps(float __w)...@@ -1944,9 +1958,8 @@ _mm_set1_ps(float __w)
1944/// A single-precision floating-point value used to initialize each vector1958/// A single-precision floating-point value used to initialize each vector
1945/// element of the result.1959/// element of the result.
1946/// \returns An initialized 128-bit floating-point vector of [4 x float].1960/// \returns An initialized 128-bit floating-point vector of [4 x float].
1947static __inline__ __m128 __DEFAULT_FN_ATTRS1961static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1948_mm_set_ps1(float __w)1962_mm_set_ps1(float __w) {
1949{
1950 return _mm_set1_ps(__w);1963 return _mm_set1_ps(__w);
1951}1964}
19521965
...@@ -1971,9 +1984,8 @@ _mm_set_ps1(float __w)...@@ -1971,9 +1984,8 @@ _mm_set_ps1(float __w)
1971/// A single-precision floating-point value used to initialize bits [31:0]1984/// A single-precision floating-point value used to initialize bits [31:0]
1972/// of the result.1985/// of the result.
1973/// \returns An initialized 128-bit floating-point vector of [4 x float].1986/// \returns An initialized 128-bit floating-point vector of [4 x float].
1974static __inline__ __m128 __DEFAULT_FN_ATTRS1987static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1975_mm_set_ps(float __z, float __y, float __x, float __w)1988_mm_set_ps(float __z, float __y, float __x, float __w) {
1976{
1977 return __extension__ (__m128){ __w, __x, __y, __z };1989 return __extension__ (__m128){ __w, __x, __y, __z };
1978}1990}
19791991
...@@ -1999,9 +2011,8 @@ _mm_set_ps(float __z, float __y, float __x, float __w)...@@ -1999,9 +2011,8 @@ _mm_set_ps(float __z, float __y, float __x, float __w)
1999/// A single-precision floating-point value used to initialize bits [127:96]2011/// A single-precision floating-point value used to initialize bits [127:96]
2000/// of the result.2012/// of the result.
2001/// \returns An initialized 128-bit floating-point vector of [4 x float].2013/// \returns An initialized 128-bit floating-point vector of [4 x float].
2002static __inline__ __m128 __DEFAULT_FN_ATTRS2014static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2003_mm_setr_ps(float __z, float __y, float __x, float __w)2015_mm_setr_ps(float __z, float __y, float __x, float __w) {
2004{
2005 return __extension__ (__m128){ __z, __y, __x, __w };2016 return __extension__ (__m128){ __z, __y, __x, __w };
2006}2017}
20072018
...@@ -2014,9 +2025,8 @@ _mm_setr_ps(float __z, float __y, float __x, float __w)...@@ -2014,9 +2025,8 @@ _mm_setr_ps(float __z, float __y, float __x, float __w)
2014///2025///
2015/// \returns An initialized 128-bit floating-point vector of [4 x float] with2026/// \returns An initialized 128-bit floating-point vector of [4 x float] with
2016/// all elements set to zero.2027/// all elements set to zero.
2017static __inline__ __m128 __DEFAULT_FN_ATTRS2028static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2018_mm_setzero_ps(void)2029_mm_setzero_ps(void) {
2019{
2020 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };2030 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };
2021}2031}
20222032
...@@ -2231,10 +2241,10 @@ _mm_storer_ps(float *__p, __m128 __a)...@@ -2231,10 +2241,10 @@ _mm_storer_ps(float *__p, __m128 __a)
2231/// A pointer to an aligned memory location used to store the register value.2241/// A pointer to an aligned memory location used to store the register value.
2232/// \param __a2242/// \param __a
2233/// A 64-bit integer containing the value to be stored.2243/// A 64-bit integer containing the value to be stored.
2234static __inline__ void __DEFAULT_FN_ATTRS_MMX2244static __inline__ void __DEFAULT_FN_ATTRS
2235_mm_stream_pi(void *__p, __m64 __a)2245_mm_stream_pi(void *__p, __m64 __a)
2236{2246{
2237 __builtin_ia32_movntq((__m64 *)__p, __a);2247 __builtin_nontemporal_store(__a, (__m64 *)__p);
2238}2248}
22392249
2240/// Moves packed float values from a 128-bit vector of [4 x float] to a2250/// Moves packed float values from a 128-bit vector of [4 x float] to a
...@@ -2296,7 +2306,7 @@ void _mm_sfence(void);...@@ -2296,7 +2306,7 @@ void _mm_sfence(void);
2296/// 3: Bits [63:48] are copied to the destination.2306/// 3: Bits [63:48] are copied to the destination.
2297/// \returns A 16-bit integer containing the extracted 16 bits of packed data.2307/// \returns A 16-bit integer containing the extracted 16 bits of packed data.
2298#define _mm_extract_pi16(a, n) \2308#define _mm_extract_pi16(a, n) \
2299 ((int)__builtin_ia32_vec_ext_v4hi((__v4hi)a, (int)n))2309 ((int)(unsigned short)__builtin_ia32_vec_ext_v4hi((__v4hi)a, (int)n))
23002310
2301/// Copies data from the 64-bit vector of [4 x i16] to the destination,2311/// Copies data from the 64-bit vector of [4 x i16] to the destination,
2302/// and inserts the lower 16-bits of an integer operand at the 16-bit offset2312/// and inserts the lower 16-bits of an integer operand at the 16-bit offset
...@@ -2342,10 +2352,10 @@ void _mm_sfence(void);...@@ -2342,10 +2352,10 @@ void _mm_sfence(void);
2342/// \param __b2352/// \param __b
2343/// A 64-bit integer vector containing one of the source operands.2353/// A 64-bit integer vector containing one of the source operands.
2344/// \returns A 64-bit integer vector containing the comparison results.2354/// \returns A 64-bit integer vector containing the comparison results.
2345static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2355static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2346_mm_max_pi16(__m64 __a, __m64 __b)2356_mm_max_pi16(__m64 __a, __m64 __b)
2347{2357{
2348 return (__m64)__builtin_ia32_pmaxsw((__v4hi)__a, (__v4hi)__b);2358 return (__m64)__builtin_elementwise_max((__v4hi)__a, (__v4hi)__b);
2349}2359}
23502360
2351/// Compares each of the corresponding packed 8-bit unsigned integer2361/// Compares each of the corresponding packed 8-bit unsigned integer
...@@ -2361,10 +2371,10 @@ _mm_max_pi16(__m64 __a, __m64 __b)...@@ -2361,10 +2371,10 @@ _mm_max_pi16(__m64 __a, __m64 __b)
2361/// \param __b2371/// \param __b
2362/// A 64-bit integer vector containing one of the source operands.2372/// A 64-bit integer vector containing one of the source operands.
2363/// \returns A 64-bit integer vector containing the comparison results.2373/// \returns A 64-bit integer vector containing the comparison results.
2364static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2374static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2365_mm_max_pu8(__m64 __a, __m64 __b)2375_mm_max_pu8(__m64 __a, __m64 __b)
2366{2376{
2367 return (__m64)__builtin_ia32_pmaxub((__v8qi)__a, (__v8qi)__b);2377 return (__m64)__builtin_elementwise_max((__v8qu)__a, (__v8qu)__b);
2368}2378}
23692379
2370/// Compares each of the corresponding packed 16-bit integer values of2380/// Compares each of the corresponding packed 16-bit integer values of
...@@ -2380,10 +2390,10 @@ _mm_max_pu8(__m64 __a, __m64 __b)...@@ -2380,10 +2390,10 @@ _mm_max_pu8(__m64 __a, __m64 __b)
2380/// \param __b2390/// \param __b
2381/// A 64-bit integer vector containing one of the source operands.2391/// A 64-bit integer vector containing one of the source operands.
2382/// \returns A 64-bit integer vector containing the comparison results.2392/// \returns A 64-bit integer vector containing the comparison results.
2383static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2393static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2384_mm_min_pi16(__m64 __a, __m64 __b)2394_mm_min_pi16(__m64 __a, __m64 __b)
2385{2395{
2386 return (__m64)__builtin_ia32_pminsw((__v4hi)__a, (__v4hi)__b);2396 return (__m64)__builtin_elementwise_min((__v4hi)__a, (__v4hi)__b);
2387}2397}
23882398
2389/// Compares each of the corresponding packed 8-bit unsigned integer2399/// Compares each of the corresponding packed 8-bit unsigned integer
...@@ -2399,10 +2409,10 @@ _mm_min_pi16(__m64 __a, __m64 __b)...@@ -2399,10 +2409,10 @@ _mm_min_pi16(__m64 __a, __m64 __b)
2399/// \param __b2409/// \param __b
2400/// A 64-bit integer vector containing one of the source operands.2410/// A 64-bit integer vector containing one of the source operands.
2401/// \returns A 64-bit integer vector containing the comparison results.2411/// \returns A 64-bit integer vector containing the comparison results.
2402static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2412static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2403_mm_min_pu8(__m64 __a, __m64 __b)2413_mm_min_pu8(__m64 __a, __m64 __b)
2404{2414{
2405 return (__m64)__builtin_ia32_pminub((__v8qi)__a, (__v8qi)__b);2415 return (__m64)__builtin_elementwise_min((__v8qu)__a, (__v8qu)__b);
2406}2416}
24072417
2408/// Takes the most significant bit from each 8-bit element in a 64-bit2418/// Takes the most significant bit from each 8-bit element in a 64-bit
...@@ -2417,10 +2427,10 @@ _mm_min_pu8(__m64 __a, __m64 __b)...@@ -2417,10 +2427,10 @@ _mm_min_pu8(__m64 __a, __m64 __b)
2417/// A 64-bit integer vector containing the values with bits to be extracted.2427/// A 64-bit integer vector containing the values with bits to be extracted.
2418/// \returns The most significant bit from each 8-bit element in \a __a,2428/// \returns The most significant bit from each 8-bit element in \a __a,
2419/// written to bits [7:0].2429/// written to bits [7:0].
2420static __inline__ int __DEFAULT_FN_ATTRS_MMX2430static __inline__ int __DEFAULT_FN_ATTRS_SSE2
2421_mm_movemask_pi8(__m64 __a)2431_mm_movemask_pi8(__m64 __a)
2422{2432{
2423 return __builtin_ia32_pmovmskb((__v8qi)__a);2433 return __builtin_ia32_pmovmskb128((__v16qi)__zext128(__a));
2424}2434}
24252435
2426/// Multiplies packed 16-bit unsigned integer values and writes the2436/// Multiplies packed 16-bit unsigned integer values and writes the
...@@ -2436,10 +2446,11 @@ _mm_movemask_pi8(__m64 __a)...@@ -2436,10 +2446,11 @@ _mm_movemask_pi8(__m64 __a)
2436/// \param __b2446/// \param __b
2437/// A 64-bit integer vector containing one of the source operands.2447/// A 64-bit integer vector containing one of the source operands.
2438/// \returns A 64-bit integer vector containing the products of both operands.2448/// \returns A 64-bit integer vector containing the products of both operands.
2439static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2449static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2440_mm_mulhi_pu16(__m64 __a, __m64 __b)2450_mm_mulhi_pu16(__m64 __a, __m64 __b)
2441{2451{
2442 return (__m64)__builtin_ia32_pmulhuw((__v4hi)__a, (__v4hi)__b);2452 return __trunc64(__builtin_ia32_pmulhuw128((__v8hi)__anyext128(__a),
2453 (__v8hi)__anyext128(__b)));
2443}2454}
24442455
2445/// Shuffles the 4 16-bit integers from a 64-bit integer vector to the2456/// Shuffles the 4 16-bit integers from a 64-bit integer vector to the
...@@ -2476,8 +2487,10 @@ _mm_mulhi_pu16(__m64 __a, __m64 __b)...@@ -2476,8 +2487,10 @@ _mm_mulhi_pu16(__m64 __a, __m64 __b)
2476/// <c>_MM_SHUFFLE(b6, b4, b2, b0)</c> can create an 8-bit mask of the form2487/// <c>_MM_SHUFFLE(b6, b4, b2, b0)</c> can create an 8-bit mask of the form
2477/// <c>[b6, b4, b2, b0]</c>.2488/// <c>[b6, b4, b2, b0]</c>.
2478/// \returns A 64-bit integer vector containing the shuffled values.2489/// \returns A 64-bit integer vector containing the shuffled values.
2479#define _mm_shuffle_pi16(a, n) \2490#define _mm_shuffle_pi16(a, n) \
2480 ((__m64)__builtin_ia32_pshufw((__v4hi)(__m64)(a), (n)))2491 ((__m64)__builtin_shufflevector((__v4hi)(__m64)(a), __extension__(__v4hi){}, \
2492 (n) & 0x3, ((n) >> 2) & 0x3, \
2493 ((n) >> 4) & 0x3, ((n) >> 6) & 0x3))
24812494
2482/// Conditionally copies the values from each 8-bit element in the first2495/// Conditionally copies the values from each 8-bit element in the first
2483/// 64-bit integer vector operand to the specified memory location, as2496/// 64-bit integer vector operand to the specified memory location, as
...@@ -2502,10 +2515,25 @@ _mm_mulhi_pu16(__m64 __a, __m64 __b)...@@ -2502,10 +2515,25 @@ _mm_mulhi_pu16(__m64 __a, __m64 __b)
2502/// A pointer to a 64-bit memory location that will receive the conditionally2515/// A pointer to a 64-bit memory location that will receive the conditionally
2503/// copied integer values. The address of the memory location does not have2516/// copied integer values. The address of the memory location does not have
2504/// to be aligned.2517/// to be aligned.
2505static __inline__ void __DEFAULT_FN_ATTRS_MMX2518static __inline__ void __DEFAULT_FN_ATTRS_SSE2
2506_mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)2519_mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)
2507{2520{
2508 __builtin_ia32_maskmovq((__v8qi)__d, (__v8qi)__n, __p);2521 // This is complex, because we need to support the case where __p is pointing
2522 // within the last 15 to 8 bytes of a page. In that case, using a 128-bit
2523 // write might cause a trap where a 64-bit maskmovq would not. (Memory
2524 // locations not selected by the mask bits might still cause traps.)
2525 __m128i __d128 = __anyext128(__d);
2526 __m128i __n128 = __zext128(__n);
2527 if (((__SIZE_TYPE__)__p & 0xfff) >= 4096-15 &&
2528 ((__SIZE_TYPE__)__p & 0xfff) <= 4096-8) {
2529 // If there's a risk of spurious trap due to a 128-bit write, back up the
2530 // pointer by 8 bytes and shift values in registers to match.
2531 __p -= 8;
2532 __d128 = __builtin_ia32_pslldqi128_byteshift((__v2di)__d128, 8);
2533 __n128 = __builtin_ia32_pslldqi128_byteshift((__v2di)__n128, 8);
2534 }
2535
2536 __builtin_ia32_maskmovdqu((__v16qi)__d128, (__v16qi)__n128, __p);
2509}2537}
25102538
2511/// Computes the rounded averages of the packed unsigned 8-bit integer2539/// Computes the rounded averages of the packed unsigned 8-bit integer
...@@ -2521,10 +2549,11 @@ _mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)...@@ -2521,10 +2549,11 @@ _mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)
2521/// \param __b2549/// \param __b
2522/// A 64-bit integer vector containing one of the source operands.2550/// A 64-bit integer vector containing one of the source operands.
2523/// \returns A 64-bit integer vector containing the averages of both operands.2551/// \returns A 64-bit integer vector containing the averages of both operands.
2524static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2552static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2525_mm_avg_pu8(__m64 __a, __m64 __b)2553_mm_avg_pu8(__m64 __a, __m64 __b)
2526{2554{
2527 return (__m64)__builtin_ia32_pavgb((__v8qi)__a, (__v8qi)__b);2555 return __trunc64(__builtin_ia32_pavgb128((__v16qi)__anyext128(__a),
2556 (__v16qi)__anyext128(__b)));
2528}2557}
25292558
2530/// Computes the rounded averages of the packed unsigned 16-bit integer2559/// Computes the rounded averages of the packed unsigned 16-bit integer
...@@ -2540,10 +2569,11 @@ _mm_avg_pu8(__m64 __a, __m64 __b)...@@ -2540,10 +2569,11 @@ _mm_avg_pu8(__m64 __a, __m64 __b)
2540/// \param __b2569/// \param __b
2541/// A 64-bit integer vector containing one of the source operands.2570/// A 64-bit integer vector containing one of the source operands.
2542/// \returns A 64-bit integer vector containing the averages of both operands.2571/// \returns A 64-bit integer vector containing the averages of both operands.
2543static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2572static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2544_mm_avg_pu16(__m64 __a, __m64 __b)2573_mm_avg_pu16(__m64 __a, __m64 __b)
2545{2574{
2546 return (__m64)__builtin_ia32_pavgw((__v4hi)__a, (__v4hi)__b);2575 return __trunc64(__builtin_ia32_pavgw128((__v8hi)__anyext128(__a),
2576 (__v8hi)__anyext128(__b)));
2547}2577}
25482578
2549/// Subtracts the corresponding 8-bit unsigned integer values of the two2579/// Subtracts the corresponding 8-bit unsigned integer values of the two
...@@ -2562,10 +2592,11 @@ _mm_avg_pu16(__m64 __a, __m64 __b)...@@ -2562,10 +2592,11 @@ _mm_avg_pu16(__m64 __a, __m64 __b)
2562/// \returns A 64-bit integer vector whose lower 16 bits contain the sums of the2592/// \returns A 64-bit integer vector whose lower 16 bits contain the sums of the
2563/// sets of absolute differences between both operands. The upper bits are2593/// sets of absolute differences between both operands. The upper bits are
2564/// cleared.2594/// cleared.
2565static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2595static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2566_mm_sad_pu8(__m64 __a, __m64 __b)2596_mm_sad_pu8(__m64 __a, __m64 __b)
2567{2597{
2568 return (__m64)__builtin_ia32_psadbw((__v8qi)__a, (__v8qi)__b);2598 return __trunc64(__builtin_ia32_psadbw128((__v16qi)__zext128(__a),
2599 (__v16qi)__zext128(__b)));
2569}2600}
25702601
2571#if defined(__cplusplus)2602#if defined(__cplusplus)
...@@ -2741,9 +2772,8 @@ void _mm_setcsr(unsigned int __i);...@@ -2741,9 +2772,8 @@ void _mm_setcsr(unsigned int __i);
2741/// Bits [95:64] are written to bits [63:32] of the destination. \n2772/// Bits [95:64] are written to bits [63:32] of the destination. \n
2742/// Bits [127:96] are written to bits [127:96] of the destination.2773/// Bits [127:96] are written to bits [127:96] of the destination.
2743/// \returns A 128-bit vector of [4 x float] containing the interleaved values.2774/// \returns A 128-bit vector of [4 x float] containing the interleaved values.
2744static __inline__ __m128 __DEFAULT_FN_ATTRS2775static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2745_mm_unpackhi_ps(__m128 __a, __m128 __b)2776_mm_unpackhi_ps(__m128 __a, __m128 __b) {
2746{
2747 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 2, 6, 3, 7);2777 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 2, 6, 3, 7);
2748}2778}
27492779
...@@ -2763,9 +2793,8 @@ _mm_unpackhi_ps(__m128 __a, __m128 __b)...@@ -2763,9 +2793,8 @@ _mm_unpackhi_ps(__m128 __a, __m128 __b)
2763/// Bits [31:0] are written to bits [63:32] of the destination. \n2793/// Bits [31:0] are written to bits [63:32] of the destination. \n
2764/// Bits [63:32] are written to bits [127:96] of the destination.2794/// Bits [63:32] are written to bits [127:96] of the destination.
2765/// \returns A 128-bit vector of [4 x float] containing the interleaved values.2795/// \returns A 128-bit vector of [4 x float] containing the interleaved values.
2766static __inline__ __m128 __DEFAULT_FN_ATTRS2796static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2767_mm_unpacklo_ps(__m128 __a, __m128 __b)2797_mm_unpacklo_ps(__m128 __a, __m128 __b) {
2768{
2769 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 0, 4, 1, 5);2798 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 0, 4, 1, 5);
2770}2799}
27712800
...@@ -2785,9 +2814,8 @@ _mm_unpacklo_ps(__m128 __a, __m128 __b)...@@ -2785,9 +2814,8 @@ _mm_unpacklo_ps(__m128 __a, __m128 __b)
2785/// A 128-bit floating-point vector of [4 x float]. The lower 32 bits are2814/// A 128-bit floating-point vector of [4 x float]. The lower 32 bits are
2786/// written to the lower 32 bits of the result.2815/// written to the lower 32 bits of the result.
2787/// \returns A 128-bit floating-point vector of [4 x float].2816/// \returns A 128-bit floating-point vector of [4 x float].
2788static __inline__ __m128 __DEFAULT_FN_ATTRS2817static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2789_mm_move_ss(__m128 __a, __m128 __b)2818_mm_move_ss(__m128 __a, __m128 __b) {
2790{
2791 __a[0] = __b[0];2819 __a[0] = __b[0];
2792 return __a;2820 return __a;
2793}2821}
...@@ -2807,9 +2835,8 @@ _mm_move_ss(__m128 __a, __m128 __b)...@@ -2807,9 +2835,8 @@ _mm_move_ss(__m128 __a, __m128 __b)
2807/// A 128-bit floating-point vector of [4 x float]. The upper 64 bits are2835/// A 128-bit floating-point vector of [4 x float]. The upper 64 bits are
2808/// written to the lower 64 bits of the result.2836/// written to the lower 64 bits of the result.
2809/// \returns A 128-bit floating-point vector of [4 x float].2837/// \returns A 128-bit floating-point vector of [4 x float].
2810static __inline__ __m128 __DEFAULT_FN_ATTRS2838static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2811_mm_movehl_ps(__m128 __a, __m128 __b)2839_mm_movehl_ps(__m128 __a, __m128 __b) {
2812{
2813 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 6, 7, 2, 3);2840 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 6, 7, 2, 3);
2814}2841}
28152842
...@@ -2828,9 +2855,8 @@ _mm_movehl_ps(__m128 __a, __m128 __b)...@@ -2828,9 +2855,8 @@ _mm_movehl_ps(__m128 __a, __m128 __b)
2828/// A 128-bit floating-point vector of [4 x float]. The lower 64 bits are2855/// A 128-bit floating-point vector of [4 x float]. The lower 64 bits are
2829/// written to the upper 64 bits of the result.2856/// written to the upper 64 bits of the result.
2830/// \returns A 128-bit floating-point vector of [4 x float].2857/// \returns A 128-bit floating-point vector of [4 x float].
2831static __inline__ __m128 __DEFAULT_FN_ATTRS2858static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2832_mm_movelh_ps(__m128 __a, __m128 __b)2859_mm_movelh_ps(__m128 __a, __m128 __b) {
2833{
2834 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 0, 1, 4, 5);2860 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 0, 1, 4, 5);
2835}2861}
28362862
...@@ -2846,22 +2872,10 @@ _mm_movelh_ps(__m128 __a, __m128 __b)...@@ -2846,22 +2872,10 @@ _mm_movelh_ps(__m128 __a, __m128 __b)
2846/// from the corresponding elements in this operand.2872/// from the corresponding elements in this operand.
2847/// \returns A 128-bit vector of [4 x float] containing the copied and converted2873/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2848/// values from the operand.2874/// values from the operand.
2849static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX2875static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
2850_mm_cvtpi16_ps(__m64 __a)2876_mm_cvtpi16_ps(__m64 __a)
2851{2877{
2852 __m64 __b, __c;2878 return __builtin_convertvector((__v4hi)__a, __v4sf);
2853 __m128 __r;
2854
2855 __b = _mm_setzero_si64();
2856 __b = _mm_cmpgt_pi16(__b, __a);
2857 __c = _mm_unpackhi_pi16(__a, __b);
2858 __r = _mm_setzero_ps();
2859 __r = _mm_cvtpi32_ps(__r, __c);
2860 __r = _mm_movelh_ps(__r, __r);
2861 __c = _mm_unpacklo_pi16(__a, __b);
2862 __r = _mm_cvtpi32_ps(__r, __c);
2863
2864 return __r;
2865}2879}
28662880
2867/// Converts a 64-bit vector of 16-bit unsigned integer values into a2881/// Converts a 64-bit vector of 16-bit unsigned integer values into a
...@@ -2876,21 +2890,10 @@ _mm_cvtpi16_ps(__m64 __a)...@@ -2876,21 +2890,10 @@ _mm_cvtpi16_ps(__m64 __a)
2876/// destination are copied from the corresponding elements in this operand.2890/// destination are copied from the corresponding elements in this operand.
2877/// \returns A 128-bit vector of [4 x float] containing the copied and converted2891/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2878/// values from the operand.2892/// values from the operand.
2879static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX2893static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
2880_mm_cvtpu16_ps(__m64 __a)2894_mm_cvtpu16_ps(__m64 __a)
2881{2895{
2882 __m64 __b, __c;2896 return __builtin_convertvector((__v4hu)__a, __v4sf);
2883 __m128 __r;
2884
2885 __b = _mm_setzero_si64();
2886 __c = _mm_unpackhi_pi16(__a, __b);
2887 __r = _mm_setzero_ps();
2888 __r = _mm_cvtpi32_ps(__r, __c);
2889 __r = _mm_movelh_ps(__r, __r);
2890 __c = _mm_unpacklo_pi16(__a, __b);
2891 __r = _mm_cvtpi32_ps(__r, __c);
2892
2893 return __r;
2894}2897}
28952898
2896/// Converts the lower four 8-bit values from a 64-bit vector of [8 x i8]2899/// Converts the lower four 8-bit values from a 64-bit vector of [8 x i8]
...@@ -2905,16 +2908,12 @@ _mm_cvtpu16_ps(__m64 __a)...@@ -2905,16 +2908,12 @@ _mm_cvtpu16_ps(__m64 __a)
2905/// from the corresponding lower 4 elements in this operand.2908/// from the corresponding lower 4 elements in this operand.
2906/// \returns A 128-bit vector of [4 x float] containing the copied and converted2909/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2907/// values from the operand.2910/// values from the operand.
2908static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX2911static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
2909_mm_cvtpi8_ps(__m64 __a)2912_mm_cvtpi8_ps(__m64 __a)
2910{2913{
2911 __m64 __b;2914 return __builtin_convertvector(
29122915 __builtin_shufflevector((__v8qs)__a, __extension__ (__v8qs){},
2913 __b = _mm_setzero_si64();2916 0, 1, 2, 3), __v4sf);
2914 __b = _mm_cmpgt_pi8(__b, __a);
2915 __b = _mm_unpacklo_pi8(__a, __b);
2916
2917 return _mm_cvtpi16_ps(__b);
2918}2917}
29192918
2920/// Converts the lower four unsigned 8-bit integer values from a 64-bit2919/// Converts the lower four unsigned 8-bit integer values from a 64-bit
...@@ -2930,15 +2929,12 @@ _mm_cvtpi8_ps(__m64 __a)...@@ -2930,15 +2929,12 @@ _mm_cvtpi8_ps(__m64 __a)
2930/// operand.2929/// operand.
2931/// \returns A 128-bit vector of [4 x float] containing the copied and converted2930/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2932/// values from the source operand.2931/// values from the source operand.
2933static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX2932static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
2934_mm_cvtpu8_ps(__m64 __a)2933_mm_cvtpu8_ps(__m64 __a)
2935{2934{
2936 __m64 __b;2935 return __builtin_convertvector(
29372936 __builtin_shufflevector((__v8qu)__a, __extension__ (__v8qu){},
2938 __b = _mm_setzero_si64();2937 0, 1, 2, 3), __v4sf);
2939 __b = _mm_unpacklo_pi8(__a, __b);
2940
2941 return _mm_cvtpi16_ps(__b);
2942}2938}
29432939
2944/// Converts the two 32-bit signed integer values from each 64-bit vector2940/// Converts the two 32-bit signed integer values from each 64-bit vector
...@@ -2957,16 +2953,12 @@ _mm_cvtpu8_ps(__m64 __a)...@@ -2957,16 +2953,12 @@ _mm_cvtpu8_ps(__m64 __a)
2957/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the2953/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
2958/// copied and converted values from the first operand. The upper 64 bits2954/// copied and converted values from the first operand. The upper 64 bits
2959/// contain the copied and converted values from the second operand.2955/// contain the copied and converted values from the second operand.
2960static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX2956static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
2961_mm_cvtpi32x2_ps(__m64 __a, __m64 __b)2957_mm_cvtpi32x2_ps(__m64 __a, __m64 __b)
2962{2958{
2963 __m128 __c;2959 return __builtin_convertvector(
29642960 __builtin_shufflevector((__v2si)__a, (__v2si)__b,
2965 __c = _mm_setzero_ps();2961 0, 1, 2, 3), __v4sf);
2966 __c = _mm_cvtpi32_ps(__c, __b);
2967 __c = _mm_movelh_ps(__c, __c);
2968
2969 return _mm_cvtpi32_ps(__c, __a);
2970}2962}
29712963
2972/// Converts each single-precision floating-point element of a 128-bit2964/// Converts each single-precision floating-point element of a 128-bit
...@@ -2986,16 +2978,11 @@ _mm_cvtpi32x2_ps(__m64 __a, __m64 __b)...@@ -2986,16 +2978,11 @@ _mm_cvtpi32x2_ps(__m64 __a, __m64 __b)
2986/// A 128-bit floating-point vector of [4 x float].2978/// A 128-bit floating-point vector of [4 x float].
2987/// \returns A 64-bit integer vector of [4 x i16] containing the converted2979/// \returns A 64-bit integer vector of [4 x i16] containing the converted
2988/// values.2980/// values.
2989static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX2981static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2990_mm_cvtps_pi16(__m128 __a)2982_mm_cvtps_pi16(__m128 __a)
2991{2983{
2992 __m64 __b, __c;2984 return __trunc64(__builtin_ia32_packssdw128(
29932985 (__v4si)__builtin_ia32_cvtps2dq((__v4sf)__a), (__v4si)_mm_setzero_ps()));
2994 __b = _mm_cvtps_pi32(__a);
2995 __a = _mm_movehl_ps(__a, __a);
2996 __c = _mm_cvtps_pi32(__a);
2997
2998 return _mm_packs_pi32(__b, __c);
2999}2986}
30002987
3001/// Converts each single-precision floating-point element of a 128-bit2988/// Converts each single-precision floating-point element of a 128-bit
...@@ -3016,7 +3003,7 @@ _mm_cvtps_pi16(__m128 __a)...@@ -3016,7 +3003,7 @@ _mm_cvtps_pi16(__m128 __a)
3016/// 128-bit floating-point vector of [4 x float].3003/// 128-bit floating-point vector of [4 x float].
3017/// \returns A 64-bit integer vector of [8 x i8]. The lower 32 bits contain the3004/// \returns A 64-bit integer vector of [8 x i8]. The lower 32 bits contain the
3018/// converted values and the uppper 32 bits are set to zero.3005/// converted values and the uppper 32 bits are set to zero.
3019static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX3006static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
3020_mm_cvtps_pi8(__m128 __a)3007_mm_cvtps_pi8(__m128 __a)
3021{3008{
3022 __m64 __b, __c;3009 __m64 __b, __c;
...@@ -3196,8 +3183,14 @@ do { \...@@ -3196,8 +3183,14 @@ do { \
3196#define _m_psadbw _mm_sad_pu83183#define _m_psadbw _mm_sad_pu8
3197#define _m_ _mm_3184#define _m_ _mm_
31983185
3186#undef __trunc64
3187#undef __zext128
3188#undef __anyext128
3189#undef __zeroupper64
3199#undef __DEFAULT_FN_ATTRS3190#undef __DEFAULT_FN_ATTRS
3200#undef __DEFAULT_FN_ATTRS_MMX3191#undef __DEFAULT_FN_ATTRS_CONSTEXPR
3192#undef __DEFAULT_FN_ATTRS_SSE2
3193#undef __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
32013194
3202/* Ugly hack for backwards-compatibility (compatible with gcc) */3195/* Ugly hack for backwards-compatibility (compatible with gcc) */
3203#if defined(__SSE2__) && !__building_module(_Builtin_intrinsics)3196#if defined(__SSE2__) && !__building_module(_Builtin_intrinsics)