Commit 8612dac225a55e09293b767aab3c81a459609bb4 | zig

lib/include/__clang_cuda_builtin_vars.h+9

lib/include/__clang_cuda_cmath.h+46-4

...	@@ -66,10 +66,38 @@ __DEVICE__ float frexp(float __arg, int *__exp) {	...	@@ -66,10 +66,38 @@ __DEVICE__ float frexp(float __arg, int *__exp) {
66	}	66	}
67		67
68	// For inscrutable reasons, the CUDA headers define these functions for us on	68	// For inscrutable reasons, the CUDA headers define these functions for us on
69	// Windows. For OpenMP we omit these as some old system headers have	69	// Windows.
70	// non-conforming `isinf(float)` and `isnan(float)` implementations that return	70	#if !defined(_MSC_VER) \|\| defined(__OPENMP_NVPTX__)
71	// an `int`. The system versions of these functions should be fine anyway.	71
72	#if !defined(_MSC_VER) && !defined(__OPENMP_NVPTX__)	72	// For OpenMP we work around some old system headers that have non-conforming
		73	// `isinf(float)` and `isnan(float)` implementations that return an `int`. We do
		74	// this by providing two versions of these functions, differing only in the
		75	// return type. To avoid conflicting definitions we disable implicit base
		76	// function generation. That means we will end up with two specializations, one
		77	// per type, but only one has a base function defined by the system header.
		78	#if defined(__OPENMP_NVPTX__)
		79	#pragma omp begin declare variant match( \
		80	implementation = {extension(disable_implicit_base)})
		81
		82	// FIXME: We lack an extension to customize the mangling of the variants, e.g.,
		83	// add a suffix. This means we would clash with the names of the variants
		84	// (note that we do not create implicit base functions here). To avoid
		85	// this clash we add a new trait to some of them that is always true
		86	// (this is LLVM after all ;)). It will only influence the mangled name
		87	// of the variants inside the inner region and avoid the clash.
		88	#pragma omp begin declare variant match(implementation = {vendor(llvm)})
		89
		90	__DEVICE__ int isinf(float __x) { return ::__isinff(__x); }
		91	__DEVICE__ int isinf(double __x) { return ::__isinf(__x); }
		92	__DEVICE__ int isfinite(float __x) { return ::__finitef(__x); }
		93	__DEVICE__ int isfinite(double __x) { return ::__isfinited(__x); }
		94	__DEVICE__ int isnan(float __x) { return ::__isnanf(__x); }
		95	__DEVICE__ int isnan(double __x) { return ::__isnan(__x); }
		96
		97	#pragma omp end declare variant
		98
		99	#endif
		100
73	__DEVICE__ bool isinf(float __x) { return ::__isinff(__x); }	101	__DEVICE__ bool isinf(float __x) { return ::__isinff(__x); }
74	__DEVICE__ bool isinf(double __x) { return ::__isinf(__x); }	102	__DEVICE__ bool isinf(double __x) { return ::__isinf(__x); }
75	__DEVICE__ bool isfinite(float __x) { return ::__finitef(__x); }	103	__DEVICE__ bool isfinite(float __x) { return ::__finitef(__x); }
...	@@ -79,6 +107,11 @@ __DEVICE__ bool isfinite(float __x) { return ::__finitef(__x); }	...	@@ -79,6 +107,11 @@ __DEVICE__ bool isfinite(float __x) { return ::__finitef(__x); }
79	__DEVICE__ bool isfinite(double __x) { return ::__isfinited(__x); }	107	__DEVICE__ bool isfinite(double __x) { return ::__isfinited(__x); }
80	__DEVICE__ bool isnan(float __x) { return ::__isnanf(__x); }	108	__DEVICE__ bool isnan(float __x) { return ::__isnanf(__x); }
81	__DEVICE__ bool isnan(double __x) { return ::__isnan(__x); }	109	__DEVICE__ bool isnan(double __x) { return ::__isnan(__x); }
		110
		111	#if defined(__OPENMP_NVPTX__)
		112	#pragma omp end declare variant
		113	#endif
		114
82	#endif	115	#endif
83		116
84	__DEVICE__ bool isgreater(float __x, float __y) {	117	__DEVICE__ bool isgreater(float __x, float __y) {
...	@@ -142,6 +175,15 @@ __DEVICE__ float sqrt(float __x) { return ::sqrtf(__x); }	...	@@ -142,6 +175,15 @@ __DEVICE__ float sqrt(float __x) { return ::sqrtf(__x); }
142	__DEVICE__ float tan(float __x) { return ::tanf(__x); }	175	__DEVICE__ float tan(float __x) { return ::tanf(__x); }
143	__DEVICE__ float tanh(float __x) { return ::tanhf(__x); }	176	__DEVICE__ float tanh(float __x) { return ::tanhf(__x); }
144		177
		178	// There was a redefinition error for this this overload in CUDA mode.
		179	// We restrict it to OpenMP mode for now, that is where it is actually needed
		180	// anyway.
		181	#ifdef __OPENMP_NVPTX__
		182	__DEVICE__ float remquo(float __n, float __d, int *__q) {
		183	return ::remquof(__n, __d, __q);
		184	}
		185	#endif
		186
145	// Notably missing above is nexttoward. We omit it because	187	// Notably missing above is nexttoward. We omit it because
146	// libdevice doesn't provide an implementation, and we don't want to be in the	188	// libdevice doesn't provide an implementation, and we don't want to be in the
147	// business of implementing tricky libm functions in this header.	189	// business of implementing tricky libm functions in this header.

lib/include/__clang_cuda_complex_builtins.h+28-2

 @@ -41,6 +41,27 @@
 #define _ABSf std::abs
 #define _LOGBd std::logb
 #define _LOGBf std::logb
+// Rather than pulling in std::max from algorithm everytime, use available ::max.
+#define _fmaxd max
+#define _fmaxf max
+#else
+#ifdef __AMDGCN__
+#define _ISNANd __ocml_isnan_f64
+#define _ISNANf __ocml_isnan_f32
+#define _ISINFd __ocml_isinf_f64
+#define _ISINFf __ocml_isinf_f32
+#define _ISFINITEd __ocml_isfinite_f64
+#define _ISFINITEf __ocml_isfinite_f32
+#define _COPYSIGNd __ocml_copysign_f64
+#define _COPYSIGNf __ocml_copysign_f32
+#define _SCALBNd __ocml_scalbn_f64
+#define _SCALBNf __ocml_scalbn_f32
+#define _ABSd __ocml_fabs_f64
+#define _ABSf __ocml_fabs_f32
+#define _LOGBd __ocml_logb_f64
+#define _LOGBf __ocml_logb_f32
+#define _fmaxd __ocml_fmax_f64
+#define _fmaxf __ocml_fmax_f32
 #else
 #define _ISNANd __nv_isnand
 #define _ISNANf __nv_isnanf
 @@ -56,6 +77,9 @@
 #define _ABSf __nv_fabsf
 #define _LOGBd __nv_logb
 #define _LOGBf __nv_logbf
+#define _fmaxd __nv_fmax
+#define _fmaxf __nv_fmaxf
+#endif
 #endif
 #if defined(__cplusplus)
 @@ -167,7 +191,7 @@ __DEVICE__ double _Complex __divdc3(double __a, double __b, double __c,
   // Can't use std::max, because that's defined in <algorithm>, and we don't
   // want to pull that in for every compile.  The CUDA headers define
   // ::max(float, float) and ::max(double, double), which is sufficient for us.
-  double __logbw = _LOGBd(max(_ABSd(__c), _ABSd(__d)));
+  double __logbw = _LOGBd(_fmaxd(_ABSd(__c), _ABSd(__d)));
   if (_ISFINITEd(__logbw)) {
     __ilogbw = (int)__logbw;
     __c = _SCALBNd(__c, -__ilogbw);
 @@ -200,7 +224,7 @@ __DEVICE__ double _Complex __divdc3(double __a, double __b, double __c,
 __DEVICE__ float _Complex __divsc3(float __a, float __b, float __c, float __d) {
   int __ilogbw = 0;
-  float __logbw = _LOGBf(max(_ABSf(__c), _ABSf(__d)));
+  float __logbw = _LOGBf(_fmaxf(_ABSf(__c), _ABSf(__d)));
   if (_ISFINITEf(__logbw)) {
     __ilogbw = (int)__logbw;
     __c = _SCALBNf(__c, -__ilogbw);
 @@ -249,6 +273,8 @@ __DEVICE__ float _Complex __divsc3(float __a, float __b, float __c, float __d) {
 #undef _ABSf
 #undef _LOGBd
 #undef _LOGBf
+#undef _fmaxd
+#undef _fmaxf
 #ifdef __OPENMP_NVPTX__
 #pragma omp end declare target

lib/include/__clang_cuda_math.h+5-4

lib/include/__clang_cuda_runtime_wrapper.h+18-10

...	@@ -377,30 +377,38 @@ __device__ static inline void *malloc(size_t __size) {	...	@@ -377,30 +377,38 @@ __device__ static inline void *malloc(size_t __size) {
377	// Out-of-line implementations from __clang_cuda_builtin_vars.h. These need to	377	// Out-of-line implementations from __clang_cuda_builtin_vars.h. These need to
378	// come after we've pulled in the definition of uint3 and dim3.	378	// come after we've pulled in the definition of uint3 and dim3.
379		379
		380	__device__ inline __cuda_builtin_threadIdx_t::operator dim3() const {
		381	return dim3(x, y, z);
		382	}
		383
380	__device__ inline __cuda_builtin_threadIdx_t::operator uint3() const {	384	__device__ inline __cuda_builtin_threadIdx_t::operator uint3() const {
381	uint3 ret;	385	return {x, y, z};
382	ret.x = x;	386	}
383	ret.y = y;	387
384	ret.z = z;	388	__device__ inline __cuda_builtin_blockIdx_t::operator dim3() const {
385	return ret;	389	return dim3(x, y, z);
386	}	390	}
387		391
388	__device__ inline __cuda_builtin_blockIdx_t::operator uint3() const {	392	__device__ inline __cuda_builtin_blockIdx_t::operator uint3() const {
389	uint3 ret;	393	return {x, y, z};
390	ret.x = x;
391	ret.y = y;
392	ret.z = z;
393	return ret;
394	}	394	}
395		395
396	__device__ inline __cuda_builtin_blockDim_t::operator dim3() const {	396	__device__ inline __cuda_builtin_blockDim_t::operator dim3() const {
397	return dim3(x, y, z);	397	return dim3(x, y, z);
398	}	398	}
399		399
		400	__device__ inline __cuda_builtin_blockDim_t::operator uint3() const {
		401	return {x, y, z};
		402	}
		403
400	__device__ inline __cuda_builtin_gridDim_t::operator dim3() const {	404	__device__ inline __cuda_builtin_gridDim_t::operator dim3() const {
401	return dim3(x, y, z);	405	return dim3(x, y, z);
402	}	406	}
403		407
		408	__device__ inline __cuda_builtin_gridDim_t::operator uint3() const {
		409	return {x, y, z};
		410	}
		411
404	#include <__clang_cuda_cmath.h>	412	#include <__clang_cuda_cmath.h>
405	#include <__clang_cuda_intrinsics.h>	413	#include <__clang_cuda_intrinsics.h>
406	#include <__clang_cuda_complex_builtins.h>	414	#include <__clang_cuda_complex_builtins.h>

lib/include/__clang_hip_cmath.h created+629

...	@@ -0,0 +1,629 @@
1	/*===---- __clang_hip_cmath.h - HIP cmath decls -----------------------------===
2	*
3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	* See https://llvm.org/LICENSE.txt for license information.
5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	*
7	*===-----------------------------------------------------------------------===
8	*/
9
10	#ifndef __CLANG_HIP_CMATH_H__
11	#define __CLANG_HIP_CMATH_H__
12
13	#if !defined(__HIP__)
14	#error "This file is for HIP and OpenMP AMDGCN device compilation only."
15	#endif
16
17	#if defined(__cplusplus)
18	#include <limits>
19	#include <type_traits>
20	#include <utility>
21	#endif
22	#include <limits.h>
23	#include <stdint.h>
24
25	#pragma push_macro("__DEVICE__")
26	#define __DEVICE__ static __device__ inline __attribute__((always_inline))
27
28	// Start with functions that cannot be defined by DEF macros below.
29	#if defined(__cplusplus)
30	__DEVICE__ double abs(double __x) { return ::fabs(__x); }
31	__DEVICE__ float abs(float __x) { return ::fabsf(__x); }
32	__DEVICE__ long long abs(long long __n) { return ::llabs(__n); }
33	__DEVICE__ long abs(long __n) { return ::labs(__n); }
34	__DEVICE__ float fma(float __x, float __y, float __z) {
35	return ::fmaf(__x, __y, __z);
36	}
37	__DEVICE__ int fpclassify(float __x) {
38	return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL,
39	FP_ZERO, __x);
40	}
41	__DEVICE__ int fpclassify(double __x) {
42	return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL,
43	FP_ZERO, __x);
44	}
45	__DEVICE__ float frexp(float __arg, int *__exp) {
46	return ::frexpf(__arg, __exp);
47	}
48	__DEVICE__ bool isfinite(float __x) { return ::__finitef(__x); }
49	__DEVICE__ bool isfinite(double __x) { return ::__finite(__x); }
50	__DEVICE__ bool isgreater(float __x, float __y) {
51	return __builtin_isgreater(__x, __y);
52	}
53	__DEVICE__ bool isgreater(double __x, double __y) {
54	return __builtin_isgreater(__x, __y);
55	}
56	__DEVICE__ bool isgreaterequal(float __x, float __y) {
57	return __builtin_isgreaterequal(__x, __y);
58	}
59	__DEVICE__ bool isgreaterequal(double __x, double __y) {
60	return __builtin_isgreaterequal(__x, __y);
61	}
62	__DEVICE__ bool isinf(float __x) { return ::__isinff(__x); }
63	__DEVICE__ bool isinf(double __x) { return ::__isinf(__x); }
64	__DEVICE__ bool isless(float __x, float __y) {
65	return __builtin_isless(__x, __y);
66	}
67	__DEVICE__ bool isless(double __x, double __y) {
68	return __builtin_isless(__x, __y);
69	}
70	__DEVICE__ bool islessequal(float __x, float __y) {
71	return __builtin_islessequal(__x, __y);
72	}
73	__DEVICE__ bool islessequal(double __x, double __y) {
74	return __builtin_islessequal(__x, __y);
75	}
76	__DEVICE__ bool islessgreater(float __x, float __y) {
77	return __builtin_islessgreater(__x, __y);
78	}
79	__DEVICE__ bool islessgreater(double __x, double __y) {
80	return __builtin_islessgreater(__x, __y);
81	}
82	__DEVICE__ bool isnan(float __x) { return ::__isnanf(__x); }
83	__DEVICE__ bool isnan(double __x) { return ::__isnan(__x); }
84	__DEVICE__ bool isnormal(float __x) { return __builtin_isnormal(__x); }
85	__DEVICE__ bool isnormal(double __x) { return __builtin_isnormal(__x); }
86	__DEVICE__ bool isunordered(float __x, float __y) {
87	return __builtin_isunordered(__x, __y);
88	}
89	__DEVICE__ bool isunordered(double __x, double __y) {
90	return __builtin_isunordered(__x, __y);
91	}
92	__DEVICE__ float modf(float __x, float *__iptr) { return ::modff(__x, __iptr); }
93	__DEVICE__ float pow(float __base, int __iexp) {
94	return ::powif(__base, __iexp);
95	}
96	__DEVICE__ double pow(double __base, int __iexp) {
97	return ::powi(__base, __iexp);
98	}
99	__DEVICE__ float remquo(float __x, float __y, int *__quo) {
100	return ::remquof(__x, __y, __quo);
101	}
102	__DEVICE__ float scalbln(float __x, long int __n) {
103	return ::scalblnf(__x, __n);
104	}
105	__DEVICE__ bool signbit(float __x) { return ::__signbitf(__x); }
106	__DEVICE__ bool signbit(double __x) { return ::__signbit(__x); }
107
108	// Notably missing above is nexttoward. We omit it because
109	// ocml doesn't provide an implementation, and we don't want to be in the
110	// business of implementing tricky libm functions in this header.
111
112	// Other functions.
113	__DEVICE__ _Float16 fma(_Float16 __x, _Float16 __y, _Float16 __z) {
114	return __ocml_fma_f16(__x, __y, __z);
115	}
116	__DEVICE__ _Float16 pow(_Float16 __base, int __iexp) {
117	return __ocml_pown_f16(__base, __iexp);
118	}
119
120	// BEGIN DEF_FUN and HIP_OVERLOAD
121
122	// BEGIN DEF_FUN
123
124	#pragma push_macro("__DEF_FUN1")
125	#pragma push_macro("__DEF_FUN2")
126	#pragma push_macro("__DEF_FUN2_FI")
127
128	// Define cmath functions with float argument and returns __retty.
129	#define __DEF_FUN1(__retty, __func) \
130	__DEVICE__ \
131	__retty __func(float __x) { return __func##f(__x); }
132
133	// Define cmath functions with two float arguments and returns __retty.
134	#define __DEF_FUN2(__retty, __func) \
135	__DEVICE__ \
136	__retty __func(float __x, float __y) { return __func##f(__x, __y); }
137
138	// Define cmath functions with a float and an int argument and returns __retty.
139	#define __DEF_FUN2_FI(__retty, __func) \
140	__DEVICE__ \
141	__retty __func(float __x, int __y) { return __func##f(__x, __y); }
142
143	__DEF_FUN1(float, acos)
144	__DEF_FUN1(float, acosh)
145	__DEF_FUN1(float, asin)
146	__DEF_FUN1(float, asinh)
147	__DEF_FUN1(float, atan)
148	__DEF_FUN2(float, atan2)
149	__DEF_FUN1(float, atanh)
150	__DEF_FUN1(float, cbrt)
151	__DEF_FUN1(float, ceil)
152	__DEF_FUN2(float, copysign)
153	__DEF_FUN1(float, cos)
154	__DEF_FUN1(float, cosh)
155	__DEF_FUN1(float, erf)
156	__DEF_FUN1(float, erfc)
157	__DEF_FUN1(float, exp)
158	__DEF_FUN1(float, exp2)
159	__DEF_FUN1(float, expm1)
160	__DEF_FUN1(float, fabs)
161	__DEF_FUN2(float, fdim)
162	__DEF_FUN1(float, floor)
163	__DEF_FUN2(float, fmax)
164	__DEF_FUN2(float, fmin)
165	__DEF_FUN2(float, fmod)
166	__DEF_FUN2(float, hypot)
167	__DEF_FUN1(int, ilogb)
168	__DEF_FUN2_FI(float, ldexp)
169	__DEF_FUN1(float, lgamma)
170	__DEF_FUN1(float, log)
171	__DEF_FUN1(float, log10)
172	__DEF_FUN1(float, log1p)
173	__DEF_FUN1(float, log2)
174	__DEF_FUN1(float, logb)
175	__DEF_FUN1(long long, llrint)
176	__DEF_FUN1(long long, llround)
177	__DEF_FUN1(long, lrint)
178	__DEF_FUN1(long, lround)
179	__DEF_FUN1(float, nearbyint)
180	__DEF_FUN2(float, nextafter)
181	__DEF_FUN2(float, pow)
182	__DEF_FUN2(float, remainder)
183	__DEF_FUN1(float, rint)
184	__DEF_FUN1(float, round)
185	__DEF_FUN2_FI(float, scalbn)
186	__DEF_FUN1(float, sin)
187	__DEF_FUN1(float, sinh)
188	__DEF_FUN1(float, sqrt)
189	__DEF_FUN1(float, tan)
190	__DEF_FUN1(float, tanh)
191	__DEF_FUN1(float, tgamma)
192	__DEF_FUN1(float, trunc)
193
194	#pragma pop_macro("__DEF_FUN1")
195	#pragma pop_macro("__DEF_FUN2")
196	#pragma pop_macro("__DEF_FUN2_FI")
197
198	// END DEF_FUN
199
200	// BEGIN HIP_OVERLOAD
201
202	#pragma push_macro("__HIP_OVERLOAD1")
203	#pragma push_macro("__HIP_OVERLOAD2")
204
205	// __hip_enable_if::type is a type function which returns __T if __B is true.
206	template <bool __B, class __T = void> struct __hip_enable_if {};
207
208	template <class __T> struct __hip_enable_if<true, __T> { typedef __T type; };
209
210	// decltype is only available in C++11 and above.
211	#if __cplusplus >= 201103L
212	// __hip_promote
213	namespace __hip {
214
215	template <class _Tp> struct __numeric_type {
216	static void __test(...);
217	static _Float16 __test(_Float16);
218	static float __test(float);
219	static double __test(char);
220	static double __test(int);
221	static double __test(unsigned);
222	static double __test(long);
223	static double __test(unsigned long);
224	static double __test(long long);
225	static double __test(unsigned long long);
226	static double __test(double);
227	// No support for long double, use double instead.
228	static double __test(long double);
229
230	typedef decltype(__test(std::declval<_Tp>())) type;
231	static const bool value = !std::is_same<type, void>::value;
232	};
233
234	template <> struct __numeric_type<void> { static const bool value = true; };
235
236	template <class _A1, class _A2 = void, class _A3 = void,
237	bool = __numeric_type<_A1>::value &&__numeric_type<_A2>::value
238	&&__numeric_type<_A3>::value>
239	class __promote_imp {
240	public:
241	static const bool value = false;
242	};
243
244	template <class _A1, class _A2, class _A3>
245	class __promote_imp<_A1, _A2, _A3, true> {
246	private:
247	typedef typename __promote_imp<_A1>::type __type1;
248	typedef typename __promote_imp<_A2>::type __type2;
249	typedef typename __promote_imp<_A3>::type __type3;
250
251	public:
252	typedef decltype(__type1() + __type2() + __type3()) type;
253	static const bool value = true;
254	};
255
256	template <class _A1, class _A2> class __promote_imp<_A1, _A2, void, true> {
257	private:
258	typedef typename __promote_imp<_A1>::type __type1;
259	typedef typename __promote_imp<_A2>::type __type2;
260
261	public:
262	typedef decltype(__type1() + __type2()) type;
263	static const bool value = true;
264	};
265
266	template <class _A1> class __promote_imp<_A1, void, void, true> {
267	public:
268	typedef typename __numeric_type<_A1>::type type;
269	static const bool value = true;
270	};
271
272	template <class _A1, class _A2 = void, class _A3 = void>
273	class __promote : public __promote_imp<_A1, _A2, _A3> {};
274
275	} // namespace __hip
276	#endif //__cplusplus >= 201103L
277
278	// __HIP_OVERLOAD1 is used to resolve function calls with integer argument to
279	// avoid compilation error due to ambibuity. e.g. floor(5) is resolved with
280	// floor(double).
281	#define __HIP_OVERLOAD1(__retty, __fn) \
282	template <typename __T> \
283	__DEVICE__ typename __hip_enable_if<std::numeric_limits<__T>::is_integer, \
284	__retty>::type \
285	__fn(__T __x) { \
286	return ::__fn((double)__x); \
287	}
288
289	// __HIP_OVERLOAD2 is used to resolve function calls with mixed float/double
290	// or integer argument to avoid compilation error due to ambibuity. e.g.
291	// max(5.0f, 6.0) is resolved with max(double, double).
292	#if __cplusplus >= 201103L
293	#define __HIP_OVERLOAD2(__retty, __fn) \
294	template <typename __T1, typename __T2> \
295	__DEVICE__ typename __hip_enable_if< \
296	std::numeric_limits<__T1>::is_specialized && \
297	std::numeric_limits<__T2>::is_specialized, \
298	typename __hip::__promote<__T1, __T2>::type>::type \
299	__fn(__T1 __x, __T2 __y) { \
300	typedef typename __hip::__promote<__T1, __T2>::type __result_type; \
301	return __fn((__result_type)__x, (__result_type)__y); \
302	}
303	#else
304	#define __HIP_OVERLOAD2(__retty, __fn) \
305	template <typename __T1, typename __T2> \
306	__DEVICE__ \
307	typename __hip_enable_if<std::numeric_limits<__T1>::is_specialized && \
308	std::numeric_limits<__T2>::is_specialized, \
309	__retty>::type \
310	__fn(__T1 __x, __T2 __y) { \
311	return __fn((double)__x, (double)__y); \
312	}
313	#endif
314
315	__HIP_OVERLOAD1(double, abs)
316	__HIP_OVERLOAD1(double, acos)
317	__HIP_OVERLOAD1(double, acosh)
318	__HIP_OVERLOAD1(double, asin)
319	__HIP_OVERLOAD1(double, asinh)
320	__HIP_OVERLOAD1(double, atan)
321	__HIP_OVERLOAD2(double, atan2)
322	__HIP_OVERLOAD1(double, atanh)
323	__HIP_OVERLOAD1(double, cbrt)
324	__HIP_OVERLOAD1(double, ceil)
325	__HIP_OVERLOAD2(double, copysign)
326	__HIP_OVERLOAD1(double, cos)
327	__HIP_OVERLOAD1(double, cosh)
328	__HIP_OVERLOAD1(double, erf)
329	__HIP_OVERLOAD1(double, erfc)
330	__HIP_OVERLOAD1(double, exp)
331	__HIP_OVERLOAD1(double, exp2)
332	__HIP_OVERLOAD1(double, expm1)
333	__HIP_OVERLOAD1(double, fabs)
334	__HIP_OVERLOAD2(double, fdim)
335	__HIP_OVERLOAD1(double, floor)
336	__HIP_OVERLOAD2(double, fmax)
337	__HIP_OVERLOAD2(double, fmin)
338	__HIP_OVERLOAD2(double, fmod)
339	__HIP_OVERLOAD1(int, fpclassify)
340	__HIP_OVERLOAD2(double, hypot)
341	__HIP_OVERLOAD1(int, ilogb)
342	__HIP_OVERLOAD1(bool, isfinite)
343	__HIP_OVERLOAD2(bool, isgreater)
344	__HIP_OVERLOAD2(bool, isgreaterequal)
345	__HIP_OVERLOAD1(bool, isinf)
346	__HIP_OVERLOAD2(bool, isless)
347	__HIP_OVERLOAD2(bool, islessequal)
348	__HIP_OVERLOAD2(bool, islessgreater)
349	__HIP_OVERLOAD1(bool, isnan)
350	__HIP_OVERLOAD1(bool, isnormal)
351	__HIP_OVERLOAD2(bool, isunordered)
352	__HIP_OVERLOAD1(double, lgamma)
353	__HIP_OVERLOAD1(double, log)
354	__HIP_OVERLOAD1(double, log10)
355	__HIP_OVERLOAD1(double, log1p)
356	__HIP_OVERLOAD1(double, log2)
357	__HIP_OVERLOAD1(double, logb)
358	__HIP_OVERLOAD1(long long, llrint)
359	__HIP_OVERLOAD1(long long, llround)
360	__HIP_OVERLOAD1(long, lrint)
361	__HIP_OVERLOAD1(long, lround)
362	__HIP_OVERLOAD1(double, nearbyint)
363	__HIP_OVERLOAD2(double, nextafter)
364	__HIP_OVERLOAD2(double, pow)
365	__HIP_OVERLOAD2(double, remainder)
366	__HIP_OVERLOAD1(double, rint)
367	__HIP_OVERLOAD1(double, round)
368	__HIP_OVERLOAD1(bool, signbit)
369	__HIP_OVERLOAD1(double, sin)
370	__HIP_OVERLOAD1(double, sinh)
371	__HIP_OVERLOAD1(double, sqrt)
372	__HIP_OVERLOAD1(double, tan)
373	__HIP_OVERLOAD1(double, tanh)
374	__HIP_OVERLOAD1(double, tgamma)
375	__HIP_OVERLOAD1(double, trunc)
376
377	// Overload these but don't add them to std, they are not part of cmath.
378	__HIP_OVERLOAD2(double, max)
379	__HIP_OVERLOAD2(double, min)
380
381	// Additional Overloads that don't quite match HIP_OVERLOAD.
382	#if __cplusplus >= 201103L
383	template <typename __T1, typename __T2, typename __T3>
384	__DEVICE__ typename __hip_enable_if<
385	std::numeric_limits<__T1>::is_specialized &&
386	std::numeric_limits<__T2>::is_specialized &&
387	std::numeric_limits<__T3>::is_specialized,
388	typename __hip::__promote<__T1, __T2, __T3>::type>::type
389	fma(__T1 __x, __T2 __y, __T3 __z) {
390	typedef typename __hip::__promote<__T1, __T2, __T3>::type __result_type;
391	return ::fma((__result_type)__x, (__result_type)__y, (__result_type)__z);
392	}
393	#else
394	template <typename __T1, typename __T2, typename __T3>
395	__DEVICE__
396	typename __hip_enable_if<std::numeric_limits<__T1>::is_specialized &&
397	std::numeric_limits<__T2>::is_specialized &&
398	std::numeric_limits<__T3>::is_specialized,
399	double>::type
400	fma(__T1 __x, __T2 __y, __T3 __z) {
401	return ::fma((double)__x, (double)__y, (double)__z);
402	}
403	#endif
404
405	template <typename __T>
406	__DEVICE__
407	typename __hip_enable_if<std::numeric_limits<__T>::is_integer, double>::type
408	frexp(__T __x, int *__exp) {
409	return ::frexp((double)__x, __exp);
410	}
411
412	template <typename __T>
413	__DEVICE__
414	typename __hip_enable_if<std::numeric_limits<__T>::is_integer, double>::type
415	ldexp(__T __x, int __exp) {
416	return ::ldexp((double)__x, __exp);
417	}
418
419	template <typename __T>
420	__DEVICE__
421	typename __hip_enable_if<std::numeric_limits<__T>::is_integer, double>::type
422	modf(__T __x, double *__exp) {
423	return ::modf((double)__x, __exp);
424	}
425
426	#if __cplusplus >= 201103L
427	template <typename __T1, typename __T2>
428	__DEVICE__
429	typename __hip_enable_if<std::numeric_limits<__T1>::is_specialized &&
430	std::numeric_limits<__T2>::is_specialized,
431	typename __hip::__promote<__T1, __T2>::type>::type
432	remquo(__T1 __x, __T2 __y, int *__quo) {
433	typedef typename __hip::__promote<__T1, __T2>::type __result_type;
434	return ::remquo((__result_type)__x, (__result_type)__y, __quo);
435	}
436	#else
437	template <typename __T1, typename __T2>
438	__DEVICE__
439	typename __hip_enable_if<std::numeric_limits<__T1>::is_specialized &&
440	std::numeric_limits<__T2>::is_specialized,
441	double>::type
442	remquo(__T1 __x, __T2 __y, int *__quo) {
443	return ::remquo((double)__x, (double)__y, __quo);
444	}
445	#endif
446
447	template <typename __T>
448	__DEVICE__
449	typename __hip_enable_if<std::numeric_limits<__T>::is_integer, double>::type
450	scalbln(__T __x, long int __exp) {
451	return ::scalbln((double)__x, __exp);
452	}
453
454	template <typename __T>
455	__DEVICE__
456	typename __hip_enable_if<std::numeric_limits<__T>::is_integer, double>::type
457	scalbn(__T __x, int __exp) {
458	return ::scalbn((double)__x, __exp);
459	}
460
461	#pragma pop_macro("__HIP_OVERLOAD1")
462	#pragma pop_macro("__HIP_OVERLOAD2")
463
464	// END HIP_OVERLOAD
465
466	// END DEF_FUN and HIP_OVERLOAD
467
468	#endif // defined(__cplusplus)
469
470	// Define these overloads inside the namespace our standard library uses.
471	#ifdef _LIBCPP_BEGIN_NAMESPACE_STD
472	_LIBCPP_BEGIN_NAMESPACE_STD
473	#else
474	namespace std {
475	#ifdef _GLIBCXX_BEGIN_NAMESPACE_VERSION
476	_GLIBCXX_BEGIN_NAMESPACE_VERSION
477	#endif
478	#endif
479
480	// Pull the new overloads we defined above into namespace std.
481	// using ::abs; - This may be considered for C++.
482	using ::acos;
483	using ::acosh;
484	using ::asin;
485	using ::asinh;
486	using ::atan;
487	using ::atan2;
488	using ::atanh;
489	using ::cbrt;
490	using ::ceil;
491	using ::copysign;
492	using ::cos;
493	using ::cosh;
494	using ::erf;
495	using ::erfc;
496	using ::exp;
497	using ::exp2;
498	using ::expm1;
499	using ::fabs;
500	using ::fdim;
501	using ::floor;
502	using ::fma;
503	using ::fmax;
504	using ::fmin;
505	using ::fmod;
506	using ::fpclassify;
507	using ::frexp;
508	using ::hypot;
509	using ::ilogb;
510	using ::isfinite;
511	using ::isgreater;
512	using ::isgreaterequal;
513	using ::isless;
514	using ::islessequal;
515	using ::islessgreater;
516	using ::isnormal;
517	using ::isunordered;
518	using ::ldexp;
519	using ::lgamma;
520	using ::llrint;
521	using ::llround;
522	using ::log;
523	using ::log10;
524	using ::log1p;
525	using ::log2;
526	using ::logb;
527	using ::lrint;
528	using ::lround;
529	using ::modf;
530	// using ::nan; - This may be considered for C++.
531	// using ::nanf; - This may be considered for C++.
532	// using ::nanl; - This is not yet defined.
533	using ::nearbyint;
534	using ::nextafter;
535	// using ::nexttoward; - Omit this since we do not have a definition.
536	using ::pow;
537	using ::remainder;
538	using ::remquo;
539	using ::rint;
540	using ::round;
541	using ::scalbln;
542	using ::scalbn;
543	using ::signbit;
544	using ::sin;
545	using ::sinh;
546	using ::sqrt;
547	using ::tan;
548	using ::tanh;
549	using ::tgamma;
550	using ::trunc;
551
552	// Well this is fun: We need to pull these symbols in for libc++, but we can't
553	// pull them in with libstdc++, because its ::isinf and ::isnan are different
554	// than its std::isinf and std::isnan.
555	#ifndef __GLIBCXX__
556	using ::isinf;
557	using ::isnan;
558	#endif
559
560	// Finally, pull the "foobarf" functions that HIP defines into std.
561	using ::acosf;
562	using ::acoshf;
563	using ::asinf;
564	using ::asinhf;
565	using ::atan2f;
566	using ::atanf;
567	using ::atanhf;
568	using ::cbrtf;
569	using ::ceilf;
570	using ::copysignf;
571	using ::cosf;
572	using ::coshf;
573	using ::erfcf;
574	using ::erff;
575	using ::exp2f;
576	using ::expf;
577	using ::expm1f;
578	using ::fabsf;
579	using ::fdimf;
580	using ::floorf;
581	using ::fmaf;
582	using ::fmaxf;
583	using ::fminf;
584	using ::fmodf;
585	using ::frexpf;
586	using ::hypotf;
587	using ::ilogbf;
588	using ::ldexpf;
589	using ::lgammaf;
590	using ::llrintf;
591	using ::llroundf;
592	using ::log10f;
593	using ::log1pf;
594	using ::log2f;
595	using ::logbf;
596	using ::logf;
597	using ::lrintf;
598	using ::lroundf;
599	using ::modff;
600	using ::nearbyintf;
601	using ::nextafterf;
602	// using ::nexttowardf; - Omit this since we do not have a definition.
603	using ::powf;
604	using ::remainderf;
605	using ::remquof;
606	using ::rintf;
607	using ::roundf;
608	using ::scalblnf;
609	using ::scalbnf;
610	using ::sinf;
611	using ::sinhf;
612	using ::sqrtf;
613	using ::tanf;
614	using ::tanhf;
615	using ::tgammaf;
616	using ::truncf;
617
618	#ifdef _LIBCPP_END_NAMESPACE_STD
619	_LIBCPP_END_NAMESPACE_STD
620	#else
621	#ifdef _GLIBCXX_BEGIN_NAMESPACE_VERSION
622	_GLIBCXX_END_NAMESPACE_VERSION
623	#endif
624	} // namespace std
625	#endif
626
627	#pragma pop_macro("__DEVICE__")
628
629	#endif // __CLANG_HIP_CMATH_H__

lib/include/__clang_hip_libdevice_declares.h+17-9

 @@ -10,7 +10,9 @@
 #ifndef __CLANG_HIP_LIBDEVICE_DECLARES_H__
 #define __CLANG_HIP_LIBDEVICE_DECLARES_H__
+#ifdef __cplusplus
 extern "C" {
+#endif
 // BEGIN FLOAT
 __device__ __attribute__((const)) float __ocml_acos_f32(float);
 @@ -78,6 +80,7 @@ __device__ __attribute__((const)) float __ocml_len4_f32(float, float, float,
 __device__ __attribute__((pure)) float __ocml_ncdf_f32(float);
 __device__ __attribute__((pure)) float __ocml_ncdfinv_f32(float);
 __device__ __attribute__((pure)) float __ocml_pow_f32(float, float);
+__device__ __attribute__((pure)) float __ocml_pown_f32(float, int);
 __device__ __attribute__((pure)) float __ocml_rcbrt_f32(float);
 __device__ __attribute__((const)) float __ocml_remainder_f32(float, float);
 __device__ float __ocml_remquo_f32(float, float,
 @@ -126,10 +129,10 @@ __device__ __attribute__((const)) float __ocml_div_rte_f32(float, float);
 __device__ __attribute__((const)) float __ocml_div_rtn_f32(float, float);
 __device__ __attribute__((const)) float __ocml_div_rtp_f32(float, float);
 __device__ __attribute__((const)) float __ocml_div_rtz_f32(float, float);
-__device__ __attribute__((const)) float __ocml_sqrt_rte_f32(float, float);
+__device__ __attribute__((const)) float __ocml_sqrt_rte_f32(float);
-__device__ __attribute__((const)) float __ocml_sqrt_rtn_f32(float, float);
+__device__ __attribute__((const)) float __ocml_sqrt_rtn_f32(float);
-__device__ __attribute__((const)) float __ocml_sqrt_rtp_f32(float, float);
+__device__ __attribute__((const)) float __ocml_sqrt_rtp_f32(float);
-__device__ __attribute__((const)) float __ocml_sqrt_rtz_f32(float, float);
+__device__ __attribute__((const)) float __ocml_sqrt_rtz_f32(float);
 __device__ __attribute__((const)) float __ocml_fma_rte_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtn_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtp_f32(float, float, float);
 @@ -205,6 +208,7 @@ __device__ __attribute__((const)) double __ocml_len4_f64(double, double, double,
 __device__ __attribute__((pure)) double __ocml_ncdf_f64(double);
 __device__ __attribute__((pure)) double __ocml_ncdfinv_f64(double);
 __device__ __attribute__((pure)) double __ocml_pow_f64(double, double);
+__device__ __attribute__((pure)) double __ocml_pown_f64(double, int);
 __device__ __attribute__((pure)) double __ocml_rcbrt_f64(double);
 __device__ __attribute__((const)) double __ocml_remainder_f64(double, double);
 __device__ double __ocml_remquo_f64(double, double,
 @@ -252,10 +256,10 @@ __device__ __attribute__((const)) double __ocml_div_rte_f64(double, double);
 __device__ __attribute__((const)) double __ocml_div_rtn_f64(double, double);
 __device__ __attribute__((const)) double __ocml_div_rtp_f64(double, double);
 __device__ __attribute__((const)) double __ocml_div_rtz_f64(double, double);
-__device__ __attribute__((const)) double __ocml_sqrt_rte_f64(double, double);
+__device__ __attribute__((const)) double __ocml_sqrt_rte_f64(double);
-__device__ __attribute__((const)) double __ocml_sqrt_rtn_f64(double, double);
+__device__ __attribute__((const)) double __ocml_sqrt_rtn_f64(double);
-__device__ __attribute__((const)) double __ocml_sqrt_rtp_f64(double, double);
+__device__ __attribute__((const)) double __ocml_sqrt_rtp_f64(double);
-__device__ __attribute__((const)) double __ocml_sqrt_rtz_f64(double, double);
+__device__ __attribute__((const)) double __ocml_sqrt_rtz_f64(double);
 __device__ __attribute__((const)) double __ocml_fma_rte_f64(double, double,
                                                             double);
 __device__ __attribute__((const)) double __ocml_fma_rtn_f64(double, double,
 @@ -290,6 +294,7 @@ __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
 __device__ _Float16 __ocml_sin_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
 typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
 typedef short __2i16 __attribute__((ext_vector_type(2)));
 @@ -313,14 +318,17 @@ __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
 __device__ inline __2f16
 __llvm_amdgcn_rcp_2f16(__2f16 __x) // Not currently exposed by ROCDL.
 {
-  return __2f16{__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y)};
+  return (__2f16)(__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y));
 }
 __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
 __device__ __2f16 __ocml_sin_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16);
+__device__ __attribute__((const)) __2f16 __ocml_pown_2f16(__2f16, __2i16);
+#ifdef __cplusplus
 } // extern "C"
+#endif
 #endif // __CLANG_HIP_LIBDEVICE_DECLARES_H__

lib/include/__clang_hip_math.h+619-529

...	@@ -1,4 +1,4 @@	...	@@ -1,4 +1,4 @@
1	/*===---- __clang_hip_math.h - HIP math decls -------------------------------===	1	/*===---- __clang_hip_math.h - Device-side HIP math support ----------------===
2	*	2	*
3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	* See https://llvm.org/LICENSE.txt for license information.	4	* See https://llvm.org/LICENSE.txt for license information.
...	@@ -6,24 +6,57 @@	...	@@ -6,24 +6,57 @@
6	*	6	*
7	*===-----------------------------------------------------------------------===	7	*===-----------------------------------------------------------------------===
8	*/	8	*/
9
10	#ifndef __CLANG_HIP_MATH_H__	9	#ifndef __CLANG_HIP_MATH_H__
11	#define __CLANG_HIP_MATH_H__	10	#define __CLANG_HIP_MATH_H__
12		11
		12	#if !defined(__HIP__)
		13	#error "This file is for HIP and OpenMP AMDGCN device compilation only."
		14	#endif
		15
		16	#if defined(__cplusplus)
13	#include <algorithm>	17	#include <algorithm>
		18	#endif
14	#include <limits.h>	19	#include <limits.h>
15	#include <limits>
16	#include <stdint.h>	20	#include <stdint.h>
17		21
18	#pragma push_macro("__DEVICE__")	22	#pragma push_macro("__DEVICE__")
19	#pragma push_macro("__RETURN_TYPE")	23	#define __DEVICE__ static __device__ inline __attribute__((always_inline))
20		24
21	// to be consistent with __clang_cuda_math_forward_declares	25	// A few functions return bool type starting only in C++11.
22	#define __DEVICE__ static __device__	26	#pragma push_macro("__RETURN_TYPE")
		27	#if defined(__cplusplus)
23	#define __RETURN_TYPE bool	28	#define __RETURN_TYPE bool
		29	#else
		30	#define __RETURN_TYPE int
		31	#endif
		32
		33	#if defined (__cplusplus) && __cplusplus < 201103L
		34	// emulate static_assert on type sizes
		35	template<bool>
		36	struct __compare_result{};
		37	template<>
		38	struct __compare_result<true> {
		39	static const bool valid;
		40	};
		41
		42	__DEVICE__
		43	void __suppress_unused_warning(bool b){};
		44	template <unsigned int S, unsigned int T>
		45	__DEVICE__ void __static_assert_equal_size() {
		46	__suppress_unused_warning(__compare_result<S == T>::valid);
		47	}
		48
		49	#define __static_assert_type_size_equal(A, B) \
		50	__static_assert_equal_size<A,B>()
		51
		52	#else
		53	#define __static_assert_type_size_equal(A,B) \
		54	static_assert((A) == (B), "")
		55
		56	#endif
24		57
25	__DEVICE__	58	__DEVICE__
26	inline uint64_t __make_mantissa_base8(const char *__tagp) {	59	uint64_t __make_mantissa_base8(const char *__tagp) {
27	uint64_t __r = 0;	60	uint64_t __r = 0;
28	while (__tagp) {	61	while (__tagp) {
29	char __tmp = *__tagp;	62	char __tmp = *__tagp;
...	@@ -40,7 +73,7 @@ inline uint64_t __make_mantissa_base8(const char *__tagp) {	...	@@ -40,7 +73,7 @@ inline uint64_t __make_mantissa_base8(const char *__tagp) {
40	}	73	}
41		74
42	__DEVICE__	75	__DEVICE__
43	inline uint64_t __make_mantissa_base10(const char *__tagp) {	76	uint64_t __make_mantissa_base10(const char *__tagp) {
44	uint64_t __r = 0;	77	uint64_t __r = 0;
45	while (__tagp) {	78	while (__tagp) {
46	char __tmp = *__tagp;	79	char __tmp = *__tagp;
...	@@ -57,7 +90,7 @@ inline uint64_t __make_mantissa_base10(const char *__tagp) {	...	@@ -57,7 +90,7 @@ inline uint64_t __make_mantissa_base10(const char *__tagp) {
57	}	90	}
58		91
59	__DEVICE__	92	__DEVICE__
60	inline uint64_t __make_mantissa_base16(const char *__tagp) {	93	uint64_t __make_mantissa_base16(const char *__tagp) {
61	uint64_t __r = 0;	94	uint64_t __r = 0;
62	while (__tagp) {	95	while (__tagp) {
63	char __tmp = *__tagp;	96	char __tmp = *__tagp;
...	@@ -78,7 +111,7 @@ inline uint64_t __make_mantissa_base16(const char *__tagp) {	...	@@ -78,7 +111,7 @@ inline uint64_t __make_mantissa_base16(const char *__tagp) {
78	}	111	}
79		112
80	__DEVICE__	113	__DEVICE__
81	inline uint64_t __make_mantissa(const char *__tagp) {	114	uint64_t __make_mantissa(const char *__tagp) {
82	if (!__tagp)	115	if (!__tagp)
83	return 0u;	116	return 0u;
84		117
...	@@ -95,78 +128,124 @@ inline uint64_t __make_mantissa(const char *__tagp) {	...	@@ -95,78 +128,124 @@ inline uint64_t __make_mantissa(const char *__tagp) {
95	}	128	}
96		129
97	// BEGIN FLOAT	130	// BEGIN FLOAT
		131	#if defined(__cplusplus)
98	__DEVICE__	132	__DEVICE__
99	inline float abs(float __x) { return __ocml_fabs_f32(__x); }	133	int abs(int __x) {
		134	int __sgn = __x >> (sizeof(int) * CHAR_BIT - 1);
		135	return (__x ^ __sgn) - __sgn;
		136	}
100	__DEVICE__	137	__DEVICE__
101	inline float acosf(float __x) { return __ocml_acos_f32(__x); }	138	long labs(long __x) {
		139	long __sgn = __x >> (sizeof(long) * CHAR_BIT - 1);
		140	return (__x ^ __sgn) - __sgn;
		141	}
102	__DEVICE__	142	__DEVICE__
103	inline float acoshf(float __x) { return __ocml_acosh_f32(__x); }	143	long long llabs(long long __x) {
		144	long long __sgn = __x >> (sizeof(long long) * CHAR_BIT - 1);
		145	return (__x ^ __sgn) - __sgn;
		146	}
		147	#endif
		148
104	__DEVICE__	149	__DEVICE__
105	inline float asinf(float __x) { return __ocml_asin_f32(__x); }	150	float acosf(float __x) { return __ocml_acos_f32(__x); }
		151
106	__DEVICE__	152	__DEVICE__
107	inline float asinhf(float __x) { return __ocml_asinh_f32(__x); }	153	float acoshf(float __x) { return __ocml_acosh_f32(__x); }
		154
108	__DEVICE__	155	__DEVICE__
109	inline float atan2f(float __x, float __y) { return __ocml_atan2_f32(__x, __y); }	156	float asinf(float __x) { return __ocml_asin_f32(__x); }
		157
110	__DEVICE__	158	__DEVICE__
111	inline float atanf(float __x) { return __ocml_atan_f32(__x); }	159	float asinhf(float __x) { return __ocml_asinh_f32(__x); }
		160
112	__DEVICE__	161	__DEVICE__
113	inline float atanhf(float __x) { return __ocml_atanh_f32(__x); }	162	float atan2f(float __x, float __y) { return __ocml_atan2_f32(__x, __y); }
		163
114	__DEVICE__	164	__DEVICE__
115	inline float cbrtf(float __x) { return __ocml_cbrt_f32(__x); }	165	float atanf(float __x) { return __ocml_atan_f32(__x); }
		166
116	__DEVICE__	167	__DEVICE__
117	inline float ceilf(float __x) { return __ocml_ceil_f32(__x); }	168	float atanhf(float __x) { return __ocml_atanh_f32(__x); }
		169
118	__DEVICE__	170	__DEVICE__
119	inline float copysignf(float __x, float __y) {	171	float cbrtf(float __x) { return __ocml_cbrt_f32(__x); }
120	return __ocml_copysign_f32(__x, __y);	172
121	}
122	__DEVICE__	173	__DEVICE__
123	inline float cosf(float __x) { return __ocml_cos_f32(__x); }	174	float ceilf(float __x) { return __ocml_ceil_f32(__x); }
		175
124	__DEVICE__	176	__DEVICE__
125	inline float coshf(float __x) { return __ocml_cosh_f32(__x); }	177	float copysignf(float __x, float __y) { return __ocml_copysign_f32(__x, __y); }
		178
126	__DEVICE__	179	__DEVICE__
127	inline float cospif(float __x) { return __ocml_cospi_f32(__x); }	180	float cosf(float __x) { return __ocml_cos_f32(__x); }
		181
128	__DEVICE__	182	__DEVICE__
129	inline float cyl_bessel_i0f(float __x) { return __ocml_i0_f32(__x); }	183	float coshf(float __x) { return __ocml_cosh_f32(__x); }
		184
130	__DEVICE__	185	__DEVICE__
131	inline float cyl_bessel_i1f(float __x) { return __ocml_i1_f32(__x); }	186	float cospif(float __x) { return __ocml_cospi_f32(__x); }
		187
132	__DEVICE__	188	__DEVICE__
133	inline float erfcf(float __x) { return __ocml_erfc_f32(__x); }	189	float cyl_bessel_i0f(float __x) { return __ocml_i0_f32(__x); }
		190
134	__DEVICE__	191	__DEVICE__
135	inline float erfcinvf(float __x) { return __ocml_erfcinv_f32(__x); }	192	float cyl_bessel_i1f(float __x) { return __ocml_i1_f32(__x); }
		193
136	__DEVICE__	194	__DEVICE__
137	inline float erfcxf(float __x) { return __ocml_erfcx_f32(__x); }	195	float erfcf(float __x) { return __ocml_erfc_f32(__x); }
		196
138	__DEVICE__	197	__DEVICE__
139	inline float erff(float __x) { return __ocml_erf_f32(__x); }	198	float erfcinvf(float __x) { return __ocml_erfcinv_f32(__x); }
		199
140	__DEVICE__	200	__DEVICE__
141	inline float erfinvf(float __x) { return __ocml_erfinv_f32(__x); }	201	float erfcxf(float __x) { return __ocml_erfcx_f32(__x); }
		202
142	__DEVICE__	203	__DEVICE__
143	inline float exp10f(float __x) { return __ocml_exp10_f32(__x); }	204	float erff(float __x) { return __ocml_erf_f32(__x); }
		205
144	__DEVICE__	206	__DEVICE__
145	inline float exp2f(float __x) { return __ocml_exp2_f32(__x); }	207	float erfinvf(float __x) { return __ocml_erfinv_f32(__x); }
		208
146	__DEVICE__	209	__DEVICE__
147	inline float expf(float __x) { return __ocml_exp_f32(__x); }	210	float exp10f(float __x) { return __ocml_exp10_f32(__x); }
		211
148	__DEVICE__	212	__DEVICE__
149	inline float expm1f(float __x) { return __ocml_expm1_f32(__x); }	213	float exp2f(float __x) { return __ocml_exp2_f32(__x); }
		214
150	__DEVICE__	215	__DEVICE__
151	inline float fabsf(float __x) { return __ocml_fabs_f32(__x); }	216	float expf(float __x) { return __ocml_exp_f32(__x); }
		217
152	__DEVICE__	218	__DEVICE__
153	inline float fdimf(float __x, float __y) { return __ocml_fdim_f32(__x, __y); }	219	float expm1f(float __x) { return __ocml_expm1_f32(__x); }
		220
154	__DEVICE__	221	__DEVICE__
155	inline float fdividef(float __x, float __y) { return __x / __y; }	222	float fabsf(float __x) { return __ocml_fabs_f32(__x); }
		223
156	__DEVICE__	224	__DEVICE__
157	inline float floorf(float __x) { return __ocml_floor_f32(__x); }	225	float fdimf(float __x, float __y) { return __ocml_fdim_f32(__x, __y); }
		226
158	__DEVICE__	227	__DEVICE__
159	inline float fmaf(float __x, float __y, float __z) {	228	float fdividef(float __x, float __y) { return __x / __y; }
		229
		230	__DEVICE__
		231	float floorf(float __x) { return __ocml_floor_f32(__x); }
		232
		233	__DEVICE__
		234	float fmaf(float __x, float __y, float __z) {
160	return __ocml_fma_f32(__x, __y, __z);	235	return __ocml_fma_f32(__x, __y, __z);
161	}	236	}
		237
162	__DEVICE__	238	__DEVICE__
163	inline float fmaxf(float __x, float __y) { return __ocml_fmax_f32(__x, __y); }	239	float fmaxf(float __x, float __y) { return __ocml_fmax_f32(__x, __y); }
		240
164	__DEVICE__	241	__DEVICE__
165	inline float fminf(float __x, float __y) { return __ocml_fmin_f32(__x, __y); }	242	float fminf(float __x, float __y) { return __ocml_fmin_f32(__x, __y); }
		243
166	__DEVICE__	244	__DEVICE__
167	inline float fmodf(float __x, float __y) { return __ocml_fmod_f32(__x, __y); }	245	float fmodf(float __x, float __y) { return __ocml_fmod_f32(__x, __y); }
		246
168	__DEVICE__	247	__DEVICE__
169	inline float frexpf(float __x, int *__nptr) {	248	float frexpf(float __x, int *__nptr) {
170	int __tmp;	249	int __tmp;
171	float __r =	250	float __r =
172	__ocml_frexp_f32(__x, (__attribute__((address_space(5))) int *)&__tmp);	251	__ocml_frexp_f32(__x, (__attribute__((address_space(5))) int *)&__tmp);
...	@@ -174,24 +253,31 @@ inline float frexpf(float __x, int *__nptr) {	...	@@ -174,24 +253,31 @@ inline float frexpf(float __x, int *__nptr) {
174		253
175	return __r;	254	return __r;
176	}	255	}
		256
177	__DEVICE__	257	__DEVICE__
178	inline float hypotf(float __x, float __y) { return __ocml_hypot_f32(__x, __y); }	258	float hypotf(float __x, float __y) { return __ocml_hypot_f32(__x, __y); }
		259
179	__DEVICE__	260	__DEVICE__
180	inline int ilogbf(float __x) { return __ocml_ilogb_f32(__x); }	261	int ilogbf(float __x) { return __ocml_ilogb_f32(__x); }
		262
181	__DEVICE__	263	__DEVICE__
182	inline __RETURN_TYPE isfinite(float __x) { return __ocml_isfinite_f32(__x); }	264	__RETURN_TYPE __finitef(float __x) { return __ocml_isfinite_f32(__x); }
		265
183	__DEVICE__	266	__DEVICE__
184	inline __RETURN_TYPE isinf(float __x) { return __ocml_isinf_f32(__x); }	267	__RETURN_TYPE __isinff(float __x) { return __ocml_isinf_f32(__x); }
		268
185	__DEVICE__	269	__DEVICE__
186	inline __RETURN_TYPE isnan(float __x) { return __ocml_isnan_f32(__x); }	270	__RETURN_TYPE __isnanf(float __x) { return __ocml_isnan_f32(__x); }
		271
187	__DEVICE__	272	__DEVICE__
188	inline float j0f(float __x) { return __ocml_j0_f32(__x); }	273	float j0f(float __x) { return __ocml_j0_f32(__x); }
		274
189	__DEVICE__	275	__DEVICE__
190	inline float j1f(float __x) { return __ocml_j1_f32(__x); }	276	float j1f(float __x) { return __ocml_j1_f32(__x); }
		277
191	__DEVICE__	278	__DEVICE__
192	inline float jnf(int __n,	279	float jnf(int __n, float __x) { // TODO: we could use Ahmes multiplication
193	float __x) { // TODO: we could use Ahmes multiplication	280	// and the Miller & Brown algorithm
194	// and the Miller & Brown algorithm
195	// for linear recurrences to get O(log n) steps, but it's unclear if	281	// for linear recurrences to get O(log n) steps, but it's unclear if
196	// it'd be beneficial in this case.	282	// it'd be beneficial in this case.
197	if (__n == 0)	283	if (__n == 0)
...	@@ -209,50 +295,61 @@ inline float jnf(int __n,	...	@@ -209,50 +295,61 @@ inline float jnf(int __n,
209		295
210	return __x1;	296	return __x1;
211	}	297	}
		298
212	__DEVICE__	299	__DEVICE__
213	inline float ldexpf(float __x, int __e) { return __ocml_ldexp_f32(__x, __e); }	300	float ldexpf(float __x, int __e) { return __ocml_ldexp_f32(__x, __e); }
		301
214	__DEVICE__	302	__DEVICE__
215	inline float lgammaf(float __x) { return __ocml_lgamma_f32(__x); }	303	float lgammaf(float __x) { return __ocml_lgamma_f32(__x); }
		304
216	__DEVICE__	305	__DEVICE__
217	inline long long int llrintf(float __x) { return __ocml_rint_f32(__x); }	306	long long int llrintf(float __x) { return __ocml_rint_f32(__x); }
		307
218	__DEVICE__	308	__DEVICE__
219	inline long long int llroundf(float __x) { return __ocml_round_f32(__x); }	309	long long int llroundf(float __x) { return __ocml_round_f32(__x); }
		310
220	__DEVICE__	311	__DEVICE__
221	inline float log10f(float __x) { return __ocml_log10_f32(__x); }	312	float log10f(float __x) { return __ocml_log10_f32(__x); }
		313
222	__DEVICE__	314	__DEVICE__
223	inline float log1pf(float __x) { return __ocml_log1p_f32(__x); }	315	float log1pf(float __x) { return __ocml_log1p_f32(__x); }
		316
224	__DEVICE__	317	__DEVICE__
225	inline float log2f(float __x) { return __ocml_log2_f32(__x); }	318	float log2f(float __x) { return __ocml_log2_f32(__x); }
		319
226	__DEVICE__	320	__DEVICE__
227	inline float logbf(float __x) { return __ocml_logb_f32(__x); }	321	float logbf(float __x) { return __ocml_logb_f32(__x); }
		322
228	__DEVICE__	323	__DEVICE__
229	inline float logf(float __x) { return __ocml_log_f32(__x); }	324	float logf(float __x) { return __ocml_log_f32(__x); }
		325
230	__DEVICE__	326	__DEVICE__
231	inline long int lrintf(float __x) { return __ocml_rint_f32(__x); }	327	long int lrintf(float __x) { return __ocml_rint_f32(__x); }
		328
232	__DEVICE__	329	__DEVICE__
233	inline long int lroundf(float __x) { return __ocml_round_f32(__x); }	330	long int lroundf(float __x) { return __ocml_round_f32(__x); }
		331
234	__DEVICE__	332	__DEVICE__
235	inline float modff(float __x, float *__iptr) {	333	float modff(float __x, float *__iptr) {
236	float __tmp;	334	float __tmp;
237	float __r =	335	float __r =
238	__ocml_modf_f32(__x, (__attribute__((address_space(5))) float *)&__tmp);	336	__ocml_modf_f32(__x, (__attribute__((address_space(5))) float *)&__tmp);
239	*__iptr = __tmp;	337	*__iptr = __tmp;
240
241	return __r;	338	return __r;
242	}	339	}
		340
243	__DEVICE__	341	__DEVICE__
244	inline float nanf(const char *__tagp) {	342	float nanf(const char *__tagp) {
245	union {	343	union {
246	float val;	344	float val;
247	struct ieee_float {	345	struct ieee_float {
248	uint32_t mantissa : 22;	346	unsigned int mantissa : 22;
249	uint32_t quiet : 1;	347	unsigned int quiet : 1;
250	uint32_t exponent : 8;	348	unsigned int exponent : 8;
251	uint32_t sign : 1;	349	unsigned int sign : 1;
252	} bits;	350	} bits;
253
254	static_assert(sizeof(float) == sizeof(ieee_float), "");
255	} __tmp;	351	} __tmp;
		352	__static_assert_type_size_equal(sizeof(__tmp.val), sizeof(__tmp.bits));
256		353
257	__tmp.bits.sign = 0u;	354	__tmp.bits.sign = 0u;
258	__tmp.bits.exponent = ~0u;	355	__tmp.bits.exponent = ~0u;
...	@@ -261,28 +358,34 @@ inline float nanf(const char *__tagp) {	...	@@ -261,28 +358,34 @@ inline float nanf(const char *__tagp) {
261		358
262	return __tmp.val;	359	return __tmp.val;
263	}	360	}
		361
264	__DEVICE__	362	__DEVICE__
265	inline float nearbyintf(float __x) { return __ocml_nearbyint_f32(__x); }	363	float nearbyintf(float __x) { return __ocml_nearbyint_f32(__x); }
		364
266	__DEVICE__	365	__DEVICE__
267	inline float nextafterf(float __x, float __y) {	366	float nextafterf(float __x, float __y) {
268	return __ocml_nextafter_f32(__x, __y);	367	return __ocml_nextafter_f32(__x, __y);
269	}	368	}
		369
270	__DEVICE__	370	__DEVICE__
271	inline float norm3df(float __x, float __y, float __z) {	371	float norm3df(float __x, float __y, float __z) {
272	return __ocml_len3_f32(__x, __y, __z);	372	return __ocml_len3_f32(__x, __y, __z);
273	}	373	}
		374
274	__DEVICE__	375	__DEVICE__
275	inline float norm4df(float __x, float __y, float __z, float __w) {	376	float norm4df(float __x, float __y, float __z, float __w) {
276	return __ocml_len4_f32(__x, __y, __z, __w);	377	return __ocml_len4_f32(__x, __y, __z, __w);
277	}	378	}
		379
278	__DEVICE__	380	__DEVICE__
279	inline float normcdff(float __x) { return __ocml_ncdf_f32(__x); }	381	float normcdff(float __x) { return __ocml_ncdf_f32(__x); }
		382
280	__DEVICE__	383	__DEVICE__
281	inline float normcdfinvf(float __x) { return __ocml_ncdfinv_f32(__x); }	384	float normcdfinvf(float __x) { return __ocml_ncdfinv_f32(__x); }
		385
282	__DEVICE__	386	__DEVICE__
283	inline float	387	float normf(int __dim,
284	normf(int __dim,	388	const float *__a) { // TODO: placeholder until OCML adds support.
285	const float *__a) { // TODO: placeholder until OCML adds support.
286	float __r = 0;	389	float __r = 0;
287	while (__dim--) {	390	while (__dim--) {
288	__r += __a[0] * __a[0];	391	__r += __a[0] * __a[0];
...	@@ -291,16 +394,23 @@ normf(int __dim,	...	@@ -291,16 +394,23 @@ normf(int __dim,
291		394
292	return __ocml_sqrt_f32(__r);	395	return __ocml_sqrt_f32(__r);
293	}	396	}
		397
294	__DEVICE__	398	__DEVICE__
295	inline float powf(float __x, float __y) { return __ocml_pow_f32(__x, __y); }	399	float powf(float __x, float __y) { return __ocml_pow_f32(__x, __y); }
		400
		401	__DEVICE__
		402	float powif(float __x, int __y) { return __ocml_pown_f32(__x, __y); }
		403
296	__DEVICE__	404	__DEVICE__
297	inline float rcbrtf(float __x) { return __ocml_rcbrt_f32(__x); }	405	float rcbrtf(float __x) { return __ocml_rcbrt_f32(__x); }
		406
298	__DEVICE__	407	__DEVICE__
299	inline float remainderf(float __x, float __y) {	408	float remainderf(float __x, float __y) {
300	return __ocml_remainder_f32(__x, __y);	409	return __ocml_remainder_f32(__x, __y);
301	}	410	}
		411
302	__DEVICE__	412	__DEVICE__
303	inline float remquof(float __x, float __y, int *__quo) {	413	float remquof(float __x, float __y, int *__quo) {
304	int __tmp;	414	int __tmp;
305	float __r = __ocml_remquo_f32(	415	float __r = __ocml_remquo_f32(
306	__x, __y, (__attribute__((address_space(5))) int *)&__tmp);	416	__x, __y, (__attribute__((address_space(5))) int *)&__tmp);
...	@@ -308,25 +418,26 @@ inline float remquof(float __x, float __y, int *__quo) {	...	@@ -308,25 +418,26 @@ inline float remquof(float __x, float __y, int *__quo) {
308		418
309	return __r;	419	return __r;
310	}	420	}
		421
311	__DEVICE__	422	__DEVICE__
312	inline float rhypotf(float __x, float __y) {	423	float rhypotf(float __x, float __y) { return __ocml_rhypot_f32(__x, __y); }
313	return __ocml_rhypot_f32(__x, __y);	424
314	}
315	__DEVICE__	425	__DEVICE__
316	inline float rintf(float __x) { return __ocml_rint_f32(__x); }	426	float rintf(float __x) { return __ocml_rint_f32(__x); }
		427
317	__DEVICE__	428	__DEVICE__
318	inline float rnorm3df(float __x, float __y, float __z) {	429	float rnorm3df(float __x, float __y, float __z) {
319	return __ocml_rlen3_f32(__x, __y, __z);	430	return __ocml_rlen3_f32(__x, __y, __z);
320	}	431	}
321		432
322	__DEVICE__	433	__DEVICE__
323	inline float rnorm4df(float __x, float __y, float __z, float __w) {	434	float rnorm4df(float __x, float __y, float __z, float __w) {
324	return __ocml_rlen4_f32(__x, __y, __z, __w);	435	return __ocml_rlen4_f32(__x, __y, __z, __w);
325	}	436	}
		437
326	__DEVICE__	438	__DEVICE__
327	inline float	439	float rnormf(int __dim,
328	rnormf(int __dim,	440	const float *__a) { // TODO: placeholder until OCML adds support.
329	const float *__a) { // TODO: placeholder until OCML adds support.
330	float __r = 0;	441	float __r = 0;
331	while (__dim--) {	442	while (__dim--) {
332	__r += __a[0] * __a[0];	443	__r += __a[0] * __a[0];
...	@@ -335,59 +446,74 @@ rnormf(int __dim,	...	@@ -335,59 +446,74 @@ rnormf(int __dim,
335		446
336	return __ocml_rsqrt_f32(__r);	447	return __ocml_rsqrt_f32(__r);
337	}	448	}
		449
338	__DEVICE__	450	__DEVICE__
339	inline float roundf(float __x) { return __ocml_round_f32(__x); }	451	float roundf(float __x) { return __ocml_round_f32(__x); }
		452
340	__DEVICE__	453	__DEVICE__
341	inline float rsqrtf(float __x) { return __ocml_rsqrt_f32(__x); }	454	float rsqrtf(float __x) { return __ocml_rsqrt_f32(__x); }
		455
342	__DEVICE__	456	__DEVICE__
343	inline float scalblnf(float __x, long int __n) {	457	float scalblnf(float __x, long int __n) {
344	return (__n < INT_MAX) ? __ocml_scalbn_f32(__x, __n)	458	return (__n < INT_MAX) ? __ocml_scalbn_f32(__x, __n)
345	: __ocml_scalb_f32(__x, __n);	459	: __ocml_scalb_f32(__x, __n);
346	}	460	}
		461
347	__DEVICE__	462	__DEVICE__
348	inline float scalbnf(float __x, int __n) { return __ocml_scalbn_f32(__x, __n); }	463	float scalbnf(float __x, int __n) { return __ocml_scalbn_f32(__x, __n); }
		464
349	__DEVICE__	465	__DEVICE__
350	inline __RETURN_TYPE signbit(float __x) { return __ocml_signbit_f32(__x); }	466	__RETURN_TYPE __signbitf(float __x) { return __ocml_signbit_f32(__x); }
		467
351	__DEVICE__	468	__DEVICE__
352	inline void sincosf(float __x, float __sinptr, float __cosptr) {	469	void sincosf(float __x, float __sinptr, float __cosptr) {
353	float __tmp;	470	float __tmp;
354
355	*__sinptr =	471	*__sinptr =
356	__ocml_sincos_f32(__x, (__attribute__((address_space(5))) float *)&__tmp);	472	__ocml_sincos_f32(__x, (__attribute__((address_space(5))) float *)&__tmp);
357	*__cosptr = __tmp;	473	*__cosptr = __tmp;
358	}	474	}
		475
359	__DEVICE__	476	__DEVICE__
360	inline void sincospif(float __x, float __sinptr, float __cosptr) {	477	void sincospif(float __x, float __sinptr, float __cosptr) {
361	float __tmp;	478	float __tmp;
362
363	*__sinptr = __ocml_sincospi_f32(	479	*__sinptr = __ocml_sincospi_f32(
364	__x, (__attribute__((address_space(5))) float *)&__tmp);	480	__x, (__attribute__((address_space(5))) float *)&__tmp);
365	*__cosptr = __tmp;	481	*__cosptr = __tmp;
366	}	482	}
		483
367	__DEVICE__	484	__DEVICE__
368	inline float sinf(float __x) { return __ocml_sin_f32(__x); }	485	float sinf(float __x) { return __ocml_sin_f32(__x); }
		486
369	__DEVICE__	487	__DEVICE__
370	inline float sinhf(float __x) { return __ocml_sinh_f32(__x); }	488	float sinhf(float __x) { return __ocml_sinh_f32(__x); }
		489
371	__DEVICE__	490	__DEVICE__
372	inline float sinpif(float __x) { return __ocml_sinpi_f32(__x); }	491	float sinpif(float __x) { return __ocml_sinpi_f32(__x); }
		492
373	__DEVICE__	493	__DEVICE__
374	inline float sqrtf(float __x) { return __ocml_sqrt_f32(__x); }	494	float sqrtf(float __x) { return __ocml_sqrt_f32(__x); }
		495
375	__DEVICE__	496	__DEVICE__
376	inline float tanf(float __x) { return __ocml_tan_f32(__x); }	497	float tanf(float __x) { return __ocml_tan_f32(__x); }
		498
377	__DEVICE__	499	__DEVICE__
378	inline float tanhf(float __x) { return __ocml_tanh_f32(__x); }	500	float tanhf(float __x) { return __ocml_tanh_f32(__x); }
		501
379	__DEVICE__	502	__DEVICE__
380	inline float tgammaf(float __x) { return __ocml_tgamma_f32(__x); }	503	float tgammaf(float __x) { return __ocml_tgamma_f32(__x); }
		504
381	__DEVICE__	505	__DEVICE__
382	inline float truncf(float __x) { return __ocml_trunc_f32(__x); }	506	float truncf(float __x) { return __ocml_trunc_f32(__x); }
		507
383	__DEVICE__	508	__DEVICE__
384	inline float y0f(float __x) { return __ocml_y0_f32(__x); }	509	float y0f(float __x) { return __ocml_y0_f32(__x); }
		510
385	__DEVICE__	511	__DEVICE__
386	inline float y1f(float __x) { return __ocml_y1_f32(__x); }	512	float y1f(float __x) { return __ocml_y1_f32(__x); }
		513
387	__DEVICE__	514	__DEVICE__
388	inline float ynf(int __n,	515	float ynf(int __n, float __x) { // TODO: we could use Ahmes multiplication
389	float __x) { // TODO: we could use Ahmes multiplication	516	// and the Miller & Brown algorithm
390	// and the Miller & Brown algorithm
391	// for linear recurrences to get O(log n) steps, but it's unclear if	517	// for linear recurrences to get O(log n) steps, but it's unclear if
392	// it'd be beneficial in this case. Placeholder until OCML adds	518	// it'd be beneficial in this case. Placeholder until OCML adds
393	// support.	519	// support.
...	@@ -408,290 +534,343 @@ inline float ynf(int __n,	...	@@ -408,290 +534,343 @@ inline float ynf(int __n,
408	}	534	}
409		535
410	// BEGIN INTRINSICS	536	// BEGIN INTRINSICS
		537
411	__DEVICE__	538	__DEVICE__
412	inline float __cosf(float __x) { return __ocml_native_cos_f32(__x); }	539	float __cosf(float __x) { return __ocml_native_cos_f32(__x); }
		540
413	__DEVICE__	541	__DEVICE__
414	inline float __exp10f(float __x) { return __ocml_native_exp10_f32(__x); }	542	float __exp10f(float __x) { return __ocml_native_exp10_f32(__x); }
		543
415	__DEVICE__	544	__DEVICE__
416	inline float __expf(float __x) { return __ocml_native_exp_f32(__x); }	545	float __expf(float __x) { return __ocml_native_exp_f32(__x); }
		546
417	#if defined OCML_BASIC_ROUNDED_OPERATIONS	547	#if defined OCML_BASIC_ROUNDED_OPERATIONS
418	__DEVICE__	548	__DEVICE__
419	inline float __fadd_rd(float __x, float __y) {	549	float __fadd_rd(float __x, float __y) { return __ocml_add_rtn_f32(__x, __y); }
420	return __ocml_add_rtn_f32(__x, __y);
421	}
422	#endif
423	__DEVICE__	550	__DEVICE__
424	inline float __fadd_rn(float __x, float __y) { return __x + __y; }	551	float __fadd_rn(float __x, float __y) { return __ocml_add_rte_f32(__x, __y); }
425	#if defined OCML_BASIC_ROUNDED_OPERATIONS
426	__DEVICE__	552	__DEVICE__
427	inline float __fadd_ru(float __x, float __y) {	553	float __fadd_ru(float __x, float __y) { return __ocml_add_rtp_f32(__x, __y); }
428	return __ocml_add_rtp_f32(__x, __y);
429	}
430	__DEVICE__	554	__DEVICE__
431	inline float __fadd_rz(float __x, float __y) {	555	float __fadd_rz(float __x, float __y) { return __ocml_add_rtz_f32(__x, __y); }
432	return __ocml_add_rtz_f32(__x, __y);	556	#else
433	}
434	__DEVICE__	557	__DEVICE__
435	inline float __fdiv_rd(float __x, float __y) {	558	float __fadd_rn(float __x, float __y) { return __x + __y; }
436	return __ocml_div_rtn_f32(__x, __y);
437	}
438	#endif	559	#endif
439	__DEVICE__	560
440	inline float __fdiv_rn(float __x, float __y) { return __x / __y; }
441	#if defined OCML_BASIC_ROUNDED_OPERATIONS	561	#if defined OCML_BASIC_ROUNDED_OPERATIONS
442	__DEVICE__	562	__DEVICE__
443	inline float __fdiv_ru(float __x, float __y) {	563	float __fdiv_rd(float __x, float __y) { return __ocml_div_rtn_f32(__x, __y); }
444	return __ocml_div_rtp_f32(__x, __y);
445	}
446	__DEVICE__	564	__DEVICE__
447	inline float __fdiv_rz(float __x, float __y) {	565	float __fdiv_rn(float __x, float __y) { return __ocml_div_rte_f32(__x, __y); }
448	return __ocml_div_rtz_f32(__x, __y);	566	__DEVICE__
449	}	567	float __fdiv_ru(float __x, float __y) { return __ocml_div_rtp_f32(__x, __y); }
		568	__DEVICE__
		569	float __fdiv_rz(float __x, float __y) { return __ocml_div_rtz_f32(__x, __y); }
		570	#else
		571	__DEVICE__
		572	float __fdiv_rn(float __x, float __y) { return __x / __y; }
450	#endif	573	#endif
		574
451	__DEVICE__	575	__DEVICE__
452	inline float __fdividef(float __x, float __y) { return __x / __y; }	576	float __fdividef(float __x, float __y) { return __x / __y; }
		577
453	#if defined OCML_BASIC_ROUNDED_OPERATIONS	578	#if defined OCML_BASIC_ROUNDED_OPERATIONS
454	__DEVICE__	579	__DEVICE__
455	inline float __fmaf_rd(float __x, float __y, float __z) {	580	float __fmaf_rd(float __x, float __y, float __z) {
456	return __ocml_fma_rtn_f32(__x, __y, __z);	581	return __ocml_fma_rtn_f32(__x, __y, __z);
457	}	582	}
458	#endif
459	__DEVICE__	583	__DEVICE__
460	inline float __fmaf_rn(float __x, float __y, float __z) {	584	float __fmaf_rn(float __x, float __y, float __z) {
461	return __ocml_fma_f32(__x, __y, __z);	585	return __ocml_fma_rte_f32(__x, __y, __z);
462	}	586	}
463	#if defined OCML_BASIC_ROUNDED_OPERATIONS
464	__DEVICE__	587	__DEVICE__
465	inline float __fmaf_ru(float __x, float __y, float __z) {	588	float __fmaf_ru(float __x, float __y, float __z) {
466	return __ocml_fma_rtp_f32(__x, __y, __z);	589	return __ocml_fma_rtp_f32(__x, __y, __z);
467	}	590	}
468	__DEVICE__	591	__DEVICE__
469	inline float __fmaf_rz(float __x, float __y, float __z) {	592	float __fmaf_rz(float __x, float __y, float __z) {
470	return __ocml_fma_rtz_f32(__x, __y, __z);	593	return __ocml_fma_rtz_f32(__x, __y, __z);
471	}	594	}
		595	#else
472	__DEVICE__	596	__DEVICE__
473	inline float __fmul_rd(float __x, float __y) {	597	float __fmaf_rn(float __x, float __y, float __z) {
474	return __ocml_mul_rtn_f32(__x, __y);	598	return __ocml_fma_f32(__x, __y, __z);
475	}	599	}
476	#endif	600	#endif
477	__DEVICE__	601
478	inline float __fmul_rn(float __x, float __y) { return __x * __y; }
479	#if defined OCML_BASIC_ROUNDED_OPERATIONS	602	#if defined OCML_BASIC_ROUNDED_OPERATIONS
480	__DEVICE__	603	__DEVICE__
481	inline float __fmul_ru(float __x, float __y) {	604	float __fmul_rd(float __x, float __y) { return __ocml_mul_rtn_f32(__x, __y); }
482	return __ocml_mul_rtp_f32(__x, __y);
483	}
484	__DEVICE__	605	__DEVICE__
485	inline float __fmul_rz(float __x, float __y) {	606	float __fmul_rn(float __x, float __y) { return __ocml_mul_rte_f32(__x, __y); }
486	return __ocml_mul_rtz_f32(__x, __y);
487	}
488	__DEVICE__	607	__DEVICE__
489	inline float __frcp_rd(float __x) { return __llvm_amdgcn_rcp_f32(__x); }	608	float __fmul_ru(float __x, float __y) { return __ocml_mul_rtp_f32(__x, __y); }
490	#endif	609	__DEVICE__
		610	float __fmul_rz(float __x, float __y) { return __ocml_mul_rtz_f32(__x, __y); }
		611	#else
491	__DEVICE__	612	__DEVICE__
492	inline float __frcp_rn(float __x) { return __llvm_amdgcn_rcp_f32(__x); }	613	float __fmul_rn(float __x, float __y) { return __x * __y; }
		614	#endif
		615
493	#if defined OCML_BASIC_ROUNDED_OPERATIONS	616	#if defined OCML_BASIC_ROUNDED_OPERATIONS
494	__DEVICE__	617	__DEVICE__
495	inline float __frcp_ru(float __x) { return __llvm_amdgcn_rcp_f32(__x); }	618	float __frcp_rd(float __x) { return __ocml_div_rtn_f32(1.0f, __x); }
496	__DEVICE__	619	__DEVICE__
497	inline float __frcp_rz(float __x) { return __llvm_amdgcn_rcp_f32(__x); }	620	float __frcp_rn(float __x) { return __ocml_div_rte_f32(1.0f, __x); }
498	#endif
499	__DEVICE__	621	__DEVICE__
500	inline float __frsqrt_rn(float __x) { return __llvm_amdgcn_rsq_f32(__x); }	622	float __frcp_ru(float __x) { return __ocml_div_rtp_f32(1.0f, __x); }
501	#if defined OCML_BASIC_ROUNDED_OPERATIONS	623	__DEVICE__
		624	float __frcp_rz(float __x) { return __ocml_div_rtz_f32(1.0f, __x); }
		625	#else
502	__DEVICE__	626	__DEVICE__
503	inline float __fsqrt_rd(float __x) { return __ocml_sqrt_rtn_f32(__x); }	627	float __frcp_rn(float __x) { return 1.0f / __x; }
504	#endif	628	#endif
		629
505	__DEVICE__	630	__DEVICE__
506	inline float __fsqrt_rn(float __x) { return __ocml_native_sqrt_f32(__x); }	631	float __frsqrt_rn(float __x) { return __llvm_amdgcn_rsq_f32(__x); }
		632
507	#if defined OCML_BASIC_ROUNDED_OPERATIONS	633	#if defined OCML_BASIC_ROUNDED_OPERATIONS
508	__DEVICE__	634	__DEVICE__
509	inline float __fsqrt_ru(float __x) { return __ocml_sqrt_rtp_f32(__x); }	635	float __fsqrt_rd(float __x) { return __ocml_sqrt_rtn_f32(__x); }
510	__DEVICE__	636	__DEVICE__
511	inline float __fsqrt_rz(float __x) { return __ocml_sqrt_rtz_f32(__x); }	637	float __fsqrt_rn(float __x) { return __ocml_sqrt_rte_f32(__x); }
512	__DEVICE__	638	__DEVICE__
513	inline float __fsub_rd(float __x, float __y) {	639	float __fsqrt_ru(float __x) { return __ocml_sqrt_rtp_f32(__x); }
514	return __ocml_sub_rtn_f32(__x, __y);	640	__DEVICE__
515	}	641	float __fsqrt_rz(float __x) { return __ocml_sqrt_rtz_f32(__x); }
516	#endif	642	#else
517	__DEVICE__	643	__DEVICE__
518	inline float __fsub_rn(float __x, float __y) { return __x - __y; }	644	float __fsqrt_rn(float __x) { return __ocml_native_sqrt_f32(__x); }
		645	#endif
		646
519	#if defined OCML_BASIC_ROUNDED_OPERATIONS	647	#if defined OCML_BASIC_ROUNDED_OPERATIONS
520	__DEVICE__	648	__DEVICE__
521	inline float __fsub_ru(float __x, float __y) {	649	float __fsub_rd(float __x, float __y) { return __ocml_sub_rtn_f32(__x, __y); }
522	return __ocml_sub_rtp_f32(__x, __y);
523	}
524	__DEVICE__	650	__DEVICE__
525	inline float __fsub_rz(float __x, float __y) {	651	float __fsub_rn(float __x, float __y) { return __ocml_sub_rte_f32(__x, __y); }
526	return __ocml_sub_rtz_f32(__x, __y);	652	__DEVICE__
527	}	653	float __fsub_ru(float __x, float __y) { return __ocml_sub_rtp_f32(__x, __y); }
		654	__DEVICE__
		655	float __fsub_rz(float __x, float __y) { return __ocml_sub_rtz_f32(__x, __y); }
		656	#else
		657	__DEVICE__
		658	float __fsub_rn(float __x, float __y) { return __x - __y; }
528	#endif	659	#endif
		660
529	__DEVICE__	661	__DEVICE__
530	inline float __log10f(float __x) { return __ocml_native_log10_f32(__x); }	662	float __log10f(float __x) { return __ocml_native_log10_f32(__x); }
		663
531	__DEVICE__	664	__DEVICE__
532	inline float __log2f(float __x) { return __ocml_native_log2_f32(__x); }	665	float __log2f(float __x) { return __ocml_native_log2_f32(__x); }
		666
533	__DEVICE__	667	__DEVICE__
534	inline float __logf(float __x) { return __ocml_native_log_f32(__x); }	668	float __logf(float __x) { return __ocml_native_log_f32(__x); }
		669
535	__DEVICE__	670	__DEVICE__
536	inline float __powf(float __x, float __y) { return __ocml_pow_f32(__x, __y); }	671	float __powf(float __x, float __y) { return __ocml_pow_f32(__x, __y); }
		672
537	__DEVICE__	673	__DEVICE__
538	inline float __saturatef(float __x) {	674	float __saturatef(float __x) { return (__x < 0) ? 0 : ((__x > 1) ? 1 : __x); }
539	return (__x < 0) ? 0 : ((__x > 1) ? 1 : __x);	675
540	}
541	__DEVICE__	676	__DEVICE__
542	inline void __sincosf(float __x, float __sinptr, float __cosptr) {	677	void __sincosf(float __x, float __sinptr, float __cosptr) {
543	*__sinptr = __ocml_native_sin_f32(__x);	678	*__sinptr = __ocml_native_sin_f32(__x);
544	*__cosptr = __ocml_native_cos_f32(__x);	679	*__cosptr = __ocml_native_cos_f32(__x);
545	}	680	}
		681
546	__DEVICE__	682	__DEVICE__
547	inline float __sinf(float __x) { return __ocml_native_sin_f32(__x); }	683	float __sinf(float __x) { return __ocml_native_sin_f32(__x); }
		684
548	__DEVICE__	685	__DEVICE__
549	inline float __tanf(float __x) { return __ocml_tan_f32(__x); }	686	float __tanf(float __x) { return __ocml_tan_f32(__x); }
550	// END INTRINSICS	687	// END INTRINSICS
551	// END FLOAT	688	// END FLOAT
552		689
553	// BEGIN DOUBLE	690	// BEGIN DOUBLE
554	__DEVICE__	691	__DEVICE__
555	inline double abs(double __x) { return __ocml_fabs_f64(__x); }	692	double acos(double __x) { return __ocml_acos_f64(__x); }
556	__DEVICE__	693
557	inline double acos(double __x) { return __ocml_acos_f64(__x); }
558	__DEVICE__	694	__DEVICE__
559	inline double acosh(double __x) { return __ocml_acosh_f64(__x); }	695	double acosh(double __x) { return __ocml_acosh_f64(__x); }
		696
560	__DEVICE__	697	__DEVICE__
561	inline double asin(double __x) { return __ocml_asin_f64(__x); }	698	double asin(double __x) { return __ocml_asin_f64(__x); }
		699
562	__DEVICE__	700	__DEVICE__
563	inline double asinh(double __x) { return __ocml_asinh_f64(__x); }	701	double asinh(double __x) { return __ocml_asinh_f64(__x); }
		702
564	__DEVICE__	703	__DEVICE__
565	inline double atan(double __x) { return __ocml_atan_f64(__x); }	704	double atan(double __x) { return __ocml_atan_f64(__x); }
		705
566	__DEVICE__	706	__DEVICE__
567	inline double atan2(double __x, double __y) {	707	double atan2(double __x, double __y) { return __ocml_atan2_f64(__x, __y); }
568	return __ocml_atan2_f64(__x, __y);	708
569	}
570	__DEVICE__	709	__DEVICE__
571	inline double atanh(double __x) { return __ocml_atanh_f64(__x); }	710	double atanh(double __x) { return __ocml_atanh_f64(__x); }
		711
572	__DEVICE__	712	__DEVICE__
573	inline double cbrt(double __x) { return __ocml_cbrt_f64(__x); }	713	double cbrt(double __x) { return __ocml_cbrt_f64(__x); }
		714
574	__DEVICE__	715	__DEVICE__
575	inline double ceil(double __x) { return __ocml_ceil_f64(__x); }	716	double ceil(double __x) { return __ocml_ceil_f64(__x); }
		717
576	__DEVICE__	718	__DEVICE__
577	inline double copysign(double __x, double __y) {	719	double copysign(double __x, double __y) {
578	return __ocml_copysign_f64(__x, __y);	720	return __ocml_copysign_f64(__x, __y);
579	}	721	}
		722
580	__DEVICE__	723	__DEVICE__
581	inline double cos(double __x) { return __ocml_cos_f64(__x); }	724	double cos(double __x) { return __ocml_cos_f64(__x); }
		725
582	__DEVICE__	726	__DEVICE__
583	inline double cosh(double __x) { return __ocml_cosh_f64(__x); }	727	double cosh(double __x) { return __ocml_cosh_f64(__x); }
		728
584	__DEVICE__	729	__DEVICE__
585	inline double cospi(double __x) { return __ocml_cospi_f64(__x); }	730	double cospi(double __x) { return __ocml_cospi_f64(__x); }
		731
586	__DEVICE__	732	__DEVICE__
587	inline double cyl_bessel_i0(double __x) { return __ocml_i0_f64(__x); }	733	double cyl_bessel_i0(double __x) { return __ocml_i0_f64(__x); }
		734
588	__DEVICE__	735	__DEVICE__
589	inline double cyl_bessel_i1(double __x) { return __ocml_i1_f64(__x); }	736	double cyl_bessel_i1(double __x) { return __ocml_i1_f64(__x); }
		737
590	__DEVICE__	738	__DEVICE__
591	inline double erf(double __x) { return __ocml_erf_f64(__x); }	739	double erf(double __x) { return __ocml_erf_f64(__x); }
		740
592	__DEVICE__	741	__DEVICE__
593	inline double erfc(double __x) { return __ocml_erfc_f64(__x); }	742	double erfc(double __x) { return __ocml_erfc_f64(__x); }
		743
594	__DEVICE__	744	__DEVICE__
595	inline double erfcinv(double __x) { return __ocml_erfcinv_f64(__x); }	745	double erfcinv(double __x) { return __ocml_erfcinv_f64(__x); }
		746
596	__DEVICE__	747	__DEVICE__
597	inline double erfcx(double __x) { return __ocml_erfcx_f64(__x); }	748	double erfcx(double __x) { return __ocml_erfcx_f64(__x); }
		749
598	__DEVICE__	750	__DEVICE__
599	inline double erfinv(double __x) { return __ocml_erfinv_f64(__x); }	751	double erfinv(double __x) { return __ocml_erfinv_f64(__x); }
		752
600	__DEVICE__	753	__DEVICE__
601	inline double exp(double __x) { return __ocml_exp_f64(__x); }	754	double exp(double __x) { return __ocml_exp_f64(__x); }
		755
602	__DEVICE__	756	__DEVICE__
603	inline double exp10(double __x) { return __ocml_exp10_f64(__x); }	757	double exp10(double __x) { return __ocml_exp10_f64(__x); }
		758
604	__DEVICE__	759	__DEVICE__
605	inline double exp2(double __x) { return __ocml_exp2_f64(__x); }	760	double exp2(double __x) { return __ocml_exp2_f64(__x); }
		761
606	__DEVICE__	762	__DEVICE__
607	inline double expm1(double __x) { return __ocml_expm1_f64(__x); }	763	double expm1(double __x) { return __ocml_expm1_f64(__x); }
		764
608	__DEVICE__	765	__DEVICE__
609	inline double fabs(double __x) { return __ocml_fabs_f64(__x); }	766	double fabs(double __x) { return __ocml_fabs_f64(__x); }
		767
610	__DEVICE__	768	__DEVICE__
611	inline double fdim(double __x, double __y) { return __ocml_fdim_f64(__x, __y); }	769	double fdim(double __x, double __y) { return __ocml_fdim_f64(__x, __y); }
		770
612	__DEVICE__	771	__DEVICE__
613	inline double floor(double __x) { return __ocml_floor_f64(__x); }	772	double floor(double __x) { return __ocml_floor_f64(__x); }
		773
614	__DEVICE__	774	__DEVICE__
615	inline double fma(double __x, double __y, double __z) {	775	double fma(double __x, double __y, double __z) {
616	return __ocml_fma_f64(__x, __y, __z);	776	return __ocml_fma_f64(__x, __y, __z);
617	}	777	}
		778
618	__DEVICE__	779	__DEVICE__
619	inline double fmax(double __x, double __y) { return __ocml_fmax_f64(__x, __y); }	780	double fmax(double __x, double __y) { return __ocml_fmax_f64(__x, __y); }
		781
620	__DEVICE__	782	__DEVICE__
621	inline double fmin(double __x, double __y) { return __ocml_fmin_f64(__x, __y); }	783	double fmin(double __x, double __y) { return __ocml_fmin_f64(__x, __y); }
		784
622	__DEVICE__	785	__DEVICE__
623	inline double fmod(double __x, double __y) { return __ocml_fmod_f64(__x, __y); }	786	double fmod(double __x, double __y) { return __ocml_fmod_f64(__x, __y); }
		787
624	__DEVICE__	788	__DEVICE__
625	inline double frexp(double __x, int *__nptr) {	789	double frexp(double __x, int *__nptr) {
626	int __tmp;	790	int __tmp;
627	double __r =	791	double __r =
628	__ocml_frexp_f64(__x, (__attribute__((address_space(5))) int *)&__tmp);	792	__ocml_frexp_f64(__x, (__attribute__((address_space(5))) int *)&__tmp);
629	*__nptr = __tmp;	793	*__nptr = __tmp;
630
631	return __r;	794	return __r;
632	}	795	}
		796
633	__DEVICE__	797	__DEVICE__
634	inline double hypot(double __x, double __y) {	798	double hypot(double __x, double __y) { return __ocml_hypot_f64(__x, __y); }
635	return __ocml_hypot_f64(__x, __y);	799
636	}
637	__DEVICE__	800	__DEVICE__
638	inline int ilogb(double __x) { return __ocml_ilogb_f64(__x); }	801	int ilogb(double __x) { return __ocml_ilogb_f64(__x); }
		802
639	__DEVICE__	803	__DEVICE__
640	inline __RETURN_TYPE isfinite(double __x) { return __ocml_isfinite_f64(__x); }	804	__RETURN_TYPE __finite(double __x) { return __ocml_isfinite_f64(__x); }
		805
641	__DEVICE__	806	__DEVICE__
642	inline __RETURN_TYPE isinf(double __x) { return __ocml_isinf_f64(__x); }	807	__RETURN_TYPE __isinf(double __x) { return __ocml_isinf_f64(__x); }
		808
643	__DEVICE__	809	__DEVICE__
644	inline __RETURN_TYPE isnan(double __x) { return __ocml_isnan_f64(__x); }	810	__RETURN_TYPE __isnan(double __x) { return __ocml_isnan_f64(__x); }
		811
645	__DEVICE__	812	__DEVICE__
646	inline double j0(double __x) { return __ocml_j0_f64(__x); }	813	double j0(double __x) { return __ocml_j0_f64(__x); }
		814
647	__DEVICE__	815	__DEVICE__
648	inline double j1(double __x) { return __ocml_j1_f64(__x); }	816	double j1(double __x) { return __ocml_j1_f64(__x); }
		817
649	__DEVICE__	818	__DEVICE__
650	inline double jn(int __n,	819	double jn(int __n, double __x) { // TODO: we could use Ahmes multiplication
651	double __x) { // TODO: we could use Ahmes multiplication	820	// and the Miller & Brown algorithm
652	// and the Miller & Brown algorithm
653	// for linear recurrences to get O(log n) steps, but it's unclear if	821	// for linear recurrences to get O(log n) steps, but it's unclear if
654	// it'd be beneficial in this case. Placeholder until OCML adds	822	// it'd be beneficial in this case. Placeholder until OCML adds
655	// support.	823	// support.
656	if (__n == 0)	824	if (__n == 0)
657	return j0f(__x);	825	return j0(__x);
658	if (__n == 1)	826	if (__n == 1)
659	return j1f(__x);	827	return j1(__x);
660		828
661	double __x0 = j0f(__x);	829	double __x0 = j0(__x);
662	double __x1 = j1f(__x);	830	double __x1 = j1(__x);
663	for (int __i = 1; __i < __n; ++__i) {	831	for (int __i = 1; __i < __n; ++__i) {
664	double __x2 = (2 * __i) / __x * __x1 - __x0;	832	double __x2 = (2 * __i) / __x * __x1 - __x0;
665	__x0 = __x1;	833	__x0 = __x1;
666	__x1 = __x2;	834	__x1 = __x2;
667	}	835	}
668
669	return __x1;	836	return __x1;
670	}	837	}
		838
671	__DEVICE__	839	__DEVICE__
672	inline double ldexp(double __x, int __e) { return __ocml_ldexp_f64(__x, __e); }	840	double ldexp(double __x, int __e) { return __ocml_ldexp_f64(__x, __e); }
		841
673	__DEVICE__	842	__DEVICE__
674	inline double lgamma(double __x) { return __ocml_lgamma_f64(__x); }	843	double lgamma(double __x) { return __ocml_lgamma_f64(__x); }
		844
675	__DEVICE__	845	__DEVICE__
676	inline long long int llrint(double __x) { return __ocml_rint_f64(__x); }	846	long long int llrint(double __x) { return __ocml_rint_f64(__x); }
		847
677	__DEVICE__	848	__DEVICE__
678	inline long long int llround(double __x) { return __ocml_round_f64(__x); }	849	long long int llround(double __x) { return __ocml_round_f64(__x); }
		850
679	__DEVICE__	851	__DEVICE__
680	inline double log(double __x) { return __ocml_log_f64(__x); }	852	double log(double __x) { return __ocml_log_f64(__x); }
		853
681	__DEVICE__	854	__DEVICE__
682	inline double log10(double __x) { return __ocml_log10_f64(__x); }	855	double log10(double __x) { return __ocml_log10_f64(__x); }
		856
683	__DEVICE__	857	__DEVICE__
684	inline double log1p(double __x) { return __ocml_log1p_f64(__x); }	858	double log1p(double __x) { return __ocml_log1p_f64(__x); }
		859
685	__DEVICE__	860	__DEVICE__
686	inline double log2(double __x) { return __ocml_log2_f64(__x); }	861	double log2(double __x) { return __ocml_log2_f64(__x); }
		862
687	__DEVICE__	863	__DEVICE__
688	inline double logb(double __x) { return __ocml_logb_f64(__x); }	864	double logb(double __x) { return __ocml_logb_f64(__x); }
		865
689	__DEVICE__	866	__DEVICE__
690	inline long int lrint(double __x) { return __ocml_rint_f64(__x); }	867	long int lrint(double __x) { return __ocml_rint_f64(__x); }
		868
691	__DEVICE__	869	__DEVICE__
692	inline long int lround(double __x) { return __ocml_round_f64(__x); }	870	long int lround(double __x) { return __ocml_round_f64(__x); }
		871
693	__DEVICE__	872	__DEVICE__
694	inline double modf(double __x, double *__iptr) {	873	double modf(double __x, double *__iptr) {
695	double __tmp;	874	double __tmp;
696	double __r =	875	double __r =
697	__ocml_modf_f64(__x, (__attribute__((address_space(5))) double *)&__tmp);	876	__ocml_modf_f64(__x, (__attribute__((address_space(5))) double *)&__tmp);
...	@@ -699,8 +878,9 @@ inline double modf(double __x, double *__iptr) {	...	@@ -699,8 +878,9 @@ inline double modf(double __x, double *__iptr) {
699		878
700	return __r;	879	return __r;
701	}	880	}
		881
702	__DEVICE__	882	__DEVICE__
703	inline double nan(const char *__tagp) {	883	double nan(const char *__tagp) {
704	#if !_WIN32	884	#if !_WIN32
705	union {	885	union {
706	double val;	886	double val;
...	@@ -710,8 +890,8 @@ inline double nan(const char *__tagp) {	...	@@ -710,8 +890,8 @@ inline double nan(const char *__tagp) {
710	uint32_t exponent : 11;	890	uint32_t exponent : 11;
711	uint32_t sign : 1;	891	uint32_t sign : 1;
712	} bits;	892	} bits;
713	static_assert(sizeof(double) == sizeof(ieee_double), "");
714	} __tmp;	893	} __tmp;
		894	__static_assert_type_size_equal(sizeof(__tmp.val), sizeof(__tmp.bits));
715		895
716	__tmp.bits.sign = 0u;	896	__tmp.bits.sign = 0u;
717	__tmp.bits.exponent = ~0u;	897	__tmp.bits.exponent = ~0u;
...	@@ -720,22 +900,24 @@ inline double nan(const char *__tagp) {	...	@@ -720,22 +900,24 @@ inline double nan(const char *__tagp) {
720		900
721	return __tmp.val;	901	return __tmp.val;
722	#else	902	#else
723	static_assert(sizeof(uint64_t) == sizeof(double));	903	__static_assert_type_size_equal(sizeof(uint64_t), sizeof(double));
724	uint64_t val = __make_mantissa(__tagp);	904	uint64_t __val = __make_mantissa(__tagp);
725	val \|= 0xFFF << 51;	905	__val \|= 0xFFF << 51;
726	return reinterpret_cast<double >(&val);	906	return reinterpret_cast<double >(&__val);
727	#endif	907	#endif
728	}	908	}
		909
729	__DEVICE__	910	__DEVICE__
730	inline double nearbyint(double __x) { return __ocml_nearbyint_f64(__x); }	911	double nearbyint(double __x) { return __ocml_nearbyint_f64(__x); }
		912
731	__DEVICE__	913	__DEVICE__
732	inline double nextafter(double __x, double __y) {	914	double nextafter(double __x, double __y) {
733	return __ocml_nextafter_f64(__x, __y);	915	return __ocml_nextafter_f64(__x, __y);
734	}	916	}
		917
735	__DEVICE__	918	__DEVICE__
736	inline double	919	double norm(int __dim,
737	norm(int __dim,	920	const double *__a) { // TODO: placeholder until OCML adds support.
738	const double *__a) { // TODO: placeholder until OCML adds support.
739	double __r = 0;	921	double __r = 0;
740	while (__dim--) {	922	while (__dim--) {
741	__r += __a[0] * __a[0];	923	__r += __a[0] * __a[0];
...	@@ -744,28 +926,39 @@ norm(int __dim,	...	@@ -744,28 +926,39 @@ norm(int __dim,
744		926
745	return __ocml_sqrt_f64(__r);	927	return __ocml_sqrt_f64(__r);
746	}	928	}
		929
747	__DEVICE__	930	__DEVICE__
748	inline double norm3d(double __x, double __y, double __z) {	931	double norm3d(double __x, double __y, double __z) {
749	return __ocml_len3_f64(__x, __y, __z);	932	return __ocml_len3_f64(__x, __y, __z);
750	}	933	}
		934
751	__DEVICE__	935	__DEVICE__
752	inline double norm4d(double __x, double __y, double __z, double __w) {	936	double norm4d(double __x, double __y, double __z, double __w) {
753	return __ocml_len4_f64(__x, __y, __z, __w);	937	return __ocml_len4_f64(__x, __y, __z, __w);
754	}	938	}
		939
		940	__DEVICE__
		941	double normcdf(double __x) { return __ocml_ncdf_f64(__x); }
		942
755	__DEVICE__	943	__DEVICE__
756	inline double normcdf(double __x) { return __ocml_ncdf_f64(__x); }	944	double normcdfinv(double __x) { return __ocml_ncdfinv_f64(__x); }
		945
757	__DEVICE__	946	__DEVICE__
758	inline double normcdfinv(double __x) { return __ocml_ncdfinv_f64(__x); }	947	double pow(double __x, double __y) { return __ocml_pow_f64(__x, __y); }
		948
759	__DEVICE__	949	__DEVICE__
760	inline double pow(double __x, double __y) { return __ocml_pow_f64(__x, __y); }	950	double powi(double __x, int __y) { return __ocml_pown_f64(__x, __y); }
		951
761	__DEVICE__	952	__DEVICE__
762	inline double rcbrt(double __x) { return __ocml_rcbrt_f64(__x); }	953	double rcbrt(double __x) { return __ocml_rcbrt_f64(__x); }
		954
763	__DEVICE__	955	__DEVICE__
764	inline double remainder(double __x, double __y) {	956	double remainder(double __x, double __y) {
765	return __ocml_remainder_f64(__x, __y);	957	return __ocml_remainder_f64(__x, __y);
766	}	958	}
		959
767	__DEVICE__	960	__DEVICE__
768	inline double remquo(double __x, double __y, int *__quo) {	961	double remquo(double __x, double __y, int *__quo) {
769	int __tmp;	962	int __tmp;
770	double __r = __ocml_remquo_f64(	963	double __r = __ocml_remquo_f64(
771	__x, __y, (__attribute__((address_space(5))) int *)&__tmp);	964	__x, __y, (__attribute__((address_space(5))) int *)&__tmp);
...	@@ -773,16 +966,16 @@ inline double remquo(double __x, double __y, int *__quo) {	...	@@ -773,16 +966,16 @@ inline double remquo(double __x, double __y, int *__quo) {
773		966
774	return __r;	967	return __r;
775	}	968	}
		969
776	__DEVICE__	970	__DEVICE__
777	inline double rhypot(double __x, double __y) {	971	double rhypot(double __x, double __y) { return __ocml_rhypot_f64(__x, __y); }
778	return __ocml_rhypot_f64(__x, __y);	972
779	}
780	__DEVICE__	973	__DEVICE__
781	inline double rint(double __x) { return __ocml_rint_f64(__x); }	974	double rint(double __x) { return __ocml_rint_f64(__x); }
		975
782	__DEVICE__	976	__DEVICE__
783	inline double	977	double rnorm(int __dim,
784	rnorm(int __dim,	978	const double *__a) { // TODO: placeholder until OCML adds support.
785	const double *__a) { // TODO: placeholder until OCML adds support.
786	double __r = 0;	979	double __r = 0;
787	while (__dim--) {	980	while (__dim--) {
788	__r += __a[0] * __a[0];	981	__r += __a[0] * __a[0];
...	@@ -791,77 +984,93 @@ rnorm(int __dim,	...	@@ -791,77 +984,93 @@ rnorm(int __dim,
791		984
792	return __ocml_rsqrt_f64(__r);	985	return __ocml_rsqrt_f64(__r);
793	}	986	}
		987
794	__DEVICE__	988	__DEVICE__
795	inline double rnorm3d(double __x, double __y, double __z) {	989	double rnorm3d(double __x, double __y, double __z) {
796	return __ocml_rlen3_f64(__x, __y, __z);	990	return __ocml_rlen3_f64(__x, __y, __z);
797	}	991	}
		992
798	__DEVICE__	993	__DEVICE__
799	inline double rnorm4d(double __x, double __y, double __z, double __w) {	994	double rnorm4d(double __x, double __y, double __z, double __w) {
800	return __ocml_rlen4_f64(__x, __y, __z, __w);	995	return __ocml_rlen4_f64(__x, __y, __z, __w);
801	}	996	}
		997
802	__DEVICE__	998	__DEVICE__
803	inline double round(double __x) { return __ocml_round_f64(__x); }	999	double round(double __x) { return __ocml_round_f64(__x); }
		1000
804	__DEVICE__	1001	__DEVICE__
805	inline double rsqrt(double __x) { return __ocml_rsqrt_f64(__x); }	1002	double rsqrt(double __x) { return __ocml_rsqrt_f64(__x); }
		1003
806	__DEVICE__	1004	__DEVICE__
807	inline double scalbln(double __x, long int __n) {	1005	double scalbln(double __x, long int __n) {
808	return (__n < INT_MAX) ? __ocml_scalbn_f64(__x, __n)	1006	return (__n < INT_MAX) ? __ocml_scalbn_f64(__x, __n)
809	: __ocml_scalb_f64(__x, __n);	1007	: __ocml_scalb_f64(__x, __n);
810	}	1008	}
811	__DEVICE__	1009	__DEVICE__
812	inline double scalbn(double __x, int __n) {	1010	double scalbn(double __x, int __n) { return __ocml_scalbn_f64(__x, __n); }
813	return __ocml_scalbn_f64(__x, __n);	1011
814	}
815	__DEVICE__	1012	__DEVICE__
816	inline __RETURN_TYPE signbit(double __x) { return __ocml_signbit_f64(__x); }	1013	__RETURN_TYPE __signbit(double __x) { return __ocml_signbit_f64(__x); }
		1014
817	__DEVICE__	1015	__DEVICE__
818	inline double sin(double __x) { return __ocml_sin_f64(__x); }	1016	double sin(double __x) { return __ocml_sin_f64(__x); }
		1017
819	__DEVICE__	1018	__DEVICE__
820	inline void sincos(double __x, double __sinptr, double __cosptr) {	1019	void sincos(double __x, double __sinptr, double __cosptr) {
821	double __tmp;	1020	double __tmp;
822	*__sinptr = __ocml_sincos_f64(	1021	*__sinptr = __ocml_sincos_f64(
823	__x, (__attribute__((address_space(5))) double *)&__tmp);	1022	__x, (__attribute__((address_space(5))) double *)&__tmp);
824	*__cosptr = __tmp;	1023	*__cosptr = __tmp;
825	}	1024	}
		1025
826	__DEVICE__	1026	__DEVICE__
827	inline void sincospi(double __x, double __sinptr, double __cosptr) {	1027	void sincospi(double __x, double __sinptr, double __cosptr) {
828	double __tmp;	1028	double __tmp;
829	*__sinptr = __ocml_sincospi_f64(	1029	*__sinptr = __ocml_sincospi_f64(
830	__x, (__attribute__((address_space(5))) double *)&__tmp);	1030	__x, (__attribute__((address_space(5))) double *)&__tmp);
831	*__cosptr = __tmp;	1031	*__cosptr = __tmp;
832	}	1032	}
		1033
833	__DEVICE__	1034	__DEVICE__
834	inline double sinh(double __x) { return __ocml_sinh_f64(__x); }	1035	double sinh(double __x) { return __ocml_sinh_f64(__x); }
		1036
835	__DEVICE__	1037	__DEVICE__
836	inline double sinpi(double __x) { return __ocml_sinpi_f64(__x); }	1038	double sinpi(double __x) { return __ocml_sinpi_f64(__x); }
		1039
837	__DEVICE__	1040	__DEVICE__
838	inline double sqrt(double __x) { return __ocml_sqrt_f64(__x); }	1041	double sqrt(double __x) { return __ocml_sqrt_f64(__x); }
		1042
839	__DEVICE__	1043	__DEVICE__
840	inline double tan(double __x) { return __ocml_tan_f64(__x); }	1044	double tan(double __x) { return __ocml_tan_f64(__x); }
		1045
841	__DEVICE__	1046	__DEVICE__
842	inline double tanh(double __x) { return __ocml_tanh_f64(__x); }	1047	double tanh(double __x) { return __ocml_tanh_f64(__x); }
		1048
843	__DEVICE__	1049	__DEVICE__
844	inline double tgamma(double __x) { return __ocml_tgamma_f64(__x); }	1050	double tgamma(double __x) { return __ocml_tgamma_f64(__x); }
		1051
845	__DEVICE__	1052	__DEVICE__
846	inline double trunc(double __x) { return __ocml_trunc_f64(__x); }	1053	double trunc(double __x) { return __ocml_trunc_f64(__x); }
		1054
847	__DEVICE__	1055	__DEVICE__
848	inline double y0(double __x) { return __ocml_y0_f64(__x); }	1056	double y0(double __x) { return __ocml_y0_f64(__x); }
		1057
849	__DEVICE__	1058	__DEVICE__
850	inline double y1(double __x) { return __ocml_y1_f64(__x); }	1059	double y1(double __x) { return __ocml_y1_f64(__x); }
		1060
851	__DEVICE__	1061	__DEVICE__
852	inline double yn(int __n,	1062	double yn(int __n, double __x) { // TODO: we could use Ahmes multiplication
853	double __x) { // TODO: we could use Ahmes multiplication	1063	// and the Miller & Brown algorithm
854	// and the Miller & Brown algorithm
855	// for linear recurrences to get O(log n) steps, but it's unclear if	1064	// for linear recurrences to get O(log n) steps, but it's unclear if
856	// it'd be beneficial in this case. Placeholder until OCML adds	1065	// it'd be beneficial in this case. Placeholder until OCML adds
857	// support.	1066	// support.
858	if (__n == 0)	1067	if (__n == 0)
859	return j0f(__x);	1068	return y0(__x);
860	if (__n == 1)	1069	if (__n == 1)
861	return j1f(__x);	1070	return y1(__x);
862		1071
863	double __x0 = j0f(__x);	1072	double __x0 = y0(__x);
864	double __x1 = j1f(__x);	1073	double __x1 = y1(__x);
865	for (int __i = 1; __i < __n; ++__i) {	1074	for (int __i = 1; __i < __n; ++__i) {
866	double __x2 = (2 * __i) / __x * __x1 - __x0;	1075	double __x2 = (2 * __i) / __x * __x1 - __x0;
867	__x0 = __x1;	1076	__x0 = __x1;
...	@@ -874,296 +1083,182 @@ inline double yn(int __n,	...	@@ -874,296 +1083,182 @@ inline double yn(int __n,
874	// BEGIN INTRINSICS	1083	// BEGIN INTRINSICS
875	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1084	#if defined OCML_BASIC_ROUNDED_OPERATIONS
876	__DEVICE__	1085	__DEVICE__
877	inline double __dadd_rd(double __x, double __y) {	1086	double __dadd_rd(double __x, double __y) {
878	return __ocml_add_rtn_f64(__x, __y);	1087	return __ocml_add_rtn_f64(__x, __y);
879	}	1088	}
880	#endif
881	__DEVICE__	1089	__DEVICE__
882	inline double __dadd_rn(double __x, double __y) { return __x + __y; }	1090	double __dadd_rn(double __x, double __y) {
883	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1091	return __ocml_add_rte_f64(__x, __y);
		1092	}
884	__DEVICE__	1093	__DEVICE__
885	inline double __dadd_ru(double __x, double __y) {	1094	double __dadd_ru(double __x, double __y) {
886	return __ocml_add_rtp_f64(__x, __y);	1095	return __ocml_add_rtp_f64(__x, __y);
887	}	1096	}
888	__DEVICE__	1097	__DEVICE__
889	inline double __dadd_rz(double __x, double __y) {	1098	double __dadd_rz(double __x, double __y) {
890	return __ocml_add_rtz_f64(__x, __y);	1099	return __ocml_add_rtz_f64(__x, __y);
891	}	1100	}
		1101	#else
892	__DEVICE__	1102	__DEVICE__
893	inline double __ddiv_rd(double __x, double __y) {	1103	double __dadd_rn(double __x, double __y) { return __x + __y; }
		1104	#endif
		1105
		1106	#if defined OCML_BASIC_ROUNDED_OPERATIONS
		1107	__DEVICE__
		1108	double __ddiv_rd(double __x, double __y) {
894	return __ocml_div_rtn_f64(__x, __y);	1109	return __ocml_div_rtn_f64(__x, __y);
895	}	1110	}
896	#endif
897	__DEVICE__	1111	__DEVICE__
898	inline double __ddiv_rn(double __x, double __y) { return __x / __y; }	1112	double __ddiv_rn(double __x, double __y) {
899	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1113	return __ocml_div_rte_f64(__x, __y);
		1114	}
900	__DEVICE__	1115	__DEVICE__
901	inline double __ddiv_ru(double __x, double __y) {	1116	double __ddiv_ru(double __x, double __y) {
902	return __ocml_div_rtp_f64(__x, __y);	1117	return __ocml_div_rtp_f64(__x, __y);
903	}	1118	}
904	__DEVICE__	1119	__DEVICE__
905	inline double __ddiv_rz(double __x, double __y) {	1120	double __ddiv_rz(double __x, double __y) {
906	return __ocml_div_rtz_f64(__x, __y);	1121	return __ocml_div_rtz_f64(__x, __y);
907	}	1122	}
		1123	#else
		1124	__DEVICE__
		1125	double __ddiv_rn(double __x, double __y) { return __x / __y; }
		1126	#endif
		1127
		1128	#if defined OCML_BASIC_ROUNDED_OPERATIONS
908	__DEVICE__	1129	__DEVICE__
909	inline double __dmul_rd(double __x, double __y) {	1130	double __dmul_rd(double __x, double __y) {
910	return __ocml_mul_rtn_f64(__x, __y);	1131	return __ocml_mul_rtn_f64(__x, __y);
911	}	1132	}
912	#endif
913	__DEVICE__	1133	__DEVICE__
914	inline double __dmul_rn(double __x, double __y) { return __x * __y; }	1134	double __dmul_rn(double __x, double __y) {
915	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1135	return __ocml_mul_rte_f64(__x, __y);
		1136	}
916	__DEVICE__	1137	__DEVICE__
917	inline double __dmul_ru(double __x, double __y) {	1138	double __dmul_ru(double __x, double __y) {
918	return __ocml_mul_rtp_f64(__x, __y);	1139	return __ocml_mul_rtp_f64(__x, __y);
919	}	1140	}
920	__DEVICE__	1141	__DEVICE__
921	inline double __dmul_rz(double __x, double __y) {	1142	double __dmul_rz(double __x, double __y) {
922	return __ocml_mul_rtz_f64(__x, __y);	1143	return __ocml_mul_rtz_f64(__x, __y);
923	}	1144	}
		1145	#else
924	__DEVICE__	1146	__DEVICE__
925	inline double __drcp_rd(double __x) { return __llvm_amdgcn_rcp_f64(__x); }	1147	double __dmul_rn(double __x, double __y) { return __x * __y; }
926	#endif	1148	#endif
927	__DEVICE__	1149
928	inline double __drcp_rn(double __x) { return __llvm_amdgcn_rcp_f64(__x); }
929	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1150	#if defined OCML_BASIC_ROUNDED_OPERATIONS
930	__DEVICE__	1151	__DEVICE__
931	inline double __drcp_ru(double __x) { return __llvm_amdgcn_rcp_f64(__x); }	1152	double __drcp_rd(double __x) { return __ocml_div_rtn_f64(1.0, __x); }
932	__DEVICE__	1153	__DEVICE__
933	inline double __drcp_rz(double __x) { return __llvm_amdgcn_rcp_f64(__x); }	1154	double __drcp_rn(double __x) { return __ocml_div_rte_f64(1.0, __x); }
934	__DEVICE__	1155	__DEVICE__
935	inline double __dsqrt_rd(double __x) { return __ocml_sqrt_rtn_f64(__x); }	1156	double __drcp_ru(double __x) { return __ocml_div_rtp_f64(1.0, __x); }
936	#endif
937	__DEVICE__	1157	__DEVICE__
938	inline double __dsqrt_rn(double __x) { return __ocml_sqrt_f64(__x); }	1158	double __drcp_rz(double __x) { return __ocml_div_rtz_f64(1.0, __x); }
		1159	#else
		1160	__DEVICE__
		1161	double __drcp_rn(double __x) { return 1.0 / __x; }
		1162	#endif
		1163
939	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1164	#if defined OCML_BASIC_ROUNDED_OPERATIONS
940	__DEVICE__	1165	__DEVICE__
941	inline double __dsqrt_ru(double __x) { return __ocml_sqrt_rtp_f64(__x); }	1166	double __dsqrt_rd(double __x) { return __ocml_sqrt_rtn_f64(__x); }
		1167	__DEVICE__
		1168	double __dsqrt_rn(double __x) { return __ocml_sqrt_rte_f64(__x); }
		1169	__DEVICE__
		1170	double __dsqrt_ru(double __x) { return __ocml_sqrt_rtp_f64(__x); }
942	__DEVICE__	1171	__DEVICE__
943	inline double __dsqrt_rz(double __x) { return __ocml_sqrt_rtz_f64(__x); }	1172	double __dsqrt_rz(double __x) { return __ocml_sqrt_rtz_f64(__x); }
		1173	#else
		1174	__DEVICE__
		1175	double __dsqrt_rn(double __x) { return __ocml_sqrt_f64(__x); }
		1176	#endif
		1177
		1178	#if defined OCML_BASIC_ROUNDED_OPERATIONS
944	__DEVICE__	1179	__DEVICE__
945	inline double __dsub_rd(double __x, double __y) {	1180	double __dsub_rd(double __x, double __y) {
946	return __ocml_sub_rtn_f64(__x, __y);	1181	return __ocml_sub_rtn_f64(__x, __y);
947	}	1182	}
948	#endif
949	__DEVICE__	1183	__DEVICE__
950	inline double __dsub_rn(double __x, double __y) { return __x - __y; }	1184	double __dsub_rn(double __x, double __y) {
951	#if defined OCML_BASIC_ROUNDED_OPERATIONS	1185	return __ocml_sub_rte_f64(__x, __y);
		1186	}
952	__DEVICE__	1187	__DEVICE__
953	inline double __dsub_ru(double __x, double __y) {	1188	double __dsub_ru(double __x, double __y) {
954	return __ocml_sub_rtp_f64(__x, __y);	1189	return __ocml_sub_rtp_f64(__x, __y);
955	}	1190	}
956	__DEVICE__	1191	__DEVICE__
957	inline double __dsub_rz(double __x, double __y) {	1192	double __dsub_rz(double __x, double __y) {
958	return __ocml_sub_rtz_f64(__x, __y);	1193	return __ocml_sub_rtz_f64(__x, __y);
959	}	1194	}
		1195	#else
		1196	__DEVICE__
		1197	double __dsub_rn(double __x, double __y) { return __x - __y; }
		1198	#endif
		1199
		1200	#if defined OCML_BASIC_ROUNDED_OPERATIONS
960	__DEVICE__	1201	__DEVICE__
961	inline double __fma_rd(double __x, double __y, double __z) {	1202	double __fma_rd(double __x, double __y, double __z) {
962	return __ocml_fma_rtn_f64(__x, __y, __z);	1203	return __ocml_fma_rtn_f64(__x, __y, __z);
963	}	1204	}
964	#endif
965	__DEVICE__	1205	__DEVICE__
966	inline double __fma_rn(double __x, double __y, double __z) {	1206	double __fma_rn(double __x, double __y, double __z) {
967	return __ocml_fma_f64(__x, __y, __z);	1207	return __ocml_fma_rte_f64(__x, __y, __z);
968	}	1208	}
969	#if defined OCML_BASIC_ROUNDED_OPERATIONS
970	__DEVICE__	1209	__DEVICE__
971	inline double __fma_ru(double __x, double __y, double __z) {	1210	double __fma_ru(double __x, double __y, double __z) {
972	return __ocml_fma_rtp_f64(__x, __y, __z);	1211	return __ocml_fma_rtp_f64(__x, __y, __z);
973	}	1212	}
974	__DEVICE__	1213	__DEVICE__
975	inline double __fma_rz(double __x, double __y, double __z) {	1214	double __fma_rz(double __x, double __y, double __z) {
976	return __ocml_fma_rtz_f64(__x, __y, __z);	1215	return __ocml_fma_rtz_f64(__x, __y, __z);
977	}	1216	}
		1217	#else
		1218	__DEVICE__
		1219	double __fma_rn(double __x, double __y, double __z) {
		1220	return __ocml_fma_f64(__x, __y, __z);
		1221	}
978	#endif	1222	#endif
979	// END INTRINSICS	1223	// END INTRINSICS
980	// END DOUBLE	1224	// END DOUBLE
981		1225
982	// BEGIN INTEGER	1226	// C only macros
983	__DEVICE__	1227	#if !defined(__cplusplus) && __STDC_VERSION__ >= 201112L
984	inline int abs(int __x) {	1228	#define isfinite(__x) _Generic((__x), float : __finitef, double : __finite)(__x)
985	int __sgn = __x >> (sizeof(int) * CHAR_BIT - 1);	1229	#define isinf(__x) _Generic((__x), float : __isinff, double : __isinf)(__x)
986	return (__x ^ __sgn) - __sgn;	1230	#define isnan(__x) _Generic((__x), float : __isnanf, double : __isnan)(__x)
987	}	1231	#define signbit(__x) \
988	__DEVICE__	1232	_Generic((__x), float : __signbitf, double : __signbit)(__x)
989	inline long labs(long __x) {	1233	#endif // !defined(__cplusplus) && __STDC_VERSION__ >= 201112L
990	long __sgn = __x >> (sizeof(long) * CHAR_BIT - 1);
991	return (__x ^ __sgn) - __sgn;
992	}
993	__DEVICE__
994	inline long long llabs(long long __x) {
995	long long __sgn = __x >> (sizeof(long long) * CHAR_BIT - 1);
996	return (__x ^ __sgn) - __sgn;
997	}
998		1234
999	#if defined(__cplusplus)	1235	#if defined(__cplusplus)
1000	__DEVICE__	1236	template <class T> __DEVICE__ T min(T __arg1, T __arg2) {
1001	inline long abs(long __x) { return labs(__x); }
1002	__DEVICE__
1003	inline long long abs(long long __x) { return llabs(__x); }
1004	#endif
1005	// END INTEGER
1006
1007	__DEVICE__
1008	inline _Float16 fma(_Float16 __x, _Float16 __y, _Float16 __z) {
1009	return __ocml_fma_f16(__x, __y, __z);
1010	}
1011
1012	__DEVICE__
1013	inline float fma(float __x, float __y, float __z) {
1014	return fmaf(__x, __y, __z);
1015	}
1016
1017	#pragma push_macro("__DEF_FUN1")
1018	#pragma push_macro("__DEF_FUN2")
1019	#pragma push_macro("__DEF_FUNI")
1020	#pragma push_macro("__DEF_FLOAT_FUN2I")
1021	#pragma push_macro("__HIP_OVERLOAD1")
1022	#pragma push_macro("__HIP_OVERLOAD2")
1023
1024	// __hip_enable_if::type is a type function which returns __T if __B is true.
1025	template <bool __B, class __T = void> struct __hip_enable_if {};
1026
1027	template <class __T> struct __hip_enable_if<true, __T> { typedef __T type; };
1028
1029	// __HIP_OVERLOAD1 is used to resolve function calls with integer argument to
1030	// avoid compilation error due to ambibuity. e.g. floor(5) is resolved with
1031	// floor(double).
1032	#define __HIP_OVERLOAD1(__retty, __fn) \
1033	template <typename __T> \
1034	__DEVICE__ typename __hip_enable_if<std::numeric_limits<__T>::is_integer, \
1035	__retty>::type \
1036	__fn(__T __x) { \
1037	return ::__fn((double)__x); \
1038	}
1039
1040	// __HIP_OVERLOAD2 is used to resolve function calls with mixed float/double
1041	// or integer argument to avoid compilation error due to ambibuity. e.g.
1042	// max(5.0f, 6.0) is resolved with max(double, double).
1043	#define __HIP_OVERLOAD2(__retty, __fn) \
1044	template <typename __T1, typename __T2> \
1045	__DEVICE__ \
1046	typename __hip_enable_if<std::numeric_limits<__T1>::is_specialized && \
1047	std::numeric_limits<__T2>::is_specialized, \
1048	__retty>::type \
1049	__fn(__T1 __x, __T2 __y) { \
1050	return __fn((double)__x, (double)__y); \
1051	}
1052
1053	// Define cmath functions with float argument and returns float.
1054	#define __DEF_FUN1(__retty, __func) \
1055	__DEVICE__ \
1056	inline float __func(float __x) { return __func##f(__x); } \
1057	__HIP_OVERLOAD1(__retty, __func)
1058
1059	// Define cmath functions with float argument and returns __retty.
1060	#define __DEF_FUNI(__retty, __func) \
1061	__DEVICE__ \
1062	inline __retty __func(float __x) { return __func##f(__x); } \
1063	__HIP_OVERLOAD1(__retty, __func)
1064
1065	// define cmath functions with two float arguments.
1066	#define __DEF_FUN2(__retty, __func) \
1067	__DEVICE__ \
1068	inline float __func(float __x, float __y) { return __func##f(__x, __y); } \
1069	__HIP_OVERLOAD2(__retty, __func)
1070
1071	__DEF_FUN1(double, acos)
1072	__DEF_FUN1(double, acosh)
1073	__DEF_FUN1(double, asin)
1074	__DEF_FUN1(double, asinh)
1075	__DEF_FUN1(double, atan)
1076	__DEF_FUN2(double, atan2);
1077	__DEF_FUN1(double, atanh)
1078	__DEF_FUN1(double, cbrt)
1079	__DEF_FUN1(double, ceil)
1080	__DEF_FUN2(double, copysign);
1081	__DEF_FUN1(double, cos)
1082	__DEF_FUN1(double, cosh)
1083	__DEF_FUN1(double, erf)
1084	__DEF_FUN1(double, erfc)
1085	__DEF_FUN1(double, exp)
1086	__DEF_FUN1(double, exp2)
1087	__DEF_FUN1(double, expm1)
1088	__DEF_FUN1(double, fabs)
1089	__DEF_FUN2(double, fdim);
1090	__DEF_FUN1(double, floor)
1091	__DEF_FUN2(double, fmax);
1092	__DEF_FUN2(double, fmin);
1093	__DEF_FUN2(double, fmod);
1094	//__HIP_OVERLOAD1(int, fpclassify)
1095	__DEF_FUN2(double, hypot);
1096	__DEF_FUNI(int, ilogb)
1097	__HIP_OVERLOAD1(bool, isfinite)
1098	__HIP_OVERLOAD2(bool, isgreater);
1099	__HIP_OVERLOAD2(bool, isgreaterequal);
1100	__HIP_OVERLOAD1(bool, isinf);
1101	__HIP_OVERLOAD2(bool, isless);
1102	__HIP_OVERLOAD2(bool, islessequal);
1103	__HIP_OVERLOAD2(bool, islessgreater);
1104	__HIP_OVERLOAD1(bool, isnan);
1105	//__HIP_OVERLOAD1(bool, isnormal)
1106	__HIP_OVERLOAD2(bool, isunordered);
1107	__DEF_FUN1(double, lgamma)
1108	__DEF_FUN1(double, log)
1109	__DEF_FUN1(double, log10)
1110	__DEF_FUN1(double, log1p)
1111	__DEF_FUN1(double, log2)
1112	__DEF_FUN1(double, logb)
1113	__DEF_FUNI(long long, llrint)
1114	__DEF_FUNI(long long, llround)
1115	__DEF_FUNI(long, lrint)
1116	__DEF_FUNI(long, lround)
1117	__DEF_FUN1(double, nearbyint);
1118	__DEF_FUN2(double, nextafter);
1119	__DEF_FUN2(double, pow);
1120	__DEF_FUN2(double, remainder);
1121	__DEF_FUN1(double, rint);
1122	__DEF_FUN1(double, round);
1123	__HIP_OVERLOAD1(bool, signbit)
1124	__DEF_FUN1(double, sin)
1125	__DEF_FUN1(double, sinh)
1126	__DEF_FUN1(double, sqrt)
1127	__DEF_FUN1(double, tan)
1128	__DEF_FUN1(double, tanh)
1129	__DEF_FUN1(double, tgamma)
1130	__DEF_FUN1(double, trunc);
1131
1132	// define cmath functions with a float and an integer argument.
1133	#define __DEF_FLOAT_FUN2I(__func) \
1134	__DEVICE__ \
1135	inline float __func(float __x, int __y) { return __func##f(__x, __y); }
1136	__DEF_FLOAT_FUN2I(scalbn)
1137
1138	template <class T> __DEVICE__ inline T min(T __arg1, T __arg2) {
1139	return (__arg1 < __arg2) ? __arg1 : __arg2;	1237	return (__arg1 < __arg2) ? __arg1 : __arg2;
1140	}	1238	}
1141		1239
1142	template <class T> __DEVICE__ inline T max(T __arg1, T __arg2) {	1240	template <class T> __DEVICE__ T max(T __arg1, T __arg2) {
1143	return (__arg1 > __arg2) ? __arg1 : __arg2;	1241	return (__arg1 > __arg2) ? __arg1 : __arg2;
1144	}	1242	}
1145		1243
1146	__DEVICE__ inline int min(int __arg1, int __arg2) {	1244	__DEVICE__ int min(int __arg1, int __arg2) {
1147	return (__arg1 < __arg2) ? __arg1 : __arg2;	1245	return (__arg1 < __arg2) ? __arg1 : __arg2;
1148	}	1246	}
1149	__DEVICE__ inline int max(int __arg1, int __arg2) {	1247	__DEVICE__ int max(int __arg1, int __arg2) {
1150	return (__arg1 > __arg2) ? __arg1 : __arg2;	1248	return (__arg1 > __arg2) ? __arg1 : __arg2;
1151	}	1249	}
1152		1250
1153	__DEVICE__	1251	__DEVICE__
1154	inline float max(float __x, float __y) { return fmaxf(__x, __y); }	1252	float max(float __x, float __y) { return fmaxf(__x, __y); }
1155		1253
1156	__DEVICE__	1254	__DEVICE__
1157	inline double max(double __x, double __y) { return fmax(__x, __y); }	1255	double max(double __x, double __y) { return fmax(__x, __y); }
1158		1256
1159	__DEVICE__	1257	__DEVICE__
1160	inline float min(float __x, float __y) { return fminf(__x, __y); }	1258	float min(float __x, float __y) { return fminf(__x, __y); }
1161		1259
1162	__DEVICE__	1260	__DEVICE__
1163	inline double min(double __x, double __y) { return fmin(__x, __y); }	1261	double min(double __x, double __y) { return fmin(__x, __y); }
1164
1165	__HIP_OVERLOAD2(double, max)
1166	__HIP_OVERLOAD2(double, min)
1167		1262
1168	__host__ inline static int min(int __arg1, int __arg2) {	1263	__host__ inline static int min(int __arg1, int __arg2) {
1169	return std::min(__arg1, __arg2);	1264	return std::min(__arg1, __arg2);
...	@@ -1172,13 +1267,8 @@ __host__ inline static int min(int __arg1, int __arg2) {	...	@@ -1172,13 +1267,8 @@ __host__ inline static int min(int __arg1, int __arg2) {
1172	__host__ inline static int max(int __arg1, int __arg2) {	1267	__host__ inline static int max(int __arg1, int __arg2) {
1173	return std::max(__arg1, __arg2);	1268	return std::max(__arg1, __arg2);
1174	}	1269	}
		1270	#endif
1175		1271
1176	#pragma pop_macro("__DEF_FUN1")
1177	#pragma pop_macro("__DEF_FUN2")
1178	#pragma pop_macro("__DEF_FUNI")
1179	#pragma pop_macro("__DEF_FLOAT_FUN2I")
1180	#pragma pop_macro("__HIP_OVERLOAD1")
1181	#pragma pop_macro("__HIP_OVERLOAD2")
1182	#pragma pop_macro("__DEVICE__")	1272	#pragma pop_macro("__DEVICE__")
1183	#pragma pop_macro("__RETURN_TYPE")	1273	#pragma pop_macro("__RETURN_TYPE")
1184		1274

lib/include/__clang_hip_runtime_wrapper.h+5

lib/include/altivec.h+1103-72

...	@@ -1709,6 +1709,20 @@ vec_cmpeq(vector double __a, vector double __b) {	...	@@ -1709,6 +1709,20 @@ vec_cmpeq(vector double __a, vector double __b) {
1709	}	1709	}
1710	#endif	1710	#endif
1711		1711
		1712	#ifdef __POWER10_VECTOR__
		1713	static __inline__ vector bool __int128 __ATTRS_o_ai
		1714	vec_cmpeq(vector signed __int128 __a, vector signed __int128 __b) {
		1715	return (vector bool __int128)__builtin_altivec_vcmpequq(
		1716	(vector bool __int128)__a, (vector bool __int128)__b);
		1717	}
		1718
		1719	static __inline__ vector bool __int128 __ATTRS_o_ai
		1720	vec_cmpeq(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		1721	return (vector bool __int128)__builtin_altivec_vcmpequq(
		1722	(vector bool __int128)__a, (vector bool __int128)__b);
		1723	}
		1724	#endif
		1725
1712	#ifdef __POWER9_VECTOR__	1726	#ifdef __POWER9_VECTOR__
1713	/* vec_cmpne */	1727	/* vec_cmpne */
1714		1728
...	@@ -1766,35 +1780,25 @@ vec_cmpne(vector unsigned int __a, vector unsigned int __b) {	...	@@ -1766,35 +1780,25 @@ vec_cmpne(vector unsigned int __a, vector unsigned int __b) {
1766	(vector int)__b);	1780	(vector int)__b);
1767	}	1781	}
1768		1782
1769	static __inline__ vector bool long long __ATTRS_o_ai
1770	vec_cmpne(vector bool long long __a, vector bool long long __b) {
1771	return (vector bool long long)
1772	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
1773	}
1774
1775	static __inline__ vector bool long long __ATTRS_o_ai
1776	vec_cmpne(vector signed long long __a, vector signed long long __b) {
1777	return (vector bool long long)
1778	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
1779	}
1780
1781	static __inline__ vector bool long long __ATTRS_o_ai
1782	vec_cmpne(vector unsigned long long __a, vector unsigned long long __b) {
1783	return (vector bool long long)
1784	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
1785	}
1786
1787	static __inline__ vector bool int __ATTRS_o_ai	1783	static __inline__ vector bool int __ATTRS_o_ai
1788	vec_cmpne(vector float __a, vector float __b) {	1784	vec_cmpne(vector float __a, vector float __b) {
1789	return (vector bool int)__builtin_altivec_vcmpnew((vector int)__a,	1785	return (vector bool int)__builtin_altivec_vcmpnew((vector int)__a,
1790	(vector int)__b);	1786	(vector int)__b);
1791	}	1787	}
1792		1788
1793	static __inline__ vector bool long long __ATTRS_o_ai	1789	#ifdef __POWER10_VECTOR__
1794	vec_cmpne(vector double __a, vector double __b) {	1790	static __inline__ vector bool __int128 __ATTRS_o_ai
1795	return (vector bool long long)	1791	vec_cmpne(vector unsigned __int128 __a, vector unsigned __int128 __b) {
1796	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));	1792	return (vector bool __int128) ~(__builtin_altivec_vcmpequq(
		1793	(vector bool __int128)__a, (vector bool __int128)__b));
		1794	}
		1795
		1796	static __inline__ vector bool __int128 __ATTRS_o_ai
		1797	vec_cmpne(vector signed __int128 __a, vector signed __int128 __b) {
		1798	return (vector bool __int128) ~(__builtin_altivec_vcmpequq(
		1799	(vector bool __int128)__a, (vector bool __int128)__b));
1797	}	1800	}
		1801	#endif
1798		1802
1799	/* vec_cmpnez */	1803	/* vec_cmpnez */
1800		1804
...	@@ -1900,6 +1904,86 @@ vec_parity_lsbb(vector signed long long __a) {	...	@@ -1900,6 +1904,86 @@ vec_parity_lsbb(vector signed long long __a) {
1900	return __builtin_altivec_vprtybd(__a);	1904	return __builtin_altivec_vprtybd(__a);
1901	}	1905	}
1902		1906
		1907	#else
		1908	/* vec_cmpne */
		1909
		1910	static __inline__ vector bool char __ATTRS_o_ai
		1911	vec_cmpne(vector bool char __a, vector bool char __b) {
		1912	return ~(vec_cmpeq(__a, __b));
		1913	}
		1914
		1915	static __inline__ vector bool char __ATTRS_o_ai
		1916	vec_cmpne(vector signed char __a, vector signed char __b) {
		1917	return ~(vec_cmpeq(__a, __b));
		1918	}
		1919
		1920	static __inline__ vector bool char __ATTRS_o_ai
		1921	vec_cmpne(vector unsigned char __a, vector unsigned char __b) {
		1922	return ~(vec_cmpeq(__a, __b));
		1923	}
		1924
		1925	static __inline__ vector bool short __ATTRS_o_ai
		1926	vec_cmpne(vector bool short __a, vector bool short __b) {
		1927	return ~(vec_cmpeq(__a, __b));
		1928	}
		1929
		1930	static __inline__ vector bool short __ATTRS_o_ai
		1931	vec_cmpne(vector signed short __a, vector signed short __b) {
		1932	return ~(vec_cmpeq(__a, __b));
		1933	}
		1934
		1935	static __inline__ vector bool short __ATTRS_o_ai
		1936	vec_cmpne(vector unsigned short __a, vector unsigned short __b) {
		1937	return ~(vec_cmpeq(__a, __b));
		1938	}
		1939
		1940	static __inline__ vector bool int __ATTRS_o_ai
		1941	vec_cmpne(vector bool int __a, vector bool int __b) {
		1942	return ~(vec_cmpeq(__a, __b));
		1943	}
		1944
		1945	static __inline__ vector bool int __ATTRS_o_ai
		1946	vec_cmpne(vector signed int __a, vector signed int __b) {
		1947	return ~(vec_cmpeq(__a, __b));
		1948	}
		1949
		1950	static __inline__ vector bool int __ATTRS_o_ai
		1951	vec_cmpne(vector unsigned int __a, vector unsigned int __b) {
		1952	return ~(vec_cmpeq(__a, __b));
		1953	}
		1954
		1955	static __inline__ vector bool int __ATTRS_o_ai
		1956	vec_cmpne(vector float __a, vector float __b) {
		1957	return ~(vec_cmpeq(__a, __b));
		1958	}
		1959	#endif
		1960
		1961	#ifdef __POWER8_VECTOR__
		1962	static __inline__ vector bool long long __ATTRS_o_ai
		1963	vec_cmpne(vector bool long long __a, vector bool long long __b) {
		1964	return (vector bool long long)
		1965	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
		1966	}
		1967
		1968	static __inline__ vector bool long long __ATTRS_o_ai
		1969	vec_cmpne(vector signed long long __a, vector signed long long __b) {
		1970	return (vector bool long long)
		1971	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
		1972	}
		1973
		1974	static __inline__ vector bool long long __ATTRS_o_ai
		1975	vec_cmpne(vector unsigned long long __a, vector unsigned long long __b) {
		1976	return (vector bool long long)
		1977	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
		1978	}
		1979	#endif
		1980
		1981	#ifdef __VSX__
		1982	static __inline__ vector bool long long __ATTRS_o_ai
		1983	vec_cmpne(vector double __a, vector double __b) {
		1984	return (vector bool long long)
		1985	~(__builtin_altivec_vcmpequd((vector long long)__a, (vector long long)__b));
		1986	}
1903	#endif	1987	#endif
1904		1988
1905	/* vec_cmpgt */	1989	/* vec_cmpgt */
...	@@ -1962,6 +2046,20 @@ vec_cmpgt(vector double __a, vector double __b) {	...	@@ -1962,6 +2046,20 @@ vec_cmpgt(vector double __a, vector double __b) {
1962	}	2046	}
1963	#endif	2047	#endif
1964		2048
		2049	#ifdef __POWER10_VECTOR__
		2050	static __inline__ vector bool __int128 __ATTRS_o_ai
		2051	vec_cmpgt(vector signed __int128 __a, vector signed __int128 __b) {
		2052	return (vector bool __int128)__builtin_altivec_vcmpgtsq(
		2053	(vector bool __int128)__a, (vector bool __int128)__b);
		2054	}
		2055
		2056	static __inline__ vector bool __int128 __ATTRS_o_ai
		2057	vec_cmpgt(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		2058	return (vector bool __int128)__builtin_altivec_vcmpgtuq(
		2059	(vector bool __int128)__a, (vector bool __int128)__b);
		2060	}
		2061	#endif
		2062
1965	/* vec_cmpge */	2063	/* vec_cmpge */
1966		2064
1967	static __inline__ vector bool char __ATTRS_o_ai	2065	static __inline__ vector bool char __ATTRS_o_ai
...	@@ -2022,6 +2120,18 @@ vec_cmpge(vector unsigned long long __a, vector unsigned long long __b) {	...	@@ -2022,6 +2120,18 @@ vec_cmpge(vector unsigned long long __a, vector unsigned long long __b) {
2022	}	2120	}
2023	#endif	2121	#endif
2024		2122
		2123	#ifdef __POWER10_VECTOR__
		2124	static __inline__ vector bool __int128 __ATTRS_o_ai
		2125	vec_cmpge(vector signed __int128 __a, vector signed __int128 __b) {
		2126	return ~(vec_cmpgt(__b, __a));
		2127	}
		2128
		2129	static __inline__ vector bool __int128 __ATTRS_o_ai
		2130	vec_cmpge(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		2131	return ~(vec_cmpgt(__b, __a));
		2132	}
		2133	#endif
		2134
2025	/* vec_vcmpgefp */	2135	/* vec_vcmpgefp */
2026		2136
2027	static __inline__ vector bool int __attribute__((__always_inline__))	2137	static __inline__ vector bool int __attribute__((__always_inline__))
...	@@ -2134,6 +2244,18 @@ vec_cmple(vector unsigned long long __a, vector unsigned long long __b) {	...	@@ -2134,6 +2244,18 @@ vec_cmple(vector unsigned long long __a, vector unsigned long long __b) {
2134	}	2244	}
2135	#endif	2245	#endif
2136		2246
		2247	#ifdef __POWER10_VECTOR__
		2248	static __inline__ vector bool __int128 __ATTRS_o_ai
		2249	vec_cmple(vector signed __int128 __a, vector signed __int128 __b) {
		2250	return vec_cmpge(__b, __a);
		2251	}
		2252
		2253	static __inline__ vector bool __int128 __ATTRS_o_ai
		2254	vec_cmple(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		2255	return vec_cmpge(__b, __a);
		2256	}
		2257	#endif
		2258
2137	/* vec_cmplt */	2259	/* vec_cmplt */
2138		2260
2139	static __inline__ vector bool char __ATTRS_o_ai	2261	static __inline__ vector bool char __ATTRS_o_ai
...	@@ -2178,6 +2300,18 @@ vec_cmplt(vector double __a, vector double __b) {	...	@@ -2178,6 +2300,18 @@ vec_cmplt(vector double __a, vector double __b) {
2178	}	2300	}
2179	#endif	2301	#endif
2180		2302
		2303	#ifdef __POWER10_VECTOR__
		2304	static __inline__ vector bool __int128 __ATTRS_o_ai
		2305	vec_cmplt(vector signed __int128 __a, vector signed __int128 __b) {
		2306	return vec_cmpgt(__b, __a);
		2307	}
		2308
		2309	static __inline__ vector bool __int128 __ATTRS_o_ai
		2310	vec_cmplt(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		2311	return vec_cmpgt(__b, __a);
		2312	}
		2313	#endif
		2314
2181	#ifdef __POWER8_VECTOR__	2315	#ifdef __POWER8_VECTOR__
2182	static __inline__ vector bool long long __ATTRS_o_ai	2316	static __inline__ vector bool long long __ATTRS_o_ai
2183	vec_cmplt(vector signed long long __a, vector signed long long __b) {	2317	vec_cmplt(vector signed long long __a, vector signed long long __b) {
...	@@ -2702,67 +2836,67 @@ vec_insert_exp(vector unsigned int __a, vector unsigned int __b) {	...	@@ -2702,67 +2836,67 @@ vec_insert_exp(vector unsigned int __a, vector unsigned int __b) {
2702	}	2836	}
2703		2837
2704	#if defined(__powerpc64__)	2838	#if defined(__powerpc64__)
2705	static __inline__ vector signed char __ATTRS_o_ai vec_xl_len(signed char *__a,	2839	static __inline__ vector signed char __ATTRS_o_ai vec_xl_len(const signed char *__a,
2706	size_t __b) {	2840	size_t __b) {
2707	return (vector signed char)__builtin_vsx_lxvl(__a, (__b << 56));	2841	return (vector signed char)__builtin_vsx_lxvl(__a, (__b << 56));
2708	}	2842	}
2709		2843
2710	static __inline__ vector unsigned char __ATTRS_o_ai	2844	static __inline__ vector unsigned char __ATTRS_o_ai
2711	vec_xl_len(unsigned char *__a, size_t __b) {	2845	vec_xl_len(const unsigned char *__a, size_t __b) {
2712	return (vector unsigned char)__builtin_vsx_lxvl(__a, (__b << 56));	2846	return (vector unsigned char)__builtin_vsx_lxvl(__a, (__b << 56));
2713	}	2847	}
2714		2848
2715	static __inline__ vector signed short __ATTRS_o_ai vec_xl_len(signed short *__a,	2849	static __inline__ vector signed short __ATTRS_o_ai vec_xl_len(const signed short *__a,
2716	size_t __b) {	2850	size_t __b) {
2717	return (vector signed short)__builtin_vsx_lxvl(__a, (__b << 56));	2851	return (vector signed short)__builtin_vsx_lxvl(__a, (__b << 56));
2718	}	2852	}
2719		2853
2720	static __inline__ vector unsigned short __ATTRS_o_ai	2854	static __inline__ vector unsigned short __ATTRS_o_ai
2721	vec_xl_len(unsigned short *__a, size_t __b) {	2855	vec_xl_len(const unsigned short *__a, size_t __b) {
2722	return (vector unsigned short)__builtin_vsx_lxvl(__a, (__b << 56));	2856	return (vector unsigned short)__builtin_vsx_lxvl(__a, (__b << 56));
2723	}	2857	}
2724		2858
2725	static __inline__ vector signed int __ATTRS_o_ai vec_xl_len(signed int *__a,	2859	static __inline__ vector signed int __ATTRS_o_ai vec_xl_len(const signed int *__a,
2726	size_t __b) {	2860	size_t __b) {
2727	return (vector signed int)__builtin_vsx_lxvl(__a, (__b << 56));	2861	return (vector signed int)__builtin_vsx_lxvl(__a, (__b << 56));
2728	}	2862	}
2729		2863
2730	static __inline__ vector unsigned int __ATTRS_o_ai vec_xl_len(unsigned int *__a,	2864	static __inline__ vector unsigned int __ATTRS_o_ai vec_xl_len(const unsigned int *__a,
2731	size_t __b) {	2865	size_t __b) {
2732	return (vector unsigned int)__builtin_vsx_lxvl(__a, (__b << 56));	2866	return (vector unsigned int)__builtin_vsx_lxvl(__a, (__b << 56));
2733	}	2867	}
2734		2868
2735	static __inline__ vector float __ATTRS_o_ai vec_xl_len(float *__a, size_t __b) {	2869	static __inline__ vector float __ATTRS_o_ai vec_xl_len(const float *__a, size_t __b) {
2736	return (vector float)__builtin_vsx_lxvl(__a, (__b << 56));	2870	return (vector float)__builtin_vsx_lxvl(__a, (__b << 56));
2737	}	2871	}
2738		2872
2739	static __inline__ vector signed __int128 __ATTRS_o_ai	2873	static __inline__ vector signed __int128 __ATTRS_o_ai
2740	vec_xl_len(signed __int128 *__a, size_t __b) {	2874	vec_xl_len(const signed __int128 *__a, size_t __b) {
2741	return (vector signed __int128)__builtin_vsx_lxvl(__a, (__b << 56));	2875	return (vector signed __int128)__builtin_vsx_lxvl(__a, (__b << 56));
2742	}	2876	}
2743		2877
2744	static __inline__ vector unsigned __int128 __ATTRS_o_ai	2878	static __inline__ vector unsigned __int128 __ATTRS_o_ai
2745	vec_xl_len(unsigned __int128 *__a, size_t __b) {	2879	vec_xl_len(const unsigned __int128 *__a, size_t __b) {
2746	return (vector unsigned __int128)__builtin_vsx_lxvl(__a, (__b << 56));	2880	return (vector unsigned __int128)__builtin_vsx_lxvl(__a, (__b << 56));
2747	}	2881	}
2748		2882
2749	static __inline__ vector signed long long __ATTRS_o_ai	2883	static __inline__ vector signed long long __ATTRS_o_ai
2750	vec_xl_len(signed long long *__a, size_t __b) {	2884	vec_xl_len(const signed long long *__a, size_t __b) {
2751	return (vector signed long long)__builtin_vsx_lxvl(__a, (__b << 56));	2885	return (vector signed long long)__builtin_vsx_lxvl(__a, (__b << 56));
2752	}	2886	}
2753		2887
2754	static __inline__ vector unsigned long long __ATTRS_o_ai	2888	static __inline__ vector unsigned long long __ATTRS_o_ai
2755	vec_xl_len(unsigned long long *__a, size_t __b) {	2889	vec_xl_len(const unsigned long long *__a, size_t __b) {
2756	return (vector unsigned long long)__builtin_vsx_lxvl(__a, (__b << 56));	2890	return (vector unsigned long long)__builtin_vsx_lxvl(__a, (__b << 56));
2757	}	2891	}
2758		2892
2759	static __inline__ vector double __ATTRS_o_ai vec_xl_len(double *__a,	2893	static __inline__ vector double __ATTRS_o_ai vec_xl_len(const double *__a,
2760	size_t __b) {	2894	size_t __b) {
2761	return (vector double)__builtin_vsx_lxvl(__a, (__b << 56));	2895	return (vector double)__builtin_vsx_lxvl(__a, (__b << 56));
2762	}	2896	}
2763		2897
2764	static __inline__ vector unsigned char __ATTRS_o_ai	2898	static __inline__ vector unsigned char __ATTRS_o_ai
2765	vec_xl_len_r(unsigned char *__a, size_t __b) {	2899	vec_xl_len_r(const unsigned char *__a, size_t __b) {
2766	vector unsigned char __res =	2900	vector unsigned char __res =
2767	(vector unsigned char)__builtin_vsx_lxvll(__a, (__b << 56));	2901	(vector unsigned char)__builtin_vsx_lxvll(__a, (__b << 56));
2768	#ifdef __LITTLE_ENDIAN__	2902	#ifdef __LITTLE_ENDIAN__
...	@@ -2862,12 +2996,12 @@ static __inline__ void __ATTRS_o_ai vec_xst_len_r(vector unsigned char __a,	...	@@ -2862,12 +2996,12 @@ static __inline__ void __ATTRS_o_ai vec_xst_len_r(vector unsigned char __a,
2862	#ifdef __VSX__	2996	#ifdef __VSX__
2863	static __inline__ vector float __ATTRS_o_ai vec_cpsgn(vector float __a,	2997	static __inline__ vector float __ATTRS_o_ai vec_cpsgn(vector float __a,
2864	vector float __b) {	2998	vector float __b) {
2865	return __builtin_vsx_xvcpsgnsp(__a, __b);	2999	return __builtin_vsx_xvcpsgnsp(__b, __a);
2866	}	3000	}
2867		3001
2868	static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,	3002	static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
2869	vector double __b) {	3003	vector double __b) {
2870	return __builtin_vsx_xvcpsgndp(__a, __b);	3004	return __builtin_vsx_xvcpsgndp(__b, __a);
2871	}	3005	}
2872	#endif	3006	#endif
2873		3007
...	@@ -2951,6 +3085,42 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,	...	@@ -2951,6 +3085,42 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
2951		3085
2952	#define vec_vctuxs __builtin_altivec_vctuxs	3086	#define vec_vctuxs __builtin_altivec_vctuxs
2953		3087
		3088	/* vec_signext */
		3089
		3090	#ifdef __POWER9_VECTOR__
		3091	static __inline__ vector signed int __ATTRS_o_ai
		3092	vec_signexti(vector signed char __a) {
		3093	return __builtin_altivec_vextsb2w(__a);
		3094	}
		3095
		3096	static __inline__ vector signed int __ATTRS_o_ai
		3097	vec_signexti(vector signed short __a) {
		3098	return __builtin_altivec_vextsh2w(__a);
		3099	}
		3100
		3101	static __inline__ vector signed long long __ATTRS_o_ai
		3102	vec_signextll(vector signed char __a) {
		3103	return __builtin_altivec_vextsb2d(__a);
		3104	}
		3105
		3106	static __inline__ vector signed long long __ATTRS_o_ai
		3107	vec_signextll(vector signed short __a) {
		3108	return __builtin_altivec_vextsh2d(__a);
		3109	}
		3110
		3111	static __inline__ vector signed long long __ATTRS_o_ai
		3112	vec_signextll(vector signed int __a) {
		3113	return __builtin_altivec_vextsw2d(__a);
		3114	}
		3115	#endif
		3116
		3117	#ifdef __POWER10_VECTOR__
		3118	static __inline__ vector signed __int128 __ATTRS_o_ai
		3119	vec_signextq(vector signed long long __a) {
		3120	return __builtin_altivec_vextsd2q(__a);
		3121	}
		3122	#endif
		3123
2954	/* vec_signed */	3124	/* vec_signed */
2955		3125
2956	static __inline__ vector signed int __ATTRS_o_ai	3126	static __inline__ vector signed int __ATTRS_o_ai
...	@@ -3288,6 +3458,66 @@ static __inline__ vector double __ATTRS_o_ai vec_div(vector double __a,	...	@@ -3288,6 +3458,66 @@ static __inline__ vector double __ATTRS_o_ai vec_div(vector double __a,
3288	}	3458	}
3289	#endif	3459	#endif
3290		3460
		3461	/* vec_dive */
		3462
		3463	#ifdef __POWER10_VECTOR__
		3464	static __inline__ vector signed int __ATTRS_o_ai
		3465	vec_dive(vector signed int __a, vector signed int __b) {
		3466	return __builtin_altivec_vdivesw(__a, __b);
		3467	}
		3468
		3469	static __inline__ vector unsigned int __ATTRS_o_ai
		3470	vec_dive(vector unsigned int __a, vector unsigned int __b) {
		3471	return __builtin_altivec_vdiveuw(__a, __b);
		3472	}
		3473
		3474	static __inline__ vector signed long long __ATTRS_o_ai
		3475	vec_dive(vector signed long long __a, vector signed long long __b) {
		3476	return __builtin_altivec_vdivesd(__a, __b);
		3477	}
		3478
		3479	static __inline__ vector unsigned long long __ATTRS_o_ai
		3480	vec_dive(vector unsigned long long __a, vector unsigned long long __b) {
		3481	return __builtin_altivec_vdiveud(__a, __b);
		3482	}
		3483
		3484	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		3485	vec_dive(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		3486	return __builtin_altivec_vdiveuq(__a, __b);
		3487	}
		3488
		3489	static __inline__ vector signed __int128 __ATTRS_o_ai
		3490	vec_dive(vector signed __int128 __a, vector signed __int128 __b) {
		3491	return __builtin_altivec_vdivesq(__a, __b);
		3492	}
		3493	#endif
		3494
		3495	#ifdef __POWER10_VECTOR__
		3496	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		3497	vec_div(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		3498	return __a / __b;
		3499	}
		3500
		3501	static __inline__ vector signed __int128 __ATTRS_o_ai
		3502	vec_div(vector signed __int128 __a, vector signed __int128 __b) {
		3503	return __a / __b;
		3504	}
		3505	#endif /* __POWER10_VECTOR__ */
		3506
		3507	/* vec_xvtdiv */
		3508
		3509	#ifdef __VSX__
		3510	static __inline__ int __ATTRS_o_ai vec_test_swdiv(vector double __a,
		3511	vector double __b) {
		3512	return __builtin_vsx_xvtdivdp(__a, __b);
		3513	}
		3514
		3515	static __inline__ int __ATTRS_o_ai vec_test_swdivs(vector float __a,
		3516	vector float __b) {
		3517	return __builtin_vsx_xvtdivsp(__a, __b);
		3518	}
		3519	#endif
		3520
3291	/* vec_dss */	3521	/* vec_dss */
3292		3522
3293	#define vec_dss __builtin_altivec_dss	3523	#define vec_dss __builtin_altivec_dss
...	@@ -3300,23 +3530,19 @@ static __inline__ void __attribute__((__always_inline__)) vec_dssall(void) {	...	@@ -3300,23 +3530,19 @@ static __inline__ void __attribute__((__always_inline__)) vec_dssall(void) {
3300		3530
3301	/* vec_dst */	3531	/* vec_dst */
3302	#define vec_dst(__PTR, __CW, __STR) \	3532	#define vec_dst(__PTR, __CW, __STR) \
3303	__extension__( \	3533	__builtin_altivec_dst((const void *)(__PTR), (__CW), (__STR))
3304	{ __builtin_altivec_dst((const void *)(__PTR), (__CW), (__STR)); })
3305		3534
3306	/* vec_dstst */	3535	/* vec_dstst */
3307	#define vec_dstst(__PTR, __CW, __STR) \	3536	#define vec_dstst(__PTR, __CW, __STR) \
3308	__extension__( \	3537	__builtin_altivec_dstst((const void *)(__PTR), (__CW), (__STR))
3309	{ __builtin_altivec_dstst((const void *)(__PTR), (__CW), (__STR)); })
3310		3538
3311	/* vec_dststt */	3539	/* vec_dststt */
3312	#define vec_dststt(__PTR, __CW, __STR) \	3540	#define vec_dststt(__PTR, __CW, __STR) \
3313	__extension__( \	3541	__builtin_altivec_dststt((const void *)(__PTR), (__CW), (__STR))
3314	{ __builtin_altivec_dststt((const void *)(__PTR), (__CW), (__STR)); })
3315		3542
3316	/* vec_dstt */	3543	/* vec_dstt */
3317	#define vec_dstt(__PTR, __CW, __STR) \	3544	#define vec_dstt(__PTR, __CW, __STR) \
3318	__extension__( \	3545	__builtin_altivec_dstt((const void *)(__PTR), (__CW), (__STR))
3319	{ __builtin_altivec_dstt((const void *)(__PTR), (__CW), (__STR)); })
3320		3546
3321	/* vec_eqv */	3547	/* vec_eqv */
3322		3548
...	@@ -5467,6 +5693,16 @@ vec_msum(vector unsigned short __a, vector unsigned short __b,	...	@@ -5467,6 +5693,16 @@ vec_msum(vector unsigned short __a, vector unsigned short __b,
5467	return __builtin_altivec_vmsumuhm(__a, __b, __c);	5693	return __builtin_altivec_vmsumuhm(__a, __b, __c);
5468	}	5694	}
5469		5695
		5696	/* vec_msumc */
		5697
		5698	#ifdef __POWER10_VECTOR__
		5699	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		5700	vec_msumc(vector unsigned long long __a, vector unsigned long long __b,
		5701	vector unsigned __int128 __c) {
		5702	return __builtin_altivec_vmsumcud(__a, __b, __c);
		5703	}
		5704	#endif
		5705
5470	/* vec_vmsummbm */	5706	/* vec_vmsummbm */
5471		5707
5472	static __inline__ vector int __attribute__((__always_inline__))	5708	static __inline__ vector int __attribute__((__always_inline__))
...	@@ -5693,6 +5929,26 @@ vec_mule(vector unsigned int __a, vector unsigned int __b) {	...	@@ -5693,6 +5929,26 @@ vec_mule(vector unsigned int __a, vector unsigned int __b) {
5693	}	5929	}
5694	#endif	5930	#endif
5695		5931
		5932	#ifdef __POWER10_VECTOR__
		5933	static __inline__ vector signed __int128 __ATTRS_o_ai
		5934	vec_mule(vector signed long long __a, vector signed long long __b) {
		5935	#ifdef __LITTLE_ENDIAN__
		5936	return __builtin_altivec_vmulosd(__a, __b);
		5937	#else
		5938	return __builtin_altivec_vmulesd(__a, __b);
		5939	#endif
		5940	}
		5941
		5942	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		5943	vec_mule(vector unsigned long long __a, vector unsigned long long __b) {
		5944	#ifdef __LITTLE_ENDIAN__
		5945	return __builtin_altivec_vmuloud(__a, __b);
		5946	#else
		5947	return __builtin_altivec_vmuleud(__a, __b);
		5948	#endif
		5949	}
		5950	#endif
		5951
5696	/* vec_vmulesb */	5952	/* vec_vmulesb */
5697		5953
5698	static __inline__ vector short __attribute__((__always_inline__))	5954	static __inline__ vector short __attribute__((__always_inline__))
...	@@ -5737,6 +5993,30 @@ vec_vmuleuh(vector unsigned short __a, vector unsigned short __b) {	...	@@ -5737,6 +5993,30 @@ vec_vmuleuh(vector unsigned short __a, vector unsigned short __b) {
5737	#endif	5993	#endif
5738	}	5994	}
5739		5995
		5996	/* vec_mulh */
		5997
		5998	#ifdef __POWER10_VECTOR__
		5999	static __inline__ vector signed int __ATTRS_o_ai
		6000	vec_mulh(vector signed int __a, vector signed int __b) {
		6001	return __builtin_altivec_vmulhsw(__a, __b);
		6002	}
		6003
		6004	static __inline__ vector unsigned int __ATTRS_o_ai
		6005	vec_mulh(vector unsigned int __a, vector unsigned int __b) {
		6006	return __builtin_altivec_vmulhuw(__a, __b);
		6007	}
		6008
		6009	static __inline__ vector signed long long __ATTRS_o_ai
		6010	vec_mulh(vector signed long long __a, vector signed long long __b) {
		6011	return __builtin_altivec_vmulhsd(__a, __b);
		6012	}
		6013
		6014	static __inline__ vector unsigned long long __ATTRS_o_ai
		6015	vec_mulh(vector unsigned long long __a, vector unsigned long long __b) {
		6016	return __builtin_altivec_vmulhud(__a, __b);
		6017	}
		6018	#endif
		6019
5740	/* vec_mulo */	6020	/* vec_mulo */
5741		6021
5742	static __inline__ vector short __ATTRS_o_ai vec_mulo(vector signed char __a,	6022	static __inline__ vector short __ATTRS_o_ai vec_mulo(vector signed char __a,
...	@@ -5795,6 +6075,26 @@ vec_mulo(vector unsigned int __a, vector unsigned int __b) {	...	@@ -5795,6 +6075,26 @@ vec_mulo(vector unsigned int __a, vector unsigned int __b) {
5795	}	6075	}
5796	#endif	6076	#endif
5797		6077
		6078	#ifdef __POWER10_VECTOR__
		6079	static __inline__ vector signed __int128 __ATTRS_o_ai
		6080	vec_mulo(vector signed long long __a, vector signed long long __b) {
		6081	#ifdef __LITTLE_ENDIAN__
		6082	return __builtin_altivec_vmulesd(__a, __b);
		6083	#else
		6084	return __builtin_altivec_vmulosd(__a, __b);
		6085	#endif
		6086	}
		6087
		6088	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		6089	vec_mulo(vector unsigned long long __a, vector unsigned long long __b) {
		6090	#ifdef __LITTLE_ENDIAN__
		6091	return __builtin_altivec_vmuleud(__a, __b);
		6092	#else
		6093	return __builtin_altivec_vmuloud(__a, __b);
		6094	#endif
		6095	}
		6096	#endif
		6097
5798	/* vec_vmulosb */	6098	/* vec_vmulosb */
5799		6099
5800	static __inline__ vector short __attribute__((__always_inline__))	6100	static __inline__ vector short __attribute__((__always_inline__))
...	@@ -7627,6 +7927,18 @@ vec_rl(vector unsigned long long __a, vector unsigned long long __b) {	...	@@ -7627,6 +7927,18 @@ vec_rl(vector unsigned long long __a, vector unsigned long long __b) {
7627	}	7927	}
7628	#endif	7928	#endif
7629		7929
		7930	#ifdef __POWER10_VECTOR__
		7931	static __inline__ vector signed __int128 __ATTRS_o_ai
		7932	vec_rl(vector signed __int128 __a, vector unsigned __int128 __b) {
		7933	return (__b << __a)\|(__b >> ((__CHAR_BIT__ * sizeof(vector signed __int128)) - __a));
		7934	}
		7935
		7936	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		7937	vec_rl(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		7938	return (__b << __a)\|(__b >> ((__CHAR_BIT__ * sizeof(vector unsigned __int128)) - __a));
		7939	}
		7940	#endif
		7941
7630	/* vec_rlmi */	7942	/* vec_rlmi */
7631	#ifdef __POWER9_VECTOR__	7943	#ifdef __POWER9_VECTOR__
7632	static __inline__ vector unsigned int __ATTRS_o_ai	7944	static __inline__ vector unsigned int __ATTRS_o_ai
...	@@ -7640,10 +7952,26 @@ vec_rlmi(vector unsigned long long __a, vector unsigned long long __b,	...	@@ -7640,10 +7952,26 @@ vec_rlmi(vector unsigned long long __a, vector unsigned long long __b,
7640	vector unsigned long long __c) {	7952	vector unsigned long long __c) {
7641	return __builtin_altivec_vrldmi(__a, __c, __b);	7953	return __builtin_altivec_vrldmi(__a, __c, __b);
7642	}	7954	}
		7955	#endif
7643		7956
7644	/* vec_rlnm */	7957	#ifdef __POWER10_VECTOR__
7645	static __inline__ vector unsigned int __ATTRS_o_ai	7958	static __inline__ vector unsigned __int128 __ATTRS_o_ai
7646	vec_rlnm(vector unsigned int __a, vector unsigned int __b,	7959	vec_rlmi(vector unsigned __int128 __a, vector unsigned __int128 __b,
		7960	vector unsigned __int128 __c) {
		7961	return __builtin_altivec_vrlqmi(__a, __c, __b);
		7962	}
		7963
		7964	static __inline__ vector signed __int128 __ATTRS_o_ai
		7965	vec_rlmi(vector signed __int128 __a, vector signed __int128 __b,
		7966	vector signed __int128 __c) {
		7967	return __builtin_altivec_vrlqmi(__a, __c, __b);
		7968	}
		7969	#endif
		7970
		7971	/* vec_rlnm */
		7972	#ifdef __POWER9_VECTOR__
		7973	static __inline__ vector unsigned int __ATTRS_o_ai
		7974	vec_rlnm(vector unsigned int __a, vector unsigned int __b,
7647	vector unsigned int __c) {	7975	vector unsigned int __c) {
7648	vector unsigned int OneByte = { 0x8, 0x8, 0x8, 0x8 };	7976	vector unsigned int OneByte = { 0x8, 0x8, 0x8, 0x8 };
7649	return __builtin_altivec_vrlwnm(__a, ((__c << OneByte) \| __b));	7977	return __builtin_altivec_vrlwnm(__a, ((__c << OneByte) \| __b));
...	@@ -7657,6 +7985,42 @@ vec_rlnm(vector unsigned long long __a, vector unsigned long long __b,	...	@@ -7657,6 +7985,42 @@ vec_rlnm(vector unsigned long long __a, vector unsigned long long __b,
7657	}	7985	}
7658	#endif	7986	#endif
7659		7987
		7988	#ifdef __POWER10_VECTOR__
		7989	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		7990	vec_rlnm(vector unsigned __int128 __a, vector unsigned __int128 __b,
		7991	vector unsigned __int128 __c) {
		7992	// Merge __b and __c using an appropriate shuffle.
		7993	vector unsigned char TmpB = (vector unsigned char)__b;
		7994	vector unsigned char TmpC = (vector unsigned char)__c;
		7995	vector unsigned char MaskAndShift =
		7996	#ifdef __LITTLE_ENDIAN__
		7997	__builtin_shufflevector(TmpB, TmpC, -1, -1, -1, -1, -1, -1, -1, -1, 16, 0,
		7998	1, -1, -1, -1, -1, -1);
		7999	#else
		8000	__builtin_shufflevector(TmpB, TmpC, -1, -1, -1, -1, -1, 31, 30, 15, -1,
		8001	-1, -1, -1, -1, -1, -1, -1);
		8002	#endif
		8003	return __builtin_altivec_vrlqnm(__a, (vector unsigned __int128) MaskAndShift);
		8004	}
		8005
		8006	static __inline__ vector signed __int128 __ATTRS_o_ai
		8007	vec_rlnm(vector signed __int128 __a, vector signed __int128 __b,
		8008	vector signed __int128 __c) {
		8009	// Merge __b and __c using an appropriate shuffle.
		8010	vector unsigned char TmpB = (vector unsigned char)__b;
		8011	vector unsigned char TmpC = (vector unsigned char)__c;
		8012	vector unsigned char MaskAndShift =
		8013	#ifdef __LITTLE_ENDIAN__
		8014	__builtin_shufflevector(TmpB, TmpC, -1, -1, -1, -1, -1, -1, -1, -1, 16, 0,
		8015	1, -1, -1, -1, -1, -1);
		8016	#else
		8017	__builtin_shufflevector(TmpB, TmpC, -1, -1, -1, -1, -1, 31, 30, 15, -1,
		8018	-1, -1, -1, -1, -1, -1, -1);
		8019	#endif
		8020	return __builtin_altivec_vrlqnm(__a, (vector unsigned __int128) MaskAndShift);
		8021	}
		8022	#endif
		8023
7660	/* vec_vrlb */	8024	/* vec_vrlb */
7661		8025
7662	static __inline__ vector signed char __ATTRS_o_ai	8026	static __inline__ vector signed char __ATTRS_o_ai
...	@@ -7771,6 +8135,18 @@ vec_vrsqrtefp(vector float __a) {	...	@@ -7771,6 +8135,18 @@ vec_vrsqrtefp(vector float __a) {
7771	return __builtin_altivec_vrsqrtefp(__a);	8135	return __builtin_altivec_vrsqrtefp(__a);
7772	}	8136	}
7773		8137
		8138	/* vec_xvtsqrt */
		8139
		8140	#ifdef __VSX__
		8141	static __inline__ int __ATTRS_o_ai vec_test_swsqrt(vector double __a) {
		8142	return __builtin_vsx_xvtsqrtdp(__a);
		8143	}
		8144
		8145	static __inline__ int __ATTRS_o_ai vec_test_swsqrts(vector float __a) {
		8146	return __builtin_vsx_xvtsqrtsp(__a);
		8147	}
		8148	#endif
		8149
7774	/* vec_sel */	8150	/* vec_sel */
7775		8151
7776	#define __builtin_altivec_vsel_4si vec_sel	8152	#define __builtin_altivec_vsel_4si vec_sel
...	@@ -13900,6 +14276,18 @@ static __inline__ int __ATTRS_o_ai vec_all_eq(vector double __a,	...	@@ -13900,6 +14276,18 @@ static __inline__ int __ATTRS_o_ai vec_all_eq(vector double __a,
13900	}	14276	}
13901	#endif	14277	#endif
13902		14278
		14279	#ifdef __POWER10_VECTOR__
		14280	static __inline__ int __ATTRS_o_ai vec_all_eq(vector signed __int128 __a,
		14281	vector signed __int128 __b) {
		14282	return __builtin_altivec_vcmpequq_p(__CR6_LT, __a, __b);
		14283	}
		14284
		14285	static __inline__ int __ATTRS_o_ai vec_all_eq(vector unsigned __int128 __a,
		14286	vector unsigned __int128 __b) {
		14287	return __builtin_altivec_vcmpequq_p(__CR6_LT, __a, __b);
		14288	}
		14289	#endif
		14290
13903	/* vec_all_ge */	14291	/* vec_all_ge */
13904		14292
13905	static __inline__ int __ATTRS_o_ai vec_all_ge(vector signed char __a,	14293	static __inline__ int __ATTRS_o_ai vec_all_ge(vector signed char __a,
...	@@ -14071,6 +14459,18 @@ static __inline__ int __ATTRS_o_ai vec_all_ge(vector double __a,	...	@@ -14071,6 +14459,18 @@ static __inline__ int __ATTRS_o_ai vec_all_ge(vector double __a,
14071	}	14459	}
14072	#endif	14460	#endif
14073		14461
		14462	#ifdef __POWER10_VECTOR__
		14463	static __inline__ int __ATTRS_o_ai vec_all_ge(vector signed __int128 __a,
		14464	vector signed __int128 __b) {
		14465	return __builtin_altivec_vcmpgtsq_p(__CR6_EQ, __b, __a);
		14466	}
		14467
		14468	static __inline__ int __ATTRS_o_ai vec_all_ge(vector unsigned __int128 __a,
		14469	vector unsigned __int128 __b) {
		14470	return __builtin_altivec_vcmpgtuq_p(__CR6_EQ, __b, __a);
		14471	}
		14472	#endif
		14473
14074	/* vec_all_gt */	14474	/* vec_all_gt */
14075		14475
14076	static __inline__ int __ATTRS_o_ai vec_all_gt(vector signed char __a,	14476	static __inline__ int __ATTRS_o_ai vec_all_gt(vector signed char __a,
...	@@ -14242,6 +14642,18 @@ static __inline__ int __ATTRS_o_ai vec_all_gt(vector double __a,	...	@@ -14242,6 +14642,18 @@ static __inline__ int __ATTRS_o_ai vec_all_gt(vector double __a,
14242	}	14642	}
14243	#endif	14643	#endif
14244		14644
		14645	#ifdef __POWER10_VECTOR__
		14646	static __inline__ int __ATTRS_o_ai vec_all_gt(vector signed __int128 __a,
		14647	vector signed __int128 __b) {
		14648	return __builtin_altivec_vcmpgtsq_p(__CR6_LT, __a, __b);
		14649	}
		14650
		14651	static __inline__ int __ATTRS_o_ai vec_all_gt(vector unsigned __int128 __a,
		14652	vector unsigned __int128 __b) {
		14653	return __builtin_altivec_vcmpgtuq_p(__CR6_LT, __a, __b);
		14654	}
		14655	#endif
		14656
14245	/* vec_all_in */	14657	/* vec_all_in */
14246		14658
14247	static __inline__ int __attribute__((__always_inline__))	14659	static __inline__ int __attribute__((__always_inline__))
...	@@ -14421,6 +14833,18 @@ static __inline__ int __ATTRS_o_ai vec_all_le(vector double __a,	...	@@ -14421,6 +14833,18 @@ static __inline__ int __ATTRS_o_ai vec_all_le(vector double __a,
14421	}	14833	}
14422	#endif	14834	#endif
14423		14835
		14836	#ifdef __POWER10_VECTOR__
		14837	static __inline__ int __ATTRS_o_ai vec_all_le(vector signed __int128 __a,
		14838	vector signed __int128 __b) {
		14839	return __builtin_altivec_vcmpgtsq_p(__CR6_EQ, __a, __b);
		14840	}
		14841
		14842	static __inline__ int __ATTRS_o_ai vec_all_le(vector unsigned __int128 __a,
		14843	vector unsigned __int128 __b) {
		14844	return __builtin_altivec_vcmpgtuq_p(__CR6_EQ, __a, __b);
		14845	}
		14846	#endif
		14847
14424	/* vec_all_lt */	14848	/* vec_all_lt */
14425		14849
14426	static __inline__ int __ATTRS_o_ai vec_all_lt(vector signed char __a,	14850	static __inline__ int __ATTRS_o_ai vec_all_lt(vector signed char __a,
...	@@ -14593,6 +15017,18 @@ static __inline__ int __ATTRS_o_ai vec_all_lt(vector double __a,	...	@@ -14593,6 +15017,18 @@ static __inline__ int __ATTRS_o_ai vec_all_lt(vector double __a,
14593	}	15017	}
14594	#endif	15018	#endif
14595		15019
		15020	#ifdef __POWER10_VECTOR__
		15021	static __inline__ int __ATTRS_o_ai vec_all_lt(vector signed __int128 __a,
		15022	vector signed __int128 __b) {
		15023	return __builtin_altivec_vcmpgtsq_p(__CR6_LT, __b, __a);
		15024	}
		15025
		15026	static __inline__ int __ATTRS_o_ai vec_all_lt(vector unsigned __int128 __a,
		15027	vector unsigned __int128 __b) {
		15028	return __builtin_altivec_vcmpgtuq_p(__CR6_LT, __b, __a);
		15029	}
		15030	#endif
		15031
14596	/* vec_all_nan */	15032	/* vec_all_nan */
14597		15033
14598	static __inline__ int __ATTRS_o_ai vec_all_nan(vector float __a) {	15034	static __inline__ int __ATTRS_o_ai vec_all_nan(vector float __a) {
...	@@ -14797,6 +15233,18 @@ static __inline__ int __ATTRS_o_ai vec_all_ne(vector double __a,	...	@@ -14797,6 +15233,18 @@ static __inline__ int __ATTRS_o_ai vec_all_ne(vector double __a,
14797	}	15233	}
14798	#endif	15234	#endif
14799		15235
		15236	#ifdef __POWER10_VECTOR__
		15237	static __inline__ int __ATTRS_o_ai vec_all_ne(vector signed __int128 __a,
		15238	vector signed __int128 __b) {
		15239	return __builtin_altivec_vcmpequq_p(__CR6_EQ, __a, __b);
		15240	}
		15241
		15242	static __inline__ int __ATTRS_o_ai vec_all_ne(vector unsigned __int128 __a,
		15243	vector unsigned __int128 __b) {
		15244	return __builtin_altivec_vcmpequq_p(__CR6_EQ, __a, __b);
		15245	}
		15246	#endif
		15247
14800	/* vec_all_nge */	15248	/* vec_all_nge */
14801		15249
14802	static __inline__ int __ATTRS_o_ai vec_all_nge(vector float __a,	15250	static __inline__ int __ATTRS_o_ai vec_all_nge(vector float __a,
...	@@ -15042,6 +15490,18 @@ static __inline__ int __ATTRS_o_ai vec_any_eq(vector double __a,	...	@@ -15042,6 +15490,18 @@ static __inline__ int __ATTRS_o_ai vec_any_eq(vector double __a,
15042	}	15490	}
15043	#endif	15491	#endif
15044		15492
		15493	#ifdef __POWER10_VECTOR__
		15494	static __inline__ int __ATTRS_o_ai vec_any_eq(vector signed __int128 __a,
		15495	vector signed __int128 __b) {
		15496	return __builtin_altivec_vcmpequq_p(__CR6_EQ_REV, __a, __b);
		15497	}
		15498
		15499	static __inline__ int __ATTRS_o_ai vec_any_eq(vector unsigned __int128 __a,
		15500	vector unsigned __int128 __b) {
		15501	return __builtin_altivec_vcmpequq_p(__CR6_EQ_REV, __a, __b);
		15502	}
		15503	#endif
		15504
15045	/* vec_any_ge */	15505	/* vec_any_ge */
15046		15506
15047	static __inline__ int __ATTRS_o_ai vec_any_ge(vector signed char __a,	15507	static __inline__ int __ATTRS_o_ai vec_any_ge(vector signed char __a,
...	@@ -15221,6 +15681,18 @@ static __inline__ int __ATTRS_o_ai vec_any_ge(vector double __a,	...	@@ -15221,6 +15681,18 @@ static __inline__ int __ATTRS_o_ai vec_any_ge(vector double __a,
15221	}	15681	}
15222	#endif	15682	#endif
15223		15683
		15684	#ifdef __POWER10_VECTOR__
		15685	static __inline__ int __ATTRS_o_ai vec_any_ge(vector signed __int128 __a,
		15686	vector signed __int128 __b) {
		15687	return __builtin_altivec_vcmpgtsq_p(__CR6_LT_REV, __b, __a);
		15688	}
		15689
		15690	static __inline__ int __ATTRS_o_ai vec_any_ge(vector unsigned __int128 __a,
		15691	vector unsigned __int128 __b) {
		15692	return __builtin_altivec_vcmpgtuq_p(__CR6_LT_REV, __b, __a);
		15693	}
		15694	#endif
		15695
15224	/* vec_any_gt */	15696	/* vec_any_gt */
15225		15697
15226	static __inline__ int __ATTRS_o_ai vec_any_gt(vector signed char __a,	15698	static __inline__ int __ATTRS_o_ai vec_any_gt(vector signed char __a,
...	@@ -15400,6 +15872,18 @@ static __inline__ int __ATTRS_o_ai vec_any_gt(vector double __a,	...	@@ -15400,6 +15872,18 @@ static __inline__ int __ATTRS_o_ai vec_any_gt(vector double __a,
15400	}	15872	}
15401	#endif	15873	#endif
15402		15874
		15875	#ifdef __POWER10_VECTOR__
		15876	static __inline__ int __ATTRS_o_ai vec_any_gt(vector signed __int128 __a,
		15877	vector signed __int128 __b) {
		15878	return __builtin_altivec_vcmpgtsq_p(__CR6_EQ_REV, __a, __b);
		15879	}
		15880
		15881	static __inline__ int __ATTRS_o_ai vec_any_gt(vector unsigned __int128 __a,
		15882	vector unsigned __int128 __b) {
		15883	return __builtin_altivec_vcmpgtuq_p(__CR6_EQ_REV, __a, __b);
		15884	}
		15885	#endif
		15886
15403	/* vec_any_le */	15887	/* vec_any_le */
15404		15888
15405	static __inline__ int __ATTRS_o_ai vec_any_le(vector signed char __a,	15889	static __inline__ int __ATTRS_o_ai vec_any_le(vector signed char __a,
...	@@ -15579,6 +16063,18 @@ static __inline__ int __ATTRS_o_ai vec_any_le(vector double __a,	...	@@ -15579,6 +16063,18 @@ static __inline__ int __ATTRS_o_ai vec_any_le(vector double __a,
15579	}	16063	}
15580	#endif	16064	#endif
15581		16065
		16066	#ifdef __POWER10_VECTOR__
		16067	static __inline__ int __ATTRS_o_ai vec_any_le(vector signed __int128 __a,
		16068	vector signed __int128 __b) {
		16069	return __builtin_altivec_vcmpgtsq_p(__CR6_LT_REV, __a, __b);
		16070	}
		16071
		16072	static __inline__ int __ATTRS_o_ai vec_any_le(vector unsigned __int128 __a,
		16073	vector unsigned __int128 __b) {
		16074	return __builtin_altivec_vcmpgtuq_p(__CR6_LT_REV, __a, __b);
		16075	}
		16076	#endif
		16077
15582	/* vec_any_lt */	16078	/* vec_any_lt */
15583		16079
15584	static __inline__ int __ATTRS_o_ai vec_any_lt(vector signed char __a,	16080	static __inline__ int __ATTRS_o_ai vec_any_lt(vector signed char __a,
...	@@ -15758,6 +16254,18 @@ static __inline__ int __ATTRS_o_ai vec_any_lt(vector double __a,	...	@@ -15758,6 +16254,18 @@ static __inline__ int __ATTRS_o_ai vec_any_lt(vector double __a,
15758	}	16254	}
15759	#endif	16255	#endif
15760		16256
		16257	#ifdef __POWER10_VECTOR__
		16258	static __inline__ int __ATTRS_o_ai vec_any_lt(vector signed __int128 __a,
		16259	vector signed __int128 __b) {
		16260	return __builtin_altivec_vcmpgtsq_p(__CR6_EQ_REV, __b, __a);
		16261	}
		16262
		16263	static __inline__ int __ATTRS_o_ai vec_any_lt(vector unsigned __int128 __a,
		16264	vector unsigned __int128 __b) {
		16265	return __builtin_altivec_vcmpgtuq_p(__CR6_EQ_REV, __b, __a);
		16266	}
		16267	#endif
		16268
15761	/* vec_any_nan */	16269	/* vec_any_nan */
15762		16270
15763	static __inline__ int __attribute__((__always_inline__))	16271	static __inline__ int __attribute__((__always_inline__))
...	@@ -15953,6 +16461,18 @@ static __inline__ int __ATTRS_o_ai vec_any_ne(vector double __a,	...	@@ -15953,6 +16461,18 @@ static __inline__ int __ATTRS_o_ai vec_any_ne(vector double __a,
15953	}	16461	}
15954	#endif	16462	#endif
15955		16463
		16464	#ifdef __POWER10_VECTOR__
		16465	static __inline__ int __ATTRS_o_ai vec_any_ne(vector signed __int128 __a,
		16466	vector signed __int128 __b) {
		16467	return __builtin_altivec_vcmpequq_p(__CR6_LT_REV, __a, __b);
		16468	}
		16469
		16470	static __inline__ int __ATTRS_o_ai vec_any_ne(vector unsigned __int128 __a,
		16471	vector unsigned __int128 __b) {
		16472	return __builtin_altivec_vcmpequq_p(__CR6_LT_REV, __a, __b);
		16473	}
		16474	#endif
		16475
15956	/* vec_any_nge */	16476	/* vec_any_nge */
15957		16477
15958	static __inline__ int __attribute__((__always_inline__))	16478	static __inline__ int __attribute__((__always_inline__))
...	@@ -16353,41 +16873,41 @@ typedef vector unsigned int unaligned_vec_uint __attribute__((aligned(1)));	...	@@ -16353,41 +16873,41 @@ typedef vector unsigned int unaligned_vec_uint __attribute__((aligned(1)));
16353	typedef vector float unaligned_vec_float __attribute__((aligned(1)));	16873	typedef vector float unaligned_vec_float __attribute__((aligned(1)));
16354		16874
16355	static inline __ATTRS_o_ai vector signed char vec_xl(signed long long __offset,	16875	static inline __ATTRS_o_ai vector signed char vec_xl(signed long long __offset,
16356	signed char *__ptr) {	16876	const signed char *__ptr) {
16357	return (unaligned_vec_schar )(__ptr + __offset);	16877	return (unaligned_vec_schar )(__ptr + __offset);
16358	}	16878	}
16359		16879
16360	static inline __ATTRS_o_ai vector unsigned char	16880	static inline __ATTRS_o_ai vector unsigned char
16361	vec_xl(signed long long __offset, unsigned char *__ptr) {	16881	vec_xl(signed long long __offset, const unsigned char *__ptr) {
16362	return (unaligned_vec_uchar)(__ptr + __offset);	16882	return (unaligned_vec_uchar)(__ptr + __offset);
16363	}	16883	}
16364		16884
16365	static inline __ATTRS_o_ai vector signed short vec_xl(signed long long __offset,	16885	static inline __ATTRS_o_ai vector signed short vec_xl(signed long long __offset,
16366	signed short *__ptr) {	16886	const signed short *__ptr) {
16367	signed char __addr = (signed char )__ptr + __offset;	16887	signed char __addr = (signed char )__ptr + __offset;
16368	return (unaligned_vec_sshort )__addr;	16888	return (unaligned_vec_sshort )__addr;
16369	}	16889	}
16370		16890
16371	static inline __ATTRS_o_ai vector unsigned short	16891	static inline __ATTRS_o_ai vector unsigned short
16372	vec_xl(signed long long __offset, unsigned short *__ptr) {	16892	vec_xl(signed long long __offset, const unsigned short *__ptr) {
16373	signed char __addr = (signed char )__ptr + __offset;	16893	signed char __addr = (signed char )__ptr + __offset;
16374	return (unaligned_vec_ushort )__addr;	16894	return (unaligned_vec_ushort )__addr;
16375	}	16895	}
16376		16896
16377	static inline __ATTRS_o_ai vector signed int vec_xl(signed long long __offset,	16897	static inline __ATTRS_o_ai vector signed int vec_xl(signed long long __offset,
16378	signed int *__ptr) {	16898	const signed int *__ptr) {
16379	signed char __addr = (signed char )__ptr + __offset;	16899	signed char __addr = (signed char )__ptr + __offset;
16380	return (unaligned_vec_sint )__addr;	16900	return (unaligned_vec_sint )__addr;
16381	}	16901	}
16382		16902
16383	static inline __ATTRS_o_ai vector unsigned int vec_xl(signed long long __offset,	16903	static inline __ATTRS_o_ai vector unsigned int vec_xl(signed long long __offset,
16384	unsigned int *__ptr) {	16904	const unsigned int *__ptr) {
16385	signed char __addr = (signed char )__ptr + __offset;	16905	signed char __addr = (signed char )__ptr + __offset;
16386	return (unaligned_vec_uint )__addr;	16906	return (unaligned_vec_uint )__addr;
16387	}	16907	}
16388		16908
16389	static inline __ATTRS_o_ai vector float vec_xl(signed long long __offset,	16909	static inline __ATTRS_o_ai vector float vec_xl(signed long long __offset,
16390	float *__ptr) {	16910	const float *__ptr) {
16391	signed char __addr = (signed char )__ptr + __offset;	16911	signed char __addr = (signed char )__ptr + __offset;
16392	return (unaligned_vec_float )__addr;	16912	return (unaligned_vec_float )__addr;
16393	}	16913	}
...	@@ -16398,19 +16918,19 @@ typedef vector unsigned long long unaligned_vec_ull __attribute__((aligned(1)));	...	@@ -16398,19 +16918,19 @@ typedef vector unsigned long long unaligned_vec_ull __attribute__((aligned(1)));
16398	typedef vector double unaligned_vec_double __attribute__((aligned(1)));	16918	typedef vector double unaligned_vec_double __attribute__((aligned(1)));
16399		16919
16400	static inline __ATTRS_o_ai vector signed long long	16920	static inline __ATTRS_o_ai vector signed long long
16401	vec_xl(signed long long __offset, signed long long *__ptr) {	16921	vec_xl(signed long long __offset, const signed long long *__ptr) {
16402	signed char __addr = (signed char )__ptr + __offset;	16922	signed char __addr = (signed char )__ptr + __offset;
16403	return (unaligned_vec_sll )__addr;	16923	return (unaligned_vec_sll )__addr;
16404	}	16924	}
16405		16925
16406	static inline __ATTRS_o_ai vector unsigned long long	16926	static inline __ATTRS_o_ai vector unsigned long long
16407	vec_xl(signed long long __offset, unsigned long long *__ptr) {	16927	vec_xl(signed long long __offset, const unsigned long long *__ptr) {
16408	signed char __addr = (signed char )__ptr + __offset;	16928	signed char __addr = (signed char )__ptr + __offset;
16409	return (unaligned_vec_ull )__addr;	16929	return (unaligned_vec_ull )__addr;
16410	}	16930	}
16411		16931
16412	static inline __ATTRS_o_ai vector double vec_xl(signed long long __offset,	16932	static inline __ATTRS_o_ai vector double vec_xl(signed long long __offset,
16413	double *__ptr) {	16933	const double *__ptr) {
16414	signed char __addr = (signed char )__ptr + __offset;	16934	signed char __addr = (signed char )__ptr + __offset;
16415	return (unaligned_vec_double )__addr;	16935	return (unaligned_vec_double )__addr;
16416	}	16936	}
...	@@ -16421,13 +16941,13 @@ typedef vector signed __int128 unaligned_vec_si128 __attribute__((aligned(1)));	...	@@ -16421,13 +16941,13 @@ typedef vector signed __int128 unaligned_vec_si128 __attribute__((aligned(1)));
16421	typedef vector unsigned __int128 unaligned_vec_ui128	16941	typedef vector unsigned __int128 unaligned_vec_ui128
16422	__attribute__((aligned(1)));	16942	__attribute__((aligned(1)));
16423	static inline __ATTRS_o_ai vector signed __int128	16943	static inline __ATTRS_o_ai vector signed __int128
16424	vec_xl(signed long long __offset, signed __int128 *__ptr) {	16944	vec_xl(signed long long __offset, const signed __int128 *__ptr) {
16425	signed char __addr = (signed char )__ptr + __offset;	16945	signed char __addr = (signed char )__ptr + __offset;
16426	return (unaligned_vec_si128 )__addr;	16946	return (unaligned_vec_si128 )__addr;
16427	}	16947	}
16428		16948
16429	static inline __ATTRS_o_ai vector unsigned __int128	16949	static inline __ATTRS_o_ai vector unsigned __int128
16430	vec_xl(signed long long __offset, unsigned __int128 *__ptr) {	16950	vec_xl(signed long long __offset, const unsigned __int128 *__ptr) {
16431	signed char __addr = (signed char )__ptr + __offset;	16951	signed char __addr = (signed char )__ptr + __offset;
16432	return (unaligned_vec_ui128 )__addr;	16952	return (unaligned_vec_ui128 )__addr;
16433	}	16953	}
...	@@ -16437,71 +16957,71 @@ vec_xl(signed long long __offset, unsigned __int128 *__ptr) {	...	@@ -16437,71 +16957,71 @@ vec_xl(signed long long __offset, unsigned __int128 *__ptr) {
16437		16957
16438	#ifdef __LITTLE_ENDIAN__	16958	#ifdef __LITTLE_ENDIAN__
16439	static __inline__ vector signed char __ATTRS_o_ai	16959	static __inline__ vector signed char __ATTRS_o_ai
16440	vec_xl_be(signed long long __offset, signed char *__ptr) {	16960	vec_xl_be(signed long long __offset, const signed char *__ptr) {
16441	vector signed char __vec = (vector signed char)__builtin_vsx_lxvd2x_be(__offset, __ptr);	16961	vector signed char __vec = (vector signed char)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16442	return __builtin_shufflevector(__vec, __vec, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14,	16962	return __builtin_shufflevector(__vec, __vec, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14,
16443	13, 12, 11, 10, 9, 8);	16963	13, 12, 11, 10, 9, 8);
16444	}	16964	}
16445		16965
16446	static __inline__ vector unsigned char __ATTRS_o_ai	16966	static __inline__ vector unsigned char __ATTRS_o_ai
16447	vec_xl_be(signed long long __offset, unsigned char *__ptr) {	16967	vec_xl_be(signed long long __offset, const unsigned char *__ptr) {
16448	vector unsigned char __vec = (vector unsigned char)__builtin_vsx_lxvd2x_be(__offset, __ptr);	16968	vector unsigned char __vec = (vector unsigned char)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16449	return __builtin_shufflevector(__vec, __vec, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14,	16969	return __builtin_shufflevector(__vec, __vec, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14,
16450	13, 12, 11, 10, 9, 8);	16970	13, 12, 11, 10, 9, 8);
16451	}	16971	}
16452		16972
16453	static __inline__ vector signed short __ATTRS_o_ai	16973	static __inline__ vector signed short __ATTRS_o_ai
16454	vec_xl_be(signed long long __offset, signed short *__ptr) {	16974	vec_xl_be(signed long long __offset, const signed short *__ptr) {
16455	vector signed short __vec = (vector signed short)__builtin_vsx_lxvd2x_be(__offset, __ptr);	16975	vector signed short __vec = (vector signed short)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16456	return __builtin_shufflevector(__vec, __vec, 3, 2, 1, 0, 7, 6, 5, 4);	16976	return __builtin_shufflevector(__vec, __vec, 3, 2, 1, 0, 7, 6, 5, 4);
16457	}	16977	}
16458		16978
16459	static __inline__ vector unsigned short __ATTRS_o_ai	16979	static __inline__ vector unsigned short __ATTRS_o_ai
16460	vec_xl_be(signed long long __offset, unsigned short *__ptr) {	16980	vec_xl_be(signed long long __offset, const unsigned short *__ptr) {
16461	vector unsigned short __vec = (vector unsigned short)__builtin_vsx_lxvd2x_be(__offset, __ptr);	16981	vector unsigned short __vec = (vector unsigned short)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16462	return __builtin_shufflevector(__vec, __vec, 3, 2, 1, 0, 7, 6, 5, 4);	16982	return __builtin_shufflevector(__vec, __vec, 3, 2, 1, 0, 7, 6, 5, 4);
16463	}	16983	}
16464		16984
16465	static __inline__ vector signed int __ATTRS_o_ai	16985	static __inline__ vector signed int __ATTRS_o_ai
16466	vec_xl_be(signed long long __offset, signed int *__ptr) {	16986	vec_xl_be(signed long long __offset, const signed int *__ptr) {
16467	return (vector signed int)__builtin_vsx_lxvw4x_be(__offset, __ptr);	16987	return (vector signed int)__builtin_vsx_lxvw4x_be(__offset, __ptr);
16468	}	16988	}
16469		16989
16470	static __inline__ vector unsigned int __ATTRS_o_ai	16990	static __inline__ vector unsigned int __ATTRS_o_ai
16471	vec_xl_be(signed long long __offset, unsigned int *__ptr) {	16991	vec_xl_be(signed long long __offset, const unsigned int *__ptr) {
16472	return (vector unsigned int)__builtin_vsx_lxvw4x_be(__offset, __ptr);	16992	return (vector unsigned int)__builtin_vsx_lxvw4x_be(__offset, __ptr);
16473	}	16993	}
16474		16994
16475	static __inline__ vector float __ATTRS_o_ai	16995	static __inline__ vector float __ATTRS_o_ai
16476	vec_xl_be(signed long long __offset, float *__ptr) {	16996	vec_xl_be(signed long long __offset, const float *__ptr) {
16477	return (vector float)__builtin_vsx_lxvw4x_be(__offset, __ptr);	16997	return (vector float)__builtin_vsx_lxvw4x_be(__offset, __ptr);
16478	}	16998	}
16479		16999
16480	#ifdef __VSX__	17000	#ifdef __VSX__
16481	static __inline__ vector signed long long __ATTRS_o_ai	17001	static __inline__ vector signed long long __ATTRS_o_ai
16482	vec_xl_be(signed long long __offset, signed long long *__ptr) {	17002	vec_xl_be(signed long long __offset, const signed long long *__ptr) {
16483	return (vector signed long long)__builtin_vsx_lxvd2x_be(__offset, __ptr);	17003	return (vector signed long long)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16484	}	17004	}
16485		17005
16486	static __inline__ vector unsigned long long __ATTRS_o_ai	17006	static __inline__ vector unsigned long long __ATTRS_o_ai
16487	vec_xl_be(signed long long __offset, unsigned long long *__ptr) {	17007	vec_xl_be(signed long long __offset, const unsigned long long *__ptr) {
16488	return (vector unsigned long long)__builtin_vsx_lxvd2x_be(__offset, __ptr);	17008	return (vector unsigned long long)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16489	}	17009	}
16490		17010
16491	static __inline__ vector double __ATTRS_o_ai	17011	static __inline__ vector double __ATTRS_o_ai
16492	vec_xl_be(signed long long __offset, double *__ptr) {	17012	vec_xl_be(signed long long __offset, const double *__ptr) {
16493	return (vector double)__builtin_vsx_lxvd2x_be(__offset, __ptr);	17013	return (vector double)__builtin_vsx_lxvd2x_be(__offset, __ptr);
16494	}	17014	}
16495	#endif	17015	#endif
16496		17016
16497	#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)	17017	#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
16498	static __inline__ vector signed __int128 __ATTRS_o_ai	17018	static __inline__ vector signed __int128 __ATTRS_o_ai
16499	vec_xl_be(signed long long __offset, signed __int128 *__ptr) {	17019	vec_xl_be(signed long long __offset, const signed __int128 *__ptr) {
16500	return vec_xl(__offset, __ptr);	17020	return vec_xl(__offset, __ptr);
16501	}	17021	}
16502		17022
16503	static __inline__ vector unsigned __int128 __ATTRS_o_ai	17023	static __inline__ vector unsigned __int128 __ATTRS_o_ai
16504	vec_xl_be(signed long long __offset, unsigned __int128 *__ptr) {	17024	vec_xl_be(signed long long __offset, const unsigned __int128 *__ptr) {
16505	return vec_xl(__offset, __ptr);	17025	return vec_xl(__offset, __ptr);
16506	}	17026	}
16507	#endif	17027	#endif
...	@@ -16509,6 +17029,54 @@ vec_xl_be(signed long long __offset, unsigned __int128 *__ptr) {	...	@@ -16509,6 +17029,54 @@ vec_xl_be(signed long long __offset, unsigned __int128 *__ptr) {
16509	#define vec_xl_be vec_xl	17029	#define vec_xl_be vec_xl
16510	#endif	17030	#endif
16511		17031
		17032	#if defined(__POWER10_VECTOR__) && defined(__VSX__)
		17033
		17034	/* vect_xl_sext */
		17035
		17036	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17037	vec_xl_sext(signed long long __offset, const signed char *__pointer) {
		17038	return (vector unsigned __int128)*(__pointer + __offset);
		17039	}
		17040
		17041	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17042	vec_xl_sext(signed long long __offset, const signed short *__pointer) {
		17043	return (vector unsigned __int128)*(__pointer + __offset);
		17044	}
		17045
		17046	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17047	vec_xl_sext(signed long long __offset, const signed int *__pointer) {
		17048	return (vector unsigned __int128)*(__pointer + __offset);
		17049	}
		17050
		17051	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17052	vec_xl_sext(signed long long __offset, const signed long long *__pointer) {
		17053	return (vector unsigned __int128)*(__pointer + __offset);
		17054	}
		17055
		17056	/* vec_xl_zext */
		17057
		17058	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17059	vec_xl_zext(signed long long __offset, const unsigned char *__pointer) {
		17060	return (vector unsigned __int128)*(__pointer + __offset);
		17061	}
		17062
		17063	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17064	vec_xl_zext(signed long long __offset, const unsigned short *__pointer) {
		17065	return (vector unsigned __int128)*(__pointer + __offset);
		17066	}
		17067
		17068	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17069	vec_xl_zext(signed long long __offset, const unsigned int *__pointer) {
		17070	return (vector unsigned __int128)*(__pointer + __offset);
		17071	}
		17072
		17073	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17074	vec_xl_zext(signed long long __offset, const unsigned long long *__pointer) {
		17075	return (vector unsigned __int128)*(__pointer + __offset);
		17076	}
		17077
		17078	#endif
		17079
16512	/* vec_xst */	17080	/* vec_xst */
16513		17081
16514	static inline __ATTRS_o_ai void vec_xst(vector signed char __vec,	17082	static inline __ATTRS_o_ai void vec_xst(vector signed char __vec,
...	@@ -16597,6 +17165,58 @@ static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec,	...	@@ -16597,6 +17165,58 @@ static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec,
16597	}	17165	}
16598	#endif	17166	#endif
16599		17167
		17168	/* vec_xst_trunc */
		17169
		17170	#if defined(__POWER10_VECTOR__) && defined(__VSX__)
		17171	static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
		17172	signed long long __offset,
		17173	signed char *__ptr) {
		17174	*(__ptr + __offset) = (signed char)__vec[0];
		17175	}
		17176
		17177	static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
		17178	signed long long __offset,
		17179	unsigned char *__ptr) {
		17180	*(__ptr + __offset) = (unsigned char)__vec[0];
		17181	}
		17182
		17183	static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
		17184	signed long long __offset,
		17185	signed short *__ptr) {
		17186	*(__ptr + __offset) = (signed short)__vec[0];
		17187	}
		17188
		17189	static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
		17190	signed long long __offset,
		17191	unsigned short *__ptr) {
		17192	*(__ptr + __offset) = (unsigned short)__vec[0];
		17193	}
		17194
		17195	static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
		17196	signed long long __offset,
		17197	signed int *__ptr) {
		17198	*(__ptr + __offset) = (signed int)__vec[0];
		17199	}
		17200
		17201	static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
		17202	signed long long __offset,
		17203	unsigned int *__ptr) {
		17204	*(__ptr + __offset) = (unsigned int)__vec[0];
		17205	}
		17206
		17207	static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
		17208	signed long long __offset,
		17209	signed long long *__ptr) {
		17210	*(__ptr + __offset) = (signed long long)__vec[0];
		17211	}
		17212
		17213	static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
		17214	signed long long __offset,
		17215	unsigned long long *__ptr) {
		17216	*(__ptr + __offset) = (unsigned long long)__vec[0];
		17217	}
		17218	#endif
		17219
16600	/* vec_xst_be */	17220	/* vec_xst_be */
16601		17221
16602	#ifdef __LITTLE_ENDIAN__	17222	#ifdef __LITTLE_ENDIAN__
...	@@ -16763,6 +17383,100 @@ static vector signed char __ATTRS_o_ai vec_nabs(vector signed char __a) {	...	@@ -16763,6 +17383,100 @@ static vector signed char __ATTRS_o_ai vec_nabs(vector signed char __a) {
16763	}	17383	}
16764		17384
16765	#ifdef __POWER10_VECTOR__	17385	#ifdef __POWER10_VECTOR__
		17386
		17387	/* vec_extractm */
		17388
		17389	static __inline__ unsigned int __ATTRS_o_ai
		17390	vec_extractm(vector unsigned char __a) {
		17391	return __builtin_altivec_vextractbm(__a);
		17392	}
		17393
		17394	static __inline__ unsigned int __ATTRS_o_ai
		17395	vec_extractm(vector unsigned short __a) {
		17396	return __builtin_altivec_vextracthm(__a);
		17397	}
		17398
		17399	static __inline__ unsigned int __ATTRS_o_ai
		17400	vec_extractm(vector unsigned int __a) {
		17401	return __builtin_altivec_vextractwm(__a);
		17402	}
		17403
		17404	static __inline__ unsigned int __ATTRS_o_ai
		17405	vec_extractm(vector unsigned long long __a) {
		17406	return __builtin_altivec_vextractdm(__a);
		17407	}
		17408
		17409	static __inline__ unsigned int __ATTRS_o_ai
		17410	vec_extractm(vector unsigned __int128 __a) {
		17411	return __builtin_altivec_vextractqm(__a);
		17412	}
		17413
		17414	/* vec_expandm */
		17415
		17416	static __inline__ vector unsigned char __ATTRS_o_ai
		17417	vec_expandm(vector unsigned char __a) {
		17418	return __builtin_altivec_vexpandbm(__a);
		17419	}
		17420
		17421	static __inline__ vector unsigned short __ATTRS_o_ai
		17422	vec_expandm(vector unsigned short __a) {
		17423	return __builtin_altivec_vexpandhm(__a);
		17424	}
		17425
		17426	static __inline__ vector unsigned int __ATTRS_o_ai
		17427	vec_expandm(vector unsigned int __a) {
		17428	return __builtin_altivec_vexpandwm(__a);
		17429	}
		17430
		17431	static __inline__ vector unsigned long long __ATTRS_o_ai
		17432	vec_expandm(vector unsigned long long __a) {
		17433	return __builtin_altivec_vexpanddm(__a);
		17434	}
		17435
		17436	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17437	vec_expandm(vector unsigned __int128 __a) {
		17438	return __builtin_altivec_vexpandqm(__a);
		17439	}
		17440
		17441	/* vec_cntm */
		17442
		17443	#define vec_cntm(__a, __mp) \
		17444	_Generic((__a), vector unsigned char \
		17445	: __builtin_altivec_vcntmbb((__a), (unsigned int)(__mp)), \
		17446	vector unsigned short \
		17447	: __builtin_altivec_vcntmbh((__a), (unsigned int)(__mp)), \
		17448	vector unsigned int \
		17449	: __builtin_altivec_vcntmbw((__a), (unsigned int)(__mp)), \
		17450	vector unsigned long long \
		17451	: __builtin_altivec_vcntmbd((__a), (unsigned int)(__mp)))
		17452
		17453	/* vec_gen[b\|h\|w\|d\|q]m */
		17454
		17455	static __inline__ vector unsigned char __ATTRS_o_ai
		17456	vec_genbm(unsigned long long __bm) {
		17457	return __builtin_altivec_mtvsrbm(__bm);
		17458	}
		17459
		17460	static __inline__ vector unsigned short __ATTRS_o_ai
		17461	vec_genhm(unsigned long long __bm) {
		17462	return __builtin_altivec_mtvsrhm(__bm);
		17463	}
		17464
		17465	static __inline__ vector unsigned int __ATTRS_o_ai
		17466	vec_genwm(unsigned long long __bm) {
		17467	return __builtin_altivec_mtvsrwm(__bm);
		17468	}
		17469
		17470	static __inline__ vector unsigned long long __ATTRS_o_ai
		17471	vec_gendm(unsigned long long __bm) {
		17472	return __builtin_altivec_mtvsrdm(__bm);
		17473	}
		17474
		17475	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17476	vec_genqm(unsigned long long __bm) {
		17477	return __builtin_altivec_mtvsrqm(__bm);
		17478	}
		17479
16766	/* vec_pdep */	17480	/* vec_pdep */
16767		17481
16768	static __inline__ vector unsigned long long __ATTRS_o_ai	17482	static __inline__ vector unsigned long long __ATTRS_o_ai
...	@@ -16881,6 +17595,38 @@ vec_cnttzm(vector unsigned long long __a, vector unsigned long long __b) {	...	@@ -16881,6 +17595,38 @@ vec_cnttzm(vector unsigned long long __a, vector unsigned long long __b) {
16881	return __builtin_altivec_vctzdm(__a, __b);	17595	return __builtin_altivec_vctzdm(__a, __b);
16882	}	17596	}
16883		17597
		17598	/* vec_mod */
		17599
		17600	static __inline__ vector signed int __ATTRS_o_ai
		17601	vec_mod(vector signed int __a, vector signed int __b) {
		17602	return __a % __b;
		17603	}
		17604
		17605	static __inline__ vector unsigned int __ATTRS_o_ai
		17606	vec_mod(vector unsigned int __a, vector unsigned int __b) {
		17607	return __a % __b;
		17608	}
		17609
		17610	static __inline__ vector signed long long __ATTRS_o_ai
		17611	vec_mod(vector signed long long __a, vector signed long long __b) {
		17612	return __a % __b;
		17613	}
		17614
		17615	static __inline__ vector unsigned long long __ATTRS_o_ai
		17616	vec_mod(vector unsigned long long __a, vector unsigned long long __b) {
		17617	return __a % __b;
		17618	}
		17619
		17620	static __inline__ vector signed __int128 __ATTRS_o_ai
		17621	vec_mod(vector signed __int128 __a, vector signed __int128 __b) {
		17622	return __a % __b;
		17623	}
		17624
		17625	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		17626	vec_mod(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		17627	return __a % __b;
		17628	}
		17629
16884	/* vec_sldbi */	17630	/* vec_sldbi */
16885		17631
16886	#define vec_sldb(__a, __b, __c) __builtin_altivec_vsldbi(__a, __b, (__c & 0x7))	17632	#define vec_sldb(__a, __b, __c) __builtin_altivec_vsldbi(__a, __b, (__c & 0x7))
...	@@ -17027,6 +17773,92 @@ vec_inserth(vector unsigned int __a, vector unsigned int __b,	...	@@ -17027,6 +17773,92 @@ vec_inserth(vector unsigned int __a, vector unsigned int __b,
17027	#endif	17773	#endif
17028	}	17774	}
17029		17775
		17776	/* vec_extractl */
		17777
		17778	static __inline__ vector unsigned long long __ATTRS_o_ai vec_extractl(
		17779	vector unsigned char __a, vector unsigned char __b, unsigned int __c) {
		17780	#ifdef __LITTLE_ENDIAN__
		17781	return __builtin_altivec_vextdubvrx(__a, __b, __c);
		17782	#else
		17783	vector unsigned long long __ret = __builtin_altivec_vextdubvlx(__a, __b, __c);
		17784	return vec_sld(__ret, __ret, 8);
		17785	#endif
		17786	}
		17787
		17788	static __inline__ vector unsigned long long __ATTRS_o_ai vec_extractl(
		17789	vector unsigned short __a, vector unsigned short __b, unsigned int __c) {
		17790	#ifdef __LITTLE_ENDIAN__
		17791	return __builtin_altivec_vextduhvrx(__a, __b, __c);
		17792	#else
		17793	vector unsigned long long __ret = __builtin_altivec_vextduhvlx(__a, __b, __c);
		17794	return vec_sld(__ret, __ret, 8);
		17795	#endif
		17796	}
		17797
		17798	static __inline__ vector unsigned long long __ATTRS_o_ai vec_extractl(
		17799	vector unsigned int __a, vector unsigned int __b, unsigned int __c) {
		17800	#ifdef __LITTLE_ENDIAN__
		17801	return __builtin_altivec_vextduwvrx(__a, __b, __c);
		17802	#else
		17803	vector unsigned long long __ret = __builtin_altivec_vextduwvlx(__a, __b, __c);
		17804	return vec_sld(__ret, __ret, 8);
		17805	#endif
		17806	}
		17807
		17808	static __inline__ vector unsigned long long __ATTRS_o_ai
		17809	vec_extractl(vector unsigned long long __a, vector unsigned long long __b,
		17810	unsigned int __c) {
		17811	#ifdef __LITTLE_ENDIAN__
		17812	return __builtin_altivec_vextddvrx(__a, __b, __c);
		17813	#else
		17814	vector unsigned long long __ret = __builtin_altivec_vextddvlx(__a, __b, __c);
		17815	return vec_sld(__ret, __ret, 8);
		17816	#endif
		17817	}
		17818
		17819	/* vec_extracth */
		17820
		17821	static __inline__ vector unsigned long long __ATTRS_o_ai vec_extracth(
		17822	vector unsigned char __a, vector unsigned char __b, unsigned int __c) {
		17823	#ifdef __LITTLE_ENDIAN__
		17824	return __builtin_altivec_vextdubvlx(__a, __b, __c);
		17825	#else
		17826	vector unsigned long long __ret = __builtin_altivec_vextdubvrx(__a, __b, __c);
		17827	return vec_sld(__ret, __ret, 8);
		17828	#endif
		17829	}
		17830
		17831	static __inline__ vector unsigned long long __ATTRS_o_ai vec_extracth(
		17832	vector unsigned short __a, vector unsigned short __b, unsigned int __c) {
		17833	#ifdef __LITTLE_ENDIAN__
		17834	return __builtin_altivec_vextduhvlx(__a, __b, __c);
		17835	#else
		17836	vector unsigned long long __ret = __builtin_altivec_vextduhvrx(__a, __b, __c);
		17837	return vec_sld(__ret, __ret, 8);
		17838	#endif
		17839	}
		17840
		17841	static __inline__ vector unsigned long long __ATTRS_o_ai vec_extracth(
		17842	vector unsigned int __a, vector unsigned int __b, unsigned int __c) {
		17843	#ifdef __LITTLE_ENDIAN__
		17844	return __builtin_altivec_vextduwvlx(__a, __b, __c);
		17845	#else
		17846	vector unsigned long long __ret = __builtin_altivec_vextduwvrx(__a, __b, __c);
		17847	return vec_sld(__ret, __ret, 8);
		17848	#endif
		17849	}
		17850
		17851	static __inline__ vector unsigned long long __ATTRS_o_ai
		17852	vec_extracth(vector unsigned long long __a, vector unsigned long long __b,
		17853	unsigned int __c) {
		17854	#ifdef __LITTLE_ENDIAN__
		17855	return __builtin_altivec_vextddvlx(__a, __b, __c);
		17856	#else
		17857	vector unsigned long long __ret = __builtin_altivec_vextddvrx(__a, __b, __c);
		17858	return vec_sld(__ret, __ret, 8);
		17859	#endif
		17860	}
		17861
17030	#ifdef __VSX__	17862	#ifdef __VSX__
17031		17863
17032	/* vec_permx */	17864	/* vec_permx */
...	@@ -17095,6 +17927,14 @@ vec_blendv(vector double __a, vector double __b,	...	@@ -17095,6 +17927,14 @@ vec_blendv(vector double __a, vector double __b,
17095	return __builtin_vsx_xxblendvd(__a, __b, __c);	17927	return __builtin_vsx_xxblendvd(__a, __b, __c);
17096	}	17928	}
17097		17929
		17930	/* vec_replace_elt */
		17931
		17932	#define vec_replace_elt __builtin_altivec_vec_replace_elt
		17933
		17934	/* vec_replace_unaligned */
		17935
		17936	#define vec_replace_unaligned __builtin_altivec_vec_replace_unaligned
		17937
17098	/* vec_splati */	17938	/* vec_splati */
17099		17939
17100	#define vec_splati(__a) \	17940	#define vec_splati(__a) \
...	@@ -17161,6 +18001,197 @@ vec_test_lsbb_all_zeros(vector unsigned char __a) {	...	@@ -17161,6 +18001,197 @@ vec_test_lsbb_all_zeros(vector unsigned char __a) {
17161	return __builtin_vsx_xvtlsbb(__a, 0);	18001	return __builtin_vsx_xvtlsbb(__a, 0);
17162	}	18002	}
17163	#endif /* __VSX__ */	18003	#endif /* __VSX__ */
		18004
		18005	/* vec_stril */
		18006
		18007	static __inline__ vector unsigned char __ATTRS_o_ai
		18008	vec_stril(vector unsigned char __a) {
		18009	#ifdef __LITTLE_ENDIAN__
		18010	return __builtin_altivec_vstribr((vector signed char)__a);
		18011	#else
		18012	return __builtin_altivec_vstribl((vector signed char)__a);
		18013	#endif
		18014	}
		18015
		18016	static __inline__ vector signed char __ATTRS_o_ai
		18017	vec_stril(vector signed char __a) {
		18018	#ifdef __LITTLE_ENDIAN__
		18019	return __builtin_altivec_vstribr(__a);
		18020	#else
		18021	return __builtin_altivec_vstribl(__a);
		18022	#endif
		18023	}
		18024
		18025	static __inline__ vector unsigned short __ATTRS_o_ai
		18026	vec_stril(vector unsigned short __a) {
		18027	#ifdef __LITTLE_ENDIAN__
		18028	return __builtin_altivec_vstrihr((vector signed short)__a);
		18029	#else
		18030	return __builtin_altivec_vstrihl((vector signed short)__a);
		18031	#endif
		18032	}
		18033
		18034	static __inline__ vector signed short __ATTRS_o_ai
		18035	vec_stril(vector signed short __a) {
		18036	#ifdef __LITTLE_ENDIAN__
		18037	return __builtin_altivec_vstrihr(__a);
		18038	#else
		18039	return __builtin_altivec_vstrihl(__a);
		18040	#endif
		18041	}
		18042
		18043	/* vec_stril_p */
		18044
		18045	static __inline__ int __ATTRS_o_ai vec_stril_p(vector unsigned char __a) {
		18046	#ifdef __LITTLE_ENDIAN__
		18047	return __builtin_altivec_vstribr_p(__CR6_EQ, (vector signed char)__a);
		18048	#else
		18049	return __builtin_altivec_vstribl_p(__CR6_EQ, (vector signed char)__a);
		18050	#endif
		18051	}
		18052
		18053	static __inline__ int __ATTRS_o_ai vec_stril_p(vector signed char __a) {
		18054	#ifdef __LITTLE_ENDIAN__
		18055	return __builtin_altivec_vstribr_p(__CR6_EQ, __a);
		18056	#else
		18057	return __builtin_altivec_vstribl_p(__CR6_EQ, __a);
		18058	#endif
		18059	}
		18060
		18061	static __inline__ int __ATTRS_o_ai vec_stril_p(vector unsigned short __a) {
		18062	#ifdef __LITTLE_ENDIAN__
		18063	return __builtin_altivec_vstrihr_p(__CR6_EQ, (vector signed short)__a);
		18064	#else
		18065	return __builtin_altivec_vstrihl_p(__CR6_EQ, (vector signed short)__a);
		18066	#endif
		18067	}
		18068
		18069	static __inline__ int __ATTRS_o_ai vec_stril_p(vector signed short __a) {
		18070	#ifdef __LITTLE_ENDIAN__
		18071	return __builtin_altivec_vstrihr_p(__CR6_EQ, __a);
		18072	#else
		18073	return __builtin_altivec_vstrihl_p(__CR6_EQ, __a);
		18074	#endif
		18075	}
		18076
		18077	/* vec_strir */
		18078
		18079	static __inline__ vector unsigned char __ATTRS_o_ai
		18080	vec_strir(vector unsigned char __a) {
		18081	#ifdef __LITTLE_ENDIAN__
		18082	return __builtin_altivec_vstribl((vector signed char)__a);
		18083	#else
		18084	return __builtin_altivec_vstribr((vector signed char)__a);
		18085	#endif
		18086	}
		18087
		18088	static __inline__ vector signed char __ATTRS_o_ai
		18089	vec_strir(vector signed char __a) {
		18090	#ifdef __LITTLE_ENDIAN__
		18091	return __builtin_altivec_vstribl(__a);
		18092	#else
		18093	return __builtin_altivec_vstribr(__a);
		18094	#endif
		18095	}
		18096
		18097	static __inline__ vector unsigned short __ATTRS_o_ai
		18098	vec_strir(vector unsigned short __a) {
		18099	#ifdef __LITTLE_ENDIAN__
		18100	return __builtin_altivec_vstrihl((vector signed short)__a);
		18101	#else
		18102	return __builtin_altivec_vstrihr((vector signed short)__a);
		18103	#endif
		18104	}
		18105
		18106	static __inline__ vector signed short __ATTRS_o_ai
		18107	vec_strir(vector signed short __a) {
		18108	#ifdef __LITTLE_ENDIAN__
		18109	return __builtin_altivec_vstrihl(__a);
		18110	#else
		18111	return __builtin_altivec_vstrihr(__a);
		18112	#endif
		18113	}
		18114
		18115	/* vec_strir_p */
		18116
		18117	static __inline__ int __ATTRS_o_ai vec_strir_p(vector unsigned char __a) {
		18118	#ifdef __LITTLE_ENDIAN__
		18119	return __builtin_altivec_vstribl_p(__CR6_EQ, (vector signed char)__a);
		18120	#else
		18121	return __builtin_altivec_vstribr_p(__CR6_EQ, (vector signed char)__a);
		18122	#endif
		18123	}
		18124
		18125	static __inline__ int __ATTRS_o_ai vec_strir_p(vector signed char __a) {
		18126	#ifdef __LITTLE_ENDIAN__
		18127	return __builtin_altivec_vstribl_p(__CR6_EQ, __a);
		18128	#else
		18129	return __builtin_altivec_vstribr_p(__CR6_EQ, __a);
		18130	#endif
		18131	}
		18132
		18133	static __inline__ int __ATTRS_o_ai vec_strir_p(vector unsigned short __a) {
		18134	#ifdef __LITTLE_ENDIAN__
		18135	return __builtin_altivec_vstrihl_p(__CR6_EQ, (vector signed short)__a);
		18136	#else
		18137	return __builtin_altivec_vstrihr_p(__CR6_EQ, (vector signed short)__a);
		18138	#endif
		18139	}
		18140
		18141	static __inline__ int __ATTRS_o_ai vec_strir_p(vector signed short __a) {
		18142	#ifdef __LITTLE_ENDIAN__
		18143	return __builtin_altivec_vstrihl_p(__CR6_EQ, __a);
		18144	#else
		18145	return __builtin_altivec_vstrihr_p(__CR6_EQ, __a);
		18146	#endif
		18147	}
		18148
		18149	/* vs[l \| r \| ra] */
		18150
		18151	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		18152	vec_sl(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		18153	return __a << (__b % (vector unsigned __int128)(sizeof(unsigned __int128) *
		18154	__CHAR_BIT__));
		18155	}
		18156
		18157	static __inline__ vector signed __int128 __ATTRS_o_ai
		18158	vec_sl(vector signed __int128 __a, vector unsigned __int128 __b) {
		18159	return __a << (__b % (vector unsigned __int128)(sizeof(unsigned __int128) *
		18160	__CHAR_BIT__));
		18161	}
		18162
		18163	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		18164	vec_sr(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		18165	return __a >> (__b % (vector unsigned __int128)(sizeof(unsigned __int128) *
		18166	__CHAR_BIT__));
		18167	}
		18168
		18169	static __inline__ vector signed __int128 __ATTRS_o_ai
		18170	vec_sr(vector signed __int128 __a, vector unsigned __int128 __b) {
		18171	return (
		18172	vector signed __int128)(((vector unsigned __int128)__a) >>
		18173	(__b %
		18174	(vector unsigned __int128)(sizeof(
		18175	unsigned __int128) *
		18176	__CHAR_BIT__)));
		18177	}
		18178
		18179	static __inline__ vector unsigned __int128 __ATTRS_o_ai
		18180	vec_sra(vector unsigned __int128 __a, vector unsigned __int128 __b) {
		18181	return (
		18182	vector unsigned __int128)(((vector signed __int128)__a) >>
		18183	(__b %
		18184	(vector unsigned __int128)(sizeof(
		18185	unsigned __int128) *
		18186	__CHAR_BIT__)));
		18187	}
		18188
		18189	static __inline__ vector signed __int128 __ATTRS_o_ai
		18190	vec_sra(vector signed __int128 __a, vector unsigned __int128 __b) {
		18191	return __a >> (__b % (vector unsigned __int128)(sizeof(unsigned __int128) *
		18192	__CHAR_BIT__));
		18193	}
		18194
17164	#endif /* __POWER10_VECTOR__ */	18195	#endif /* __POWER10_VECTOR__ */
17165		18196
17166	#undef __ATTRS_o_ai	18197	#undef __ATTRS_o_ai

lib/include/amxintrin.h+70-22

 @@ -15,8 +15,8 @@
 #define __AMXINTRIN_H
 #ifdef __x86_64__
-#define __DEFAULT_FN_ATTRS \
+#define __DEFAULT_FN_ATTRS_TILE                                                \
-  __attribute__((__always_inline__, __nodebug__,  __target__("amx-tile")))
+  __attribute__((__always_inline__, __nodebug__, __target__("amx-tile")))
 /// Load tile configuration from a 64-byte memory location specified by
 /// "mem_addr". The tile configuration includes the tile type palette, the
 @@ -31,9 +31,8 @@
 ///
 /// \param __config
 ///    A pointer to 512-bits configuration
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS_TILE
-_tile_loadconfig(const void *__config)
+_tile_loadconfig(const void *__config) {
-{
   __builtin_ia32_tile_loadconfig(__config);
 }
 @@ -48,9 +47,8 @@ _tile_loadconfig(const void *__config)
 ///
 /// \param __config
 ///    A pointer to 512-bits configuration
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS_TILE
-_tile_storeconfig(void *__config)
+_tile_storeconfig(void *__config) {
-{
   __builtin_ia32_tile_storeconfig(__config);
 }
 @@ -60,9 +58,7 @@ _tile_storeconfig(void *__config)
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> TILERELEASE </c> instruction.
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS_TILE _tile_release(void) {
-_tile_release(void)
-{
   __builtin_ia32_tilerelease();
 }
 @@ -80,8 +76,9 @@ _tile_release(void)
 ///    A pointer to base address.
 /// \param stride
 ///    The stride between the rows' data to be loaded in memory.
-#define _tile_loadd(dst, base, stride) \
+#define _tile_loadd(dst, base, stride)                                         \
-  __builtin_ia32_tileloadd64((dst), ((const void *)(base)), (__SIZE_TYPE__)(stride))
+  __builtin_ia32_tileloadd64((dst), ((const void *)(base)),                    \
+                             (__SIZE_TYPE__)(stride))
 /// Load tile rows from memory specifieid by "base" address and "stride" into
 /// destination tile "dst" using the tile configuration previously configured
 @@ -99,8 +96,9 @@ _tile_release(void)
 ///    A pointer to base address.
 /// \param stride
 ///    The stride between the rows' data to be loaded in memory.
-#define _tile_stream_loadd(dst, base, stride) \
+#define _tile_stream_loadd(dst, base, stride)                                  \
-  __builtin_ia32_tileloaddt164((dst), ((const void *)(base)), (__SIZE_TYPE__)(stride))
+  __builtin_ia32_tileloaddt164((dst), ((const void *)(base)),                  \
+                               (__SIZE_TYPE__)(stride))
 /// Store the tile specified by "src" to memory specifieid by "base" address and
 /// "stride" using the tile configuration previously configured via
 @@ -116,7 +114,7 @@ _tile_release(void)
 ///    A pointer to base address.
 /// \param stride
 ///    The stride between the rows' data to be stored in memory.
-#define _tile_stored(dst, base, stride) \
+#define _tile_stored(dst, base, stride)                                        \
   __builtin_ia32_tilestored64((dst), ((void *)(base)), (__SIZE_TYPE__)(stride))
 /// Zero the tile specified by "tdest".
 @@ -145,7 +143,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbssd(dst, src0, src1) __builtin_ia32_tdpbssd((dst), (src0), (src1))
+#define _tile_dpbssd(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbssd((dst), (src0), (src1))
 /// Compute dot-product of bytes in tiles with a source/destination accumulator.
 /// Multiply groups of 4 adjacent pairs of signed 8-bit integers in src0 with
 @@ -163,7 +162,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbsud(dst, src0, src1) __builtin_ia32_tdpbsud((dst), (src0), (src1))
+#define _tile_dpbsud(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbsud((dst), (src0), (src1))
 /// Compute dot-product of bytes in tiles with a source/destination accumulator.
 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in src0 with
 @@ -181,7 +181,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbusd(dst, src0, src1) __builtin_ia32_tdpbusd((dst), (src0), (src1))
+#define _tile_dpbusd(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbusd((dst), (src0), (src1))
 /// Compute dot-product of bytes in tiles with a source/destination accumulator.
 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in src0 with
 @@ -199,7 +200,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbuud(dst, src0, src1) __builtin_ia32_tdpbuud((dst), (src0), (src1))
+#define _tile_dpbuud(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbuud((dst), (src0), (src1))
 /// Compute dot-product of BF16 (16-bit) floating-point pairs in tiles src0 and
 /// src1, accumulating the intermediate single-precision (32-bit) floating-point
 @@ -216,10 +218,56 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbf16ps(dst, src0, src1) \
+#define _tile_dpbf16ps(dst, src0, src1)                                        \
   __builtin_ia32_tdpbf16ps((dst), (src0), (src1))
-#undef __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS_INT8                                                \
+  __attribute__((__always_inline__, __nodebug__, __target__("amx-int8")))
+typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
+_tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
+                     __SIZE_TYPE__ stride) {
+  return __builtin_ia32_tileloadd64_internal(m, n, base,
+                                             (__SIZE_TYPE__)(stride));
+}
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
+_tile_dpbssd_internal(unsigned short m, unsigned short n, unsigned short k,
+                      _tile1024i dst, _tile1024i src1, _tile1024i src2) {
+  return __builtin_ia32_tdpbssd_internal(m, n, k, dst, src1, src2);
+}
+static __inline__ void __DEFAULT_FN_ATTRS_INT8
+_tile_stored_internal(unsigned short m, unsigned short n, void *base,
+                      __SIZE_TYPE__ stride, _tile1024i tile) {
+  return __builtin_ia32_tilestored64_internal(m, n, base,
+                                              (__SIZE_TYPE__)(stride), tile);
+}
+typedef struct __tile1024i_str {
+  const unsigned short row;
+  const unsigned short col;
+  _tile1024i tile;
+} __tile1024i;
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_loadd(__tile1024i *dst, const void *base,
+                         __SIZE_TYPE__ stride) {
+  dst->tile = _tile_loadd_internal(dst->row, dst->col, base, stride);
+}
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_dpbsud(__tile1024i *dst, __tile1024i src1,
+                          __tile1024i src2) {
+  dst->tile = _tile_dpbssd_internal(src1.row, src2.col, src1.col, dst->tile,
+                                    src1.tile, src2.tile);
+}
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_stored(void *base, __SIZE_TYPE__ stride, __tile1024i src) {
+  _tile_stored_internal(src.row, src.col, base, stride, src.tile);
+}
 #endif /* __x86_64__ */
 #endif /* __AMXINTRIN_H */

lib/include/arm_neon.h+441-127

...	@@ -40429,6 +40429,150 @@ __ai float32x4_t vcaddq_rot90_f32(float32x4_t __p0, float32x4_t __p1) {	...	@@ -40429,6 +40429,150 @@ __ai float32x4_t vcaddq_rot90_f32(float32x4_t __p0, float32x4_t __p1) {
40429	}	40429	}
40430	#endif	40430	#endif
40431		40431
		40432	#ifdef __LITTLE_ENDIAN__
		40433	__ai float32x4_t vcmlaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40434	float32x4_t __ret;
		40435	__ret = (float32x4_t) __builtin_neon_vcmlaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
		40436	return __ret;
		40437	}
		40438	#else
		40439	__ai float32x4_t vcmlaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40440	float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40441	float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40442	float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40443	float32x4_t __ret;
		40444	__ret = (float32x4_t) __builtin_neon_vcmlaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41);
		40445	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40446	return __ret;
		40447	}
		40448	#endif
		40449
		40450	#ifdef __LITTLE_ENDIAN__
		40451	__ai float32x2_t vcmla_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40452	float32x2_t __ret;
		40453	__ret = (float32x2_t) __builtin_neon_vcmla_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
		40454	return __ret;
		40455	}
		40456	#else
		40457	__ai float32x2_t vcmla_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40458	float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40459	float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40460	float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40461	float32x2_t __ret;
		40462	__ret = (float32x2_t) __builtin_neon_vcmla_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9);
		40463	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40464	return __ret;
		40465	}
		40466	#endif
		40467
		40468	#ifdef __LITTLE_ENDIAN__
		40469	__ai float32x4_t vcmlaq_rot180_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40470	float32x4_t __ret;
		40471	__ret = (float32x4_t) __builtin_neon_vcmlaq_rot180_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
		40472	return __ret;
		40473	}
		40474	#else
		40475	__ai float32x4_t vcmlaq_rot180_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40476	float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40477	float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40478	float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40479	float32x4_t __ret;
		40480	__ret = (float32x4_t) __builtin_neon_vcmlaq_rot180_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41);
		40481	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40482	return __ret;
		40483	}
		40484	#endif
		40485
		40486	#ifdef __LITTLE_ENDIAN__
		40487	__ai float32x2_t vcmla_rot180_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40488	float32x2_t __ret;
		40489	__ret = (float32x2_t) __builtin_neon_vcmla_rot180_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
		40490	return __ret;
		40491	}
		40492	#else
		40493	__ai float32x2_t vcmla_rot180_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40494	float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40495	float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40496	float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40497	float32x2_t __ret;
		40498	__ret = (float32x2_t) __builtin_neon_vcmla_rot180_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9);
		40499	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40500	return __ret;
		40501	}
		40502	#endif
		40503
		40504	#ifdef __LITTLE_ENDIAN__
		40505	__ai float32x4_t vcmlaq_rot270_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40506	float32x4_t __ret;
		40507	__ret = (float32x4_t) __builtin_neon_vcmlaq_rot270_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
		40508	return __ret;
		40509	}
		40510	#else
		40511	__ai float32x4_t vcmlaq_rot270_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40512	float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40513	float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40514	float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40515	float32x4_t __ret;
		40516	__ret = (float32x4_t) __builtin_neon_vcmlaq_rot270_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41);
		40517	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40518	return __ret;
		40519	}
		40520	#endif
		40521
		40522	#ifdef __LITTLE_ENDIAN__
		40523	__ai float32x2_t vcmla_rot270_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40524	float32x2_t __ret;
		40525	__ret = (float32x2_t) __builtin_neon_vcmla_rot270_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
		40526	return __ret;
		40527	}
		40528	#else
		40529	__ai float32x2_t vcmla_rot270_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40530	float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40531	float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40532	float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40533	float32x2_t __ret;
		40534	__ret = (float32x2_t) __builtin_neon_vcmla_rot270_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9);
		40535	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40536	return __ret;
		40537	}
		40538	#endif
		40539
		40540	#ifdef __LITTLE_ENDIAN__
		40541	__ai float32x4_t vcmlaq_rot90_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40542	float32x4_t __ret;
		40543	__ret = (float32x4_t) __builtin_neon_vcmlaq_rot90_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
		40544	return __ret;
		40545	}
		40546	#else
		40547	__ai float32x4_t vcmlaq_rot90_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) {
		40548	float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40549	float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40550	float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40551	float32x4_t __ret;
		40552	__ret = (float32x4_t) __builtin_neon_vcmlaq_rot90_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41);
		40553	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40554	return __ret;
		40555	}
		40556	#endif
		40557
		40558	#ifdef __LITTLE_ENDIAN__
		40559	__ai float32x2_t vcmla_rot90_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40560	float32x2_t __ret;
		40561	__ret = (float32x2_t) __builtin_neon_vcmla_rot90_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
		40562	return __ret;
		40563	}
		40564	#else
		40565	__ai float32x2_t vcmla_rot90_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) {
		40566	float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40567	float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40568	float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40569	float32x2_t __ret;
		40570	__ret = (float32x2_t) __builtin_neon_vcmla_rot90_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9);
		40571	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40572	return __ret;
		40573	}
		40574	#endif
		40575
40432	#endif	40576	#endif
40433	#if defined(__ARM_FEATURE_COMPLEX) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)	40577	#if defined(__ARM_FEATURE_COMPLEX) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
40434	#ifdef __LITTLE_ENDIAN__	40578	#ifdef __LITTLE_ENDIAN__
...	@@ -40499,6 +40643,150 @@ __ai float16x8_t vcaddq_rot90_f16(float16x8_t __p0, float16x8_t __p1) {	...	@@ -40499,6 +40643,150 @@ __ai float16x8_t vcaddq_rot90_f16(float16x8_t __p0, float16x8_t __p1) {
40499	}	40643	}
40500	#endif	40644	#endif
40501		40645
		40646	#ifdef __LITTLE_ENDIAN__
		40647	__ai float16x8_t vcmlaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40648	float16x8_t __ret;
		40649	__ret = (float16x8_t) __builtin_neon_vcmlaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40);
		40650	return __ret;
		40651	}
		40652	#else
		40653	__ai float16x8_t vcmlaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40654	float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
		40655	float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
		40656	float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
		40657	float16x8_t __ret;
		40658	__ret = (float16x8_t) __builtin_neon_vcmlaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40);
		40659	__ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
		40660	return __ret;
		40661	}
		40662	#endif
		40663
		40664	#ifdef __LITTLE_ENDIAN__
		40665	__ai float16x4_t vcmla_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40666	float16x4_t __ret;
		40667	__ret = (float16x4_t) __builtin_neon_vcmla_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8);
		40668	return __ret;
		40669	}
		40670	#else
		40671	__ai float16x4_t vcmla_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40672	float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40673	float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40674	float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40675	float16x4_t __ret;
		40676	__ret = (float16x4_t) __builtin_neon_vcmla_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8);
		40677	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40678	return __ret;
		40679	}
		40680	#endif
		40681
		40682	#ifdef __LITTLE_ENDIAN__
		40683	__ai float16x8_t vcmlaq_rot180_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40684	float16x8_t __ret;
		40685	__ret = (float16x8_t) __builtin_neon_vcmlaq_rot180_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40);
		40686	return __ret;
		40687	}
		40688	#else
		40689	__ai float16x8_t vcmlaq_rot180_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40690	float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
		40691	float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
		40692	float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
		40693	float16x8_t __ret;
		40694	__ret = (float16x8_t) __builtin_neon_vcmlaq_rot180_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40);
		40695	__ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
		40696	return __ret;
		40697	}
		40698	#endif
		40699
		40700	#ifdef __LITTLE_ENDIAN__
		40701	__ai float16x4_t vcmla_rot180_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40702	float16x4_t __ret;
		40703	__ret = (float16x4_t) __builtin_neon_vcmla_rot180_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8);
		40704	return __ret;
		40705	}
		40706	#else
		40707	__ai float16x4_t vcmla_rot180_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40708	float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40709	float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40710	float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40711	float16x4_t __ret;
		40712	__ret = (float16x4_t) __builtin_neon_vcmla_rot180_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8);
		40713	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40714	return __ret;
		40715	}
		40716	#endif
		40717
		40718	#ifdef __LITTLE_ENDIAN__
		40719	__ai float16x8_t vcmlaq_rot270_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40720	float16x8_t __ret;
		40721	__ret = (float16x8_t) __builtin_neon_vcmlaq_rot270_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40);
		40722	return __ret;
		40723	}
		40724	#else
		40725	__ai float16x8_t vcmlaq_rot270_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40726	float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
		40727	float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
		40728	float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
		40729	float16x8_t __ret;
		40730	__ret = (float16x8_t) __builtin_neon_vcmlaq_rot270_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40);
		40731	__ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
		40732	return __ret;
		40733	}
		40734	#endif
		40735
		40736	#ifdef __LITTLE_ENDIAN__
		40737	__ai float16x4_t vcmla_rot270_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40738	float16x4_t __ret;
		40739	__ret = (float16x4_t) __builtin_neon_vcmla_rot270_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8);
		40740	return __ret;
		40741	}
		40742	#else
		40743	__ai float16x4_t vcmla_rot270_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40744	float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40745	float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40746	float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40747	float16x4_t __ret;
		40748	__ret = (float16x4_t) __builtin_neon_vcmla_rot270_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8);
		40749	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40750	return __ret;
		40751	}
		40752	#endif
		40753
		40754	#ifdef __LITTLE_ENDIAN__
		40755	__ai float16x8_t vcmlaq_rot90_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40756	float16x8_t __ret;
		40757	__ret = (float16x8_t) __builtin_neon_vcmlaq_rot90_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40);
		40758	return __ret;
		40759	}
		40760	#else
		40761	__ai float16x8_t vcmlaq_rot90_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) {
		40762	float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
		40763	float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
		40764	float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
		40765	float16x8_t __ret;
		40766	__ret = (float16x8_t) __builtin_neon_vcmlaq_rot90_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40);
		40767	__ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
		40768	return __ret;
		40769	}
		40770	#endif
		40771
		40772	#ifdef __LITTLE_ENDIAN__
		40773	__ai float16x4_t vcmla_rot90_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40774	float16x4_t __ret;
		40775	__ret = (float16x4_t) __builtin_neon_vcmla_rot90_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8);
		40776	return __ret;
		40777	}
		40778	#else
		40779	__ai float16x4_t vcmla_rot90_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) {
		40780	float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
		40781	float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
		40782	float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
		40783	float16x4_t __ret;
		40784	__ret = (float16x4_t) __builtin_neon_vcmla_rot90_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8);
		40785	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
		40786	return __ret;
		40787	}
		40788	#endif
		40789
40502	#endif	40790	#endif
40503	#if defined(__ARM_FEATURE_COMPLEX) && defined(__aarch64__)	40791	#if defined(__ARM_FEATURE_COMPLEX) && defined(__aarch64__)
40504	#ifdef __LITTLE_ENDIAN__	40792	#ifdef __LITTLE_ENDIAN__
...	@@ -40535,6 +40823,98 @@ __ai float64x2_t vcaddq_rot90_f64(float64x2_t __p0, float64x2_t __p1) {	...	@@ -40535,6 +40823,98 @@ __ai float64x2_t vcaddq_rot90_f64(float64x2_t __p0, float64x2_t __p1) {
40535	}	40823	}
40536	#endif	40824	#endif
40537		40825
		40826	#ifdef __LITTLE_ENDIAN__
		40827	__ai float64x2_t vcmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40828	float64x2_t __ret;
		40829	__ret = (float64x2_t) __builtin_neon_vcmlaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
		40830	return __ret;
		40831	}
		40832	#else
		40833	__ai float64x2_t vcmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40834	float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40835	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40836	float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40837	float64x2_t __ret;
		40838	__ret = (float64x2_t) __builtin_neon_vcmlaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42);
		40839	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40840	return __ret;
		40841	}
		40842	#endif
		40843
		40844	__ai float64x1_t vcmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
		40845	float64x1_t __ret;
		40846	__ret = (float64x1_t) __builtin_neon_vcmla_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
		40847	return __ret;
		40848	}
		40849	#ifdef __LITTLE_ENDIAN__
		40850	__ai float64x2_t vcmlaq_rot180_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40851	float64x2_t __ret;
		40852	__ret = (float64x2_t) __builtin_neon_vcmlaq_rot180_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
		40853	return __ret;
		40854	}
		40855	#else
		40856	__ai float64x2_t vcmlaq_rot180_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40857	float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40858	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40859	float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40860	float64x2_t __ret;
		40861	__ret = (float64x2_t) __builtin_neon_vcmlaq_rot180_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42);
		40862	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40863	return __ret;
		40864	}
		40865	#endif
		40866
		40867	__ai float64x1_t vcmla_rot180_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
		40868	float64x1_t __ret;
		40869	__ret = (float64x1_t) __builtin_neon_vcmla_rot180_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
		40870	return __ret;
		40871	}
		40872	#ifdef __LITTLE_ENDIAN__
		40873	__ai float64x2_t vcmlaq_rot270_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40874	float64x2_t __ret;
		40875	__ret = (float64x2_t) __builtin_neon_vcmlaq_rot270_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
		40876	return __ret;
		40877	}
		40878	#else
		40879	__ai float64x2_t vcmlaq_rot270_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40880	float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40881	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40882	float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40883	float64x2_t __ret;
		40884	__ret = (float64x2_t) __builtin_neon_vcmlaq_rot270_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42);
		40885	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40886	return __ret;
		40887	}
		40888	#endif
		40889
		40890	__ai float64x1_t vcmla_rot270_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
		40891	float64x1_t __ret;
		40892	__ret = (float64x1_t) __builtin_neon_vcmla_rot270_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
		40893	return __ret;
		40894	}
		40895	#ifdef __LITTLE_ENDIAN__
		40896	__ai float64x2_t vcmlaq_rot90_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40897	float64x2_t __ret;
		40898	__ret = (float64x2_t) __builtin_neon_vcmlaq_rot90_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
		40899	return __ret;
		40900	}
		40901	#else
		40902	__ai float64x2_t vcmlaq_rot90_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
		40903	float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
		40904	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
		40905	float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0);
		40906	float64x2_t __ret;
		40907	__ret = (float64x2_t) __builtin_neon_vcmlaq_rot90_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42);
		40908	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
		40909	return __ret;
		40910	}
		40911	#endif
		40912
		40913	__ai float64x1_t vcmla_rot90_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
		40914	float64x1_t __ret;
		40915	__ret = (float64x1_t) __builtin_neon_vcmla_rot90_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
		40916	return __ret;
		40917	}
40538	#endif	40918	#endif
40539	#if defined(__ARM_FEATURE_DOTPROD)	40919	#if defined(__ARM_FEATURE_DOTPROD)
40540	#ifdef __LITTLE_ENDIAN__	40920	#ifdef __LITTLE_ENDIAN__
...	@@ -45860,9 +46240,9 @@ __ai uint64_t vceqd_u64(uint64_t __p0, uint64_t __p1) {	...	@@ -45860,9 +46240,9 @@ __ai uint64_t vceqd_u64(uint64_t __p0, uint64_t __p1) {
45860	__ret = (uint64_t) __builtin_neon_vceqd_u64(__p0, __p1);	46240	__ret = (uint64_t) __builtin_neon_vceqd_u64(__p0, __p1);
45861	return __ret;	46241	return __ret;
45862	}	46242	}
45863	__ai int64_t vceqd_s64(int64_t __p0, int64_t __p1) {	46243	__ai uint64_t vceqd_s64(int64_t __p0, int64_t __p1) {
45864	int64_t __ret;	46244	uint64_t __ret;
45865	__ret = (int64_t) __builtin_neon_vceqd_s64(__p0, __p1);	46245	__ret = (uint64_t) __builtin_neon_vceqd_s64(__p0, __p1);
45866	return __ret;	46246	return __ret;
45867	}	46247	}
45868	__ai uint64_t vceqd_f64(float64_t __p0, float64_t __p1) {	46248	__ai uint64_t vceqd_f64(float64_t __p0, float64_t __p1) {
...	@@ -45896,22 +46276,6 @@ __ai uint64x1_t vceqz_p64(poly64x1_t __p0) {	...	@@ -45896,22 +46276,6 @@ __ai uint64x1_t vceqz_p64(poly64x1_t __p0) {
45896	__ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19);	46276	__ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19);
45897	return __ret;	46277	return __ret;
45898	}	46278	}
45899	#ifdef __LITTLE_ENDIAN__
45900	__ai uint16x4_t vceqz_p16(poly16x4_t __p0) {
45901	uint16x4_t __ret;
45902	__ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17);
45903	return __ret;
45904	}
45905	#else
45906	__ai uint16x4_t vceqz_p16(poly16x4_t __p0) {
45907	poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
45908	uint16x4_t __ret;
45909	__ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17);
45910	__ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
45911	return __ret;
45912	}
45913	#endif
45914
45915	#ifdef __LITTLE_ENDIAN__	46279	#ifdef __LITTLE_ENDIAN__
45916	__ai uint8x16_t vceqzq_p8(poly8x16_t __p0) {	46280	__ai uint8x16_t vceqzq_p8(poly8x16_t __p0) {
45917	uint8x16_t __ret;	46281	uint8x16_t __ret;
...	@@ -45944,22 +46308,6 @@ __ai uint64x2_t vceqzq_p64(poly64x2_t __p0) {	...	@@ -45944,22 +46308,6 @@ __ai uint64x2_t vceqzq_p64(poly64x2_t __p0) {
45944	}	46308	}
45945	#endif	46309	#endif
45946		46310
45947	#ifdef __LITTLE_ENDIAN__
45948	__ai uint16x8_t vceqzq_p16(poly16x8_t __p0) {
45949	uint16x8_t __ret;
45950	__ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49);
45951	return __ret;
45952	}
45953	#else
45954	__ai uint16x8_t vceqzq_p16(poly16x8_t __p0) {
45955	poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
45956	uint16x8_t __ret;
45957	__ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49);
45958	__ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
45959	return __ret;
45960	}
45961	#endif
45962
45963	#ifdef __LITTLE_ENDIAN__	46311	#ifdef __LITTLE_ENDIAN__
45964	__ai uint8x16_t vceqzq_u8(uint8x16_t __p0) {	46312	__ai uint8x16_t vceqzq_u8(uint8x16_t __p0) {
45965	uint8x16_t __ret;	46313	uint8x16_t __ret;
...	@@ -46252,9 +46600,9 @@ __ai uint64_t vceqzd_u64(uint64_t __p0) {	...	@@ -46252,9 +46600,9 @@ __ai uint64_t vceqzd_u64(uint64_t __p0) {
46252	__ret = (uint64_t) __builtin_neon_vceqzd_u64(__p0);	46600	__ret = (uint64_t) __builtin_neon_vceqzd_u64(__p0);
46253	return __ret;	46601	return __ret;
46254	}	46602	}
46255	__ai int64_t vceqzd_s64(int64_t __p0) {	46603	__ai uint64_t vceqzd_s64(int64_t __p0) {
46256	int64_t __ret;	46604	uint64_t __ret;
46257	__ret = (int64_t) __builtin_neon_vceqzd_s64(__p0);	46605	__ret = (uint64_t) __builtin_neon_vceqzd_s64(__p0);
46258	return __ret;	46606	return __ret;
46259	}	46607	}
46260	__ai uint64_t vceqzd_f64(float64_t __p0) {	46608	__ai uint64_t vceqzd_f64(float64_t __p0) {
...	@@ -46333,9 +46681,9 @@ __ai uint64x1_t vcge_s64(int64x1_t __p0, int64x1_t __p1) {	...	@@ -46333,9 +46681,9 @@ __ai uint64x1_t vcge_s64(int64x1_t __p0, int64x1_t __p1) {
46333	__ret = (uint64x1_t)(__p0 >= __p1);	46681	__ret = (uint64x1_t)(__p0 >= __p1);
46334	return __ret;	46682	return __ret;
46335	}	46683	}
46336	__ai int64_t vcged_s64(int64_t __p0, int64_t __p1) {	46684	__ai uint64_t vcged_s64(int64_t __p0, int64_t __p1) {
46337	int64_t __ret;	46685	uint64_t __ret;
46338	__ret = (int64_t) __builtin_neon_vcged_s64(__p0, __p1);	46686	__ret = (uint64_t) __builtin_neon_vcged_s64(__p0, __p1);
46339	return __ret;	46687	return __ret;
46340	}	46688	}
46341	__ai uint64_t vcged_u64(uint64_t __p0, uint64_t __p1) {	46689	__ai uint64_t vcged_u64(uint64_t __p0, uint64_t __p1) {
...	@@ -46523,9 +46871,9 @@ __ai uint16x4_t vcgez_s16(int16x4_t __p0) {	...	@@ -46523,9 +46871,9 @@ __ai uint16x4_t vcgez_s16(int16x4_t __p0) {
46523	}	46871	}
46524	#endif	46872	#endif
46525		46873
46526	__ai int64_t vcgezd_s64(int64_t __p0) {	46874	__ai uint64_t vcgezd_s64(int64_t __p0) {
46527	int64_t __ret;	46875	uint64_t __ret;
46528	__ret = (int64_t) __builtin_neon_vcgezd_s64(__p0);	46876	__ret = (uint64_t) __builtin_neon_vcgezd_s64(__p0);
46529	return __ret;	46877	return __ret;
46530	}	46878	}
46531	__ai uint64_t vcgezd_f64(float64_t __p0) {	46879	__ai uint64_t vcgezd_f64(float64_t __p0) {
...	@@ -46604,9 +46952,9 @@ __ai uint64x1_t vcgt_s64(int64x1_t __p0, int64x1_t __p1) {	...	@@ -46604,9 +46952,9 @@ __ai uint64x1_t vcgt_s64(int64x1_t __p0, int64x1_t __p1) {
46604	__ret = (uint64x1_t)(__p0 > __p1);	46952	__ret = (uint64x1_t)(__p0 > __p1);
46605	return __ret;	46953	return __ret;
46606	}	46954	}
46607	__ai int64_t vcgtd_s64(int64_t __p0, int64_t __p1) {	46955	__ai uint64_t vcgtd_s64(int64_t __p0, int64_t __p1) {
46608	int64_t __ret;	46956	uint64_t __ret;
46609	__ret = (int64_t) __builtin_neon_vcgtd_s64(__p0, __p1);	46957	__ret = (uint64_t) __builtin_neon_vcgtd_s64(__p0, __p1);
46610	return __ret;	46958	return __ret;
46611	}	46959	}
46612	__ai uint64_t vcgtd_u64(uint64_t __p0, uint64_t __p1) {	46960	__ai uint64_t vcgtd_u64(uint64_t __p0, uint64_t __p1) {
...	@@ -46794,9 +47142,9 @@ __ai uint16x4_t vcgtz_s16(int16x4_t __p0) {	...	@@ -46794,9 +47142,9 @@ __ai uint16x4_t vcgtz_s16(int16x4_t __p0) {
46794	}	47142	}
46795	#endif	47143	#endif
46796		47144
46797	__ai int64_t vcgtzd_s64(int64_t __p0) {	47145	__ai uint64_t vcgtzd_s64(int64_t __p0) {
46798	int64_t __ret;	47146	uint64_t __ret;
46799	__ret = (int64_t) __builtin_neon_vcgtzd_s64(__p0);	47147	__ret = (uint64_t) __builtin_neon_vcgtzd_s64(__p0);
46800	return __ret;	47148	return __ret;
46801	}	47149	}
46802	__ai uint64_t vcgtzd_f64(float64_t __p0) {	47150	__ai uint64_t vcgtzd_f64(float64_t __p0) {
...	@@ -46880,9 +47228,9 @@ __ai uint64_t vcled_u64(uint64_t __p0, uint64_t __p1) {	...	@@ -46880,9 +47228,9 @@ __ai uint64_t vcled_u64(uint64_t __p0, uint64_t __p1) {
46880	__ret = (uint64_t) __builtin_neon_vcled_u64(__p0, __p1);	47228	__ret = (uint64_t) __builtin_neon_vcled_u64(__p0, __p1);
46881	return __ret;	47229	return __ret;
46882	}	47230	}
46883	__ai int64_t vcled_s64(int64_t __p0, int64_t __p1) {	47231	__ai uint64_t vcled_s64(int64_t __p0, int64_t __p1) {
46884	int64_t __ret;	47232	uint64_t __ret;
46885	__ret = (int64_t) __builtin_neon_vcled_s64(__p0, __p1);	47233	__ret = (uint64_t) __builtin_neon_vcled_s64(__p0, __p1);
46886	return __ret;	47234	return __ret;
46887	}	47235	}
46888	__ai uint64_t vcled_f64(float64_t __p0, float64_t __p1) {	47236	__ai uint64_t vcled_f64(float64_t __p0, float64_t __p1) {
...	@@ -47065,9 +47413,9 @@ __ai uint16x4_t vclez_s16(int16x4_t __p0) {	...	@@ -47065,9 +47413,9 @@ __ai uint16x4_t vclez_s16(int16x4_t __p0) {
47065	}	47413	}
47066	#endif	47414	#endif
47067		47415
47068	__ai int64_t vclezd_s64(int64_t __p0) {	47416	__ai uint64_t vclezd_s64(int64_t __p0) {
47069	int64_t __ret;	47417	uint64_t __ret;
47070	__ret = (int64_t) __builtin_neon_vclezd_s64(__p0);	47418	__ret = (uint64_t) __builtin_neon_vclezd_s64(__p0);
47071	return __ret;	47419	return __ret;
47072	}	47420	}
47073	__ai uint64_t vclezd_f64(float64_t __p0) {	47421	__ai uint64_t vclezd_f64(float64_t __p0) {
...	@@ -47151,9 +47499,9 @@ __ai uint64_t vcltd_u64(uint64_t __p0, uint64_t __p1) {	...	@@ -47151,9 +47499,9 @@ __ai uint64_t vcltd_u64(uint64_t __p0, uint64_t __p1) {
47151	__ret = (uint64_t) __builtin_neon_vcltd_u64(__p0, __p1);	47499	__ret = (uint64_t) __builtin_neon_vcltd_u64(__p0, __p1);
47152	return __ret;	47500	return __ret;
47153	}	47501	}
47154	__ai int64_t vcltd_s64(int64_t __p0, int64_t __p1) {	47502	__ai uint64_t vcltd_s64(int64_t __p0, int64_t __p1) {
47155	int64_t __ret;	47503	uint64_t __ret;
47156	__ret = (int64_t) __builtin_neon_vcltd_s64(__p0, __p1);	47504	__ret = (uint64_t) __builtin_neon_vcltd_s64(__p0, __p1);
47157	return __ret;	47505	return __ret;
47158	}	47506	}
47159	__ai uint64_t vcltd_f64(float64_t __p0, float64_t __p1) {	47507	__ai uint64_t vcltd_f64(float64_t __p0, float64_t __p1) {
...	@@ -47336,9 +47684,9 @@ __ai uint16x4_t vcltz_s16(int16x4_t __p0) {	...	@@ -47336,9 +47684,9 @@ __ai uint16x4_t vcltz_s16(int16x4_t __p0) {
47336	}	47684	}
47337	#endif	47685	#endif
47338		47686
47339	__ai int64_t vcltzd_s64(int64_t __p0) {	47687	__ai uint64_t vcltzd_s64(int64_t __p0) {
47340	int64_t __ret;	47688	uint64_t __ret;
47341	__ret = (int64_t) __builtin_neon_vcltzd_s64(__p0);	47689	__ret = (uint64_t) __builtin_neon_vcltzd_s64(__p0);
47342	return __ret;	47690	return __ret;
47343	}	47691	}
47344	__ai uint64_t vcltzd_f64(float64_t __p0) {	47692	__ai uint64_t vcltzd_f64(float64_t __p0) {
...	@@ -52787,23 +53135,6 @@ __ai float64x1_t vmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2)	...	@@ -52787,23 +53135,6 @@ __ai float64x1_t vmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2)
52787	})	53135	})
52788	#endif	53136	#endif
52789		53137
52790	#ifdef __LITTLE_ENDIAN__
52791	__ai float64x2_t vmlaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) {
52792	float64x2_t __ret;
52793	__ret = __p0 + __p1 * (float64x2_t) {__p2, __p2};
52794	return __ret;
52795	}
52796	#else
52797	__ai float64x2_t vmlaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) {
52798	float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
52799	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
52800	float64x2_t __ret;
52801	__ret = __rev0 + __rev1 * (float64x2_t) {__p2, __p2};
52802	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
52803	return __ret;
52804	}
52805	#endif
52806
52807	#ifdef __LITTLE_ENDIAN__	53138	#ifdef __LITTLE_ENDIAN__
52808	#define vmlal_high_lane_u32(__p0_443, __p1_443, __p2_443, __p3_443) __extension__ ({ \	53139	#define vmlal_high_lane_u32(__p0_443, __p1_443, __p2_443, __p3_443) __extension__ ({ \
52809	uint64x2_t __s0_443 = __p0_443; \	53140	uint64x2_t __s0_443 = __p0_443; \
...	@@ -53355,23 +53686,6 @@ __ai float64x1_t vmls_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2)	...	@@ -53355,23 +53686,6 @@ __ai float64x1_t vmls_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2)
53355	})	53686	})
53356	#endif	53687	#endif
53357		53688
53358	#ifdef __LITTLE_ENDIAN__
53359	__ai float64x2_t vmlsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) {
53360	float64x2_t __ret;
53361	__ret = __p0 - __p1 * (float64x2_t) {__p2, __p2};
53362	return __ret;
53363	}
53364	#else
53365	__ai float64x2_t vmlsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) {
53366	float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
53367	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
53368	float64x2_t __ret;
53369	__ret = __rev0 - __rev1 * (float64x2_t) {__p2, __p2};
53370	__ret = __builtin_shufflevector(__ret, __ret, 1, 0);
53371	return __ret;
53372	}
53373	#endif
53374
53375	#ifdef __LITTLE_ENDIAN__	53689	#ifdef __LITTLE_ENDIAN__
53376	#define vmlsl_high_lane_u32(__p0_487, __p1_487, __p2_487, __p3_487) __extension__ ({ \	53690	#define vmlsl_high_lane_u32(__p0_487, __p1_487, __p2_487, __p3_487) __extension__ ({ \
53377	uint64x2_t __s0_487 = __p0_487; \	53691	uint64x2_t __s0_487 = __p0_487; \
...	@@ -57188,30 +57502,30 @@ __ai int8x16_t vqmovn_high_s16(int8x8_t __p0, int16x8_t __p1) {	...	@@ -57188,30 +57502,30 @@ __ai int8x16_t vqmovn_high_s16(int8x8_t __p0, int16x8_t __p1) {
57188	}	57502	}
57189	#endif	57503	#endif
57190		57504
57191	__ai int16_t vqmovuns_s32(int32_t __p0) {	57505	__ai uint16_t vqmovuns_s32(int32_t __p0) {
57192	int16_t __ret;	57506	uint16_t __ret;
57193	__ret = (int16_t) __builtin_neon_vqmovuns_s32(__p0);	57507	__ret = (uint16_t) __builtin_neon_vqmovuns_s32(__p0);
57194	return __ret;	57508	return __ret;
57195	}	57509	}
57196	__ai int32_t vqmovund_s64(int64_t __p0) {	57510	__ai uint32_t vqmovund_s64(int64_t __p0) {
57197	int32_t __ret;	57511	uint32_t __ret;
57198	__ret = (int32_t) __builtin_neon_vqmovund_s64(__p0);	57512	__ret = (uint32_t) __builtin_neon_vqmovund_s64(__p0);
57199	return __ret;	57513	return __ret;
57200	}	57514	}
57201	__ai int8_t vqmovunh_s16(int16_t __p0) {	57515	__ai uint8_t vqmovunh_s16(int16_t __p0) {
57202	int8_t __ret;	57516	uint8_t __ret;
57203	__ret = (int8_t) __builtin_neon_vqmovunh_s16(__p0);	57517	__ret = (uint8_t) __builtin_neon_vqmovunh_s16(__p0);
57204	return __ret;	57518	return __ret;
57205	}	57519	}
57206	#ifdef __LITTLE_ENDIAN__	57520	#ifdef __LITTLE_ENDIAN__
57207	__ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) {	57521	__ai uint16x8_t vqmovun_high_s32(uint16x4_t __p0, int32x4_t __p1) {
57208	uint16x8_t __ret;	57522	uint16x8_t __ret;
57209	__ret = vcombine_u16((uint16x4_t)(__p0), vqmovun_s32(__p1));	57523	__ret = vcombine_u16((uint16x4_t)(__p0), vqmovun_s32(__p1));
57210	return __ret;	57524	return __ret;
57211	}	57525	}
57212	#else	57526	#else
57213	__ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) {	57527	__ai uint16x8_t vqmovun_high_s32(uint16x4_t __p0, int32x4_t __p1) {
57214	int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);	57528	uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
57215	int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);	57529	int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
57216	uint16x8_t __ret;	57530	uint16x8_t __ret;
57217	__ret = __noswap_vcombine_u16((uint16x4_t)(__rev0), __noswap_vqmovun_s32(__rev1));	57531	__ret = __noswap_vcombine_u16((uint16x4_t)(__rev0), __noswap_vqmovun_s32(__rev1));
...	@@ -57221,14 +57535,14 @@ __ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) {	...	@@ -57221,14 +57535,14 @@ __ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) {
57221	#endif	57535	#endif
57222		57536
57223	#ifdef __LITTLE_ENDIAN__	57537	#ifdef __LITTLE_ENDIAN__
57224	__ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) {	57538	__ai uint32x4_t vqmovun_high_s64(uint32x2_t __p0, int64x2_t __p1) {
57225	uint32x4_t __ret;	57539	uint32x4_t __ret;
57226	__ret = vcombine_u32((uint32x2_t)(__p0), vqmovun_s64(__p1));	57540	__ret = vcombine_u32((uint32x2_t)(__p0), vqmovun_s64(__p1));
57227	return __ret;	57541	return __ret;
57228	}	57542	}
57229	#else	57543	#else
57230	__ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) {	57544	__ai uint32x4_t vqmovun_high_s64(uint32x2_t __p0, int64x2_t __p1) {
57231	int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);	57545	uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
57232	int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);	57546	int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
57233	uint32x4_t __ret;	57547	uint32x4_t __ret;
57234	__ret = __noswap_vcombine_u32((uint32x2_t)(__rev0), __noswap_vqmovun_s64(__rev1));	57548	__ret = __noswap_vcombine_u32((uint32x2_t)(__rev0), __noswap_vqmovun_s64(__rev1));
...	@@ -57238,14 +57552,14 @@ __ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) {	...	@@ -57238,14 +57552,14 @@ __ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) {
57238	#endif	57552	#endif
57239		57553
57240	#ifdef __LITTLE_ENDIAN__	57554	#ifdef __LITTLE_ENDIAN__
57241	__ai uint8x16_t vqmovun_high_s16(int8x8_t __p0, int16x8_t __p1) {	57555	__ai uint8x16_t vqmovun_high_s16(uint8x8_t __p0, int16x8_t __p1) {
57242	uint8x16_t __ret;	57556	uint8x16_t __ret;
57243	__ret = vcombine_u8((uint8x8_t)(__p0), vqmovun_s16(__p1));	57557	__ret = vcombine_u8((uint8x8_t)(__p0), vqmovun_s16(__p1));
57244	return __ret;	57558	return __ret;
57245	}	57559	}
57246	#else	57560	#else
57247	__ai uint8x16_t vqmovun_high_s16(int8x8_t __p0, int16x8_t __p1) {	57561	__ai uint8x16_t vqmovun_high_s16(uint8x8_t __p0, int16x8_t __p1) {
57248	int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);	57562	uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
57249	int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);	57563	int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
57250	uint8x16_t __ret;	57564	uint8x16_t __ret;
57251	__ret = __noswap_vcombine_u8((uint8x8_t)(__rev0), __noswap_vqmovun_s16(__rev1));	57565	__ret = __noswap_vcombine_u8((uint8x8_t)(__rev0), __noswap_vqmovun_s16(__rev1));
...	@@ -57549,22 +57863,22 @@ __ai int16_t vqrdmulhh_s16(int16_t __p0, int16_t __p1) {	...	@@ -57549,22 +57863,22 @@ __ai int16_t vqrdmulhh_s16(int16_t __p0, int16_t __p1) {
57549	})	57863	})
57550	#endif	57864	#endif
57551		57865
57552	__ai uint8_t vqrshlb_u8(uint8_t __p0, uint8_t __p1) {	57866	__ai uint8_t vqrshlb_u8(uint8_t __p0, int8_t __p1) {
57553	uint8_t __ret;	57867	uint8_t __ret;
57554	__ret = (uint8_t) __builtin_neon_vqrshlb_u8(__p0, __p1);	57868	__ret = (uint8_t) __builtin_neon_vqrshlb_u8(__p0, __p1);
57555	return __ret;	57869	return __ret;
57556	}	57870	}
57557	__ai uint32_t vqrshls_u32(uint32_t __p0, uint32_t __p1) {	57871	__ai uint32_t vqrshls_u32(uint32_t __p0, int32_t __p1) {
57558	uint32_t __ret;	57872	uint32_t __ret;
57559	__ret = (uint32_t) __builtin_neon_vqrshls_u32(__p0, __p1);	57873	__ret = (uint32_t) __builtin_neon_vqrshls_u32(__p0, __p1);
57560	return __ret;	57874	return __ret;
57561	}	57875	}
57562	__ai uint64_t vqrshld_u64(uint64_t __p0, uint64_t __p1) {	57876	__ai uint64_t vqrshld_u64(uint64_t __p0, int64_t __p1) {
57563	uint64_t __ret;	57877	uint64_t __ret;
57564	__ret = (uint64_t) __builtin_neon_vqrshld_u64(__p0, __p1);	57878	__ret = (uint64_t) __builtin_neon_vqrshld_u64(__p0, __p1);
57565	return __ret;	57879	return __ret;
57566	}	57880	}
57567	__ai uint16_t vqrshlh_u16(uint16_t __p0, uint16_t __p1) {	57881	__ai uint16_t vqrshlh_u16(uint16_t __p0, int16_t __p1) {
57568	uint16_t __ret;	57882	uint16_t __ret;
57569	__ret = (uint16_t) __builtin_neon_vqrshlh_u16(__p0, __p1);	57883	__ret = (uint16_t) __builtin_neon_vqrshlh_u16(__p0, __p1);
57570	return __ret;	57884	return __ret;
...	@@ -57832,22 +58146,22 @@ __ai int16_t vqrshlh_s16(int16_t __p0, int16_t __p1) {	...	@@ -57832,22 +58146,22 @@ __ai int16_t vqrshlh_s16(int16_t __p0, int16_t __p1) {
57832	__ret = (int8_t) __builtin_neon_vqrshrunh_n_s16(__s0, __p1); \	58146	__ret = (int8_t) __builtin_neon_vqrshrunh_n_s16(__s0, __p1); \
57833	__ret; \	58147	__ret; \
57834	})	58148	})
57835	__ai uint8_t vqshlb_u8(uint8_t __p0, uint8_t __p1) {	58149	__ai uint8_t vqshlb_u8(uint8_t __p0, int8_t __p1) {
57836	uint8_t __ret;	58150	uint8_t __ret;
57837	__ret = (uint8_t) __builtin_neon_vqshlb_u8(__p0, __p1);	58151	__ret = (uint8_t) __builtin_neon_vqshlb_u8(__p0, __p1);
57838	return __ret;	58152	return __ret;
57839	}	58153	}
57840	__ai uint32_t vqshls_u32(uint32_t __p0, uint32_t __p1) {	58154	__ai uint32_t vqshls_u32(uint32_t __p0, int32_t __p1) {
57841	uint32_t __ret;	58155	uint32_t __ret;
57842	__ret = (uint32_t) __builtin_neon_vqshls_u32(__p0, __p1);	58156	__ret = (uint32_t) __builtin_neon_vqshls_u32(__p0, __p1);
57843	return __ret;	58157	return __ret;
57844	}	58158	}
57845	__ai uint64_t vqshld_u64(uint64_t __p0, uint64_t __p1) {	58159	__ai uint64_t vqshld_u64(uint64_t __p0, int64_t __p1) {
57846	uint64_t __ret;	58160	uint64_t __ret;
57847	__ret = (uint64_t) __builtin_neon_vqshld_u64(__p0, __p1);	58161	__ret = (uint64_t) __builtin_neon_vqshld_u64(__p0, __p1);
57848	return __ret;	58162	return __ret;
57849	}	58163	}
57850	__ai uint16_t vqshlh_u16(uint16_t __p0, uint16_t __p1) {	58164	__ai uint16_t vqshlh_u16(uint16_t __p0, int16_t __p1) {
57851	uint16_t __ret;	58165	uint16_t __ret;
57852	__ret = (uint16_t) __builtin_neon_vqshlh_u16(__p0, __p1);	58166	__ret = (uint16_t) __builtin_neon_vqshlh_u16(__p0, __p1);
57853	return __ret;	58167	return __ret;
...	@@ -59452,7 +59766,7 @@ __ai float32_t vrecpxs_f32(float32_t __p0) {	...	@@ -59452,7 +59766,7 @@ __ai float32_t vrecpxs_f32(float32_t __p0) {
59452	__ret = (float32_t) __builtin_neon_vrecpxs_f32(__p0);	59766	__ret = (float32_t) __builtin_neon_vrecpxs_f32(__p0);
59453	return __ret;	59767	return __ret;
59454	}	59768	}
59455	__ai uint64_t vrshld_u64(uint64_t __p0, uint64_t __p1) {	59769	__ai uint64_t vrshld_u64(uint64_t __p0, int64_t __p1) {
59456	uint64_t __ret;	59770	uint64_t __ret;
59457	__ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1);	59771	__ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1);
59458	return __ret;	59772	return __ret;
...	@@ -59853,7 +60167,7 @@ __ai int8x16_t vrsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) {	...	@@ -59853,7 +60167,7 @@ __ai int8x16_t vrsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) {
59853	__ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (float64x1_t)__s1, __p2); \	60167	__ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (float64x1_t)__s1, __p2); \
59854	__ret; \	60168	__ret; \
59855	})	60169	})
59856	__ai uint64_t vshld_u64(uint64_t __p0, uint64_t __p1) {	60170	__ai uint64_t vshld_u64(uint64_t __p0, int64_t __p1) {
59857	uint64_t __ret;	60171	uint64_t __ret;
59858	__ret = (uint64_t) __builtin_neon_vshld_u64(__p0, __p1);	60172	__ret = (uint64_t) __builtin_neon_vshld_u64(__p0, __p1);
59859	return __ret;	60173	return __ret;
...	@@ -62423,9 +62737,9 @@ __ai uint64_t vtstd_u64(uint64_t __p0, uint64_t __p1) {	...	@@ -62423,9 +62737,9 @@ __ai uint64_t vtstd_u64(uint64_t __p0, uint64_t __p1) {
62423	__ret = (uint64_t) __builtin_neon_vtstd_u64(__p0, __p1);	62737	__ret = (uint64_t) __builtin_neon_vtstd_u64(__p0, __p1);
62424	return __ret;	62738	return __ret;
62425	}	62739	}
62426	__ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) {	62740	__ai uint64_t vtstd_s64(int64_t __p0, int64_t __p1) {
62427	int64_t __ret;	62741	uint64_t __ret;
62428	__ret = (int64_t) __builtin_neon_vtstd_s64(__p0, __p1);	62742	__ret = (uint64_t) __builtin_neon_vtstd_s64(__p0, __p1);
62429	return __ret;	62743	return __ret;
62430	}	62744	}
62431	__ai int8_t vuqaddb_s8(int8_t __p0, uint8_t __p1) {	62745	__ai int8_t vuqaddb_s8(int8_t __p0, uint8_t __p1) {

lib/include/arm_sve.h+80-80

lib/include/avx512fintrin.h+40-117

 @@ -9305,295 +9305,218 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)
  * This takes log2(n) steps where n is the number of elements in the vector.
  */
-#define _mm512_mask_reduce_operator(op) \
-  __v4du __t1 = (__v4du)_mm512_extracti64x4_epi64(__W, 0); \
-  __v4du __t2 = (__v4du)_mm512_extracti64x4_epi64(__W, 1); \
-  __m256i __t3 = (__m256i)(__t1 op __t2); \
-  __v2du __t4 = (__v2du)_mm256_extracti128_si256(__t3, 0); \
-  __v2du __t5 = (__v2du)_mm256_extracti128_si256(__t3, 1); \
-  __v2du __t6 = __t4 op __t5; \
-  __v2du __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
-  __v2du __t8 = __t6 op __t7; \
-  return __t8[0]
 static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) {
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_add_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_epi64(__m512i __W) {
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_mul_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_and_epi64(__m512i __W) {
-  _mm512_mask_reduce_operator(&);
+  return __builtin_ia32_reduce_and_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_or_epi64(__m512i __W) {
-  _mm512_mask_reduce_operator(|);
+  return __builtin_ia32_reduce_or_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_add_epi64(__mmask8 __M, __m512i __W) {
   __W = _mm512_maskz_mov_epi64(__M, __W);
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_add_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_mul_epi64(__mmask8 __M, __m512i __W) {
   __W = _mm512_mask_mov_epi64(_mm512_set1_epi64(1), __M, __W);
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_mul_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_and_epi64(__mmask8 __M, __m512i __W) {
   __W = _mm512_mask_mov_epi64(_mm512_set1_epi64(~0ULL), __M, __W);
-  _mm512_mask_reduce_operator(&);
+  return __builtin_ia32_reduce_and_q512(__W);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) {
   __W = _mm512_maskz_mov_epi64(__M, __W);
-  _mm512_mask_reduce_operator(|);
+  return __builtin_ia32_reduce_or_q512(__W);
 }
-#undef _mm512_mask_reduce_operator
-#define _mm512_mask_reduce_operator(op) \
-  __m256d __t1 = _mm512_extractf64x4_pd(__W, 0); \
-  __m256d __t2 = _mm512_extractf64x4_pd(__W, 1); \
-  __m256d __t3 = __t1 op __t2; \
-  __m128d __t4 = _mm256_extractf128_pd(__t3, 0); \
-  __m128d __t5 = _mm256_extractf128_pd(__t3, 1); \
-  __m128d __t6 = __t4 op __t5; \
-  __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
-  __m128d __t8 = __t6 op __t7; \
-  return __t8[0]
 static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_add_pd(__m512d __W) {
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_pd512(0.0, __W);
 }
 static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_pd(__m512d __W) {
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_pd512(1.0, __W);
 }
 static __inline__ double __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W) {
   __W = _mm512_maskz_mov_pd(__M, __W);
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_pd512(0.0, __W);
 }
 static __inline__ double __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {
   __W = _mm512_mask_mov_pd(_mm512_set1_pd(1.0), __M, __W);
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_pd512(1.0, __W);
 }
-#undef _mm512_mask_reduce_operator
-#define _mm512_mask_reduce_operator(op) \
-  __v8su __t1 = (__v8su)_mm512_extracti64x4_epi64(__W, 0); \
-  __v8su __t2 = (__v8su)_mm512_extracti64x4_epi64(__W, 1); \
-  __m256i __t3 = (__m256i)(__t1 op __t2); \
-  __v4su __t4 = (__v4su)_mm256_extracti128_si256(__t3, 0); \
-  __v4su __t5 = (__v4su)_mm256_extracti128_si256(__t3, 1); \
-  __v4su __t6 = __t4 op __t5; \
-  __v4su __t7 = __builtin_shufflevector(__t6, __t6, 2, 3, 0, 1); \
-  __v4su __t8 = __t6 op __t7; \
-  __v4su __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
-  __v4su __t10 = __t8 op __t9; \
-  return __t10[0]
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_add_epi32(__m512i __W) {
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_add_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_mul_epi32(__m512i __W) {
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_mul_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_and_epi32(__m512i __W) {
-  _mm512_mask_reduce_operator(&);
+  return __builtin_ia32_reduce_and_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_or_epi32(__m512i __W) {
-  _mm512_mask_reduce_operator(|);
+  return __builtin_ia32_reduce_or_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_add_epi32( __mmask16 __M, __m512i __W) {
   __W = _mm512_maskz_mov_epi32(__M, __W);
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_add_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_mul_epi32( __mmask16 __M, __m512i __W) {
   __W = _mm512_mask_mov_epi32(_mm512_set1_epi32(1), __M, __W);
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_mul_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_and_epi32( __mmask16 __M, __m512i __W) {
   __W = _mm512_mask_mov_epi32(_mm512_set1_epi32(~0U), __M, __W);
-  _mm512_mask_reduce_operator(&);
+  return __builtin_ia32_reduce_and_d512((__v16si)__W);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W) {
   __W = _mm512_maskz_mov_epi32(__M, __W);
-  _mm512_mask_reduce_operator(|);
+  return __builtin_ia32_reduce_or_d512((__v16si)__W);
 }
-#undef _mm512_mask_reduce_operator
-#define _mm512_mask_reduce_operator(op) \
-  __m256 __t1 = (__m256)_mm512_extractf64x4_pd((__m512d)__W, 0); \
-  __m256 __t2 = (__m256)_mm512_extractf64x4_pd((__m512d)__W, 1); \
-  __m256 __t3 = __t1 op __t2; \
-  __m128 __t4 = _mm256_extractf128_ps(__t3, 0); \
-  __m128 __t5 = _mm256_extractf128_ps(__t3, 1); \
-  __m128 __t6 = __t4 op __t5; \
-  __m128 __t7 = __builtin_shufflevector(__t6, __t6, 2, 3, 0, 1); \
-  __m128 __t8 = __t6 op __t7; \
-  __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
-  __m128 __t10 = __t8 op __t9; \
-  return __t10[0]
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_reduce_add_ps(__m512 __W) {
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_ps512(0.0f, __W);
 }
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_reduce_mul_ps(__m512 __W) {
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_ps512(1.0f, __W);
 }
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_add_ps(__mmask16 __M, __m512 __W) {
   __W = _mm512_maskz_mov_ps(__M, __W);
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_ps512(0.0f, __W);
 }
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {
   __W = _mm512_mask_mov_ps(_mm512_set1_ps(1.0f), __M, __W);
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_ps512(1.0f, __W);
 }
-#undef _mm512_mask_reduce_operator
-#define _mm512_mask_reduce_operator(op) \
-  __m512i __t1 = (__m512i)__builtin_shufflevector((__v8di)__V, (__v8di)__V, 4, 5, 6, 7, 0, 1, 2, 3); \
-  __m512i __t2 = _mm512_##op(__V, __t1); \
-  __m512i __t3 = (__m512i)__builtin_shufflevector((__v8di)__t2, (__v8di)__t2, 2, 3, 0, 1, 6, 7, 4, 5); \
-  __m512i __t4 = _mm512_##op(__t2, __t3); \
-  __m512i __t5 = (__m512i)__builtin_shufflevector((__v8di)__t4, (__v8di)__t4, 1, 0, 3, 2, 5, 4, 7, 6); \
-  __v8di __t6 = (__v8di)_mm512_##op(__t4, __t5); \
-  return __t6[0]
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_reduce_max_epi64(__m512i __V) {
-  _mm512_mask_reduce_operator(max_epi64);
+  return __builtin_ia32_reduce_smax_q512(__V);
 }
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
 _mm512_reduce_max_epu64(__m512i __V) {
-  _mm512_mask_reduce_operator(max_epu64);
+  return __builtin_ia32_reduce_umax_q512(__V);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_reduce_min_epi64(__m512i __V) {
-  _mm512_mask_reduce_operator(min_epi64);
+  return __builtin_ia32_reduce_smin_q512(__V);
 }
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
 _mm512_reduce_min_epu64(__m512i __V) {
-  _mm512_mask_reduce_operator(min_epu64);
+  return __builtin_ia32_reduce_umin_q512(__V);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_max_epi64(__mmask8 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(-__LONG_LONG_MAX__ - 1LL), __M, __V);
-  _mm512_mask_reduce_operator(max_epi64);
+  return __builtin_ia32_reduce_smax_q512(__V);
 }
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_max_epu64(__mmask8 __M, __m512i __V) {
   __V = _mm512_maskz_mov_epi64(__M, __V);
-  _mm512_mask_reduce_operator(max_epu64);
+  return __builtin_ia32_reduce_umax_q512(__V);
 }
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_min_epi64(__mmask8 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(__LONG_LONG_MAX__), __M, __V);
-  _mm512_mask_reduce_operator(min_epi64);
+  return __builtin_ia32_reduce_smin_q512(__V);
 }
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(~0ULL), __M, __V);
-  _mm512_mask_reduce_operator(min_epu64);
+  return __builtin_ia32_reduce_umin_q512(__V);
 }
-#undef _mm512_mask_reduce_operator
-#define _mm512_mask_reduce_operator(op) \
-  __m256i __t1 = _mm512_extracti64x4_epi64(__V, 0); \
-  __m256i __t2 = _mm512_extracti64x4_epi64(__V, 1); \
-  __m256i __t3 = _mm256_##op(__t1, __t2); \
-  __m128i __t4 = _mm256_extracti128_si256(__t3, 0); \
-  __m128i __t5 = _mm256_extracti128_si256(__t3, 1); \
-  __m128i __t6 = _mm_##op(__t4, __t5); \
-  __m128i __t7 = (__m128i)__builtin_shufflevector((__v4si)__t6, (__v4si)__t6, 2, 3, 0, 1); \
-  __m128i __t8 = _mm_##op(__t6, __t7); \
-  __m128i __t9 = (__m128i)__builtin_shufflevector((__v4si)__t8, (__v4si)__t8, 1, 0, 3, 2); \
-  __v4si __t10 = (__v4si)_mm_##op(__t8, __t9); \
-  return __t10[0]
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_max_epi32(__m512i __V) {
-  _mm512_mask_reduce_operator(max_epi32);
+  return __builtin_ia32_reduce_smax_d512((__v16si)__V);
 }
 static __inline__ unsigned int __DEFAULT_FN_ATTRS512
 _mm512_reduce_max_epu32(__m512i __V) {
-  _mm512_mask_reduce_operator(max_epu32);
+  return __builtin_ia32_reduce_umax_d512((__v16si)__V);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_min_epi32(__m512i __V) {
-  _mm512_mask_reduce_operator(min_epi32);
+  return __builtin_ia32_reduce_smin_d512((__v16si)__V);
 }
 static __inline__ unsigned int __DEFAULT_FN_ATTRS512
 _mm512_reduce_min_epu32(__m512i __V) {
-  _mm512_mask_reduce_operator(min_epu32);
+  return __builtin_ia32_reduce_umin_d512((__v16si)__V);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_max_epi32(__mmask16 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi32(_mm512_set1_epi32(-__INT_MAX__ - 1), __M, __V);
-  _mm512_mask_reduce_operator(max_epi32);
+  return __builtin_ia32_reduce_smax_d512((__v16si)__V);
 }
 static __inline__ unsigned int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_max_epu32(__mmask16 __M, __m512i __V) {
   __V = _mm512_maskz_mov_epi32(__M, __V);
-  _mm512_mask_reduce_operator(max_epu32);
+  return __builtin_ia32_reduce_umax_d512((__v16si)__V);
 }
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_min_epi32(__mmask16 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi32(_mm512_set1_epi32(__INT_MAX__), __M, __V);
-  _mm512_mask_reduce_operator(min_epi32);
+  return __builtin_ia32_reduce_smin_d512((__v16si)__V);
 }
 static __inline__ unsigned int __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_min_epu32(__mmask16 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi32(_mm512_set1_epi32(~0U), __M, __V);
-  _mm512_mask_reduce_operator(min_epu32);
+  return __builtin_ia32_reduce_umin_d512((__v16si)__V);
 }
-#undef _mm512_mask_reduce_operator
 #define _mm512_mask_reduce_operator(op) \
   __m256d __t1 = _mm512_extractf64x4_pd(__V, 0); \

lib/include/avx512vlvnniintrin.h+150-55

...	@@ -18,13 +18,157 @@	...	@@ -18,13 +18,157 @@
18	#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vnni"), __min_vector_width__(128)))	18	#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vnni"), __min_vector_width__(128)))
19	#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vnni"), __min_vector_width__(256)))	19	#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vnni"), __min_vector_width__(256)))
20		20
		21	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
		22	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
		23	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
		24	/// in \a S, and store the packed 32-bit results in DST.
		25	///
		26	/// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.
		27	///
		28	/// \operation
		29	/// FOR j := 0 to 7
		30	/// tmp1.word := Signed(ZeroExtend16(A.byte[4j]) SignExtend16(B.byte[4*j]))
		31	/// tmp2.word := Signed(ZeroExtend16(A.byte[4j+1]) SignExtend16(B.byte[4*j+1]))
		32	/// tmp3.word := Signed(ZeroExtend16(A.byte[4j+2]) SignExtend16(B.byte[4*j+2]))
		33	/// tmp4.word := Signed(ZeroExtend16(A.byte[4j+3]) SignExtend16(B.byte[4*j+3]))
		34	/// DST.dword[j] := S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4
		35	/// ENDFOR
		36	/// DST[MAX:256] := 0
		37	/// \endoperation
		38	#define _mm256_dpbusd_epi32(S, A, B) \
		39	(__m256i)__builtin_ia32_vpdpbusd256((__v8si)(S), (__v8si)(A), (__v8si)(B))
21		40
22	static __inline__ __m256i __DEFAULT_FN_ATTRS256	41	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
23	_mm256_dpbusd_epi32(__m256i __S, __m256i __A, __m256i __B)	42	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
24	{	43	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
25	return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v8si)__A,	44	/// in \a S using signed saturation, and store the packed 32-bit results in DST.
26	(__v8si)__B);	45	///
27	}	46	/// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.
		47	///
		48	/// \operation
		49	/// FOR j := 0 to 7
		50	/// tmp1.word := Signed(ZeroExtend16(A.byte[4j]) SignExtend16(B.byte[4*j]))
		51	/// tmp2.word := Signed(ZeroExtend16(A.byte[4j+1]) SignExtend16(B.byte[4*j+1]))
		52	/// tmp3.word := Signed(ZeroExtend16(A.byte[4j+2]) SignExtend16(B.byte[4*j+2]))
		53	/// tmp4.word := Signed(ZeroExtend16(A.byte[4j+3]) SignExtend16(B.byte[4*j+3]))
		54	/// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)
		55	/// ENDFOR
		56	/// DST[MAX:256] := 0
		57	/// \endoperation
		58	#define _mm256_dpbusds_epi32(S, A, B) \
		59	(__m256i)__builtin_ia32_vpdpbusds256((__v8si)(S), (__v8si)(A), (__v8si)(B))
		60
		61	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
		62	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
		63	/// results. Sum these 2 results with the corresponding 32-bit integer in \a S,
		64	/// and store the packed 32-bit results in DST.
		65	///
		66	/// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.
		67	///
		68	/// \operation
		69	/// FOR j := 0 to 7
		70	/// tmp1.dword := SignExtend32(A.word[2j]) SignExtend32(B.word[2*j])
		71	/// tmp2.dword := SignExtend32(A.word[2j+1]) SignExtend32(B.word[2*j+1])
		72	/// DST.dword[j] := S.dword[j] + tmp1 + tmp2
		73	/// ENDFOR
		74	/// DST[MAX:256] := 0
		75	/// \endoperation
		76	#define _mm256_dpwssd_epi32(S, A, B) \
		77	(__m256i)__builtin_ia32_vpdpwssd256((__v8si)(S), (__v8si)(A), (__v8si)(B))
		78
		79	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
		80	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
		81	/// results. Sum these 2 results with the corresponding 32-bit integer in \a S
		82	/// using signed saturation, and store the packed 32-bit results in DST.
		83	///
		84	/// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.
		85	///
		86	/// \operation
		87	/// FOR j := 0 to 7
		88	/// tmp1.dword := SignExtend32(A.word[2j]) SignExtend32(B.word[2*j])
		89	/// tmp2.dword := SignExtend32(A.word[2j+1]) SignExtend32(B.word[2*j+1])
		90	/// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2)
		91	/// ENDFOR
		92	/// DST[MAX:256] := 0
		93	/// \endoperation
		94	#define _mm256_dpwssds_epi32(S, A, B) \
		95	(__m256i)__builtin_ia32_vpdpwssds256((__v8si)(S), (__v8si)(A), (__v8si)(B))
		96
		97	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
		98	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
		99	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
		100	/// in \a S, and store the packed 32-bit results in DST.
		101	///
		102	/// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.
		103	///
		104	/// \operation
		105	/// FOR j := 0 to 3
		106	/// tmp1.word := Signed(ZeroExtend16(A.byte[4j]) SignExtend16(B.byte[4*j]))
		107	/// tmp2.word := Signed(ZeroExtend16(A.byte[4j+1]) SignExtend16(B.byte[4*j+1]))
		108	/// tmp3.word := Signed(ZeroExtend16(A.byte[4j+2]) SignExtend16(B.byte[4*j+2]))
		109	/// tmp4.word := Signed(ZeroExtend16(A.byte[4j+3]) SignExtend16(B.byte[4*j+3]))
		110	/// DST.dword[j] := S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4
		111	/// ENDFOR
		112	/// DST[MAX:128] := 0
		113	/// \endoperation
		114	#define _mm_dpbusd_epi32(S, A, B) \
		115	(__m128i)__builtin_ia32_vpdpbusd128((__v4si)(S), (__v4si)(A), (__v4si)(B))
		116
		117	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
		118	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
		119	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
		120	/// in \a S using signed saturation, and store the packed 32-bit results in DST.
		121	///
		122	/// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.
		123	///
		124	/// \operation
		125	/// FOR j := 0 to 3
		126	/// tmp1.word := Signed(ZeroExtend16(A.byte[4j]) SignExtend16(B.byte[4*j]))
		127	/// tmp2.word := Signed(ZeroExtend16(A.byte[4j+1]) SignExtend16(B.byte[4*j+1]))
		128	/// tmp3.word := Signed(ZeroExtend16(A.byte[4j+2]) SignExtend16(B.byte[4*j+2]))
		129	/// tmp4.word := Signed(ZeroExtend16(A.byte[4j+3]) SignExtend16(B.byte[4*j+3]))
		130	/// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)
		131	/// ENDFOR
		132	/// DST[MAX:128] := 0
		133	/// \endoperation
		134	#define _mm_dpbusds_epi32(S, A, B) \
		135	(__m128i)__builtin_ia32_vpdpbusds128((__v4si)(S), (__v4si)(A), (__v4si)(B))
		136
		137	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
		138	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
		139	/// results. Sum these 2 results with the corresponding 32-bit integer in \a S,
		140	/// and store the packed 32-bit results in DST.
		141	///
		142	/// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.
		143	///
		144	/// \operation
		145	/// FOR j := 0 to 3
		146	/// tmp1.dword := SignExtend32(A.word[2j]) SignExtend32(B.word[2*j])
		147	/// tmp2.dword := SignExtend32(A.word[2j+1]) SignExtend32(B.word[2*j+1])
		148	/// DST.dword[j] := S.dword[j] + tmp1 + tmp2
		149	/// ENDFOR
		150	/// DST[MAX:128] := 0
		151	/// \endoperation
		152	#define _mm_dpwssd_epi32(S, A, B) \
		153	(__m128i)__builtin_ia32_vpdpwssd128((__v4si)(S), (__v4si)(A), (__v4si)(B))
		154
		155	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
		156	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
		157	/// results. Sum these 2 results with the corresponding 32-bit integer in \a S
		158	/// using signed saturation, and store the packed 32-bit results in DST.
		159	///
		160	/// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.
		161	///
		162	/// \operation
		163	/// FOR j := 0 to 3
		164	/// tmp1.dword := SignExtend32(A.word[2j]) SignExtend32(B.word[2*j])
		165	/// tmp2.dword := SignExtend32(A.word[2j+1]) SignExtend32(B.word[2*j+1])
		166	/// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2)
		167	/// ENDFOR
		168	/// DST[MAX:128] := 0
		169	/// \endoperation
		170	#define _mm_dpwssds_epi32(S, A, B) \
		171	(__m128i)__builtin_ia32_vpdpwssds128((__v4si)(S), (__v4si)(A), (__v4si)(B))
28		172
29	static __inline__ __m256i __DEFAULT_FN_ATTRS256	173	static __inline__ __m256i __DEFAULT_FN_ATTRS256
30	_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)	174	_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
...	@@ -42,13 +186,6 @@ _mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)	...	@@ -42,13 +186,6 @@ _mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
42	(__v8si)_mm256_setzero_si256());	186	(__v8si)_mm256_setzero_si256());
43	}	187	}
44		188
45	static __inline__ __m256i __DEFAULT_FN_ATTRS256
46	_mm256_dpbusds_epi32(__m256i __S, __m256i __A, __m256i __B)
47	{
48	return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v8si)__A,
49	(__v8si)__B);
50	}
51
52	static __inline__ __m256i __DEFAULT_FN_ATTRS256	189	static __inline__ __m256i __DEFAULT_FN_ATTRS256
53	_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)	190	_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
54	{	191	{
...	@@ -65,13 +202,6 @@ _mm256_maskz_dpbusds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)	...	@@ -65,13 +202,6 @@ _mm256_maskz_dpbusds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
65	(__v8si)_mm256_setzero_si256());	202	(__v8si)_mm256_setzero_si256());
66	}	203	}
67		204
68	static __inline__ __m256i __DEFAULT_FN_ATTRS256
69	_mm256_dpwssd_epi32(__m256i __S, __m256i __A, __m256i __B)
70	{
71	return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v8si)__A,
72	(__v8si)__B);
73	}
74
75	static __inline__ __m256i __DEFAULT_FN_ATTRS256	205	static __inline__ __m256i __DEFAULT_FN_ATTRS256
76	_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)	206	_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
77	{	207	{
...	@@ -88,13 +218,6 @@ _mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)	...	@@ -88,13 +218,6 @@ _mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
88	(__v8si)_mm256_setzero_si256());	218	(__v8si)_mm256_setzero_si256());
89	}	219	}
90		220
91	static __inline__ __m256i __DEFAULT_FN_ATTRS256
92	_mm256_dpwssds_epi32(__m256i __S, __m256i __A, __m256i __B)
93	{
94	return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v8si)__A,
95	(__v8si)__B);
96	}
97
98	static __inline__ __m256i __DEFAULT_FN_ATTRS256	221	static __inline__ __m256i __DEFAULT_FN_ATTRS256
99	_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)	222	_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
100	{	223	{
...	@@ -111,13 +234,6 @@ _mm256_maskz_dpwssds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)	...	@@ -111,13 +234,6 @@ _mm256_maskz_dpwssds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
111	(__v8si)_mm256_setzero_si256());	234	(__v8si)_mm256_setzero_si256());
112	}	235	}
113		236
114	static __inline__ __m128i __DEFAULT_FN_ATTRS128
115	_mm_dpbusd_epi32(__m128i __S, __m128i __A, __m128i __B)
116	{
117	return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v4si)__A,
118	(__v4si)__B);
119	}
120
121	static __inline__ __m128i __DEFAULT_FN_ATTRS128	237	static __inline__ __m128i __DEFAULT_FN_ATTRS128
122	_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)	238	_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
123	{	239	{
...	@@ -134,13 +250,6 @@ _mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)	...	@@ -134,13 +250,6 @@ _mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
134	(__v4si)_mm_setzero_si128());	250	(__v4si)_mm_setzero_si128());
135	}	251	}
136		252
137	static __inline__ __m128i __DEFAULT_FN_ATTRS128
138	_mm_dpbusds_epi32(__m128i __S, __m128i __A, __m128i __B)
139	{
140	return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v4si)__A,
141	(__v4si)__B);
142	}
143
144	static __inline__ __m128i __DEFAULT_FN_ATTRS128	253	static __inline__ __m128i __DEFAULT_FN_ATTRS128
145	_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)	254	_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
146	{	255	{
...	@@ -157,13 +266,6 @@ _mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)	...	@@ -157,13 +266,6 @@ _mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
157	(__v4si)_mm_setzero_si128());	266	(__v4si)_mm_setzero_si128());
158	}	267	}
159		268
160	static __inline__ __m128i __DEFAULT_FN_ATTRS128
161	_mm_dpwssd_epi32(__m128i __S, __m128i __A, __m128i __B)
162	{
163	return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v4si)__A,
164	(__v4si)__B);
165	}
166
167	static __inline__ __m128i __DEFAULT_FN_ATTRS128	269	static __inline__ __m128i __DEFAULT_FN_ATTRS128
168	_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)	270	_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
169	{	271	{
...	@@ -180,13 +282,6 @@ _mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)	...	@@ -180,13 +282,6 @@ _mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
180	(__v4si)_mm_setzero_si128());	282	(__v4si)_mm_setzero_si128());
181	}	283	}
182		284
183	static __inline__ __m128i __DEFAULT_FN_ATTRS128
184	_mm_dpwssds_epi32(__m128i __S, __m128i __A, __m128i __B)
185	{
186	return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v4si)__A,
187	(__v4si)__B);
188	}
189
190	static __inline__ __m128i __DEFAULT_FN_ATTRS128	285	static __inline__ __m128i __DEFAULT_FN_ATTRS128
191	_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)	286	_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
192	{	287	{

lib/include/avxvnniintrin.h created+225

...	@@ -0,0 +1,225 @@
1	/*===--------------- avxvnniintrin.h - VNNI intrinsics --------------------===
2	*
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a copy
5	* of this software and associated documentation files (the "Software"), to deal
6	* in the Software without restriction, including without limitation the rights
7	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8	* copies of the Software, and to permit persons to whom the Software is
9	* furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20	* THE SOFTWARE.
21	*
22	*===-----------------------------------------------------------------------===
23	*/
24	#ifndef __IMMINTRIN_H
25	#error "Never use <avxvnniintrin.h> directly; include <immintrin.h> instead."
26	#endif
27
28	#ifndef __AVXVNNIINTRIN_H
29	#define __AVXVNNIINTRIN_H
30
31	/* Below intrinsics defined in avx512vlvnniintrin.h can be used for AVXVNNI */
32	/// \fn __m256i _mm256_dpbusd_epi32(__m256i __S, __m256i __A, __m256i __B)
33	/// \fn __m256i _mm256_dpbusds_epi32(__m256i __S, __m256i __A, __m256i __B)
34	/// \fn __m256i _mm256_dpwssd_epi32(__m256i __S, __m256i __A, __m256i __B)
35	/// \fn __m256i _mm256_dpwssds_epi32(__m256i __S, __m256i __A, __m256i __B)
36	/// \fn __m128i _mm_dpbusd_epi32(__m128i __S, __m128i __A, __m128i __B)
37	/// \fn __m128i _mm_dpbusds_epi32(__m128i __S, __m128i __A, __m128i __B)
38	/// \fn __m128i _mm_dpwssd_epi32(__m128i __S, __m128i __A, __m128i __B)
39	/// \fn __m128i _mm_dpwssds_epi32(__m128i __S, __m128i __A, __m128i __B)
40
41	/* Intrinsics with _avx_ prefix are for compatibility with msvc. */
42	/* Define the default attributes for the functions in this file. */
43	#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(256)))
44	#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(128)))
45
46	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
47	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed
48	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
49	/// in \a __S, and store the packed 32-bit results in DST.
50	///
51	/// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.
52	///
53	/// \operation
54	/// FOR j := 0 to 7
55	/// tmp1.word := Signed(ZeroExtend16(__A.byte[4j]) SignExtend16(__B.byte[4*j]))
56	/// tmp2.word := Signed(ZeroExtend16(__A.byte[4j+1]) SignExtend16(__B.byte[4*j+1]))
57	/// tmp3.word := Signed(ZeroExtend16(__A.byte[4j+2]) SignExtend16(__B.byte[4*j+2]))
58	/// tmp4.word := Signed(ZeroExtend16(__A.byte[4j+3]) SignExtend16(__B.byte[4*j+3]))
59	/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4
60	/// ENDFOR
61	/// DST[MAX:256] := 0
62	/// \endoperation
63	static __inline__ __m256i __DEFAULT_FN_ATTRS256
64	_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
65	{
66	return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v8si)__A, (__v8si)__B);
67	}
68
69	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
70	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed
71	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
72	/// in \a __S using signed saturation, and store the packed 32-bit results in DST.
73	///
74	/// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.
75	///
76	/// \operation
77	/// FOR j := 0 to 7
78	/// tmp1.word := Signed(ZeroExtend16(__A.byte[4j]) SignExtend16(__B.byte[4*j]))
79	/// tmp2.word := Signed(ZeroExtend16(__A.byte[4j+1]) SignExtend16(__B.byte[4*j+1]))
80	/// tmp3.word := Signed(ZeroExtend16(__A.byte[4j+2]) SignExtend16(__B.byte[4*j+2]))
81	/// tmp4.word := Signed(ZeroExtend16(__A.byte[4j+3]) SignExtend16(__B.byte[4*j+3]))
82	/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)
83	/// ENDFOR
84	/// DST[MAX:256] := 0
85	/// \endoperation
86	static __inline__ __m256i __DEFAULT_FN_ATTRS256
87	_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
88	{
89	return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v8si)__A, (__v8si)__B);
90	}
91
92	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
93	/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit
94	/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S,
95	/// and store the packed 32-bit results in DST.
96	///
97	/// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.
98	///
99	/// \operation
100	/// FOR j := 0 to 7
101	/// tmp1.dword := SignExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
102	/// tmp2.dword := SignExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
103	/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2
104	/// ENDFOR
105	/// DST[MAX:256] := 0
106	/// \endoperation
107	static __inline__ __m256i __DEFAULT_FN_ATTRS256
108	_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
109	{
110	return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v8si)__A, (__v8si)__B);
111	}
112
113	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
114	/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit
115	/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S
116	/// using signed saturation, and store the packed 32-bit results in DST.
117	///
118	/// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.
119	///
120	/// \operation
121	/// FOR j := 0 to 7
122	/// tmp1.dword := SignExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
123	/// tmp2.dword := SignExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
124	/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2)
125	/// ENDFOR
126	/// DST[MAX:256] := 0
127	/// \endoperation
128	static __inline__ __m256i __DEFAULT_FN_ATTRS256
129	_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
130	{
131	return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v8si)__A, (__v8si)__B);
132	}
133
134	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
135	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed
136	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
137	/// in \a __S, and store the packed 32-bit results in DST.
138	///
139	/// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.
140	///
141	/// \operation
142	/// FOR j := 0 to 3
143	/// tmp1.word := Signed(ZeroExtend16(__A.byte[4j]) SignExtend16(__B.byte[4*j]))
144	/// tmp2.word := Signed(ZeroExtend16(__A.byte[4j+1]) SignExtend16(__B.byte[4*j+1]))
145	/// tmp3.word := Signed(ZeroExtend16(__A.byte[4j+2]) SignExtend16(__B.byte[4*j+2]))
146	/// tmp4.word := Signed(ZeroExtend16(__A.byte[4j+3]) SignExtend16(__B.byte[4*j+3]))
147	/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4
148	/// ENDFOR
149	/// DST[MAX:128] := 0
150	/// \endoperation
151	static __inline__ __m128i __DEFAULT_FN_ATTRS128
152	_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
153	{
154	return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v4si)__A, (__v4si)__B);
155	}
156
157	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
158	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed
159	/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
160	/// in \a __S using signed saturation, and store the packed 32-bit results in DST.
161	///
162	/// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.
163	///
164	/// \operation
165	/// FOR j := 0 to 3
166	/// tmp1.word := Signed(ZeroExtend16(__A.byte[4j]) SignExtend16(__B.byte[4*j]))
167	/// tmp2.word := Signed(ZeroExtend16(__A.byte[4j+1]) SignExtend16(__B.byte[4*j+1]))
168	/// tmp3.word := Signed(ZeroExtend16(__A.byte[4j+2]) SignExtend16(__B.byte[4*j+2]))
169	/// tmp4.word := Signed(ZeroExtend16(__A.byte[4j+3]) SignExtend16(__B.byte[4*j+3]))
170	/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)
171	/// ENDFOR
172	/// DST[MAX:128] := 0
173	/// \endoperation
174	static __inline__ __m128i __DEFAULT_FN_ATTRS128
175	_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
176	{
177	return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v4si)__A, (__v4si)__B);
178	}
179
180	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
181	/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit
182	/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S,
183	/// and store the packed 32-bit results in DST.
184	///
185	/// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.
186	///
187	/// \operation
188	/// FOR j := 0 to 3
189	/// tmp1.dword := SignExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
190	/// tmp2.dword := SignExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
191	/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2
192	/// ENDFOR
193	/// DST[MAX:128] := 0
194	/// \endoperation
195	static __inline__ __m128i __DEFAULT_FN_ATTRS128
196	_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
197	{
198	return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v4si)__A, (__v4si)__B);
199	}
200
201	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
202	/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit
203	/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S
204	/// using signed saturation, and store the packed 32-bit results in DST.
205	///
206	/// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.
207	///
208	/// \operation
209	/// FOR j := 0 to 3
210	/// tmp1.dword := SignExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
211	/// tmp2.dword := SignExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
212	/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2)
213	/// ENDFOR
214	/// DST[MAX:128] := 0
215	/// \endoperation
216	static __inline__ __m128i __DEFAULT_FN_ATTRS128
217	_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
218	{
219	return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v4si)__A, (__v4si)__B);
220	}
221
222	#undef __DEFAULT_FN_ATTRS128
223	#undef __DEFAULT_FN_ATTRS256
224
225	#endif // __AVXVNNIINTRIN_H

lib/include/cpuid.h+8

...	@@ -7,6 +7,9 @@	...	@@ -7,6 +7,9 @@
7	*===-----------------------------------------------------------------------===	7	*===-----------------------------------------------------------------------===
8	*/	8	*/
9		9
		10	#ifndef __CPUID_H
		11	#define __CPUID_H
		12
10	#if !(__x86_64__ \|\| __i386__)	13	#if !(__x86_64__ \|\| __i386__)
11	#error this header is for x86 only	14	#error this header is for x86 only
12	#endif	15	#endif
...	@@ -186,6 +189,7 @@	...	@@ -186,6 +189,7 @@
186	/* Features in %edx for leaf 7 sub-leaf 0 */	189	/* Features in %edx for leaf 7 sub-leaf 0 */
187	#define bit_AVX5124VNNIW 0x00000004	190	#define bit_AVX5124VNNIW 0x00000004
188	#define bit_AVX5124FMAPS 0x00000008	191	#define bit_AVX5124FMAPS 0x00000008
		192	#define bit_UINTR 0x00000020
189	#define bit_SERIALIZE 0x00004000	193	#define bit_SERIALIZE 0x00004000
190	#define bit_TSXLDTRK 0x00010000	194	#define bit_TSXLDTRK 0x00010000
191	#define bit_PCONFIG 0x00040000	195	#define bit_PCONFIG 0x00040000
...	@@ -195,7 +199,9 @@	...	@@ -195,7 +199,9 @@
195	#define bit_AMXINT8 0x02000000	199	#define bit_AMXINT8 0x02000000
196		200
197	/* Features in %eax for leaf 7 sub-leaf 1 */	201	/* Features in %eax for leaf 7 sub-leaf 1 */
		202	#define bit_AVXVNNI 0x00000008
198	#define bit_AVX512BF16 0x00000020	203	#define bit_AVX512BF16 0x00000020
		204	#define bit_HRESET 0x00400000
199		205
200	/* Features in %eax for leaf 13 sub-leaf 1 */	206	/* Features in %eax for leaf 13 sub-leaf 1 */
201	#define bit_XSAVEOPT 0x00000001	207	#define bit_XSAVEOPT 0x00000001
...	@@ -309,3 +315,5 @@ static __inline int __get_cpuid_count (unsigned int __leaf,	...	@@ -309,3 +315,5 @@ static __inline int __get_cpuid_count (unsigned int __leaf,
309	__cpuid_count(__leaf, __subleaf, __eax, __ebx, __ecx, __edx);	315	__cpuid_count(__leaf, __subleaf, __eax, __ebx, __ecx, __edx);
310	return 1;	316	return 1;
311	}	317	}
		318
		319	#endif /* __CPUID_H */

lib/include/cuda_wrappers/new+8

 @@ -26,6 +26,13 @@
 #include_next <new>
+#if !defined(__device__)
+// The header has been included too early from the standard C++ library
+// and CUDA-specific macros are not available yet.
+// Undo the include guard and try again later.
+#undef __CLANG_CUDA_WRAPPERS_NEW
+#else
 #pragma push_macro("CUDA_NOEXCEPT")
 #if __cplusplus >= 201103L
 #define CUDA_NOEXCEPT noexcept
 @@ -95,4 +102,5 @@ __device__ inline void operator delete[](void *, void *) CUDA_NOEXCEPT {}
 #pragma pop_macro("CUDA_NOEXCEPT")
+#endif // __device__
 #endif // include guard

lib/include/gfniintrin.h+90-91

...	@@ -14,38 +14,56 @@	...	@@ -14,38 +14,56 @@
14	#ifndef __GFNIINTRIN_H	14	#ifndef __GFNIINTRIN_H
15	#define __GFNIINTRIN_H	15	#define __GFNIINTRIN_H
16		16
		17	/* Default attributes for simple form (no masking). */
		18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("gfni"), __min_vector_width__(128)))
		19
		20	/* Default attributes for YMM unmasked form. */
		21	#define __DEFAULT_FN_ATTRS_Y __attribute__((__always_inline__, __nodebug__, __target__("avx,gfni"), __min_vector_width__(256)))
		22
		23	/* Default attributes for ZMM forms. */
		24	#define __DEFAULT_FN_ATTRS_Z __attribute__((__always_inline__, __nodebug__, __target__("avx512bw,gfni"), __min_vector_width__(512)))
		25
		26	/* Default attributes for VLX forms. */
		27	#define __DEFAULT_FN_ATTRS_VL128 __attribute__((__always_inline__, __nodebug__, __target__("avx512bw,avx512vl,gfni"), __min_vector_width__(128)))
		28	#define __DEFAULT_FN_ATTRS_VL256 __attribute__((__always_inline__, __nodebug__, __target__("avx512bw,avx512vl,gfni"), __min_vector_width__(256)))
17		29
18	#define _mm_gf2p8affineinv_epi64_epi8(A, B, I) \	30	#define _mm_gf2p8affineinv_epi64_epi8(A, B, I) \
19	(__m128i)__builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \	31	(__m128i)__builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \
20	(__v16qi)(__m128i)(B), \	32	(__v16qi)(__m128i)(B), \
21	(char)(I))	33	(char)(I))
22		34
23	#define _mm_mask_gf2p8affineinv_epi64_epi8(S, U, A, B, I) \	35	#define _mm_gf2p8affine_epi64_epi8(A, B, I) \
24	(__m128i)__builtin_ia32_selectb_128((__mmask16)(U), \	36	(__m128i)__builtin_ia32_vgf2p8affineqb_v16qi((__v16qi)(__m128i)(A), \
25	(__v16qi)_mm_gf2p8affineinv_epi64_epi8(A, B, I), \	37	(__v16qi)(__m128i)(B), \
26	(__v16qi)(__m128i)(S))	38	(char)(I))
27
28
29	#define _mm_maskz_gf2p8affineinv_epi64_epi8(U, A, B, I) \
30	(__m128i)_mm_mask_gf2p8affineinv_epi64_epi8((__m128i)_mm_setzero_si128(), \
31	U, A, B, I)
32		39
		40	static __inline__ __m128i __DEFAULT_FN_ATTRS
		41	_mm_gf2p8mul_epi8(__m128i __A, __m128i __B)
		42	{
		43	return (__m128i) __builtin_ia32_vgf2p8mulb_v16qi((__v16qi) __A,
		44	(__v16qi) __B);
		45	}
33		46
		47	#ifdef __AVXINTRIN_H
34	#define _mm256_gf2p8affineinv_epi64_epi8(A, B, I) \	48	#define _mm256_gf2p8affineinv_epi64_epi8(A, B, I) \
35	(__m256i)__builtin_ia32_vgf2p8affineinvqb_v32qi((__v32qi)(__m256i)(A), \	49	(__m256i)__builtin_ia32_vgf2p8affineinvqb_v32qi((__v32qi)(__m256i)(A), \
36	(__v32qi)(__m256i)(B), \	50	(__v32qi)(__m256i)(B), \
37	(char)(I))	51	(char)(I))
38		52
39	#define _mm256_mask_gf2p8affineinv_epi64_epi8(S, U, A, B, I) \	53	#define _mm256_gf2p8affine_epi64_epi8(A, B, I) \
40	(__m256i)__builtin_ia32_selectb_256((__mmask32)(U), \	54	(__m256i)__builtin_ia32_vgf2p8affineqb_v32qi((__v32qi)(__m256i)(A), \
41	(__v32qi)_mm256_gf2p8affineinv_epi64_epi8(A, B, I), \	55	(__v32qi)(__m256i)(B), \
42	(__v32qi)(__m256i)(S))	56	(char)(I))
43
44	#define _mm256_maskz_gf2p8affineinv_epi64_epi8(U, A, B, I) \
45	(__m256i)_mm256_mask_gf2p8affineinv_epi64_epi8((__m256i)_mm256_setzero_si256(), \
46	U, A, B, I)
47		57
		58	static __inline__ __m256i __DEFAULT_FN_ATTRS_Y
		59	_mm256_gf2p8mul_epi8(__m256i __A, __m256i __B)
		60	{
		61	return (__m256i) __builtin_ia32_vgf2p8mulb_v32qi((__v32qi) __A,
		62	(__v32qi) __B);
		63	}
		64	#endif /* __AVXINTRIN_H */
48		65
		66	#ifdef __AVX512BWINTRIN_H
49	#define _mm512_gf2p8affineinv_epi64_epi8(A, B, I) \	67	#define _mm512_gf2p8affineinv_epi64_epi8(A, B, I) \
50	(__m512i)__builtin_ia32_vgf2p8affineinvqb_v64qi((__v64qi)(__m512i)(A), \	68	(__m512i)__builtin_ia32_vgf2p8affineinvqb_v64qi((__v64qi)(__m512i)(A), \
51	(__v64qi)(__m512i)(B), \	69	(__v64qi)(__m512i)(B), \
...	@@ -60,27 +78,71 @@	...	@@ -60,27 +78,71 @@
60	(__m512i)_mm512_mask_gf2p8affineinv_epi64_epi8((__m512i)_mm512_setzero_si512(), \	78	(__m512i)_mm512_mask_gf2p8affineinv_epi64_epi8((__m512i)_mm512_setzero_si512(), \
61	U, A, B, I)	79	U, A, B, I)
62		80
63	#define _mm_gf2p8affine_epi64_epi8(A, B, I) \	81	#define _mm512_gf2p8affine_epi64_epi8(A, B, I) \
64	(__m128i)__builtin_ia32_vgf2p8affineqb_v16qi((__v16qi)(__m128i)(A), \	82	(__m512i)__builtin_ia32_vgf2p8affineqb_v64qi((__v64qi)(__m512i)(A), \
65	(__v16qi)(__m128i)(B), \	83	(__v64qi)(__m512i)(B), \
66	(char)(I))	84	(char)(I))
67		85
		86	#define _mm512_mask_gf2p8affine_epi64_epi8(S, U, A, B, I) \
		87	(__m512i)__builtin_ia32_selectb_512((__mmask64)(U), \
		88	(__v64qi)_mm512_gf2p8affine_epi64_epi8(A, B, I), \
		89	(__v64qi)(__m512i)(S))
		90
		91	#define _mm512_maskz_gf2p8affine_epi64_epi8(U, A, B, I) \
		92	(__m512i)_mm512_mask_gf2p8affine_epi64_epi8((__m512i)_mm512_setzero_si512(), \
		93	U, A, B, I)
		94
		95	static __inline__ __m512i __DEFAULT_FN_ATTRS_Z
		96	_mm512_gf2p8mul_epi8(__m512i __A, __m512i __B)
		97	{
		98	return (__m512i) __builtin_ia32_vgf2p8mulb_v64qi((__v64qi) __A,
		99	(__v64qi) __B);
		100	}
		101
		102	static __inline__ __m512i __DEFAULT_FN_ATTRS_Z
		103	_mm512_mask_gf2p8mul_epi8(__m512i __S, __mmask64 __U, __m512i __A, __m512i __B)
		104	{
		105	return (__m512i) __builtin_ia32_selectb_512(__U,
		106	(__v64qi) _mm512_gf2p8mul_epi8(__A, __B),
		107	(__v64qi) __S);
		108	}
		109
		110	static __inline__ __m512i __DEFAULT_FN_ATTRS_Z
		111	_mm512_maskz_gf2p8mul_epi8(__mmask64 __U, __m512i __A, __m512i __B)
		112	{
		113	return _mm512_mask_gf2p8mul_epi8((__m512i)_mm512_setzero_si512(),
		114	__U, __A, __B);
		115	}
		116	#endif /* __AVX512BWINTRIN_H */
		117
		118	#ifdef __AVX512VLBWINTRIN_H
		119	#define _mm_mask_gf2p8affineinv_epi64_epi8(S, U, A, B, I) \
		120	(__m128i)__builtin_ia32_selectb_128((__mmask16)(U), \
		121	(__v16qi)_mm_gf2p8affineinv_epi64_epi8(A, B, I), \
		122	(__v16qi)(__m128i)(S))
		123
		124	#define _mm_maskz_gf2p8affineinv_epi64_epi8(U, A, B, I) \
		125	(__m128i)_mm_mask_gf2p8affineinv_epi64_epi8((__m128i)_mm_setzero_si128(), \
		126	U, A, B, I)
		127
		128	#define _mm256_mask_gf2p8affineinv_epi64_epi8(S, U, A, B, I) \
		129	(__m256i)__builtin_ia32_selectb_256((__mmask32)(U), \
		130	(__v32qi)_mm256_gf2p8affineinv_epi64_epi8(A, B, I), \
		131	(__v32qi)(__m256i)(S))
		132
		133	#define _mm256_maskz_gf2p8affineinv_epi64_epi8(U, A, B, I) \
		134	(__m256i)_mm256_mask_gf2p8affineinv_epi64_epi8((__m256i)_mm256_setzero_si256(), \
		135	U, A, B, I)
		136
68	#define _mm_mask_gf2p8affine_epi64_epi8(S, U, A, B, I) \	137	#define _mm_mask_gf2p8affine_epi64_epi8(S, U, A, B, I) \
69	(__m128i)__builtin_ia32_selectb_128((__mmask16)(U), \	138	(__m128i)__builtin_ia32_selectb_128((__mmask16)(U), \
70	(__v16qi)_mm_gf2p8affine_epi64_epi8(A, B, I), \	139	(__v16qi)_mm_gf2p8affine_epi64_epi8(A, B, I), \
71	(__v16qi)(__m128i)(S))	140	(__v16qi)(__m128i)(S))
72		141
73
74	#define _mm_maskz_gf2p8affine_epi64_epi8(U, A, B, I) \	142	#define _mm_maskz_gf2p8affine_epi64_epi8(U, A, B, I) \
75	(__m128i)_mm_mask_gf2p8affine_epi64_epi8((__m128i)_mm_setzero_si128(), \	143	(__m128i)_mm_mask_gf2p8affine_epi64_epi8((__m128i)_mm_setzero_si128(), \
76	U, A, B, I)	144	U, A, B, I)
77		145
78
79	#define _mm256_gf2p8affine_epi64_epi8(A, B, I) \
80	(__m256i)__builtin_ia32_vgf2p8affineqb_v32qi((__v32qi)(__m256i)(A), \
81	(__v32qi)(__m256i)(B), \
82	(char)(I))
83
84	#define _mm256_mask_gf2p8affine_epi64_epi8(S, U, A, B, I) \	146	#define _mm256_mask_gf2p8affine_epi64_epi8(S, U, A, B, I) \
85	(__m256i)__builtin_ia32_selectb_256((__mmask32)(U), \	147	(__m256i)__builtin_ia32_selectb_256((__mmask32)(U), \
86	(__v32qi)_mm256_gf2p8affine_epi64_epi8(A, B, I), \	148	(__v32qi)_mm256_gf2p8affine_epi64_epi8(A, B, I), \
...	@@ -90,41 +152,6 @@	...	@@ -90,41 +152,6 @@
90	(__m256i)_mm256_mask_gf2p8affine_epi64_epi8((__m256i)_mm256_setzero_si256(), \	152	(__m256i)_mm256_mask_gf2p8affine_epi64_epi8((__m256i)_mm256_setzero_si256(), \
91	U, A, B, I)	153	U, A, B, I)
92		154
93
94	#define _mm512_gf2p8affine_epi64_epi8(A, B, I) \
95	(__m512i)__builtin_ia32_vgf2p8affineqb_v64qi((__v64qi)(__m512i)(A), \
96	(__v64qi)(__m512i)(B), \
97	(char)(I))
98
99	#define _mm512_mask_gf2p8affine_epi64_epi8(S, U, A, B, I) \
100	(__m512i)__builtin_ia32_selectb_512((__mmask64)(U), \
101	(__v64qi)_mm512_gf2p8affine_epi64_epi8(A, B, I), \
102	(__v64qi)(__m512i)(S))
103
104	#define _mm512_maskz_gf2p8affine_epi64_epi8(U, A, B, I) \
105	(__m512i)_mm512_mask_gf2p8affine_epi64_epi8((__m512i)_mm512_setzero_si512(), \
106	U, A, B, I)
107
108	/* Default attributes for simple form (no masking). */
109	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("gfni"), __min_vector_width__(128)))
110
111	/* Default attributes for YMM unmasked form. */
112	#define __DEFAULT_FN_ATTRS_Y __attribute__((__always_inline__, __nodebug__, __target__("avx,gfni"), __min_vector_width__(256)))
113
114	/* Default attributes for ZMM forms. */
115	#define __DEFAULT_FN_ATTRS_Z __attribute__((__always_inline__, __nodebug__, __target__("avx512bw,gfni"), __min_vector_width__(512)))
116
117	/* Default attributes for VLX forms. */
118	#define __DEFAULT_FN_ATTRS_VL128 __attribute__((__always_inline__, __nodebug__, __target__("avx512bw,avx512vl,gfni"), __min_vector_width__(128)))
119	#define __DEFAULT_FN_ATTRS_VL256 __attribute__((__always_inline__, __nodebug__, __target__("avx512bw,avx512vl,gfni"), __min_vector_width__(256)))
120
121	static __inline__ __m128i __DEFAULT_FN_ATTRS
122	_mm_gf2p8mul_epi8(__m128i __A, __m128i __B)
123	{
124	return (__m128i) __builtin_ia32_vgf2p8mulb_v16qi((__v16qi) __A,
125	(__v16qi) __B);
126	}
127
128	static __inline__ __m128i __DEFAULT_FN_ATTRS_VL128	155	static __inline__ __m128i __DEFAULT_FN_ATTRS_VL128
129	_mm_mask_gf2p8mul_epi8(__m128i __S, __mmask16 __U, __m128i __A, __m128i __B)	156	_mm_mask_gf2p8mul_epi8(__m128i __S, __mmask16 __U, __m128i __A, __m128i __B)
130	{	157	{
...	@@ -140,13 +167,6 @@ _mm_maskz_gf2p8mul_epi8(__mmask16 __U, __m128i __A, __m128i __B)	...	@@ -140,13 +167,6 @@ _mm_maskz_gf2p8mul_epi8(__mmask16 __U, __m128i __A, __m128i __B)
140	__U, __A, __B);	167	__U, __A, __B);
141	}	168	}
142		169
143	static __inline__ __m256i __DEFAULT_FN_ATTRS_Y
144	_mm256_gf2p8mul_epi8(__m256i __A, __m256i __B)
145	{
146	return (__m256i) __builtin_ia32_vgf2p8mulb_v32qi((__v32qi) __A,
147	(__v32qi) __B);
148	}
149
150	static __inline__ __m256i __DEFAULT_FN_ATTRS_VL256	170	static __inline__ __m256i __DEFAULT_FN_ATTRS_VL256
151	_mm256_mask_gf2p8mul_epi8(__m256i __S, __mmask32 __U, __m256i __A, __m256i __B)	171	_mm256_mask_gf2p8mul_epi8(__m256i __S, __mmask32 __U, __m256i __A, __m256i __B)
152	{	172	{
...	@@ -161,28 +181,7 @@ _mm256_maskz_gf2p8mul_epi8(__mmask32 __U, __m256i __A, __m256i __B)	...	@@ -161,28 +181,7 @@ _mm256_maskz_gf2p8mul_epi8(__mmask32 __U, __m256i __A, __m256i __B)
161	return _mm256_mask_gf2p8mul_epi8((__m256i)_mm256_setzero_si256(),	181	return _mm256_mask_gf2p8mul_epi8((__m256i)_mm256_setzero_si256(),
162	__U, __A, __B);	182	__U, __A, __B);
163	}	183	}
164		184	#endif /* __AVX512VLBWINTRIN_H */
165	static __inline__ __m512i __DEFAULT_FN_ATTRS_Z
166	_mm512_gf2p8mul_epi8(__m512i __A, __m512i __B)
167	{
168	return (__m512i) __builtin_ia32_vgf2p8mulb_v64qi((__v64qi) __A,
169	(__v64qi) __B);
170	}
171
172	static __inline__ __m512i __DEFAULT_FN_ATTRS_Z
173	_mm512_mask_gf2p8mul_epi8(__m512i __S, __mmask64 __U, __m512i __A, __m512i __B)
174	{
175	return (__m512i) __builtin_ia32_selectb_512(__U,
176	(__v64qi) _mm512_gf2p8mul_epi8(__A, __B),
177	(__v64qi) __S);
178	}
179
180	static __inline__ __m512i __DEFAULT_FN_ATTRS_Z
181	_mm512_maskz_gf2p8mul_epi8(__mmask64 __U, __m512i __A, __m512i __B)
182	{
183	return _mm512_mask_gf2p8mul_epi8((__m512i)_mm512_setzero_si512(),
184	__U, __A, __B);
185	}
186		185
187	#undef __DEFAULT_FN_ATTRS	186	#undef __DEFAULT_FN_ATTRS
188	#undef __DEFAULT_FN_ATTRS_Y	187	#undef __DEFAULT_FN_ATTRS_Y

lib/include/hresetintrin.h created+49

...	@@ -0,0 +1,49 @@
1	/*===---------------- hresetintrin.h - HRESET intrinsics -------------------===
2	*
3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	* See https://llvm.org/LICENSE.txt for license information.
5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	*
7	*===-----------------------------------------------------------------------===
8	*/
9	#ifndef __X86GPRINTRIN_H
10	#error "Never use <hresetintrin.h> directly; include <x86gprintrin.h> instead."
11	#endif
12
13	#ifndef __HRESETINTRIN_H
14	#define __HRESETINTRIN_H
15
16	#if __has_extension(gnu_asm)
17
18	/* Define the default attributes for the functions in this file. */
19	#define __DEFAULT_FN_ATTRS \
20	__attribute__((__always_inline__, __nodebug__, __target__("hreset")))
21
22	/// Provides a hint to the processor to selectively reset the prediction
23	/// history of the current logical processor specified by a 32-bit integer
24	/// value \a __eax.
25	///
26	/// This intrinsic corresponds to the <c> HRESET </c> instruction.
27	///
28	/// \operation
29	/// IF __eax == 0
30	/// // nop
31	/// ELSE
32	/// FOR i := 0 to 31
33	/// IF __eax[i]
34	/// ResetPredictionFeature(i)
35	/// FI
36	/// ENDFOR
37	/// FI
38	/// \endoperation
39	static __inline void __DEFAULT_FN_ATTRS
40	_hreset(int __eax)
41	{
42	__asm__ ("hreset $0" :: "a"(__eax));
43	}
44
45	#undef __DEFAULT_FN_ATTRS
46
47	#endif /* __has_extension(gnu_asm) */
48
49	#endif /* __HRESETINTRIN_H */

lib/include/ia32intrin.h+53-44

 @@ -14,6 +14,18 @@
 #ifndef __IA32INTRIN_H
 #define __IA32INTRIN_H
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
+#define __DEFAULT_FN_ATTRS_SSE42 __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS_CAST __attribute__((__always_inline__)) constexpr
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS_CAST __attribute__((__always_inline__))
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
 /** Find the first set bit starting from the lsb. Result is undefined if
  *  input is 0.
  *
 @@ -26,7 +38,7 @@
  *     A 32-bit integer operand.
  *  \returns A 32-bit integer containing the bit number.
  */
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsfd(int __A) {
   return __builtin_ctz(__A);
 }
 @@ -43,7 +55,7 @@ __bsfd(int __A) {
  *     A 32-bit integer operand.
  *  \returns A 32-bit integer containing the bit number.
  */
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsrd(int __A) {
   return 31 - __builtin_clz(__A);
 }
 @@ -59,12 +71,12 @@ __bsrd(int __A) {
  *     A 32-bit integer operand.
  *  \returns A 32-bit integer containing the swapped bytes.
  */
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bswapd(int __A) {
   return __builtin_bswap32(__A);
 }
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 _bswap(int __A) {
   return __builtin_bswap32(__A);
 }
 @@ -85,7 +97,7 @@ _bswap(int __A) {
  *     A 64-bit integer operand.
  *  \returns A 32-bit integer containing the bit number.
  */
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsfq(long long __A) {
   return __builtin_ctzll(__A);
 }
 @@ -102,7 +114,7 @@ __bsfq(long long __A) {
  *     A 64-bit integer operand.
  *  \returns A 32-bit integer containing the bit number.
  */
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsrq(long long __A) {
   return 63 - __builtin_clzll(__A);
 }
 @@ -118,7 +130,7 @@ __bsrq(long long __A) {
  *     A 64-bit integer operand.
  *  \returns A 64-bit integer containing the swapped bytes.
  */
-static __inline__ long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __bswapq(long long __A) {
   return __builtin_bswap64(__A);
 }
 @@ -138,7 +150,7 @@ __bswapq(long long __A) {
  *  \returns A 32-bit integer containing the number of bits with value 1 in the
  *     source operand.
  */
-static __inline__ int __attribute__((__always_inline__, __nodebug__))
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __popcntd(unsigned int __A)
 {
   return __builtin_popcount(__A);
 @@ -159,7 +171,7 @@ __popcntd(unsigned int __A)
  *  \returns A 64-bit integer containing the number of bits with value 1 in the
  *     source operand.
  */
-static __inline__ long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __popcntq(unsigned long long __A)
 {
   return __builtin_popcountll(__A);
 @@ -169,26 +181,26 @@ __popcntq(unsigned long long __A)
 #endif /* __x86_64__ */
 #ifdef __x86_64__
-static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __readeflags(void)
 {
   return __builtin_ia32_readeflags_u64();
 }
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
+static __inline__ void __DEFAULT_FN_ATTRS
 __writeeflags(unsigned long long __f)
 {
   __builtin_ia32_writeeflags_u64(__f);
 }
 #else /* !__x86_64__ */
-static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS
 __readeflags(void)
 {
   return __builtin_ia32_readeflags_u32();
 }
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
+static __inline__ void __DEFAULT_FN_ATTRS
 __writeeflags(unsigned int __f)
 {
   __builtin_ia32_writeeflags_u32(__f);
 @@ -205,11 +217,9 @@ __writeeflags(unsigned int __f)
  *     A 32-bit float value.
  *  \returns a 32-bit unsigned integer containing the converted value.
  */
-static __inline__ unsigned int __attribute__((__always_inline__))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS_CAST
 _castf32_u32(float __A) {
-  unsigned int D;
+  return __builtin_bit_cast(unsigned int, __A);
-  __builtin_memcpy(&D, &__A, sizeof(__A));
-  return D;
 }
 /** Cast a 64-bit float value to a 64-bit unsigned integer value
 @@ -222,11 +232,9 @@ _castf32_u32(float __A) {
  *     A 64-bit float value.
  *  \returns a 64-bit unsigned integer containing the converted value.
  */
-static __inline__ unsigned long long __attribute__((__always_inline__))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CAST
 _castf64_u64(double __A) {
-  unsigned long long D;
+  return __builtin_bit_cast(unsigned long long, __A);
-  __builtin_memcpy(&D, &__A, sizeof(__A));
-  return D;
 }
 /** Cast a 32-bit unsigned integer value to a 32-bit float value
 @@ -239,11 +247,9 @@ _castf64_u64(double __A) {
  *     A 32-bit unsigned integer value.
  *  \returns a 32-bit float value containing the converted value.
  */
-static __inline__ float __attribute__((__always_inline__))
+static __inline__ float __DEFAULT_FN_ATTRS_CAST
 _castu32_f32(unsigned int __A) {
-  float D;
+  return __builtin_bit_cast(float, __A);
-  __builtin_memcpy(&D, &__A, sizeof(__A));
-  return D;
 }
 /** Cast a 64-bit unsigned integer value to a 64-bit float value
 @@ -256,11 +262,9 @@ _castu32_f32(unsigned int __A) {
  *     A 64-bit unsigned integer value.
  *  \returns a 64-bit float value containing the converted value.
  */
-static __inline__ double __attribute__((__always_inline__))
+static __inline__ double __DEFAULT_FN_ATTRS_CAST
 _castu64_f64(unsigned long long __A) {
-  double D;
+  return __builtin_bit_cast(double, __A);
-  __builtin_memcpy(&D, &__A, sizeof(__A));
-  return D;
 }
 /** Adds the unsigned integer operand to the CRC-32C checksum of the
 @@ -278,7 +282,7 @@ _castu64_f64(unsigned long long __A) {
  *  \returns The result of adding operand \a __C to the CRC-32C checksum of
  *     operand \a __D.
  */
-static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS_SSE42
 __crc32b(unsigned int __C, unsigned char __D)
 {
   return __builtin_ia32_crc32qi(__C, __D);
 @@ -299,7 +303,7 @@ __crc32b(unsigned int __C, unsigned char __D)
  *  \returns The result of adding operand \a __C to the CRC-32C checksum of
  *     operand \a __D.
  */
-static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS_SSE42
 __crc32w(unsigned int __C, unsigned short __D)
 {
   return __builtin_ia32_crc32hi(__C, __D);
 @@ -320,7 +324,7 @@ __crc32w(unsigned int __C, unsigned short __D)
  *  \returns The result of adding operand \a __C to the CRC-32C checksum of
  *     operand \a __D.
  */
-static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS_SSE42
 __crc32d(unsigned int __C, unsigned int __D)
 {
   return __builtin_ia32_crc32si(__C, __D);
 @@ -342,20 +346,20 @@ __crc32d(unsigned int __C, unsigned int __D)
  *  \returns The result of adding operand \a __C to the CRC-32C checksum of
  *     operand \a __D.
  */
-static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS_SSE42
 __crc32q(unsigned long long __C, unsigned long long __D)
 {
   return __builtin_ia32_crc32di(__C, __D);
 }
 #endif /* __x86_64__ */
-static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __rdpmc(int __A) {
   return __builtin_ia32_rdpmc(__A);
 }
 /* __rdtscp */
-static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __rdtscp(unsigned int *__A) {
   return __builtin_ia32_rdtscp(__A);
 }
 @@ -364,48 +368,48 @@ __rdtscp(unsigned int *__A) {
 #define _rdpmc(A) __rdpmc(A)
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
+static __inline__ void __DEFAULT_FN_ATTRS
 _wbinvd(void) {
   __builtin_ia32_wbinvd();
 }
-static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
 __rolb(unsigned char __X, int __C) {
   return __builtin_rotateleft8(__X, __C);
 }
-static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
 __rorb(unsigned char __X, int __C) {
   return __builtin_rotateright8(__X, __C);
 }
-static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned short __DEFAULT_FN_ATTRS_CONSTEXPR
 __rolw(unsigned short __X, int __C) {
   return __builtin_rotateleft16(__X, __C);
 }
-static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned short __DEFAULT_FN_ATTRS_CONSTEXPR
 __rorw(unsigned short __X, int __C) {
   return __builtin_rotateright16(__X, __C);
 }
-static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS_CONSTEXPR
 __rold(unsigned int __X, int __C) {
   return __builtin_rotateleft32(__X, __C);
 }
-static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __DEFAULT_FN_ATTRS_CONSTEXPR
 __rord(unsigned int __X, int __C) {
   return __builtin_rotateright32(__X, __C);
 }
 #ifdef __x86_64__
-static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __rolq(unsigned long long __X, int __C) {
   return __builtin_rotateleft64(__X, __C);
 }
-static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __rorq(unsigned long long __X, int __C) {
   return __builtin_rotateright64(__X, __C);
 }
 @@ -429,4 +433,9 @@ __rorq(unsigned long long __X, int __C) {
 #define _rotwl(a,b) __rolw((a), (b))
 #define _rotwr(a,b) __rorw((a), (b))
+#undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS_CAST
+#undef __DEFAULT_FN_ATTRS_SSE42
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
 #endif /* __IA32INTRIN_H */

lib/include/immintrin.h+12

 @@ -10,6 +10,8 @@
 #ifndef __IMMINTRIN_H
 #define __IMMINTRIN_H
+#include <x86gprintrin.h>
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__MMX__)
 #include <mmintrin.h>
 @@ -143,6 +145,11 @@
 #include <avx512vlvnniintrin.h>
 #endif
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__AVXVNNI__)
+#include <avxvnniintrin.h>
+#endif
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__AVX512DQ__)
 #include <avx512dqintrin.h>
 @@ -471,6 +478,11 @@ _storebe_i64(void * __P, long long __D) {
 #include <invpcidintrin.h>
 #endif
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__KL__) || defined(__WIDEKL__)
+#include <keylockerintrin.h>
+#endif
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
 #include <amxintrin.h>

lib/include/intrin.h+78-91

 @@ -57,16 +57,11 @@ void __addfsbyte(unsigned long, unsigned char);
 void __addfsdword(unsigned long, unsigned long);
 void __addfsword(unsigned long, unsigned short);
 void __code_seg(const char *);
-static __inline__
 void __cpuid(int[4], int);
-static __inline__
 void __cpuidex(int[4], int, int);
-static __inline__
 __int64 __emul(int, int);
-static __inline__
 unsigned __int64 __emulu(unsigned int, unsigned int);
 unsigned int __getcallerseflags(void);
-static __inline__
 void __halt(void);
 unsigned char __inbyte(unsigned short);
 void __inbytestring(unsigned short, unsigned char *, unsigned long);
 @@ -82,13 +77,9 @@ void __inwordstring(unsigned short, unsigned short *, unsigned long);
 void __lidt(void *);
 unsigned __int64 __ll_lshift(unsigned __int64, int);
 __int64 __ll_rshift(__int64, int);
-static __inline__
 void __movsb(unsigned char *, unsigned char const *, size_t);
-static __inline__
 void __movsd(unsigned long *, unsigned long const *, size_t);
-static __inline__
 void __movsw(unsigned short *, unsigned short const *, size_t);
-static __inline__
 void __nop(void);
 void __nvreg_restore_fence(void);
 void __nvreg_save_fence(void);
 @@ -105,23 +96,16 @@ unsigned long __readcr4(void);
 unsigned long __readcr8(void);
 unsigned int __readdr(unsigned int);
 #ifdef __i386__
-static __inline__
 unsigned char __readfsbyte(unsigned long);
-static __inline__
 unsigned __int64 __readfsqword(unsigned long);
-static __inline__
 unsigned short __readfsword(unsigned long);
 #endif
-static __inline__
 unsigned __int64 __readmsr(unsigned long);
 unsigned __int64 __readpmc(unsigned long);
 unsigned long __segmentlimit(unsigned long);
 void __sidt(void *);
-static __inline__
 void __stosb(unsigned char *, unsigned char, size_t);
-static __inline__
 void __stosd(unsigned long *, unsigned long, size_t);
-static __inline__
 void __stosw(unsigned short *, unsigned short, size_t);
 void __svm_clgi(void);
 void __svm_invlpga(void *, int);
 @@ -136,7 +120,6 @@ void __vmx_off(void);
 void __vmx_vmptrst(unsigned __int64 *);
 void __wbinvd(void);
 void __writecr0(unsigned int);
-static __inline__
 void __writecr3(unsigned __INTPTR_TYPE__);
 void __writecr4(unsigned int);
 void __writecr8(unsigned int);
 @@ -146,11 +129,8 @@ void __writefsdword(unsigned long, unsigned long);
 void __writefsqword(unsigned long, unsigned __int64);
 void __writefsword(unsigned long, unsigned short);
 void __writemsr(unsigned long, unsigned __int64);
-static __inline__
 void *_AddressOfReturnAddress(void);
-static __inline__
 unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
-static __inline__
 unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
 unsigned char _bittest(long const *, long);
 unsigned char _bittestandcomplement(long *, long);
 @@ -169,12 +149,10 @@ long _InterlockedExchangeAdd_HLEAcquire(long volatile *, long);
 long _InterlockedExchangeAdd_HLERelease(long volatile *, long);
 __int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64);
 __int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64);
-static __inline__ void
+void __attribute__((__deprecated__(
-__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead")))
+    "use other intrinsics or C++11 atomics instead"))) _ReadBarrier(void);
-_ReadBarrier(void);
+void __attribute__((__deprecated__(
-static __inline__ void
+    "use other intrinsics or C++11 atomics instead"))) _ReadWriteBarrier(void);
-__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead")))
-_ReadWriteBarrier(void);
 unsigned int _rorx_u32(unsigned int, const unsigned int);
 int _sarx_i32(int, unsigned int);
 #if __STDC_HOSTED__
 @@ -185,9 +163,8 @@ unsigned int _shrx_u32(unsigned int, unsigned int);
 void _Store_HLERelease(long volatile *, long);
 void _Store64_HLERelease(__int64 volatile *, __int64);
 void _StorePointer_HLERelease(void *volatile *, void *);
-static __inline__ void
+void __attribute__((__deprecated__(
-__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead")))
+    "use other intrinsics or C++11 atomics instead"))) _WriteBarrier(void);
-_WriteBarrier(void);
 unsigned __int32 xbegin(void);
 void _xend(void);
 @@ -197,19 +174,14 @@ void __addgsbyte(unsigned long, unsigned char);
 void __addgsdword(unsigned long, unsigned long);
 void __addgsqword(unsigned long, unsigned __int64);
 void __addgsword(unsigned long, unsigned short);
-static __inline__
 void __faststorefence(void);
 void __incgsbyte(unsigned long);
 void __incgsdword(unsigned long);
 void __incgsqword(unsigned long);
 void __incgsword(unsigned long);
-static __inline__
 void __movsq(unsigned long long *, unsigned long long const *, size_t);
-static __inline__
 unsigned char __readgsbyte(unsigned long);
-static __inline__
 unsigned long __readgsdword(unsigned long);
-static __inline__
 unsigned __int64 __readgsqword(unsigned long);
 unsigned short __readgsword(unsigned long);
 unsigned __int64 __shiftleft128(unsigned __int64 _LowPart,
 @@ -218,7 +190,6 @@ unsigned __int64 __shiftleft128(unsigned __int64 _LowPart,
 unsigned __int64 __shiftright128(unsigned __int64 _LowPart,
                                  unsigned __int64 _HighPart,
                                  unsigned char _Shift);
-static __inline__
 void __stosq(unsigned __int64 *, unsigned __int64, size_t);
 unsigned char __vmx_on(unsigned __int64 *);
 unsigned char __vmx_vmclear(unsigned __int64 *);
 @@ -243,10 +214,6 @@ unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64);
 unsigned char _interlockedbittestandset64(__int64 volatile *, __int64);
 long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange,
                                     long _Comparand);
-unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination,
-                                             __int64 _ExchangeHigh,
-                                             __int64 _ExchangeLow,
-                                             __int64 *_CompareandResult);
 unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination,
                                                 __int64 _ExchangeHigh,
                                                 __int64 _ExchangeLow,
 @@ -269,13 +236,9 @@ unsigned __int64 _rorx_u64(unsigned __int64, const unsigned int);
 __int64 _sarx_i64(__int64, unsigned int);
 unsigned __int64 _shlx_u64(unsigned __int64, unsigned int);
 unsigned __int64 _shrx_u64(unsigned __int64, unsigned int);
-static __inline__
 __int64 __mulh(__int64, __int64);
-static __inline__
 unsigned __int64 __umulh(unsigned __int64, unsigned __int64);
-static __inline__
 __int64 _mul128(__int64, __int64, __int64*);
-static __inline__
 unsigned __int64 _umul128(unsigned __int64,
                           unsigned __int64,
                           unsigned __int64*);
 @@ -284,29 +247,19 @@ unsigned __int64 _umul128(unsigned __int64,
 #if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__)
-static __inline__
 unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
-static __inline__
 unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
 #endif
 #if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__aarch64__)
-static __inline__
 __int64 _InterlockedDecrement64(__int64 volatile *_Addend);
-static __inline__
 __int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value);
-static __inline__
 __int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value);
-static __inline__
 __int64 _InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value);
-static __inline__
 __int64 _InterlockedIncrement64(__int64 volatile *_Addend);
-static __inline__
 __int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask);
-static __inline__
 __int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask);
-static __inline__
 __int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask);
 #endif
 @@ -470,45 +423,81 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
 __int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
                               __int64 _Exchange, __int64 _Comparand);
 #endif
+#if defined(__x86_64__) || defined(__aarch64__)
+unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination,
+                                             __int64 _ExchangeHigh,
+                                             __int64 _ExchangeLow,
+                                             __int64 *_ComparandResult);
+#endif
+#if defined(__aarch64__)
+unsigned char _InterlockedCompareExchange128_acq(__int64 volatile *_Destination,
+                                                 __int64 _ExchangeHigh,
+                                                 __int64 _ExchangeLow,
+                                                 __int64 *_ComparandResult);
+unsigned char _InterlockedCompareExchange128_nf(__int64 volatile *_Destination,
+                                                __int64 _ExchangeHigh,
+                                                __int64 _ExchangeLow,
+                                                __int64 *_ComparandResult);
+unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination,
+                                                 __int64 _ExchangeHigh,
+                                                 __int64 _ExchangeLow,
+                                                 __int64 *_ComparandResult);
+#endif
 /*----------------------------------------------------------------------------*\
 |* movs, stos
 \*----------------------------------------------------------------------------*/
 #if defined(__i386__) || defined(__x86_64__)
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __movsb(unsigned char *__dst,
-__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) {
+                                                  unsigned char const *__src,
+                                                  size_t __n) {
   __asm__ __volatile__("rep movsb" : "+D"(__dst), "+S"(__src), "+c"(__n)
                        : : "memory");
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __movsd(unsigned long *__dst,
-__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) {
+                                                  unsigned long const *__src,
-  __asm__ __volatile__("rep movsl" : "+D"(__dst), "+S"(__src), "+c"(__n)
+                                                  size_t __n) {
-                       : : "memory");
+  __asm__ __volatile__("rep movsl"
+                       : "+D"(__dst), "+S"(__src), "+c"(__n)
+                       :
+                       : "memory");
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __movsw(unsigned short *__dst,
-__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) {
+                                                  unsigned short const *__src,
-  __asm__ __volatile__("rep movsw" : "+D"(__dst), "+S"(__src), "+c"(__n)
+                                                  size_t __n) {
-                       : : "memory");
+  __asm__ __volatile__("rep movsw"
+                       : "+D"(__dst), "+S"(__src), "+c"(__n)
+                       :
+                       : "memory");
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __stosd(unsigned long *__dst,
-__stosd(unsigned long *__dst, unsigned long __x, size_t __n) {
+                                                  unsigned long __x,
-  __asm__ __volatile__("rep stosl" : "+D"(__dst), "+c"(__n) : "a"(__x)
+                                                  size_t __n) {
+  __asm__ __volatile__("rep stosl"
+                       : "+D"(__dst), "+c"(__n)
+                       : "a"(__x)
                        : "memory");
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __stosw(unsigned short *__dst,
-__stosw(unsigned short *__dst, unsigned short __x, size_t __n) {
+                                                  unsigned short __x,
-  __asm__ __volatile__("rep stosw" : "+D"(__dst), "+c"(__n) : "a"(__x)
+                                                  size_t __n) {
+  __asm__ __volatile__("rep stosw"
+                       : "+D"(__dst), "+c"(__n)
+                       : "a"(__x)
                        : "memory");
 }
 #endif
 #ifdef __x86_64__
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __movsq(
-__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) {
+    unsigned long long *__dst, unsigned long long const *__src, size_t __n) {
-  __asm__ __volatile__("rep movsq" : "+D"(__dst), "+S"(__src), "+c"(__n)
+  __asm__ __volatile__("rep movsq"
-                       : : "memory");
+                       : "+D"(__dst), "+S"(__src), "+c"(__n)
+                       :
+                       : "memory");
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __stosq(unsigned __int64 *__dst,
-__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) {
+                                                  unsigned __int64 __x,
+                                                  size_t __n) {
   __asm__ __volatile__("rep stosq" : "+D"(__dst), "+c"(__n) : "a"(__x)
                        : "memory");
 }
 @@ -518,26 +507,25 @@ __stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) {
 |* Misc
 \*----------------------------------------------------------------------------*/
 #if defined(__i386__) || defined(__x86_64__)
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __cpuid(int __info[4], int __level) {
-__cpuid(int __info[4], int __level) {
+  __asm__("cpuid"
-  __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3])
+          : "=a"(__info[0]), "=b"(__info[1]), "=c"(__info[2]), "=d"(__info[3])
-                   : "a"(__level), "c"(0));
+          : "a"(__level), "c"(0));
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __cpuidex(int __info[4], int __level,
-__cpuidex(int __info[4], int __level, int __ecx) {
+                                                    int __ecx) {
-  __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3])
+  __asm__("cpuid"
-                   : "a"(__level), "c"(__ecx));
+          : "=a"(__info[0]), "=b"(__info[1]), "=c"(__info[2]), "=d"(__info[3])
+          : "a"(__level), "c"(__ecx));
 }
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __halt(void) {
-__halt(void) {
+  __asm__ volatile("hlt");
-  __asm__ volatile ("hlt");
 }
 #endif
 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__)
-static __inline__ void __DEFAULT_FN_ATTRS
+static __inline__ void __DEFAULT_FN_ATTRS __nop(void) {
-__nop(void) {
+  __asm__ volatile("nop");
-  __asm__ volatile ("nop");
 }
 #endif
 @@ -574,8 +562,7 @@ __readmsr(unsigned long __register) {
 }
 #endif
-static __inline__ unsigned __LPTRINT_TYPE__ __DEFAULT_FN_ATTRS
+static __inline__ unsigned __LPTRINT_TYPE__ __DEFAULT_FN_ATTRS __readcr3(void) {
-__readcr3(void) {
   unsigned __LPTRINT_TYPE__ __cr3_val;
   __asm__ __volatile__ ("mov %%cr3, %0" : "=r"(__cr3_val) : : "memory");
   return __cr3_val;

lib/include/keylockerintrin.h created+506

...	@@ -0,0 +1,506 @@
1	/*===----------------- keylockerintrin.h - KL Intrinsics -------------------===
2	*
3	* Permission is hereby granted, free of charge, to any person obtaining a copy
4	* of this software and associated documentation files (the "Software"), to deal
5	* in the Software without restriction, including without limitation the rights
6	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7	* copies of the Software, and to permit persons to whom the Software is
8	* furnished to do so, subject to the following conditions:
9	*
10	* The above copyright notice and this permission notice shall be included in
11	* all copies or substantial portions of the Software.
12	*
13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19	* THE SOFTWARE.
20	*
21	*===-----------------------------------------------------------------------===
22	*/
23
24	#ifndef __IMMINTRIN_H
25	#error "Never use <keylockerintrin.h> directly; include <immintrin.h> instead."
26	#endif
27
28	#ifndef _KEYLOCKERINTRIN_H
29	#define _KEYLOCKERINTRIN_H
30
31	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
32	defined(__KL__)
33
34	/* Define the default attributes for the functions in this file. */
35	#define __DEFAULT_FN_ATTRS \
36	__attribute__((__always_inline__, __nodebug__, __target__("kl"),\
37	__min_vector_width__(128)))
38
39	/// Load internal wrapping key from __intkey, __enkey_lo and __enkey_hi. __ctl
40	/// will assigned to EAX, whch specifies the KeySource and whether backing up
41	/// the key is permitted. The 256-bit encryption key is loaded from the two
42	/// explicit operands (__enkey_lo and __enkey_hi). The 128-bit integrity key is
43	/// loaded from the implicit operand XMM0 which assigned by __intkey.
44	///
45	/// \headerfile <x86intrin.h>
46	///
47	/// This intrinsic corresponds to the <c> LOADIWKEY </c> instructions.
48	///
49	/// \operation
50	/// IF CPL > 0 // LOADKWKEY only allowed at ring 0 (supervisor mode)
51	/// GP (0)
52	/// FI
53	/// IF “LOADIWKEY exiting” VM execution control set
54	/// VMexit
55	/// FI
56	/// IF __ctl[4:1] > 1 // Reserved KeySource encoding used
57	/// GP (0)
58	/// FI
59	/// IF __ctl[31:5] != 0 // Reserved bit in __ctl is set
60	/// GP (0)
61	/// FI
62	/// IF __ctl[0] AND (CPUID.19H.ECX[0] == 0) // NoBackup is not supported on this part
63	/// GP (0)
64	/// FI
65	/// IF (__ctl[4:1] == 1) AND (CPUID.19H.ECX[1] == 0) // KeySource of 1 is not supported on this part
66	/// GP (0)
67	/// FI
68	/// IF (__ctl[4:1] == 0) // KeySource of 0.
69	/// IWKey.Encryption Key[127:0] := __enkey_hi[127:0]:
70	/// IWKey.Encryption Key[255:128] := __enkey_lo[127:0]
71	/// IWKey.IntegrityKey[127:0] := __intkey[127:0]
72	/// IWKey.NoBackup := __ctl[0]
73	/// IWKey.KeySource := __ctl[4:1]
74	/// ZF := 0
75	/// ELSE // KeySource of 1. See RDSEED definition for details of randomness
76	/// IF HW_NRND_GEN.ready == 1 // Full-entropy random data from RDSEED was received
77	/// IWKey.Encryption Key[127:0] := __enkey_hi[127:0] XOR HW_NRND_GEN.data[127:0]
78	/// IWKey.Encryption Key[255:128] := __enkey_lo[127:0] XOR HW_NRND_GEN.data[255:128]
79	/// IWKey.Encryption Key[255:0] := __enkey_hi[127:0]:__enkey_lo[127:0] XOR HW_NRND_GEN.data[255:0]
80	/// IWKey.IntegrityKey[127:0] := __intkey[127:0] XOR HW_NRND_GEN.data[383:256]
81	/// IWKey.NoBackup := __ctl[0]
82	/// IWKey.KeySource := __ctl[4:1]
83	/// ZF := 0
84	/// ELSE // Random data was not returned from RDSEED. IWKey was not loaded
85	/// ZF := 1
86	/// FI
87	/// FI
88	/// dst := ZF
89	/// OF := 0
90	/// SF := 0
91	/// AF := 0
92	/// PF := 0
93	/// CF := 0
94	/// \endoperation
95	static __inline__ void __DEFAULT_FN_ATTRS
96	_mm_loadiwkey (unsigned int __ctl, __m128i __intkey,
97	__m128i __enkey_lo, __m128i __enkey_hi) {
98	__builtin_ia32_loadiwkey (__intkey, __enkey_lo, __enkey_hi, __ctl);
99	}
100
101	/// Wrap a 128-bit AES key from __key into a key handle and output in
102	/// ((__m128i)__h) to ((__m128i)__h) + 5 and a 32-bit value as return.
103	/// The explicit source operand __htype specifies handle restrictions.
104	///
105	/// \headerfile <x86intrin.h>
106	///
107	/// This intrinsic corresponds to the <c> ENCODEKEY128 </c> instructions.
108	///
109	/// \operation
110	/// InputKey[127:0] := __key[127:0]
111	/// KeyMetadata[2:0] := __htype[2:0]
112	/// KeyMetadata[23:3] := 0 // Reserved for future usage
113	/// KeyMetadata[27:24] := 0 // KeyType is AES-128 (value of 0)
114	/// KeyMetadata[127:28] := 0 // Reserved for future usage
115	/// Handle[383:0] := WrapKey128(InputKey[127:0], KeyMetadata[127:0],
116	/// IWKey.Integrity Key[127:0], IWKey.Encryption Key[255:0])
117	/// dst[0] := IWKey.NoBackup
118	/// dst[4:1] := IWKey.KeySource[3:0]
119	/// dst[31:5] := 0
120	/// MEM[__h+127:__h] := Handle[127:0] // AAD
121	/// MEM[__h+255:__h+128] := Handle[255:128] // Integrity Tag
122	/// MEM[__h+383:__h+256] := Handle[383:256] // CipherText
123	/// MEM[__h+511:__h+384] := 0 // Reserved for future usage
124	/// MEM[__h+639:__h+512] := 0 // Reserved for future usage
125	/// MEM[__h+767:__h+640] := 0 // Reserved for future usage
126	/// OF := 0
127	/// SF := 0
128	/// ZF := 0
129	/// AF := 0
130	/// PF := 0
131	/// CF := 0
132	/// \endoperation
133	static __inline__ unsigned int __DEFAULT_FN_ATTRS
134	_mm_encodekey128_u32(unsigned int __htype, __m128i __key, void *__h) {
135	return __builtin_ia32_encodekey128_u32(__htype, (__v2di)__key, __h);
136	}
137
138	/// Wrap a 256-bit AES key from __key_hi:__key_lo into a key handle, then
139	/// output handle in ((__m128i)__h) to ((__m128i)__h) + 6 and
140	/// a 32-bit value as return.
141	/// The explicit source operand __htype specifies handle restrictions.
142	///
143	/// \headerfile <x86intrin.h>
144	///
145	/// This intrinsic corresponds to the <c> ENCODEKEY256 </c> instructions.
146	///
147	/// \operation
148	/// InputKey[127:0] := __key_lo[127:0]
149	/// InputKey[255:128] := __key_hi[255:128]
150	/// KeyMetadata[2:0] := __htype[2:0]
151	/// KeyMetadata[23:3] := 0 // Reserved for future usage
152	/// KeyMetadata[27:24] := 1 // KeyType is AES-256 (value of 1)
153	/// KeyMetadata[127:28] := 0 // Reserved for future usage
154	/// Handle[511:0] := WrapKey256(InputKey[255:0], KeyMetadata[127:0],
155	/// IWKey.Integrity Key[127:0], IWKey.Encryption Key[255:0])
156	/// dst[0] := IWKey.NoBackup
157	/// dst[4:1] := IWKey.KeySource[3:0]
158	/// dst[31:5] := 0
159	/// MEM[__h+127:__h] := Handle[127:0] // AAD
160	/// MEM[__h+255:__h+128] := Handle[255:128] // Tag
161	/// MEM[__h+383:__h+256] := Handle[383:256] // CipherText[127:0]
162	/// MEM[__h+511:__h+384] := Handle[511:384] // CipherText[255:128]
163	/// MEM[__h+639:__h+512] := 0 // Reserved for future usage
164	/// MEM[__h+767:__h+640] := 0 // Reserved for future usage
165	/// MEM[__h+895:__h+768] := 0 Integrity// Reserved for future usage
166	/// OF := 0
167	/// SF := 0
168	/// ZF := 0
169	/// AF := 0
170	/// PF := 0
171	/// CF := 0
172	/// \endoperation
173	static __inline__ unsigned int __DEFAULT_FN_ATTRS
174	_mm_encodekey256_u32(unsigned int __htype, __m128i __key_lo, __m128i __key_hi,
175	void *__h) {
176	return __builtin_ia32_encodekey256_u32(__htype, (__v2di)__key_lo,
177	(__v2di)__key_hi, __h);
178	}
179
180	/// The AESENC128KL performs 10 rounds of AES to encrypt the __idata using
181	/// the 128-bit key in the handle from the __h. It stores the result in the
182	/// __odata. And return the affected ZF flag status.
183	///
184	/// \headerfile <x86intrin.h>
185	///
186	/// This intrinsic corresponds to the <c> AESENC128KL </c> instructions.
187	///
188	/// \operation
189	/// Handle[383:0] := MEM[__h+383:__h] // Load is not guaranteed to be atomic.
190	/// IllegalHandle := ( HandleReservedBitSet (Handle[383:0]) \|\|
191	/// (Handle[127:0] AND (CPL > 0)) \|\|
192	/// Handle[383:256] \|\|
193	/// HandleKeyType (Handle[383:0]) != HANDLE_KEY_TYPE_AES128 )
194	/// IF (IllegalHandle)
195	/// ZF := 1
196	/// ELSE
197	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate384 (Handle[383:0], IWKey)
198	/// IF (Authentic == 0)
199	/// ZF := 1
200	/// ELSE
201	/// MEM[__odata+127:__odata] := AES128Encrypt (__idata[127:0], UnwrappedKey)
202	/// ZF := 0
203	/// FI
204	/// FI
205	/// dst := ZF
206	/// OF := 0
207	/// SF := 0
208	/// AF := 0
209	/// PF := 0
210	/// CF := 0
211	/// \endoperation
212	static __inline__ unsigned char __DEFAULT_FN_ATTRS
213	_mm_aesenc128kl_u8(__m128i* __odata, __m128i __idata, const void *__h) {
214	return __builtin_ia32_aesenc128kl_u8((__v2di *)__odata, (__v2di)__idata, __h);
215	}
216
217	/// The AESENC256KL performs 14 rounds of AES to encrypt the __idata using
218	/// the 256-bit key in the handle from the __h. It stores the result in the
219	/// __odata. And return the affected ZF flag status.
220	///
221	/// \headerfile <x86intrin.h>
222	///
223	/// This intrinsic corresponds to the <c> AESENC256KL </c> instructions.
224	///
225	/// \operation
226	/// Handle[511:0] := MEM[__h+511:__h] // Load is not guaranteed to be atomic.
227	/// IllegalHandle := ( HandleReservedBitSet (Handle[511:0]) \|\|
228	/// (Handle[127:0] AND (CPL > 0)) \|\|
229	/// Handle[255:128] \|\|
230	/// HandleKeyType (Handle[511:0]) != HANDLE_KEY_TYPE_AES256 )
231	/// IF (IllegalHandle)
232	/// ZF := 1
233	/// ELSE
234	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate512 (Handle[511:0], IWKey)
235	/// IF (Authentic == 0)
236	/// ZF := 1
237	/// ELSE
238	/// MEM[__odata+127:__odata] := AES256Encrypt (__idata[127:0], UnwrappedKey)
239	/// ZF := 0
240	/// FI
241	/// FI
242	/// dst := ZF
243	/// OF := 0
244	/// SF := 0
245	/// AF := 0
246	/// PF := 0
247	/// CF := 0
248	/// \endoperation
249	static __inline__ unsigned char __DEFAULT_FN_ATTRS
250	_mm_aesenc256kl_u8(__m128i* __odata, __m128i __idata, const void *__h) {
251	return __builtin_ia32_aesenc256kl_u8((__v2di *)__odata, (__v2di)__idata, __h);
252	}
253
254	/// The AESDEC128KL performs 10 rounds of AES to decrypt the __idata using
255	/// the 128-bit key in the handle from the __h. It stores the result in the
256	/// __odata. And return the affected ZF flag status.
257	///
258	/// \headerfile <x86intrin.h>
259	///
260	/// This intrinsic corresponds to the <c> AESDEC128KL </c> instructions.
261	///
262	/// \operation
263	/// Handle[383:0] := MEM[__h+383:__h] // Load is not guaranteed to be atomic.
264	/// IllegalHandle := (HandleReservedBitSet (Handle[383:0]) \|\|
265	/// (Handle[127:0] AND (CPL > 0)) \|\|
266	/// Handle[383:256] \|\|
267	/// HandleKeyType (Handle[383:0]) != HANDLE_KEY_TYPE_AES128)
268	/// IF (IllegalHandle)
269	/// ZF := 1
270	/// ELSE
271	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate384 (Handle[383:0], IWKey)
272	/// IF (Authentic == 0)
273	/// ZF := 1
274	/// ELSE
275	/// MEM[__odata+127:__odata] := AES128Decrypt (__idata[127:0], UnwrappedKey)
276	/// ZF := 0
277	/// FI
278	/// FI
279	/// dst := ZF
280	/// OF := 0
281	/// SF := 0
282	/// AF := 0
283	/// PF := 0
284	/// CF := 0
285	/// \endoperation
286	static __inline__ unsigned char __DEFAULT_FN_ATTRS
287	_mm_aesdec128kl_u8(__m128i* __odata, __m128i __idata, const void *__h) {
288	return __builtin_ia32_aesdec128kl_u8((__v2di *)__odata, (__v2di)__idata, __h);
289	}
290
291	/// The AESDEC256KL performs 10 rounds of AES to decrypt the __idata using
292	/// the 256-bit key in the handle from the __h. It stores the result in the
293	/// __odata. And return the affected ZF flag status.
294	///
295	/// \headerfile <x86intrin.h>
296	///
297	/// This intrinsic corresponds to the <c> AESDEC256KL </c> instructions.
298	///
299	/// \operation
300	/// Handle[511:0] := MEM[__h+511:__h]
301	/// IllegalHandle := (HandleReservedBitSet (Handle[511:0]) \|\|
302	/// (Handle[127:0] AND (CPL > 0)) \|\|
303	/// Handle[383:256] \|\|
304	/// HandleKeyType (Handle[511:0]) != HANDLE_KEY_TYPE_AES256)
305	/// IF (IllegalHandle)
306	/// ZF := 1
307	/// ELSE
308	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate512 (Handle[511:0], IWKey)
309	/// IF (Authentic == 0)
310	/// ZF := 1
311	/// ELSE
312	/// MEM[__odata+127:__odata] := AES256Decrypt (__idata[127:0], UnwrappedKey)
313	/// ZF := 0
314	/// FI
315	/// FI
316	/// dst := ZF
317	/// OF := 0
318	/// SF := 0
319	/// AF := 0
320	/// PF := 0
321	/// CF := 0
322	/// \endoperation
323	static __inline__ unsigned char __DEFAULT_FN_ATTRS
324	_mm_aesdec256kl_u8(__m128i* __odata, __m128i __idata, const void *__h) {
325	return __builtin_ia32_aesdec256kl_u8((__v2di *)__odata, (__v2di)__idata, __h);
326	}
327
328	#undef __DEFAULT_FN_ATTRS
329
330	#endif /* !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \
331	\|\| defined(__KL__) */
332
333	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
334	defined(__WIDEKL__)
335
336	/* Define the default attributes for the functions in this file. */
337	#define __DEFAULT_FN_ATTRS \
338	__attribute__((__always_inline__, __nodebug__, __target__("kl,widekl"),\
339	__min_vector_width__(128)))
340
341	/// Encrypt __idata[0] to __idata[7] using 128-bit AES key indicated by handle
342	/// at __h and store each resultant block back from __odata to __odata+7. And
343	/// return the affected ZF flag status.
344	///
345	/// \headerfile <x86intrin.h>
346	///
347	/// This intrinsic corresponds to the <c> AESENCWIDE128KL </c> instructions.
348	///
349	/// \operation
350	/// Handle := MEM[__h+383:__h]
351	/// IllegalHandle := ( HandleReservedBitSet (Handle[383:0]) \|\|
352	/// (Handle[127:0] AND (CPL > 0)) \|\|
353	/// Handle[255:128] \|\|
354	/// HandleKeyType (Handle[383:0]) != HANDLE_KEY_TYPE_AES128 )
355	/// IF (IllegalHandle)
356	/// ZF := 1
357	/// ELSE
358	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate384 (Handle[383:0], IWKey)
359	/// IF Authentic == 0
360	/// ZF := 1
361	/// ELSE
362	/// FOR i := 0 to 7
363	/// __odata[i] := AES128Encrypt (__idata[i], UnwrappedKey)
364	/// ENDFOR
365	/// ZF := 0
366	/// FI
367	/// FI
368	/// dst := ZF
369	/// OF := 0
370	/// SF := 0
371	/// AF := 0
372	/// PF := 0
373	/// CF := 0
374	/// \endoperation
375	static __inline__ unsigned char __DEFAULT_FN_ATTRS
376	_mm_aesencwide128kl_u8(__m128i __odata[8], const __m128i __idata[8], const void* __h) {
377	return __builtin_ia32_aesencwide128kl_u8((__v2di *)__odata,
378	(const __v2di *)__idata, __h);
379	}
380
381	/// Encrypt __idata[0] to __idata[7] using 256-bit AES key indicated by handle
382	/// at __h and store each resultant block back from __odata to __odata+7. And
383	/// return the affected ZF flag status.
384	///
385	/// \headerfile <x86intrin.h>
386	///
387	/// This intrinsic corresponds to the <c> AESENCWIDE256KL </c> instructions.
388	///
389	/// \operation
390	/// Handle[511:0] := MEM[__h+511:__h]
391	/// IllegalHandle := ( HandleReservedBitSet (Handle[511:0]) \|\|
392	/// (Handle[127:0] AND (CPL > 0)) \|\|
393	/// Handle[255:128] \|\|
394	/// HandleKeyType (Handle[511:0]) != HANDLE_KEY_TYPE_AES512 )
395	/// IF (IllegalHandle)
396	/// ZF := 1
397	/// ELSE
398	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate512 (Handle[511:0], IWKey)
399	/// IF Authentic == 0
400	/// ZF := 1
401	/// ELSE
402	/// FOR i := 0 to 7
403	/// __odata[i] := AES256Encrypt (__idata[i], UnwrappedKey)
404	/// ENDFOR
405	/// ZF := 0
406	/// FI
407	/// FI
408	/// dst := ZF
409	/// OF := 0
410	/// SF := 0
411	/// AF := 0
412	/// PF := 0
413	/// CF := 0
414	/// \endoperation
415	static __inline__ unsigned char __DEFAULT_FN_ATTRS
416	_mm_aesencwide256kl_u8(__m128i __odata[8], const __m128i __idata[8], const void* __h) {
417	return __builtin_ia32_aesencwide256kl_u8((__v2di *)__odata,
418	(const __v2di *)__idata, __h);
419	}
420
421	/// Decrypt __idata[0] to __idata[7] using 128-bit AES key indicated by handle
422	/// at __h and store each resultant block back from __odata to __odata+7. And
423	/// return the affected ZF flag status.
424	///
425	/// \headerfile <x86intrin.h>
426	///
427	/// This intrinsic corresponds to the <c> AESDECWIDE128KL </c> instructions.
428	///
429	/// \operation
430	/// Handle[383:0] := MEM[__h+383:__h]
431	/// IllegalHandle := ( HandleReservedBitSet (Handle[383:0]) \|\|
432	/// (Handle[127:0] AND (CPL > 0)) \|\|
433	/// Handle[255:128] \|\|
434	/// HandleKeyType (Handle) != HANDLE_KEY_TYPE_AES128 )
435	/// IF (IllegalHandle)
436	/// ZF := 1
437	/// ELSE
438	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate384 (Handle[383:0], IWKey)
439	/// IF Authentic == 0
440	/// ZF := 1
441	/// ELSE
442	/// FOR i := 0 to 7
443	/// __odata[i] := AES128Decrypt (__idata[i], UnwrappedKey)
444	/// ENDFOR
445	/// ZF := 0
446	/// FI
447	/// FI
448	/// dst := ZF
449	/// OF := 0
450	/// SF := 0
451	/// AF := 0
452	/// PF := 0
453	/// CF := 0
454	/// \endoperation
455	static __inline__ unsigned char __DEFAULT_FN_ATTRS
456	_mm_aesdecwide128kl_u8(__m128i __odata[8], const __m128i __idata[8], const void* __h) {
457	return __builtin_ia32_aesdecwide128kl_u8((__v2di *)__odata,
458	(const __v2di *)__idata, __h);
459	}
460
461	/// Decrypt __idata[0] to __idata[7] using 256-bit AES key indicated by handle
462	/// at __h and store each resultant block back from __odata to __odata+7. And
463	/// return the affected ZF flag status.
464	///
465	/// \headerfile <x86intrin.h>
466	///
467	/// This intrinsic corresponds to the <c> AESDECWIDE256KL </c> instructions.
468	///
469	/// \operation
470	/// Handle[511:0] := MEM[__h+511:__h]
471	/// IllegalHandle = ( HandleReservedBitSet (Handle[511:0]) \|\|
472	/// (Handle[127:0] AND (CPL > 0)) \|\|
473	/// Handle[255:128] \|\|
474	/// HandleKeyType (Handle) != HANDLE_KEY_TYPE_AES512 )
475	/// If (IllegalHandle)
476	/// ZF := 1
477	/// ELSE
478	/// (UnwrappedKey, Authentic) := UnwrapKeyAndAuthenticate512 (Handle[511:0], IWKey)
479	/// IF Authentic == 0
480	/// ZF := 1
481	/// ELSE
482	/// FOR i := 0 to 7
483	/// __odata[i] := AES256Decrypt (__idata[i], UnwrappedKey)
484	/// ENDFOR
485	/// ZF := 0
486	/// FI
487	/// FI
488	/// dst := ZF
489	/// OF := 0
490	/// SF := 0
491	/// AF := 0
492	/// PF := 0
493	/// CF := 0
494	/// \endoperation
495	static __inline__ unsigned char __DEFAULT_FN_ATTRS
496	_mm_aesdecwide256kl_u8(__m128i __odata[8], const __m128i __idata[8], const void* __h) {
497	return __builtin_ia32_aesdecwide256kl_u8((__v2di *)__odata,
498	(const __v2di *)__idata, __h);
499	}
500
501	#undef __DEFAULT_FN_ATTRS
502
503	#endif /* !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \
504	\|\| defined(__WIDEKL__) */
505
506	#endif /* _KEYLOCKERINTRIN_H */

lib/include/mm_malloc.h+6

 @@ -54,7 +54,13 @@ _mm_malloc(size_t __size, size_t __align)
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_free(void *__p)
 {
+#if defined(__MINGW32__)
+  __mingw_aligned_free(__p);
+#elif defined(_WIN32)
+  _aligned_free(__p);
+#else
   free(__p);
+#endif
 }
 #endif

lib/include/opencl-c-base.h+18

...	@@ -9,6 +9,21 @@	...	@@ -9,6 +9,21 @@
9	#ifndef _OPENCL_BASE_H_	9	#ifndef _OPENCL_BASE_H_
10	#define _OPENCL_BASE_H_	10	#define _OPENCL_BASE_H_
11		11
		12	// Define extension macros
		13
		14	#if (defined(__OPENCL_CPP_VERSION__) \|\| __OPENCL_C_VERSION__ >= 200)
		15	// For SPIR all extensions are supported.
		16	#if defined(__SPIR__)
		17	#define cl_khr_subgroup_extended_types 1
		18	#define cl_khr_subgroup_non_uniform_vote 1
		19	#define cl_khr_subgroup_ballot 1
		20	#define cl_khr_subgroup_non_uniform_arithmetic 1
		21	#define cl_khr_subgroup_shuffle 1
		22	#define cl_khr_subgroup_shuffle_relative 1
		23	#define cl_khr_subgroup_clustered_reduce 1
		24	#endif // defined(__SPIR__)
		25	#endif // (defined(__OPENCL_CPP_VERSION__) \|\| __OPENCL_C_VERSION__ >= 200)
		26
12	// built-in scalar data types:	27	// built-in scalar data types:
13		28
14	/**	29	/**
...	@@ -568,4 +583,7 @@ typedef struct {	...	@@ -568,4 +583,7 @@ typedef struct {
568	#pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : end	583	#pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : end
569	#endif // cl_intel_device_side_avc_motion_estimation	584	#endif // cl_intel_device_side_avc_motion_estimation
570		585
		586	// Disable any extensions we may have enabled previously.
		587	#pragma OPENCL EXTENSION all : disable
		588
571	#endif //_OPENCL_BASE_H_	589	#endif //_OPENCL_BASE_H_

lib/include/opencl-c.h+2

...	@@ -4633,6 +4633,7 @@ float16 __ovld __cnfn convert_float16(float16);	...	@@ -4633,6 +4633,7 @@ float16 __ovld __cnfn convert_float16(float16);
4633	// Conversions with double data type parameters or return value.	4633	// Conversions with double data type parameters or return value.
4634		4634
4635	#ifdef cl_khr_fp64	4635	#ifdef cl_khr_fp64
		4636	#pragma OPENCL EXTENSION cl_khr_fp64 : enable
4636	char __ovld __cnfn convert_char(double);	4637	char __ovld __cnfn convert_char(double);
4637	char __ovld __cnfn convert_char_rte(double);	4638	char __ovld __cnfn convert_char_rte(double);
4638	char __ovld __cnfn convert_char_rtn(double);	4639	char __ovld __cnfn convert_char_rtn(double);
...	@@ -5455,6 +5456,7 @@ double16 __ovld __cnfn convert_double16_rtz(ushort16);	...	@@ -5455,6 +5456,7 @@ double16 __ovld __cnfn convert_double16_rtz(ushort16);
5455	#endif //cl_khr_fp64	5456	#endif //cl_khr_fp64
5456		5457
5457	#ifdef cl_khr_fp16	5458	#ifdef cl_khr_fp16
		5459	#pragma OPENCL EXTENSION cl_khr_fp16 : enable
5458	// Convert half types to non-double types.	5460	// Convert half types to non-double types.
5459	uchar __ovld __cnfn convert_uchar(half);	5461	uchar __ovld __cnfn convert_uchar(half);
5460	uchar __ovld __cnfn convert_uchar_rte(half);	5462	uchar __ovld __cnfn convert_uchar_rte(half);

lib/include/openmp_wrappers/cmath+4-1

 @@ -24,8 +24,11 @@
 // which might live in cstdlib.
 #include <cstdlib>
+// We need limits because __clang_cuda_cmath.h below uses `std::numeric_limit`.
+#include <limits>
 #pragma omp begin declare variant match(                                       \
-    device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any)})
+    device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any, allow_templates)})
 #define __CUDA__
 #define __OPENMP_NVPTX__

lib/include/openmp_wrappers/complex+25

...	@@ -25,3 +25,28 @@	...	@@ -25,3 +25,28 @@
25		25
26	// Grab the host header too.	26	// Grab the host header too.
27	#include_next <complex>	27	#include_next <complex>
		28
		29
		30	#ifdef __cplusplus
		31
		32	// If we are compiling against libc++, the macro _LIBCPP_STD_VER should be set
		33	// after including <cmath> above. Since the complex header we use is a
		34	// simplified version of the libc++, we don't need it in this case. If we
		35	// compile against libstdc++, or any other standard library, we will overload
		36	// the (hopefully template) functions in the <complex> header with the ones we
		37	// got from libc++ which decomposes math functions, like `std::sin`, into
		38	// arithmetic and calls to non-complex functions, all of which we can then
		39	// handle.
		40	#ifndef _LIBCPP_STD_VER
		41
		42	#pragma omp begin declare variant match( \
		43	device = {arch(nvptx, nvptx64)}, \
		44	implementation = {extension(match_any, allow_templates)})
		45
		46	#include <complex_cmath.h>
		47
		48	#pragma omp end declare variant
		49
		50	#endif
		51
		52	#endif

lib/include/openmp_wrappers/complex_cmath.h created+388

...	@@ -0,0 +1,388 @@
1	//===------------------------- __complex_cmath.h --------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// std::complex header copied from the libcxx source and simplified for use in
10	// OpenMP target offload regions.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef _OPENMP
15	#error "This file is for OpenMP compilation only."
16	#endif
17
18	#ifndef __cplusplus
19	#error "This file is for C++ compilation only."
20	#endif
21
22	#ifndef _LIBCPP_COMPLEX
23	#define _LIBCPP_COMPLEX
24
25	#include <cmath>
26	#include <type_traits>
27
28	#define __DEVICE__ static constexpr __attribute__((nothrow))
29
30	namespace std {
31
32	// abs
33
34	template <class _Tp> __DEVICE__ _Tp abs(const std::complex<_Tp> &__c) {
35	return hypot(__c.real(), __c.imag());
36	}
37
38	// arg
39
40	template <class _Tp> __DEVICE__ _Tp arg(const std::complex<_Tp> &__c) {
41	return atan2(__c.imag(), __c.real());
42	}
43
44	template <class _Tp>
45	typename enable_if<is_integral<_Tp>::value \|\| is_same<_Tp, double>::value,
46	double>::type
47	arg(_Tp __re) {
48	return atan2(0., __re);
49	}
50
51	template <class _Tp>
52	typename enable_if<is_same<_Tp, float>::value, float>::type arg(_Tp __re) {
53	return atan2f(0.F, __re);
54	}
55
56	// norm
57
58	template <class _Tp> __DEVICE__ _Tp norm(const std::complex<_Tp> &__c) {
59	if (std::isinf(__c.real()))
60	return abs(__c.real());
61	if (std::isinf(__c.imag()))
62	return abs(__c.imag());
63	return __c.real() * __c.real() + __c.imag() * __c.imag();
64	}
65
66	// conj
67
68	template <class _Tp> std::complex<_Tp> conj(const std::complex<_Tp> &__c) {
69	return std::complex<_Tp>(__c.real(), -__c.imag());
70	}
71
72	// proj
73
74	template <class _Tp> std::complex<_Tp> proj(const std::complex<_Tp> &__c) {
75	std::complex<_Tp> __r = __c;
76	if (std::isinf(__c.real()) \|\| std::isinf(__c.imag()))
77	__r = std::complex<_Tp>(INFINITY, copysign(_Tp(0), __c.imag()));
78	return __r;
79	}
80
81	// polar
82
83	template <class _Tp>
84	complex<_Tp> polar(const _Tp &__rho, const _Tp &__theta = _Tp()) {
85	if (std::isnan(__rho) \|\| signbit(__rho))
86	return std::complex<_Tp>(_Tp(NAN), _Tp(NAN));
87	if (std::isnan(__theta)) {
88	if (std::isinf(__rho))
89	return std::complex<_Tp>(__rho, __theta);
90	return std::complex<_Tp>(__theta, __theta);
91	}
92	if (std::isinf(__theta)) {
93	if (std::isinf(__rho))
94	return std::complex<_Tp>(__rho, _Tp(NAN));
95	return std::complex<_Tp>(_Tp(NAN), _Tp(NAN));
96	}
97	_Tp __x = __rho * cos(__theta);
98	if (std::isnan(__x))
99	__x = 0;
100	_Tp __y = __rho * sin(__theta);
101	if (std::isnan(__y))
102	__y = 0;
103	return std::complex<_Tp>(__x, __y);
104	}
105
106	// log
107
108	template <class _Tp> std::complex<_Tp> log(const std::complex<_Tp> &__x) {
109	return std::complex<_Tp>(log(abs(__x)), arg(__x));
110	}
111
112	// log10
113
114	template <class _Tp> std::complex<_Tp> log10(const std::complex<_Tp> &__x) {
115	return log(__x) / log(_Tp(10));
116	}
117
118	// sqrt
119
120	template <class _Tp>
121	__DEVICE__ std::complex<_Tp> sqrt(const std::complex<_Tp> &__x) {
122	if (std::isinf(__x.imag()))
123	return std::complex<_Tp>(_Tp(INFINITY), __x.imag());
124	if (std::isinf(__x.real())) {
125	if (__x.real() > _Tp(0))
126	return std::complex<_Tp>(__x.real(), std::isnan(__x.imag())
127	? __x.imag()
128	: copysign(_Tp(0), __x.imag()));
129	return std::complex<_Tp>(std::isnan(__x.imag()) ? __x.imag() : _Tp(0),
130	copysign(__x.real(), __x.imag()));
131	}
132	return polar(sqrt(abs(__x)), arg(__x) / _Tp(2));
133	}
134
135	// exp
136
137	template <class _Tp>
138	__DEVICE__ std::complex<_Tp> exp(const std::complex<_Tp> &__x) {
139	_Tp __i = __x.imag();
140	if (std::isinf(__x.real())) {
141	if (__x.real() < _Tp(0)) {
142	if (!std::isfinite(__i))
143	__i = _Tp(1);
144	} else if (__i == 0 \|\| !std::isfinite(__i)) {
145	if (std::isinf(__i))
146	__i = _Tp(NAN);
147	return std::complex<_Tp>(__x.real(), __i);
148	}
149	} else if (std::isnan(__x.real()) && __x.imag() == 0)
150	return __x;
151	_Tp __e = exp(__x.real());
152	return std::complex<_Tp>(__e * cos(__i), __e * sin(__i));
153	}
154
155	// pow
156
157	template <class _Tp>
158	std::complex<_Tp> pow(const std::complex<_Tp> &__x,
159	const std::complex<_Tp> &__y) {
160	return exp(__y * log(__x));
161	}
162
163	// __sqr, computes pow(x, 2)
164
165	template <class _Tp> std::complex<_Tp> __sqr(const std::complex<_Tp> &__x) {
166	return std::complex<_Tp>((__x.real() - __x.imag()) *
167	(__x.real() + __x.imag()),
168	_Tp(2) * __x.real() * __x.imag());
169	}
170
171	// asinh
172
173	template <class _Tp>
174	__DEVICE__ std::complex<_Tp> asinh(const std::complex<_Tp> &__x) {
175	const _Tp __pi(atan2(+0., -0.));
176	if (std::isinf(__x.real())) {
177	if (std::isnan(__x.imag()))
178	return __x;
179	if (std::isinf(__x.imag()))
180	return std::complex<_Tp>(__x.real(),
181	copysign(__pi * _Tp(0.25), __x.imag()));
182	return std::complex<_Tp>(__x.real(), copysign(_Tp(0), __x.imag()));
183	}
184	if (std::isnan(__x.real())) {
185	if (std::isinf(__x.imag()))
186	return std::complex<_Tp>(__x.imag(), __x.real());
187	if (__x.imag() == 0)
188	return __x;
189	return std::complex<_Tp>(__x.real(), __x.real());
190	}
191	if (std::isinf(__x.imag()))
192	return std::complex<_Tp>(copysign(__x.imag(), __x.real()),
193	copysign(__pi / _Tp(2), __x.imag()));
194	std::complex<_Tp> __z = log(__x + sqrt(__sqr(__x) + _Tp(1)));
195	return std::complex<_Tp>(copysign(__z.real(), __x.real()),
196	copysign(__z.imag(), __x.imag()));
197	}
198
199	// acosh
200
201	template <class _Tp>
202	__DEVICE__ std::complex<_Tp> acosh(const std::complex<_Tp> &__x) {
203	const _Tp __pi(atan2(+0., -0.));
204	if (std::isinf(__x.real())) {
205	if (std::isnan(__x.imag()))
206	return std::complex<_Tp>(abs(__x.real()), __x.imag());
207	if (std::isinf(__x.imag())) {
208	if (__x.real() > 0)
209	return std::complex<_Tp>(__x.real(),
210	copysign(__pi * _Tp(0.25), __x.imag()));
211	else
212	return std::complex<_Tp>(-__x.real(),
213	copysign(__pi * _Tp(0.75), __x.imag()));
214	}
215	if (__x.real() < 0)
216	return std::complex<_Tp>(-__x.real(), copysign(__pi, __x.imag()));
217	return std::complex<_Tp>(__x.real(), copysign(_Tp(0), __x.imag()));
218	}
219	if (std::isnan(__x.real())) {
220	if (std::isinf(__x.imag()))
221	return std::complex<_Tp>(abs(__x.imag()), __x.real());
222	return std::complex<_Tp>(__x.real(), __x.real());
223	}
224	if (std::isinf(__x.imag()))
225	return std::complex<_Tp>(abs(__x.imag()),
226	copysign(__pi / _Tp(2), __x.imag()));
227	std::complex<_Tp> __z = log(__x + sqrt(__sqr(__x) - _Tp(1)));
228	return std::complex<_Tp>(copysign(__z.real(), _Tp(0)),
229	copysign(__z.imag(), __x.imag()));
230	}
231
232	// atanh
233
234	template <class _Tp>
235	__DEVICE__ std::complex<_Tp> atanh(const std::complex<_Tp> &__x) {
236	const _Tp __pi(atan2(+0., -0.));
237	if (std::isinf(__x.imag())) {
238	return std::complex<_Tp>(copysign(_Tp(0), __x.real()),
239	copysign(__pi / _Tp(2), __x.imag()));
240	}
241	if (std::isnan(__x.imag())) {
242	if (std::isinf(__x.real()) \|\| __x.real() == 0)
243	return std::complex<_Tp>(copysign(_Tp(0), __x.real()), __x.imag());
244	return std::complex<_Tp>(__x.imag(), __x.imag());
245	}
246	if (std::isnan(__x.real())) {
247	return std::complex<_Tp>(__x.real(), __x.real());
248	}
249	if (std::isinf(__x.real())) {
250	return std::complex<_Tp>(copysign(_Tp(0), __x.real()),
251	copysign(__pi / _Tp(2), __x.imag()));
252	}
253	if (abs(__x.real()) == _Tp(1) && __x.imag() == _Tp(0)) {
254	return std::complex<_Tp>(copysign(_Tp(INFINITY), __x.real()),
255	copysign(_Tp(0), __x.imag()));
256	}
257	std::complex<_Tp> __z = log((_Tp(1) + __x) / (_Tp(1) - __x)) / _Tp(2);
258	return std::complex<_Tp>(copysign(__z.real(), __x.real()),
259	copysign(__z.imag(), __x.imag()));
260	}
261
262	// sinh
263
264	template <class _Tp>
265	__DEVICE__ std::complex<_Tp> sinh(const std::complex<_Tp> &__x) {
266	if (std::isinf(__x.real()) && !std::isfinite(__x.imag()))
267	return std::complex<_Tp>(__x.real(), _Tp(NAN));
268	if (__x.real() == 0 && !std::isfinite(__x.imag()))
269	return std::complex<_Tp>(__x.real(), _Tp(NAN));
270	if (__x.imag() == 0 && !std::isfinite(__x.real()))
271	return __x;
272	return std::complex<_Tp>(sinh(__x.real()) * cos(__x.imag()),
273	cosh(__x.real()) * sin(__x.imag()));
274	}
275
276	// cosh
277
278	template <class _Tp>
279	__DEVICE__ std::complex<_Tp> cosh(const std::complex<_Tp> &__x) {
280	if (std::isinf(__x.real()) && !std::isfinite(__x.imag()))
281	return std::complex<_Tp>(abs(__x.real()), _Tp(NAN));
282	if (__x.real() == 0 && !std::isfinite(__x.imag()))
283	return std::complex<_Tp>(_Tp(NAN), __x.real());
284	if (__x.real() == 0 && __x.imag() == 0)
285	return std::complex<_Tp>(_Tp(1), __x.imag());
286	if (__x.imag() == 0 && !std::isfinite(__x.real()))
287	return std::complex<_Tp>(abs(__x.real()), __x.imag());
288	return std::complex<_Tp>(cosh(__x.real()) * cos(__x.imag()),
289	sinh(__x.real()) * sin(__x.imag()));
290	}
291
292	// tanh
293
294	template <class _Tp>
295	__DEVICE__ std::complex<_Tp> tanh(const std::complex<_Tp> &__x) {
296	if (std::isinf(__x.real())) {
297	if (!std::isfinite(__x.imag()))
298	return std::complex<_Tp>(_Tp(1), _Tp(0));
299	return std::complex<_Tp>(_Tp(1),
300	copysign(_Tp(0), sin(_Tp(2) * __x.imag())));
301	}
302	if (std::isnan(__x.real()) && __x.imag() == 0)
303	return __x;
304	_Tp __2r(_Tp(2) * __x.real());
305	_Tp __2i(_Tp(2) * __x.imag());
306	_Tp __d(cosh(__2r) + cos(__2i));
307	_Tp __2rsh(sinh(__2r));
308	if (std::isinf(__2rsh) && std::isinf(__d))
309	return std::complex<_Tp>(__2rsh > _Tp(0) ? _Tp(1) : _Tp(-1),
310	__2i > _Tp(0) ? _Tp(0) : _Tp(-0.));
311	return std::complex<_Tp>(__2rsh / __d, sin(__2i) / __d);
312	}
313
314	// asin
315
316	template <class _Tp>
317	__DEVICE__ std::complex<_Tp> asin(const std::complex<_Tp> &__x) {
318	std::complex<_Tp> __z = asinh(complex<_Tp>(-__x.imag(), __x.real()));
319	return std::complex<_Tp>(__z.imag(), -__z.real());
320	}
321
322	// acos
323
324	template <class _Tp>
325	__DEVICE__ std::complex<_Tp> acos(const std::complex<_Tp> &__x) {
326	const _Tp __pi(atan2(+0., -0.));
327	if (std::isinf(__x.real())) {
328	if (std::isnan(__x.imag()))
329	return std::complex<_Tp>(__x.imag(), __x.real());
330	if (std::isinf(__x.imag())) {
331	if (__x.real() < _Tp(0))
332	return std::complex<_Tp>(_Tp(0.75) * __pi, -__x.imag());
333	return std::complex<_Tp>(_Tp(0.25) * __pi, -__x.imag());
334	}
335	if (__x.real() < _Tp(0))
336	return std::complex<_Tp>(__pi,
337	signbit(__x.imag()) ? -__x.real() : __x.real());
338	return std::complex<_Tp>(_Tp(0),
339	signbit(__x.imag()) ? __x.real() : -__x.real());
340	}
341	if (std::isnan(__x.real())) {
342	if (std::isinf(__x.imag()))
343	return std::complex<_Tp>(__x.real(), -__x.imag());
344	return std::complex<_Tp>(__x.real(), __x.real());
345	}
346	if (std::isinf(__x.imag()))
347	return std::complex<_Tp>(__pi / _Tp(2), -__x.imag());
348	if (__x.real() == 0 && (__x.imag() == 0 \|\| isnan(__x.imag())))
349	return std::complex<_Tp>(__pi / _Tp(2), -__x.imag());
350	std::complex<_Tp> __z = log(__x + sqrt(__sqr(__x) - _Tp(1)));
351	if (signbit(__x.imag()))
352	return std::complex<_Tp>(abs(__z.imag()), abs(__z.real()));
353	return std::complex<_Tp>(abs(__z.imag()), -abs(__z.real()));
354	}
355
356	// atan
357
358	template <class _Tp>
359	__DEVICE__ std::complex<_Tp> atan(const std::complex<_Tp> &__x) {
360	std::complex<_Tp> __z = atanh(complex<_Tp>(-__x.imag(), __x.real()));
361	return std::complex<_Tp>(__z.imag(), -__z.real());
362	}
363
364	// sin
365
366	template <class _Tp>
367	__DEVICE__ std::complex<_Tp> sin(const std::complex<_Tp> &__x) {
368	std::complex<_Tp> __z = sinh(complex<_Tp>(-__x.imag(), __x.real()));
369	return std::complex<_Tp>(__z.imag(), -__z.real());
370	}
371
372	// cos
373
374	template <class _Tp> std::complex<_Tp> cos(const std::complex<_Tp> &__x) {
375	return cosh(complex<_Tp>(-__x.imag(), __x.real()));
376	}
377
378	// tan
379
380	template <class _Tp>
381	__DEVICE__ std::complex<_Tp> tan(const std::complex<_Tp> &__x) {
382	std::complex<_Tp> __z = tanh(complex<_Tp>(-__x.imag(), __x.real()));
383	return std::complex<_Tp>(__z.imag(), -__z.real());
384	}
385
386	} // namespace std
387
388	#endif

lib/include/popcntintrin.h+9-2

...	@@ -13,6 +13,12 @@	...	@@ -13,6 +13,12 @@
13	/* Define the default attributes for the functions in this file. */	13	/* Define the default attributes for the functions in this file. */
14	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("popcnt")))	14	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("popcnt")))
15		15
		16	#if defined(__cplusplus) && (__cplusplus >= 201103L)
		17	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
		18	#else
		19	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
		20	#endif
		21
16	/// Counts the number of bits in the source operand having a value of 1.	22	/// Counts the number of bits in the source operand having a value of 1.
17	///	23	///
18	/// \headerfile <x86intrin.h>	24	/// \headerfile <x86intrin.h>
...	@@ -23,7 +29,7 @@	...	@@ -23,7 +29,7 @@
23	/// An unsigned 32-bit integer operand.	29	/// An unsigned 32-bit integer operand.
24	/// \returns A 32-bit integer containing the number of bits with value 1 in the	30	/// \returns A 32-bit integer containing the number of bits with value 1 in the
25	/// source operand.	31	/// source operand.
26	static __inline__ int __DEFAULT_FN_ATTRS	32	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
27	_mm_popcnt_u32(unsigned int __A)	33	_mm_popcnt_u32(unsigned int __A)
28	{	34	{
29	return __builtin_popcount(__A);	35	return __builtin_popcount(__A);
...	@@ -40,7 +46,7 @@ _mm_popcnt_u32(unsigned int __A)	...	@@ -40,7 +46,7 @@ _mm_popcnt_u32(unsigned int __A)
40	/// An unsigned 64-bit integer operand.	46	/// An unsigned 64-bit integer operand.
41	/// \returns A 64-bit integer containing the number of bits with value 1 in the	47	/// \returns A 64-bit integer containing the number of bits with value 1 in the
42	/// source operand.	48	/// source operand.
43	static __inline__ long long __DEFAULT_FN_ATTRS	49	static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
44	_mm_popcnt_u64(unsigned long long __A)	50	_mm_popcnt_u64(unsigned long long __A)
45	{	51	{
46	return __builtin_popcountll(__A);	52	return __builtin_popcountll(__A);
...	@@ -48,5 +54,6 @@ _mm_popcnt_u64(unsigned long long __A)	...	@@ -48,5 +54,6 @@ _mm_popcnt_u64(unsigned long long __A)
48	#endif /* __x86_64__ */	54	#endif /* __x86_64__ */
49		55
50	#undef __DEFAULT_FN_ATTRS	56	#undef __DEFAULT_FN_ATTRS
		57	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
51		58
52	#endif /* __POPCNTINTRIN_H */	59	#endif /* __POPCNTINTRIN_H */

lib/include/ppc_wrappers/smmintrin.h+24

...	@@ -78,6 +78,30 @@ extern __inline __m128i	...	@@ -78,6 +78,30 @@ extern __inline __m128i
78	return (__m128i)vec_sel((__v16qu)__A, (__v16qu)__B, __lmask);	78	return (__m128i)vec_sel((__v16qu)__A, (__v16qu)__B, __lmask);
79	}	79	}
80		80
		81	extern __inline __m128i
		82	__attribute__((__gnu_inline__, __always_inline__, __artificial__))
		83	_mm_insert_epi8(__m128i const __A, int const __D, int const __N) {
		84	__v16qi result = (__v16qi)__A;
		85	result[__N & 0xf] = __D;
		86	return (__m128i)result;
		87	}
		88
		89	extern __inline __m128i
		90	__attribute__((__gnu_inline__, __always_inline__, __artificial__))
		91	_mm_insert_epi32(__m128i const __A, int const __D, int const __N) {
		92	__v4si result = (__v4si)__A;
		93	result[__N & 3] = __D;
		94	return (__m128i)result;
		95	}
		96
		97	extern __inline __m128i
		98	__attribute__((__gnu_inline__, __always_inline__, __artificial__))
		99	_mm_insert_epi64(__m128i const __A, long long const __D, int const __N) {
		100	__v2di result = (__v2di)__A;
		101	result[__N & 1] = __D;
		102	return (__m128i)result;
		103	}
		104
81	#else	105	#else
82	#include_next <smmintrin.h>	106	#include_next <smmintrin.h>
83	#endif /* defined(__linux__) && defined(__ppc64__) */	107	#endif /* defined(__linux__) && defined(__ppc64__) */

lib/include/uintrintrin.h created+150

...	@@ -0,0 +1,150 @@
1	/*===------------------ uintrintrin.h - UINTR intrinsics -------------------===
2	*
3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	* See https://llvm.org/LICENSE.txt for license information.
5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	*
7	*===-----------------------------------------------------------------------===
8	*/
9
10	#ifndef __X86GPRINTRIN_H
11	#error "Never use <uintrintrin.h> directly; include <x86gprintrin.h> instead."
12	#endif
13
14	#ifndef __UINTRINTRIN_H
15	#define __UINTRINTRIN_H
16
17	/* Define the default attributes for the functions in this file */
18	#define __DEFAULT_FN_ATTRS \
19	__attribute__((__always_inline__, __nodebug__, __target__("uintr")))
20
21	#ifdef __x86_64__
22
23	/// Clears the user interrupt flag (UIF). Its effect takes place immediately: a
24	/// user interrupt cannot be delivered on the instruction boundary following
25	/// CLUI. Can be executed only if CR4.UINT = 1, the logical processor is in
26	/// 64-bit mode, and software is not executing inside an enclave; otherwise,
27	/// each causes an invalid-opcode exception. Causes a transactional abort if
28	/// executed inside a transactional region; the abort loads EAX as it would
29	/// had it been due to an execution of CLI.
30	///
31	/// \headerfile <x86gprintrin.h>
32	///
33	/// This intrinsic corresponds to the <c> CLUI </c> instruction.
34	///
35	/// \operation
36	/// UIF := 0
37	/// \endoperation
38	static __inline__ void __DEFAULT_FN_ATTRS
39	_clui (void)
40	{
41	__builtin_ia32_clui();
42	}
43
44	/// Sets the user interrupt flag (UIF). Its effect takes place immediately; a
45	/// user interrupt may be delivered on the instruction boundary following
46	/// STUI. Can be executed only if CR4.UINT = 1, the logical processor is in
47	/// 64-bit mode, and software is not executing inside an enclave; otherwise,
48	/// each causes an invalid-opcode exception. Causes a transactional abort if
49	/// executed inside a transactional region; the abort loads EAX as it would
50	/// had it been due to an execution of STI.
51	///
52	/// \headerfile <x86gprintrin.h>
53	///
54	/// This intrinsic corresponds to the <c> STUI </c> instruction.
55	///
56	/// \operation
57	/// UIF := 1
58	/// \endoperation
59	static __inline__ void __DEFAULT_FN_ATTRS
60	_stui (void)
61	{
62	__builtin_ia32_stui();
63	}
64
65	/// Get the current value of the user interrupt flag (UIF). Can be executed
66	/// regardless of CPL and inside a transactional region. Can be executed only
67	/// if CR4.UINT = 1, the logical processor is in 64-bit mode, and software is
68	/// not executing inside an enclave; otherwise, it causes an invalid-opcode
69	/// exception.
70	///
71	/// \headerfile <x86gprintrin.h>
72	///
73	/// This intrinsic corresponds to the <c> TESTUI </c> instruction.
74	///
75	/// \returns The current value of the user interrupt flag (UIF).
76	///
77	/// \operation
78	/// CF := UIF
79	/// ZF := 0
80	/// AF := 0
81	/// OF := 0
82	/// PF := 0
83	/// SF := 0
84	/// dst := CF
85	/// \endoperation
86	static __inline__ unsigned char __DEFAULT_FN_ATTRS
87	_testui (void)
88	{
89	return __builtin_ia32_testui();
90	}
91
92	/// Send interprocessor user interrupt. Can be executed only if
93	/// CR4.UINT = IA32_UINT_TT[0] = 1, the logical processor is in 64-bit mode,
94	/// and software is not executing inside an enclave; otherwise, it causes an
95	/// invalid-opcode exception. May be executed at any privilege level, all of
96	/// its memory accesses are performed with supervisor privilege.
97	///
98	/// \headerfile <x86gprintrin.h>
99	///
100	/// This intrinsic corresponds to the <c> SENDUIPI </c> instruction
101	///
102	/// \param __a
103	/// Index of user-interrupt target table entry in user-interrupt target
104	/// table.
105	///
106	/// \operation
107	/// IF __a > UITTSZ
108	/// GP (0)
109	/// FI
110	/// tempUITTE := MEM[UITTADDR + (a<<4)]
111	/// // tempUITTE must be valid, and can't have any reserved bit set
112	/// IF (tempUITTE.V == 0 OR tempUITTE[7:1] != 0)
113	/// GP (0)
114	/// FI
115	/// tempUPID := MEM[tempUITTE.UPIDADDR] // under lock
116	/// // tempUPID can't have any reserved bit set
117	/// IF (tempUPID[15:2] != 0 OR tempUPID[31:24] != 0)
118	/// GP (0) // release lock
119	/// FI
120	/// tempUPID.PIR[tempUITTE.UV] := 1;
121	/// IF (tempUPID.SN == 0 AND tempUPID.ON == 0)
122	/// tempUPID.ON := 1
123	/// sendNotify := 1
124	/// ELSE
125	/// sendNotify := 0
126	/// FI
127	/// MEM[tempUITTE.UPIDADDR] := tempUPID // release lock
128	/// IF sendNotify == 1
129	/// IF IA32_APIC_BASE[10] == 1 // local APIC is in x2APIC mode
130	/// // send ordinary IPI with vector tempUPID.NV to 32-bit physical APIC
131	/// // ID tempUPID.NDST
132	/// SendOrdinaryIPI(tempUPID.NV, tempUPID.NDST)
133	/// ELSE
134	/// // send ordinary IPI with vector tempUPID.NV to 8-bit physical APIC
135	/// // ID tempUPID.NDST[15:8]
136	/// SendOrdinaryIPI(tempUPID.NV, tempUPID.NDST[15:8])
137	/// FI
138	/// FI
139	/// \endoperation
140	static __inline__ void __DEFAULT_FN_ATTRS
141	_senduipi (unsigned long long __a)
142	{
143	__builtin_ia32_senduipi(__a);
144	}
145
146	#endif /* __x86_64__ */
147
148	#undef __DEFAULT_FN_ATTRS
149
150	#endif /* __UINTRINTRIN_H */

lib/include/wasm_simd128.h+73-39

 @@ -18,8 +18,7 @@ typedef int32_t v128_t __attribute__((__vector_size__(16), __aligned__(16)));
 // Internal types determined by clang builtin definitions
 typedef int32_t __v128_u __attribute__((__vector_size__(16), __aligned__(1)));
-typedef char __i8x16 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef signed char __i8x16
-typedef signed char __s8x16
     __attribute__((__vector_size__(16), __aligned__(16)));
 typedef unsigned char __u8x16
     __attribute__((__vector_size__(16), __aligned__(16)));
 @@ -35,6 +34,13 @@ typedef unsigned long long __u64x2
 typedef float __f32x4 __attribute__((__vector_size__(16), __aligned__(16)));
 typedef double __f64x2 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef signed char __i8x8 __attribute__((__vector_size__(8), __aligned__(8)));
+typedef unsigned char __u8x8
+    __attribute__((__vector_size__(8), __aligned__(8)));
+typedef short __i16x4 __attribute__((__vector_size__(8), __aligned__(8)));
+typedef unsigned short __u16x4
+    __attribute__((__vector_size__(8), __aligned__(8)));
 #define __DEFAULT_FN_ATTRS                                                     \
   __attribute__((__always_inline__, __nodebug__, __target__("simd128"),        \
                  __min_vector_width__(128)))
 @@ -273,7 +279,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_splat(int8_t __a) {
   (__builtin_wasm_extract_lane_s_i8x16((__i8x16)(__a), __i))
 #define wasm_u8x16_extract_lane(__a, __i)                                      \
-  (__builtin_wasm_extract_lane_u_i8x16((__i8x16)(__a), __i))
+  (__builtin_wasm_extract_lane_u_i8x16((__u8x16)(__a), __i))
 #define wasm_i8x16_replace_lane(__a, __i, __b)                                 \
   ((v128_t)__builtin_wasm_replace_lane_i8x16((__i8x16)(__a), __i, __b))
 @@ -286,7 +292,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_splat(int16_t __a) {
   (__builtin_wasm_extract_lane_s_i16x8((__i16x8)(__a), __i))
 #define wasm_u16x8_extract_lane(__a, __i)                                      \
-  (__builtin_wasm_extract_lane_u_i16x8((__i16x8)(__a), __i))
+  (__builtin_wasm_extract_lane_u_i16x8((__u16x8)(__a), __i))
 #define wasm_i16x8_replace_lane(__a, __i, __b)                                 \
   ((v128_t)__builtin_wasm_replace_lane_i16x8((__i16x8)(__a), __i, __b))
 @@ -333,17 +339,17 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_splat(double __a) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_eq(v128_t __a,
                                                           v128_t __b) {
-  return (v128_t)((__s8x16)__a == (__s8x16)__b);
+  return (v128_t)((__i8x16)__a == (__i8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_ne(v128_t __a,
                                                           v128_t __b) {
-  return (v128_t)((__s8x16)__a != (__s8x16)__b);
+  return (v128_t)((__i8x16)__a != (__i8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_lt(v128_t __a,
                                                           v128_t __b) {
-  return (v128_t)((__s8x16)__a < (__s8x16)__b);
+  return (v128_t)((__i8x16)__a < (__i8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_lt(v128_t __a,
 @@ -353,7 +359,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_lt(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_gt(v128_t __a,
                                                           v128_t __b) {
-  return (v128_t)((__s8x16)__a > (__s8x16)__b);
+  return (v128_t)((__i8x16)__a > (__i8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_gt(v128_t __a,
 @@ -363,7 +369,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_gt(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_le(v128_t __a,
                                                           v128_t __b) {
-  return (v128_t)((__s8x16)__a <= (__s8x16)__b);
+  return (v128_t)((__i8x16)__a <= (__i8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_le(v128_t __a,
 @@ -373,7 +379,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_le(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_ge(v128_t __a,
                                                           v128_t __b) {
-  return (v128_t)((__s8x16)__a >= (__s8x16)__b);
+  return (v128_t)((__i8x16)__a >= (__i8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_ge(v128_t __a,
 @@ -595,7 +601,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shr(v128_t __a,
                                                            int32_t __b) {
-  return (v128_t)((__s8x16)__a >> __b);
+  return (v128_t)((__i8x16)__a >> __b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_shr(v128_t __a,
 @@ -616,8 +622,8 @@ wasm_i8x16_add_saturate(v128_t __a, v128_t __b) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u8x16_add_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_add_saturate_u_i8x16((__i8x16)__a,
+  return (v128_t)__builtin_wasm_add_saturate_u_i8x16((__u8x16)__a,
-                                                     (__i8x16)__b);
+                                                     (__u8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
 @@ -633,8 +639,8 @@ wasm_i8x16_sub_saturate(v128_t __a, v128_t __b) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u8x16_sub_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_sub_saturate_u_i8x16((__i8x16)__a,
+  return (v128_t)__builtin_wasm_sub_saturate_u_i8x16((__u8x16)__a,
-                                                     (__i8x16)__b);
+                                                     (__u8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,
 @@ -644,7 +650,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_min(v128_t __a,
                                                            v128_t __b) {
-  return (v128_t)__builtin_wasm_min_u_i8x16((__i8x16)__a, (__i8x16)__b);
+  return (v128_t)__builtin_wasm_min_u_i8x16((__u8x16)__a, (__u8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_max(v128_t __a,
 @@ -654,12 +660,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_max(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_max(v128_t __a,
                                                            v128_t __b) {
-  return (v128_t)__builtin_wasm_max_u_i8x16((__i8x16)__a, (__i8x16)__b);
+  return (v128_t)__builtin_wasm_max_u_i8x16((__u8x16)__a, (__u8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_avgr(v128_t __a,
                                                             v128_t __b) {
-  return (v128_t)__builtin_wasm_avgr_u_i8x16((__i8x16)__a, (__i8x16)__b);
+  return (v128_t)__builtin_wasm_avgr_u_i8x16((__u8x16)__a, (__u8x16)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_abs(v128_t __a) {
 @@ -706,8 +712,8 @@ wasm_i16x8_add_saturate(v128_t __a, v128_t __b) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u16x8_add_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_add_saturate_u_i16x8((__i16x8)__a,
+  return (v128_t)__builtin_wasm_add_saturate_u_i16x8((__u16x8)__a,
-                                                     (__i16x8)__b);
+                                                     (__u16x8)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
 @@ -723,8 +729,8 @@ wasm_i16x8_sub_saturate(v128_t __a, v128_t __b) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u16x8_sub_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_sub_saturate_u_i16x8((__i16x8)__a,
+  return (v128_t)__builtin_wasm_sub_saturate_u_i16x8((__u16x8)__a,
-                                                     (__i16x8)__b);
+                                                     (__u16x8)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
 @@ -739,7 +745,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_min(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_min(v128_t __a,
                                                            v128_t __b) {
-  return (v128_t)__builtin_wasm_min_u_i16x8((__i16x8)__a, (__i16x8)__b);
+  return (v128_t)__builtin_wasm_min_u_i16x8((__u16x8)__a, (__u16x8)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_max(v128_t __a,
 @@ -749,12 +755,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_max(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_max(v128_t __a,
                                                            v128_t __b) {
-  return (v128_t)__builtin_wasm_max_u_i16x8((__i16x8)__a, (__i16x8)__b);
+  return (v128_t)__builtin_wasm_max_u_i16x8((__u16x8)__a, (__u16x8)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_avgr(v128_t __a,
                                                             v128_t __b) {
-  return (v128_t)__builtin_wasm_avgr_u_i16x8((__i16x8)__a, (__i16x8)__b);
+  return (v128_t)__builtin_wasm_avgr_u_i16x8((__u16x8)__a, (__u16x8)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_abs(v128_t __a) {
 @@ -810,7 +816,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_min(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_min(v128_t __a,
                                                            v128_t __b) {
-  return (v128_t)__builtin_wasm_min_u_i32x4((__i32x4)__a, (__i32x4)__b);
+  return (v128_t)__builtin_wasm_min_u_i32x4((__u32x4)__a, (__u32x4)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_max(v128_t __a,
 @@ -820,7 +826,7 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_max(v128_t __a,
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_max(v128_t __a,
                                                            v128_t __b) {
-  return (v128_t)__builtin_wasm_max_u_i32x4((__i32x4)__a, (__i32x4)__b);
+  return (v128_t)__builtin_wasm_max_u_i32x4((__u32x4)__a, (__u32x4)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_neg(v128_t __a) {
 @@ -1071,8 +1077,8 @@ wasm_i8x16_narrow_i16x8(v128_t __a, v128_t __b) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u8x16_narrow_i16x8(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_narrow_u_i8x16_i16x8((__i16x8)__a,
+  return (v128_t)__builtin_wasm_narrow_u_i8x16_i16x8((__u16x8)__a,
-                                                     (__i16x8)__b);
+                                                     (__u16x8)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 @@ -1083,48 +1089,76 @@ wasm_i16x8_narrow_i32x4(v128_t __a, v128_t __b) {
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u16x8_narrow_i32x4(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_narrow_u_i16x8_i32x4((__i32x4)__a,
+  return (v128_t)__builtin_wasm_narrow_u_i16x8_i32x4((__u32x4)__a,
-                                                     (__i32x4)__b);
+                                                     (__u32x4)__b);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_low_i8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_s_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i8x8){((__i8x16)__a)[0], ((__i8x16)__a)[1], ((__i8x16)__a)[2],
+               ((__i8x16)__a)[3], ((__i8x16)__a)[4], ((__i8x16)__a)[5],
+               ((__i8x16)__a)[6], ((__i8x16)__a)[7]},
+      __i16x8);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_high_i8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_s_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i8x8){((__i8x16)__a)[8], ((__i8x16)__a)[9], ((__i8x16)__a)[10],
+               ((__i8x16)__a)[11], ((__i8x16)__a)[12], ((__i8x16)__a)[13],
+               ((__i8x16)__a)[14], ((__i8x16)__a)[15]},
+      __i16x8);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_low_u8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_u_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u8x8){((__u8x16)__a)[0], ((__u8x16)__a)[1], ((__u8x16)__a)[2],
+               ((__u8x16)__a)[3], ((__u8x16)__a)[4], ((__u8x16)__a)[5],
+               ((__u8x16)__a)[6], ((__u8x16)__a)[7]},
+      __u16x8);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_high_u8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_u_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u8x8){((__u8x16)__a)[8], ((__u8x16)__a)[9], ((__u8x16)__a)[10],
+               ((__u8x16)__a)[11], ((__u8x16)__a)[12], ((__u8x16)__a)[13],
+               ((__u8x16)__a)[14], ((__u8x16)__a)[15]},
+      __u16x8);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_low_i16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_s_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i16x4){((__i16x8)__a)[0], ((__i16x8)__a)[1], ((__i16x8)__a)[2],
+                ((__i16x8)__a)[3]},
+      __i32x4);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_high_i16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_s_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i16x4){((__i16x8)__a)[4], ((__i16x8)__a)[5], ((__i16x8)__a)[6],
+                ((__i16x8)__a)[7]},
+      __i32x4);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_low_u16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_u_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u16x4){((__u16x8)__a)[0], ((__u16x8)__a)[1], ((__u16x8)__a)[2],
+                ((__u16x8)__a)[3]},
+      __u32x4);
 }
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_high_u16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_u_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u16x4){((__u16x8)__a)[4], ((__u16x8)__a)[5], ((__u16x8)__a)[6],
+                ((__u16x8)__a)[7]},
+      __u32x4);
 }
 // Undefine helper macros

lib/include/x86gprintrin.h created+23

...	@@ -0,0 +1,23 @@
1	/*===--------------- x86gprintrin.h - X86 GPR intrinsics ------------------===
2	*
3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	* See https://llvm.org/LICENSE.txt for license information.
5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	*
7	*===-----------------------------------------------------------------------===
8	*/
9
10	#ifndef __X86GPRINTRIN_H
11	#define __X86GPRINTRIN_H
12
13	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
14	defined(__HRESET__)
15	#include <hresetintrin.h>
16	#endif
17
18	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
19	defined(__UINTR__)
20	#include <uintrintrin.h>
21	#endif
22
23	#endif /* __X86GPRINTRIN_H */

author	Jakub Konka <kubkon@jakubkonka.com> 2020-12-16 10:43:57+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2020-12-16 12:18:33+01:00
log	8612dac225a55e09293b767aab3c81a459609bb4
tree	7dd26a69e4730dab5b3bcfb545bce507053c4c0d
parent	83ff94406e13e18c8826cd48a68c2c8d676feaac

...	@@ -55,7 +55,9 @@ struct __cuda_builtin_threadIdx_t {	...	@@ -55,7 +55,9 @@ struct __cuda_builtin_threadIdx_t {
55	__CUDA_DEVICE_BUILTIN(z,__nvvm_read_ptx_sreg_tid_z());	55	__CUDA_DEVICE_BUILTIN(z,__nvvm_read_ptx_sreg_tid_z());
56	// threadIdx should be convertible to uint3 (in fact in nvcc, it is a	56	// threadIdx should be convertible to uint3 (in fact in nvcc, it is a
57	// uint3). This function is defined after we pull in vector_types.h.	57	// uint3). This function is defined after we pull in vector_types.h.
		58	__attribute__((device)) operator dim3() const;
58	__attribute__((device)) operator uint3() const;	59	__attribute__((device)) operator uint3() const;
		60
59	private:	61	private:
60	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_threadIdx_t);	62	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_threadIdx_t);
61	};	63	};
...	@@ -66,7 +68,9 @@ struct __cuda_builtin_blockIdx_t {	...	@@ -66,7 +68,9 @@ struct __cuda_builtin_blockIdx_t {
66	__CUDA_DEVICE_BUILTIN(z,__nvvm_read_ptx_sreg_ctaid_z());	68	__CUDA_DEVICE_BUILTIN(z,__nvvm_read_ptx_sreg_ctaid_z());
67	// blockIdx should be convertible to uint3 (in fact in nvcc, it is a	69	// blockIdx should be convertible to uint3 (in fact in nvcc, it is a
68	// uint3). This function is defined after we pull in vector_types.h.	70	// uint3). This function is defined after we pull in vector_types.h.
		71	__attribute__((device)) operator dim3() const;
69	__attribute__((device)) operator uint3() const;	72	__attribute__((device)) operator uint3() const;
		73
70	private:	74	private:
71	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_blockIdx_t);	75	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_blockIdx_t);
72	};	76	};
...	@@ -78,6 +82,8 @@ struct __cuda_builtin_blockDim_t {	...	@@ -78,6 +82,8 @@ struct __cuda_builtin_blockDim_t {
78	// blockDim should be convertible to dim3 (in fact in nvcc, it is a	82	// blockDim should be convertible to dim3 (in fact in nvcc, it is a
79	// dim3). This function is defined after we pull in vector_types.h.	83	// dim3). This function is defined after we pull in vector_types.h.
80	__attribute__((device)) operator dim3() const;	84	__attribute__((device)) operator dim3() const;
		85	__attribute__((device)) operator uint3() const;
		86
81	private:	87	private:
82	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_blockDim_t);	88	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_blockDim_t);
83	};	89	};
...	@@ -89,6 +95,8 @@ struct __cuda_builtin_gridDim_t {	...	@@ -89,6 +95,8 @@ struct __cuda_builtin_gridDim_t {
89	// gridDim should be convertible to dim3 (in fact in nvcc, it is a	95	// gridDim should be convertible to dim3 (in fact in nvcc, it is a
90	// dim3). This function is defined after we pull in vector_types.h.	96	// dim3). This function is defined after we pull in vector_types.h.
91	__attribute__((device)) operator dim3() const;	97	__attribute__((device)) operator dim3() const;
		98	__attribute__((device)) operator uint3() const;
		99
92	private:	100	private:
93	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_gridDim_t);	101	__CUDA_DISALLOW_BUILTINVAR_ACCESS(__cuda_builtin_gridDim_t);
94	};	102	};
...	@@ -108,5 +116,6 @@ __attribute__((device)) const int warpSize = 32;	...	@@ -108,5 +116,6 @@ __attribute__((device)) const int warpSize = 32;
108	#undef __CUDA_DEVICE_BUILTIN	116	#undef __CUDA_DEVICE_BUILTIN
109	#undef __CUDA_BUILTIN_VAR	117	#undef __CUDA_BUILTIN_VAR
110	#undef __CUDA_DISALLOW_BUILTINVAR_ACCESS	118	#undef __CUDA_DISALLOW_BUILTINVAR_ACCESS
		119	#undef __DELETE
111		120
112	#endif /* __CUDA_BUILTIN_VARS_H */	121	#endif /* __CUDA_BUILTIN_VARS_H */

...	@@ -195,8 +195,8 @@ __DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }	...	@@ -195,8 +195,8 @@ __DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }
195	__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }	195	__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }
196	__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }	196	__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }
197	__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }	197	__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }
198	__DEVICE__ double nearbyint(double __a) { return __nv_nearbyint(__a); }	198	__DEVICE__ double nearbyint(double __a) { return __builtin_nearbyint(__a); }
199	__DEVICE__ float nearbyintf(float __a) { return __nv_nearbyintf(__a); }	199	__DEVICE__ float nearbyintf(float __a) { return __builtin_nearbyintf(__a); }
200	__DEVICE__ double nextafter(double __a, double __b) {	200	__DEVICE__ double nextafter(double __a, double __b) {
201	return __nv_nextafter(__a, __b);	201	return __nv_nextafter(__a, __b);
202	}	202	}
...	@@ -249,8 +249,9 @@ __DEVICE__ double rhypot(double __a, double __b) {	...	@@ -249,8 +249,9 @@ __DEVICE__ double rhypot(double __a, double __b) {
249	__DEVICE__ float rhypotf(float __a, float __b) {	249	__DEVICE__ float rhypotf(float __a, float __b) {
250	return __nv_rhypotf(__a, __b);	250	return __nv_rhypotf(__a, __b);
251	}	251	}
252	__DEVICE__ double rint(double __a) { return __nv_rint(__a); }	252	// __nv_rint* in libdevice is buggy and produces incorrect results.
253	__DEVICE__ float rintf(float __a) { return __nv_rintf(__a); }	253	__DEVICE__ double rint(double __a) { return __builtin_rint(__a); }
		254	__DEVICE__ float rintf(float __a) { return __builtin_rintf(__a); }
254	__DEVICE__ double rnorm(int __a, const double *__b) {	255	__DEVICE__ double rnorm(int __a, const double *__b) {
255	return __nv_rnorm(__a, __b);	256	return __nv_rnorm(__a, __b);
256	}	257	}

...	@@ -28,6 +28,10 @@	...	@@ -28,6 +28,10 @@
28	#define __shared__ __attribute__((shared))	28	#define __shared__ __attribute__((shared))
29	#define __constant__ __attribute__((constant))	29	#define __constant__ __attribute__((constant))
30		30
		31	#if !defined(__cplusplus) \|\| __cplusplus < 201103L
		32	#define nullptr NULL;
		33	#endif
		34
31	#if __HIP_ENABLE_DEVICE_MALLOC__	35	#if __HIP_ENABLE_DEVICE_MALLOC__
32	extern "C" __device__ void *__hip_malloc(size_t __size);	36	extern "C" __device__ void *__hip_malloc(size_t __size);
33	extern "C" __device__ void __hip_free(void __ptr);	37	extern "C" __device__ void __hip_free(void __ptr);
...	@@ -51,6 +55,7 @@ static inline __device__ void free(void __ptr) {	...	@@ -51,6 +55,7 @@ static inline __device__ void free(void __ptr) {
51		55
52	#if !_OPENMP \|\| __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__	56	#if !_OPENMP \|\| __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
53	#include <__clang_cuda_math_forward_declares.h>	57	#include <__clang_cuda_math_forward_declares.h>
		58	#include <__clang_hip_cmath.h>
54	#include <__clang_cuda_complex_builtins.h>	59	#include <__clang_cuda_complex_builtins.h>
55		60
56	#include <algorithm>	61	#include <algorithm>

...	@@ -94,7 +94,7 @@ typedef __clang_svbfloat16x2_t svbfloat16x2_t;	...	@@ -94,7 +94,7 @@ typedef __clang_svbfloat16x2_t svbfloat16x2_t;
94	typedef __clang_svbfloat16x3_t svbfloat16x3_t;	94	typedef __clang_svbfloat16x3_t svbfloat16x3_t;
95	typedef __clang_svbfloat16x4_t svbfloat16x4_t;	95	typedef __clang_svbfloat16x4_t svbfloat16x4_t;
96	#endif	96	#endif
97	typedef enum	97	enum svpattern
98	{	98	{
99	SV_POW2 = 0,	99	SV_POW2 = 0,
100	SV_VL1 = 1,	100	SV_VL1 = 1,
...	@@ -113,9 +113,9 @@ typedef enum	...	@@ -113,9 +113,9 @@ typedef enum
113	SV_MUL4 = 29,	113	SV_MUL4 = 29,
114	SV_MUL3 = 30,	114	SV_MUL3 = 30,
115	SV_ALL = 31	115	SV_ALL = 31
116	} sv_pattern;	116	};
117		117
118	typedef enum	118	enum svprfop
119	{	119	{
120	SV_PLDL1KEEP = 0,	120	SV_PLDL1KEEP = 0,
121	SV_PLDL1STRM = 1,	121	SV_PLDL1STRM = 1,
...	@@ -129,7 +129,7 @@ typedef enum	...	@@ -129,7 +129,7 @@ typedef enum
129	SV_PSTL2STRM = 11,	129	SV_PSTL2STRM = 11,
130	SV_PSTL3KEEP = 12,	130	SV_PSTL3KEEP = 12,
131	SV_PSTL3STRM = 13	131	SV_PSTL3STRM = 13
132	} sv_prfop;	132	};
133		133
134	/* Function attributes */	134	/* Function attributes */
135	#define __aio static inline __attribute__((__always_inline__, __nodebug__, __overloadable__))	135	#define __aio static inline __attribute__((__always_inline__, __nodebug__, __overloadable__))
...	@@ -10013,69 +10013,69 @@ int16_t svorv(svbool_t, svint16_t);	...	@@ -10013,69 +10013,69 @@ int16_t svorv(svbool_t, svint16_t);
10013	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpfirst_b)))	10013	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpfirst_b)))
10014	svbool_t svpfirst(svbool_t, svbool_t);	10014	svbool_t svpfirst(svbool_t, svbool_t);
10015	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u32base)))	10015	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u32base)))
10016	void svprfb_gather(svbool_t, svuint32_t, sv_prfop);	10016	void svprfb_gather(svbool_t, svuint32_t, enum svprfop);
10017	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u64base)))	10017	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u64base)))
10018	void svprfb_gather(svbool_t, svuint64_t, sv_prfop);	10018	void svprfb_gather(svbool_t, svuint64_t, enum svprfop);
10019	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u32base_offset)))	10019	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u32base_offset)))
10020	void svprfb_gather_offset(svbool_t, svuint32_t, int64_t, sv_prfop);	10020	void svprfb_gather_offset(svbool_t, svuint32_t, int64_t, enum svprfop);
10021	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u64base_offset)))	10021	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u64base_offset)))
10022	void svprfb_gather_offset(svbool_t, svuint64_t, int64_t, sv_prfop);	10022	void svprfb_gather_offset(svbool_t, svuint64_t, int64_t, enum svprfop);
10023	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_s32offset)))	10023	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_s32offset)))
10024	void svprfb_gather_offset(svbool_t, void const *, svint32_t, sv_prfop);	10024	void svprfb_gather_offset(svbool_t, void const *, svint32_t, enum svprfop);
10025	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u32offset)))	10025	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svprfb_gather_u32offset)))

Update clang headers