1/**
2 * This file has no copyright assigned and is placed in the Public Domain.
3 * This file is part of the mingw-w64 runtime package.
4 * No warranty is given; refer to the file DISCLAIMER.PD within this package.
5 */
6#ifndef _DVEC_H_INCLUDED
7#define _DVEC_H_INCLUDED
8#ifndef RC_INVOKED
9
10#if !defined __cplusplus
11#error This file is only supported in C++ compilations!
12#endif
13
14#include <intrin.h>
15#include <assert.h>
16#include <fvec.h>
17#include <crtdefs.h>
18
19#pragma pack(push,_CRT_PACKING)
20
21#if defined(_ENABLE_VEC_DEBUG)
22#include <iostream>
23#endif
24
25#ifdef __SSE__
26
27#pragma pack(push,16)
28
29#define EXPLICIT explicit
30
31class I8vec16;
32class Is8vec16;
33class Iu8vec16;
34class I16vec8;
35class Is16vec8;
36class Iu16vec8;
37class I32vec4;
38class Is32vec4;
39class Iu32vec4;
40class I64vec2;
41class I128vec1;
42
43#define _MM_16UB(element,vector) (*((unsigned char*)&(vector) + (element)))
44#define _MM_16B(element,vector) (*((signed char*)&(vector) + (element)))
45
46#define _MM_8UW(element,vector) (*((unsigned short*)&(vector) + (element)))
47#define _MM_8W(element,vector) (*((short*)&(vector) + (element)))
48
49#define _MM_4UDW(element,vector) (*((unsigned int*)&(vector) + (element)))
50#define _MM_4DW(element,vector) (*((int*)&(vector) + (element)))
51
52#define _MM_2QW(element,vector) (*((__int64*)&(vector) + (element)))
53
54__MINGW_EXTENSION inline const __m128i get_mask128()
55{
56 static const __m128i mask128 = _mm_set1_epi64(M64((__int64)0xffffffffffffffffll));
57 return mask128;
58}
59
60class M128
61{
62protected:
63 __m128i vec;
64
65public:
66 M128() { }
67 M128(__m128i mm) { vec = mm; }
68
69 operator __m128i() const { return vec; }
70
71 M128& operator&=(const M128 &a) { return *this = (M128) _mm_and_si128(vec,a); }
72 M128& operator|=(const M128 &a) { return *this = (M128) _mm_or_si128(vec,a); }
73 M128& operator^=(const M128 &a) { return *this = (M128) _mm_xor_si128(vec,a); }
74
75};
76
77inline M128 operator&(const M128 &a,const M128 &b) { return _mm_and_si128(a,b); }
78inline M128 operator|(const M128 &a,const M128 &b) { return _mm_or_si128(a,b); }
79inline M128 operator^(const M128 &a,const M128 &b) { return _mm_xor_si128(a,b); }
80inline M128 andnot(const M128 &a,const M128 &b) { return _mm_andnot_si128(a,b); }
81
82class I128vec1 : public M128
83{
84public:
85 I128vec1() { }
86 I128vec1(__m128i mm) : M128(mm) { }
87
88 I128vec1& operator= (const M128 &a) { return *this = (I128vec1) a; }
89 I128vec1& operator&=(const M128 &a) { return *this = (I128vec1) _mm_and_si128(vec,a); }
90 I128vec1& operator|=(const M128 &a) { return *this = (I128vec1) _mm_or_si128(vec,a); }
91 I128vec1& operator^=(const M128 &a) { return *this = (I128vec1) _mm_xor_si128(vec,a); }
92
93};
94
95class I64vec2 : public M128
96{
97public:
98 I64vec2() { }
99 I64vec2(__m128i mm) : M128(mm) { }
100
101 __MINGW_EXTENSION I64vec2(__m64 q1,__m64 q0)
102 {
103 _MM_2QW(0,vec) = *(__int64*)&q0;
104 _MM_2QW(1,vec) = *(__int64*)&q1;
105 }
106
107 I64vec2& operator= (const M128 &a) { return *this = (I64vec2) a; }
108
109 I64vec2& operator&=(const M128 &a) { return *this = (I64vec2) _mm_and_si128(vec,a); }
110 I64vec2& operator|=(const M128 &a) { return *this = (I64vec2) _mm_or_si128(vec,a); }
111 I64vec2& operator^=(const M128 &a) { return *this = (I64vec2) _mm_xor_si128(vec,a); }
112
113 I64vec2& operator +=(const I64vec2 &a) { return *this = (I64vec2) _mm_add_epi64(vec,a); }
114 I64vec2& operator -=(const I64vec2 &a) { return *this = (I64vec2) _mm_sub_epi64(vec,a); }
115
116 I64vec2 operator<<(const I64vec2 &a) { return _mm_sll_epi64(vec,a); }
117 I64vec2 operator<<(int count) { return _mm_slli_epi64(vec,count); }
118 I64vec2& operator<<=(const I64vec2 &a) { return *this = (I64vec2) _mm_sll_epi64(vec,a); }
119 I64vec2& operator<<=(int count) { return *this = (I64vec2) _mm_slli_epi64(vec,count); }
120 I64vec2 operator>>(const I64vec2 &a) { return _mm_srl_epi64(vec,a); }
121 I64vec2 operator>>(int count) { return _mm_srli_epi64(vec,count); }
122 I64vec2& operator>>=(const I64vec2 &a) { return *this = (I64vec2) _mm_srl_epi64(vec,a); }
123 I64vec2& operator>>=(int count) { return *this = (I64vec2) _mm_srli_epi64(vec,count); }
124
125 __MINGW_EXTENSION const __int64& operator[](int i)const
126 {
127 assert(static_cast<unsigned int>(i) < 2);
128 return _MM_2QW(i,vec);
129 }
130
131 __MINGW_EXTENSION __int64& operator[](int i)
132 {
133 assert(static_cast<unsigned int>(i) < 2);
134 return _MM_2QW(i,vec);
135 }
136
137};
138
139inline I64vec2 unpack_low(const I64vec2 &a,const I64vec2 &b) {return _mm_unpacklo_epi64(a,b); }
140inline I64vec2 unpack_high(const I64vec2 &a,const I64vec2 &b) {return _mm_unpackhi_epi64(a,b); }
141
142class I32vec4 : public M128
143{
144public:
145 I32vec4() { }
146 I32vec4(__m128i mm) : M128(mm) { }
147
148 I32vec4& operator= (const M128 &a) { return *this = (I32vec4) a; }
149
150 I32vec4& operator&=(const M128 &a) { return *this = (I32vec4) _mm_and_si128(vec,a); }
151 I32vec4& operator|=(const M128 &a) { return *this = (I32vec4) _mm_or_si128(vec,a); }
152 I32vec4& operator^=(const M128 &a) { return *this = (I32vec4) _mm_xor_si128(vec,a); }
153
154 I32vec4& operator +=(const I32vec4 &a) { return *this = (I32vec4)_mm_add_epi32(vec,a); }
155 I32vec4& operator -=(const I32vec4 &a) { return *this = (I32vec4)_mm_sub_epi32(vec,a); }
156
157 I32vec4 operator<<(const I32vec4 &a) { return _mm_sll_epi32(vec,a); }
158 I32vec4 operator<<(int count) { return _mm_slli_epi32(vec,count); }
159 I32vec4& operator<<=(const I32vec4 &a) { return *this = (I32vec4)_mm_sll_epi32(vec,a); }
160 I32vec4& operator<<=(int count) { return *this = (I32vec4)_mm_slli_epi32(vec,count); }
161
162};
163
164inline I32vec4 cmpeq(const I32vec4 &a,const I32vec4 &b) { return _mm_cmpeq_epi32(a,b); }
165inline I32vec4 cmpneq(const I32vec4 &a,const I32vec4 &b) { return _mm_andnot_si128(_mm_cmpeq_epi32(a,b),get_mask128()); }
166
167inline I32vec4 unpack_low(const I32vec4 &a,const I32vec4 &b) { return _mm_unpacklo_epi32(a,b); }
168inline I32vec4 unpack_high(const I32vec4 &a,const I32vec4 &b) { return _mm_unpackhi_epi32(a,b); }
169
170class Is32vec4 : public I32vec4
171{
172public:
173 Is32vec4() { }
174 Is32vec4(__m128i mm) : I32vec4(mm) { }
175 Is32vec4(int i3,int i2,int i1,int i0)
176 {
177 _MM_4DW(0,vec) = i0;
178 _MM_4DW(1,vec) = i1;
179 _MM_4DW(2,vec) = i2;
180 _MM_4DW(3,vec) = i3;
181 }
182
183 Is32vec4& operator= (const M128 &a) { return *this = (Is32vec4) a; }
184
185 Is32vec4& operator&=(const M128 &a) { return *this = (Is32vec4) _mm_and_si128(vec,a); }
186 Is32vec4& operator|=(const M128 &a) { return *this = (Is32vec4) _mm_or_si128(vec,a); }
187 Is32vec4& operator^=(const M128 &a) { return *this = (Is32vec4) _mm_xor_si128(vec,a); }
188
189 Is32vec4& operator +=(const I32vec4 &a) { return *this = (Is32vec4)_mm_add_epi32(vec,a); }
190 Is32vec4& operator -=(const I32vec4 &a) { return *this = (Is32vec4)_mm_sub_epi32(vec,a); }
191
192 Is32vec4 operator<<(const M128 &a) { return _mm_sll_epi32(vec,a); }
193 Is32vec4 operator<<(int count) { return _mm_slli_epi32(vec,count); }
194 Is32vec4& operator<<=(const M128 &a) { return *this = (Is32vec4)_mm_sll_epi32(vec,a); }
195 Is32vec4& operator<<=(int count) { return *this = (Is32vec4)_mm_slli_epi32(vec,count); }
196
197 Is32vec4 operator>>(const M128 &a) { return _mm_sra_epi32(vec,a); }
198 Is32vec4 operator>>(int count) { return _mm_srai_epi32(vec,count); }
199 Is32vec4& operator>>=(const M128 &a) { return *this = (Is32vec4) _mm_sra_epi32(vec,a); }
200 Is32vec4& operator>>=(int count) { return *this = (Is32vec4) _mm_srai_epi32(vec,count); }
201
202#if defined(_ENABLE_VEC_DEBUG)
203
204 friend std::ostream& operator<< (std::ostream &os,const Is32vec4 &a)
205 {
206 os << "[3]:" << _MM_4DW(3,a)
207 << " [2]:" << _MM_4DW(2,a)
208 << " [1]:" << _MM_4DW(1,a)
209 << " [0]:" << _MM_4DW(0,a);
210 return os;
211 }
212#endif
213
214 const int& operator[](int i)const
215 {
216 assert(static_cast<unsigned int>(i) < 4);
217 return _MM_4DW(i,vec);
218 }
219
220 int& operator[](int i)
221 {
222 assert(static_cast<unsigned int>(i) < 4);
223 return _MM_4DW(i,vec);
224 }
225};
226
227inline Is32vec4 cmpeq(const Is32vec4 &a,const Is32vec4 &b) { return _mm_cmpeq_epi32(a,b); }
228inline Is32vec4 cmpneq(const Is32vec4 &a,const Is32vec4 &b) { return _mm_andnot_si128(_mm_cmpeq_epi32(a,b),get_mask128()); }
229inline Is32vec4 cmpgt(const Is32vec4 &a,const Is32vec4 &b) { return _mm_cmpgt_epi32(a,b); }
230inline Is32vec4 cmplt(const Is32vec4 &a,const Is32vec4 &b) { return _mm_cmpgt_epi32(b,a); }
231
232inline Is32vec4 unpack_low(const Is32vec4 &a,const Is32vec4 &b) { return _mm_unpacklo_epi32(a,b); }
233inline Is32vec4 unpack_high(const Is32vec4 &a,const Is32vec4 &b) { return _mm_unpackhi_epi32(a,b); }
234
235class Iu32vec4 : public I32vec4
236{
237public:
238 Iu32vec4() { }
239 Iu32vec4(__m128i mm) : I32vec4(mm) { }
240 Iu32vec4(unsigned int ui3,unsigned int ui2,unsigned int ui1,unsigned int ui0)
241 {
242 _MM_4UDW(0,vec) = ui0;
243 _MM_4UDW(1,vec) = ui1;
244 _MM_4UDW(2,vec) = ui2;
245 _MM_4UDW(3,vec) = ui3;
246 }
247
248 Iu32vec4& operator= (const M128 &a) { return *this = (Iu32vec4) a; }
249
250 Iu32vec4& operator&=(const M128 &a) { return *this = (Iu32vec4) _mm_and_si128(vec,a); }
251 Iu32vec4& operator|=(const M128 &a) { return *this = (Iu32vec4) _mm_or_si128(vec,a); }
252 Iu32vec4& operator^=(const M128 &a) { return *this = (Iu32vec4) _mm_xor_si128(vec,a); }
253
254 Iu32vec4& operator +=(const I32vec4 &a) { return *this = (Iu32vec4)_mm_add_epi32(vec,a); }
255 Iu32vec4& operator -=(const I32vec4 &a) { return *this = (Iu32vec4)_mm_sub_epi32(vec,a); }
256
257 Iu32vec4 operator<<(const M128 &a) { return _mm_sll_epi32(vec,a); }
258 Iu32vec4 operator<<(int count) { return _mm_slli_epi32(vec,count); }
259 Iu32vec4& operator<<=(const M128 &a) { return *this = (Iu32vec4)_mm_sll_epi32(vec,a); }
260 Iu32vec4& operator<<=(int count) { return *this = (Iu32vec4)_mm_slli_epi32(vec,count); }
261 Iu32vec4 operator>>(const M128 &a) { return _mm_srl_epi32(vec,a); }
262 Iu32vec4 operator>>(int count) { return _mm_srli_epi32(vec,count); }
263 Iu32vec4& operator>>=(const M128 &a) { return *this = (Iu32vec4) _mm_srl_epi32(vec,a); }
264 Iu32vec4& operator>>=(int count) { return *this = (Iu32vec4) _mm_srli_epi32(vec,count); }
265
266#if defined(_ENABLE_VEC_DEBUG)
267
268 friend std::ostream& operator<< (std::ostream &os,const Iu32vec4 &a)
269 {
270 os << "[3]:" << _MM_4UDW(3,a)
271 << " [2]:" << _MM_4UDW(2,a)
272 << " [1]:" << _MM_4UDW(1,a)
273 << " [0]:" << _MM_4UDW(0,a);
274 return os;
275 }
276#endif
277
278 const unsigned int& operator[](int i)const
279 {
280 assert(static_cast<unsigned int>(i) < 4);
281 return _MM_4UDW(i,vec);
282 }
283
284 unsigned int& operator[](int i)
285 {
286 assert(static_cast<unsigned int>(i) < 4);
287 return _MM_4UDW(i,vec);
288 }
289};
290
291inline I64vec2 operator*(const Iu32vec4 &a,const Iu32vec4 &b) { return _mm_mul_epu32(a,b); }
292inline Iu32vec4 cmpeq(const Iu32vec4 &a,const Iu32vec4 &b) { return _mm_cmpeq_epi32(a,b); }
293inline Iu32vec4 cmpneq(const Iu32vec4 &a,const Iu32vec4 &b) { return _mm_andnot_si128(_mm_cmpeq_epi32(a,b),get_mask128()); }
294
295inline Iu32vec4 unpack_low(const Iu32vec4 &a,const Iu32vec4 &b) { return _mm_unpacklo_epi32(a,b); }
296inline Iu32vec4 unpack_high(const Iu32vec4 &a,const Iu32vec4 &b) { return _mm_unpackhi_epi32(a,b); }
297
298class I16vec8 : public M128
299{
300public:
301 I16vec8() { }
302 I16vec8(__m128i mm) : M128(mm) { }
303
304 I16vec8& operator= (const M128 &a) { return *this = (I16vec8) a; }
305
306 I16vec8& operator&=(const M128 &a) { return *this = (I16vec8) _mm_and_si128(vec,a); }
307 I16vec8& operator|=(const M128 &a) { return *this = (I16vec8) _mm_or_si128(vec,a); }
308 I16vec8& operator^=(const M128 &a) { return *this = (I16vec8) _mm_xor_si128(vec,a); }
309
310 I16vec8& operator +=(const I16vec8 &a) { return *this = (I16vec8) _mm_add_epi16(vec,a); }
311 I16vec8& operator -=(const I16vec8 &a) { return *this = (I16vec8) _mm_sub_epi16(vec,a); }
312 I16vec8& operator *=(const I16vec8 &a) { return *this = (I16vec8) _mm_mullo_epi16(vec,a); }
313
314 I16vec8 operator<<(const M128 &a) { return _mm_sll_epi16(vec,a); }
315 I16vec8 operator<<(int count) { return _mm_slli_epi16(vec,count); }
316 I16vec8& operator<<=(const M128 &a) { return *this = (I16vec8)_mm_sll_epi16(vec,a); }
317 I16vec8& operator<<=(int count) { return *this = (I16vec8)_mm_slli_epi16(vec,count); }
318
319};
320
321inline I16vec8 operator*(const I16vec8 &a,const I16vec8 &b) { return _mm_mullo_epi16(a,b); }
322
323inline I16vec8 cmpeq(const I16vec8 &a,const I16vec8 &b) { return _mm_cmpeq_epi16(a,b); }
324inline I16vec8 cmpneq(const I16vec8 &a,const I16vec8 &b) { return _mm_andnot_si128(_mm_cmpeq_epi16(a,b),get_mask128()); }
325
326inline I16vec8 unpack_low(const I16vec8 &a,const I16vec8 &b) { return _mm_unpacklo_epi16(a,b); }
327inline I16vec8 unpack_high(const I16vec8 &a,const I16vec8 &b) { return _mm_unpackhi_epi16(a,b); }
328
329class Is16vec8 : public I16vec8
330{
331public:
332 Is16vec8() { }
333 Is16vec8(__m128i mm) : I16vec8(mm) { }
334 Is16vec8(signed short s7,signed short s6,signed short s5,signed short s4,signed short s3,signed short s2,signed short s1,signed short s0)
335 {
336 _MM_8W(0,vec) = s0;
337 _MM_8W(1,vec) = s1;
338 _MM_8W(2,vec) = s2;
339 _MM_8W(3,vec) = s3;
340 _MM_8W(4,vec) = s4;
341 _MM_8W(5,vec) = s5;
342 _MM_8W(6,vec) = s6;
343 _MM_8W(7,vec) = s7;
344 }
345
346 Is16vec8& operator= (const M128 &a) { return *this = (Is16vec8) a; }
347
348 Is16vec8& operator&=(const M128 &a) { return *this = (Is16vec8) _mm_and_si128(vec,a); }
349 Is16vec8& operator|=(const M128 &a) { return *this = (Is16vec8) _mm_or_si128(vec,a); }
350 Is16vec8& operator^=(const M128 &a) { return *this = (Is16vec8) _mm_xor_si128(vec,a); }
351
352 Is16vec8& operator +=(const I16vec8 &a) { return *this = (Is16vec8) _mm_add_epi16(vec,a); }
353 Is16vec8& operator -=(const I16vec8 &a) { return *this = (Is16vec8) _mm_sub_epi16(vec,a); }
354 Is16vec8& operator *=(const I16vec8 &a) { return *this = (Is16vec8) _mm_mullo_epi16(vec,a); }
355
356 Is16vec8 operator<<(const M128 &a) { return _mm_sll_epi16(vec,a); }
357 Is16vec8 operator<<(int count) { return _mm_slli_epi16(vec,count); }
358 Is16vec8& operator<<=(const M128 &a) { return *this = (Is16vec8)_mm_sll_epi16(vec,a); }
359 Is16vec8& operator<<=(int count) { return *this = (Is16vec8)_mm_slli_epi16(vec,count); }
360
361 Is16vec8 operator>>(const M128 &a) { return _mm_sra_epi16(vec,a); }
362 Is16vec8 operator>>(int count) { return _mm_srai_epi16(vec,count); }
363 Is16vec8& operator>>=(const M128 &a) { return *this = (Is16vec8)_mm_sra_epi16(vec,a); }
364 Is16vec8& operator>>=(int count) { return *this = (Is16vec8)_mm_srai_epi16(vec,count); }
365
366#if defined(_ENABLE_VEC_DEBUG)
367
368 friend std::ostream& operator<< (std::ostream &os,const Is16vec8 &a)
369 {
370 os << "[7]:" << _MM_8W(7,a)
371 << " [6]:" << _MM_8W(6,a)
372 << " [5]:" << _MM_8W(5,a)
373 << " [4]:" << _MM_8W(4,a)
374 << " [3]:" << _MM_8W(3,a)
375 << " [2]:" << _MM_8W(2,a)
376 << " [1]:" << _MM_8W(1,a)
377 << " [0]:" << _MM_8W(0,a);
378 return os;
379 }
380#endif
381
382 const signed short& operator[](int i)const
383 {
384 assert(static_cast<unsigned int>(i) < 8);
385 return _MM_8W(i,vec);
386 }
387
388 signed short& operator[](int i)
389 {
390 assert(static_cast<unsigned int>(i) < 8);
391 return _MM_8W(i,vec);
392 }
393};
394
395inline Is16vec8 operator*(const Is16vec8 &a,const Is16vec8 &b) { return _mm_mullo_epi16(a,b); }
396
397inline Is16vec8 cmpeq(const Is16vec8 &a,const Is16vec8 &b) { return _mm_cmpeq_epi16(a,b); }
398inline Is16vec8 cmpneq(const Is16vec8 &a,const Is16vec8 &b) { return _mm_andnot_si128(_mm_cmpeq_epi16(a,b),get_mask128()); }
399inline Is16vec8 cmpgt(const Is16vec8 &a,const Is16vec8 &b) { return _mm_cmpgt_epi16(a,b); }
400inline Is16vec8 cmplt(const Is16vec8 &a,const Is16vec8 &b) { return _mm_cmpgt_epi16(b,a); }
401
402inline Is16vec8 unpack_low(const Is16vec8 &a,const Is16vec8 &b) { return _mm_unpacklo_epi16(a,b); }
403inline Is16vec8 unpack_high(const Is16vec8 &a,const Is16vec8 &b) { return _mm_unpackhi_epi16(a,b); }
404
405inline Is16vec8 mul_high(const Is16vec8 &a,const Is16vec8 &b) { return _mm_mulhi_epi16(a,b); }
406inline Is32vec4 mul_add(const Is16vec8 &a,const Is16vec8 &b) { return _mm_madd_epi16(a,b);}
407
408inline Is16vec8 sat_add(const Is16vec8 &a,const Is16vec8 &b) { return _mm_adds_epi16(a,b); }
409inline Is16vec8 sat_sub(const Is16vec8 &a,const Is16vec8 &b) { return _mm_subs_epi16(a,b); }
410
411inline Is16vec8 simd_max(const Is16vec8 &a,const Is16vec8 &b) { return _mm_max_epi16(a,b); }
412inline Is16vec8 simd_min(const Is16vec8 &a,const Is16vec8 &b) { return _mm_min_epi16(a,b); }
413
414class Iu16vec8 : public I16vec8
415{
416public:
417 Iu16vec8() { }
418 Iu16vec8(__m128i mm) : I16vec8(mm) { }
419 Iu16vec8(unsigned short s7,unsigned short s6,unsigned short s5,unsigned short s4,unsigned short s3,unsigned short s2,unsigned short s1,unsigned short s0)
420 {
421 _MM_8UW(0,vec) = s0;
422 _MM_8UW(1,vec) = s1;
423 _MM_8UW(2,vec) = s2;
424 _MM_8UW(3,vec) = s3;
425 _MM_8UW(4,vec) = s4;
426 _MM_8UW(5,vec) = s5;
427 _MM_8UW(6,vec) = s6;
428 _MM_8UW(7,vec) = s7;
429 }
430
431 Iu16vec8& operator= (const M128 &a) { return *this = (Iu16vec8) a; }
432
433 Iu16vec8& operator&=(const M128 &a) { return *this = (Iu16vec8) _mm_and_si128(vec,a); }
434 Iu16vec8& operator|=(const M128 &a) { return *this = (Iu16vec8) _mm_or_si128(vec,a); }
435 Iu16vec8& operator^=(const M128 &a) { return *this = (Iu16vec8) _mm_xor_si128(vec,a); }
436
437 Iu16vec8& operator +=(const I16vec8 &a) { return *this = (Iu16vec8) _mm_add_epi16(vec,a); }
438 Iu16vec8& operator -=(const I16vec8 &a) { return *this = (Iu16vec8) _mm_sub_epi16(vec,a); }
439 Iu16vec8& operator *=(const I16vec8 &a) { return *this = (Iu16vec8) _mm_mullo_epi16(vec,a); }
440
441 Iu16vec8 operator<<(const M128 &a) { return _mm_sll_epi16(vec,a); }
442 Iu16vec8 operator<<(int count) { return _mm_slli_epi16(vec,count); }
443 Iu16vec8& operator<<=(const M128 &a) { return *this = (Iu16vec8)_mm_sll_epi16(vec,a); }
444 Iu16vec8& operator<<=(int count) { return *this = (Iu16vec8)_mm_slli_epi16(vec,count); }
445 Iu16vec8 operator>>(const M128 &a) { return _mm_srl_epi16(vec,a); }
446 Iu16vec8 operator>>(int count) { return _mm_srli_epi16(vec,count); }
447 Iu16vec8& operator>>=(const M128 &a) { return *this = (Iu16vec8) _mm_srl_epi16(vec,a); }
448 Iu16vec8& operator>>=(int count) { return *this = (Iu16vec8) _mm_srli_epi16(vec,count); }
449
450#if defined(_ENABLE_VEC_DEBUG)
451
452 friend std::ostream& operator << (std::ostream &os,const Iu16vec8 &a)
453 {
454 os << "[7]:" << unsigned short(_MM_8UW(7,a))
455 << " [6]:" << unsigned short(_MM_8UW(6,a))
456 << " [5]:" << unsigned short(_MM_8UW(5,a))
457 << " [4]:" << unsigned short(_MM_8UW(4,a))
458 << " [3]:" << unsigned short(_MM_8UW(3,a))
459 << " [2]:" << unsigned short(_MM_8UW(2,a))
460 << " [1]:" << unsigned short(_MM_8UW(1,a))
461 << " [0]:" << unsigned short(_MM_8UW(0,a));
462 return os;
463 }
464#endif
465
466 const unsigned short& operator[](int i)const
467 {
468 assert(static_cast<unsigned int>(i) < 8);
469 return _MM_8UW(i,vec);
470 }
471
472 unsigned short& operator[](int i)
473 {
474 assert(static_cast<unsigned int>(i) < 8);
475 return _MM_8UW(i,vec);
476 }
477};
478
479inline Iu16vec8 operator*(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_mullo_epi16(a,b); }
480
481inline Iu16vec8 cmpeq(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_cmpeq_epi16(a,b); }
482inline Iu16vec8 cmpneq(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_andnot_si128(_mm_cmpeq_epi16(a,b),get_mask128()); }
483
484inline Iu16vec8 unpack_low(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_unpacklo_epi16(a,b); }
485inline Iu16vec8 unpack_high(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_unpackhi_epi16(a,b); }
486
487inline Iu16vec8 sat_add(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_adds_epu16(a,b); }
488inline Iu16vec8 sat_sub(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_subs_epu16(a,b); }
489
490inline Iu16vec8 simd_avg(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_avg_epu16(a,b); }
491inline I16vec8 mul_high(const Iu16vec8 &a,const Iu16vec8 &b) { return _mm_mulhi_epu16(a,b); }
492
493class I8vec16 : public M128
494{
495public:
496 I8vec16() { }
497 I8vec16(__m128i mm) : M128(mm) { }
498
499 I8vec16& operator= (const M128 &a) { return *this = (I8vec16) a; }
500
501 I8vec16& operator&=(const M128 &a) { return *this = (I8vec16) _mm_and_si128(vec,a); }
502 I8vec16& operator|=(const M128 &a) { return *this = (I8vec16) _mm_or_si128(vec,a); }
503 I8vec16& operator^=(const M128 &a) { return *this = (I8vec16) _mm_xor_si128(vec,a); }
504
505 I8vec16& operator +=(const I8vec16 &a) { return *this = (I8vec16) _mm_add_epi8(vec,a); }
506 I8vec16& operator -=(const I8vec16 &a) { return *this = (I8vec16) _mm_sub_epi8(vec,a); }
507
508};
509
510inline I8vec16 cmpeq(const I8vec16 &a,const I8vec16 &b) { return _mm_cmpeq_epi8(a,b); }
511inline I8vec16 cmpneq(const I8vec16 &a,const I8vec16 &b) { return _mm_andnot_si128(_mm_cmpeq_epi8(a,b),get_mask128()); }
512
513inline I8vec16 unpack_low(const I8vec16 &a,const I8vec16 &b) { return _mm_unpacklo_epi8(a,b); }
514inline I8vec16 unpack_high(const I8vec16 &a,const I8vec16 &b) { return _mm_unpackhi_epi8(a,b); }
515
516class Is8vec16 : public I8vec16
517{
518public:
519 Is8vec16() { }
520 Is8vec16(__m128i mm) : I8vec16(mm) { }
521
522 Is8vec16& operator= (const M128 &a) { return *this = (Is8vec16) a; }
523
524 Is8vec16& operator&=(const M128 &a) { return *this = (Is8vec16) _mm_and_si128(vec,a); }
525 Is8vec16& operator|=(const M128 &a) { return *this = (Is8vec16) _mm_or_si128(vec,a); }
526 Is8vec16& operator^=(const M128 &a) { return *this = (Is8vec16) _mm_xor_si128(vec,a); }
527
528 Is8vec16& operator +=(const I8vec16 &a) { return *this = (Is8vec16) _mm_add_epi8(vec,a); }
529 Is8vec16& operator -=(const I8vec16 &a) { return *this = (Is8vec16) _mm_sub_epi8(vec,a); }
530
531#if defined(_ENABLE_VEC_DEBUG)
532
533 friend std::ostream& operator << (std::ostream &os,const Is8vec16 &a)
534 {
535 os << "[15]:" << short(_MM_16B(15,a))
536 << " [14]:" << short(_MM_16B(14,a))
537 << " [13]:" << short(_MM_16B(13,a))
538 << " [12]:" << short(_MM_16B(12,a))
539 << " [11]:" << short(_MM_16B(11,a))
540 << " [10]:" << short(_MM_16B(10,a))
541 << " [9]:" << short(_MM_16B(9,a))
542 << " [8]:" << short(_MM_16B(8,a))
543 << " [7]:" << short(_MM_16B(7,a))
544 << " [6]:" << short(_MM_16B(6,a))
545 << " [5]:" << short(_MM_16B(5,a))
546 << " [4]:" << short(_MM_16B(4,a))
547 << " [3]:" << short(_MM_16B(3,a))
548 << " [2]:" << short(_MM_16B(2,a))
549 << " [1]:" << short(_MM_16B(1,a))
550 << " [0]:" << short(_MM_16B(0,a));
551 return os;
552 }
553#endif
554
555 const signed char& operator[](int i)const
556 {
557 assert(static_cast<unsigned int>(i) < 16);
558 return _MM_16B(i,vec);
559 }
560
561 signed char& operator[](int i)
562 {
563 assert(static_cast<unsigned int>(i) < 16);
564 return _MM_16B(i,vec);
565 }
566
567};
568
569inline Is8vec16 cmpeq(const Is8vec16 &a,const Is8vec16 &b) { return _mm_cmpeq_epi8(a,b); }
570inline Is8vec16 cmpneq(const Is8vec16 &a,const Is8vec16 &b) { return _mm_andnot_si128(_mm_cmpeq_epi8(a,b),get_mask128()); }
571inline Is8vec16 cmpgt(const Is8vec16 &a,const Is8vec16 &b) { return _mm_cmpgt_epi8(a,b); }
572inline Is8vec16 cmplt(const Is8vec16 &a,const Is8vec16 &b) { return _mm_cmplt_epi8(a,b); }
573
574inline Is8vec16 unpack_low(const Is8vec16 &a,const Is8vec16 &b) { return _mm_unpacklo_epi8(a,b); }
575inline Is8vec16 unpack_high(const Is8vec16 &a,const Is8vec16 &b) { return _mm_unpackhi_epi8(a,b); }
576
577inline Is8vec16 sat_add(const Is8vec16 &a,const Is8vec16 &b) { return _mm_adds_epi8(a,b); }
578inline Is8vec16 sat_sub(const Is8vec16 &a,const Is8vec16 &b) { return _mm_subs_epi8(a,b); }
579
580class Iu8vec16 : public I8vec16
581{
582public:
583 Iu8vec16() { }
584 Iu8vec16(__m128i mm) : I8vec16(mm) { }
585
586 Iu8vec16& operator= (const M128 &a) { return *this = (Iu8vec16) a; }
587
588 Iu8vec16& operator&=(const M128 &a) { return *this = (Iu8vec16) _mm_and_si128(vec,a); }
589 Iu8vec16& operator|=(const M128 &a) { return *this = (Iu8vec16) _mm_or_si128(vec,a); }
590 Iu8vec16& operator^=(const M128 &a) { return *this = (Iu8vec16) _mm_xor_si128(vec,a); }
591
592 Iu8vec16& operator +=(const I8vec16 &a) { return *this = (Iu8vec16) _mm_add_epi8(vec,a); }
593 Iu8vec16& operator -=(const I8vec16 &a) { return *this = (Iu8vec16) _mm_sub_epi8(vec,a); }
594
595#if defined(_ENABLE_VEC_DEBUG)
596
597 friend std::ostream& operator << (std::ostream &os,const Iu8vec16 &a)
598 {
599 os << "[15]:" << unsigned short(_MM_16UB(15,a))
600 << " [14]:" << unsigned short(_MM_16UB(14,a))
601 << " [13]:" << unsigned short(_MM_16UB(13,a))
602 << " [12]:" << unsigned short(_MM_16UB(12,a))
603 << " [11]:" << unsigned short(_MM_16UB(11,a))
604 << " [10]:" << unsigned short(_MM_16UB(10,a))
605 << " [9]:" << unsigned short(_MM_16UB(9,a))
606 << " [8]:" << unsigned short(_MM_16UB(8,a))
607 << " [7]:" << unsigned short(_MM_16UB(7,a))
608 << " [6]:" << unsigned short(_MM_16UB(6,a))
609 << " [5]:" << unsigned short(_MM_16UB(5,a))
610 << " [4]:" << unsigned short(_MM_16UB(4,a))
611 << " [3]:" << unsigned short(_MM_16UB(3,a))
612 << " [2]:" << unsigned short(_MM_16UB(2,a))
613 << " [1]:" << unsigned short(_MM_16UB(1,a))
614 << " [0]:" << unsigned short(_MM_16UB(0,a));
615 return os;
616 }
617#endif
618
619 const unsigned char& operator[](int i)const
620 {
621 assert(static_cast<unsigned int>(i) < 16);
622 return _MM_16UB(i,vec);
623 }
624
625 unsigned char& operator[](int i)
626 {
627 assert(static_cast<unsigned int>(i) < 16);
628 return _MM_16UB(i,vec);
629 }
630
631};
632
633inline Iu8vec16 cmpeq(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_cmpeq_epi8(a,b); }
634inline Iu8vec16 cmpneq(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_andnot_si128(_mm_cmpeq_epi8(a,b),get_mask128()); }
635
636inline Iu8vec16 unpack_low(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_unpacklo_epi8(a,b); }
637inline Iu8vec16 unpack_high(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_unpackhi_epi8(a,b); }
638
639inline Iu8vec16 sat_add(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_adds_epu8(a,b); }
640inline Iu8vec16 sat_sub(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_subs_epu8(a,b); }
641
642inline I64vec2 sum_abs(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_sad_epu8(a,b); }
643
644inline Iu8vec16 simd_avg(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_avg_epu8(a,b); }
645inline Iu8vec16 simd_max(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_max_epu8(a,b); }
646inline Iu8vec16 simd_min(const Iu8vec16 &a,const Iu8vec16 &b) { return _mm_min_epu8(a,b); }
647
648inline Is16vec8 pack_sat(const Is32vec4 &a,const Is32vec4 &b) { return _mm_packs_epi32(a,b); }
649inline Is8vec16 pack_sat(const Is16vec8 &a,const Is16vec8 &b) { return _mm_packs_epi16(a,b); }
650inline Iu8vec16 packu_sat(const Is16vec8 &a,const Is16vec8 &b) { return _mm_packus_epi16(a,b);}
651
652#define IVEC128_LOGICALS(vect,element) inline I##vect##vec##element operator& (const I##vect##vec##element &a,const I##vect##vec##element &b) { return _mm_and_si128(a,b); } inline I##vect##vec##element operator| (const I##vect##vec##element &a,const I##vect##vec##element &b) { return _mm_or_si128(a,b); } inline I##vect##vec##element operator^ (const I##vect##vec##element &a,const I##vect##vec##element &b) { return _mm_xor_si128(a,b); } inline I##vect##vec##element andnot (const I##vect##vec##element &a,const I##vect##vec##element &b) { return _mm_andnot_si128(a,b); }
653
654IVEC128_LOGICALS(8,16)
655IVEC128_LOGICALS(u8,16)
656IVEC128_LOGICALS(s8,16)
657IVEC128_LOGICALS(16,8)
658IVEC128_LOGICALS(u16,8)
659IVEC128_LOGICALS(s16,8)
660IVEC128_LOGICALS(32,4)
661IVEC128_LOGICALS(u32,4)
662IVEC128_LOGICALS(s32,4)
663IVEC128_LOGICALS(64,2)
664IVEC128_LOGICALS(128,1)
665#undef IVEC128_LOGICALS
666
667#define IVEC128_ADD_SUB(vect,element,opsize) inline I##vect##vec##element operator+ (const I##vect##vec##element &a,const I##vect##vec##element &b) { return _mm_add_##opsize(a,b); } inline I##vect##vec##element operator- (const I##vect##vec##element &a,const I##vect##vec##element &b) { return _mm_sub_##opsize(a,b); }
668
669IVEC128_ADD_SUB(8,16,epi8)
670IVEC128_ADD_SUB(u8,16,epi8)
671IVEC128_ADD_SUB(s8,16,epi8)
672IVEC128_ADD_SUB(16,8,epi16)
673IVEC128_ADD_SUB(u16,8,epi16)
674IVEC128_ADD_SUB(s16,8,epi16)
675IVEC128_ADD_SUB(32,4,epi32)
676IVEC128_ADD_SUB(u32,4,epi32)
677IVEC128_ADD_SUB(s32,4,epi32)
678IVEC128_ADD_SUB(64,2,epi64)
679#undef IVEC128_ADD_SUB
680
681#define IVEC128_SELECT(vect12,vect34,element,selop,arg1,arg2) inline I##vect34##vec##element select_##selop (const I##vect12##vec##element &a,const I##vect12##vec##element &b,const I##vect34##vec##element &c,const I##vect34##vec##element &d) { I##vect12##vec##element mask = cmp##selop(a,b); return(I##vect34##vec##element ((mask & arg1) | I##vect12##vec##element ((_mm_andnot_si128(mask,arg2))))); }
682IVEC128_SELECT(8,s8,16,eq,c,d)
683IVEC128_SELECT(8,u8,16,eq,c,d)
684IVEC128_SELECT(8,8,16,eq,c,d)
685IVEC128_SELECT(8,s8,16,neq,c,d)
686IVEC128_SELECT(8,u8,16,neq,c,d)
687IVEC128_SELECT(8,8,16,neq,c,d)
688
689IVEC128_SELECT(16,s16,8,eq,c,d)
690IVEC128_SELECT(16,u16,8,eq,c,d)
691IVEC128_SELECT(16,16,8,eq,c,d)
692IVEC128_SELECT(16,s16,8,neq,c,d)
693IVEC128_SELECT(16,u16,8,neq,c,d)
694IVEC128_SELECT(16,16,8,neq,c,d)
695
696IVEC128_SELECT(32,s32,4,eq,c,d)
697IVEC128_SELECT(32,u32,4,eq,c,d)
698IVEC128_SELECT(32,32,4,eq,c,d)
699IVEC128_SELECT(32,s32,4,neq,c,d)
700IVEC128_SELECT(32,u32,4,neq,c,d)
701IVEC128_SELECT(32,32,4,neq,c,d)
702
703IVEC128_SELECT(s8,s8,16,gt,c,d)
704IVEC128_SELECT(s8,u8,16,gt,c,d)
705IVEC128_SELECT(s8,8,16,gt,c,d)
706IVEC128_SELECT(s8,s8,16,lt,c,d)
707IVEC128_SELECT(s8,u8,16,lt,c,d)
708IVEC128_SELECT(s8,8,16,lt,c,d)
709
710IVEC128_SELECT(s16,s16,8,gt,c,d)
711IVEC128_SELECT(s16,u16,8,gt,c,d)
712IVEC128_SELECT(s16,16,8,gt,c,d)
713IVEC128_SELECT(s16,s16,8,lt,c,d)
714IVEC128_SELECT(s16,u16,8,lt,c,d)
715IVEC128_SELECT(s16,16,8,lt,c,d)
716
717#undef IVEC128_SELECT
718
719class F64vec2
720{
721protected:
722 __m128d vec;
723public:
724
725 F64vec2() {}
726
727 F64vec2(__m128d m) { vec = m;}
728
729 F64vec2(double d1,double d0) { vec= _mm_set_pd(d1,d0); }
730
731 EXPLICIT F64vec2(double d) { vec = _mm_set1_pd(d); }
732
733 operator __m128d() const { return vec; }
734
735 friend F64vec2 operator &(const F64vec2 &a,const F64vec2 &b) { return _mm_and_pd(a,b); }
736 friend F64vec2 operator |(const F64vec2 &a,const F64vec2 &b) { return _mm_or_pd(a,b); }
737 friend F64vec2 operator ^(const F64vec2 &a,const F64vec2 &b) { return _mm_xor_pd(a,b); }
738
739 friend F64vec2 operator +(const F64vec2 &a,const F64vec2 &b) { return _mm_add_pd(a,b); }
740 friend F64vec2 operator -(const F64vec2 &a,const F64vec2 &b) { return _mm_sub_pd(a,b); }
741 friend F64vec2 operator *(const F64vec2 &a,const F64vec2 &b) { return _mm_mul_pd(a,b); }
742 friend F64vec2 operator /(const F64vec2 &a,const F64vec2 &b) { return _mm_div_pd(a,b); }
743
744 F64vec2& operator +=(F64vec2 &a) { return *this = _mm_add_pd(vec,a); }
745 F64vec2& operator -=(F64vec2 &a) { return *this = _mm_sub_pd(vec,a); }
746 F64vec2& operator *=(F64vec2 &a) { return *this = _mm_mul_pd(vec,a); }
747 F64vec2& operator /=(F64vec2 &a) { return *this = _mm_div_pd(vec,a); }
748 F64vec2& operator &=(F64vec2 &a) { return *this = _mm_and_pd(vec,a); }
749 F64vec2& operator |=(F64vec2 &a) { return *this = _mm_or_pd(vec,a); }
750 F64vec2& operator ^=(F64vec2 &a) { return *this = _mm_xor_pd(vec,a); }
751
752 friend double add_horizontal(F64vec2 &a)
753 {
754 F64vec2 ftemp = _mm_add_sd(a,_mm_shuffle_pd(a,a,1));
755 return ftemp[0];
756 }
757
758 friend F64vec2 andnot(const F64vec2 &a,const F64vec2 &b) { return _mm_andnot_pd(a,b); }
759
760 friend F64vec2 sqrt(const F64vec2 &a) { return _mm_sqrt_pd(a); }
761
762#define F64vec2_COMP(op) friend F64vec2 cmp##op (const F64vec2 &a,const F64vec2 &b) { return _mm_cmp##op##_pd(a,b); }
763 F64vec2_COMP(eq)
764 F64vec2_COMP(lt)
765 F64vec2_COMP(le)
766 F64vec2_COMP(gt)
767 F64vec2_COMP(ge)
768 F64vec2_COMP(ngt)
769 F64vec2_COMP(nge)
770 F64vec2_COMP(neq)
771 F64vec2_COMP(nlt)
772 F64vec2_COMP(nle)
773#undef F64vec2_COMP
774
775 friend F64vec2 simd_min(const F64vec2 &a,const F64vec2 &b) { return _mm_min_pd(a,b); }
776 friend F64vec2 simd_max(const F64vec2 &a,const F64vec2 &b) { return _mm_max_pd(a,b); }
777
778#define F64vec2_COMI(op) friend int comi##op (const F64vec2 &a,const F64vec2 &b) { return _mm_comi##op##_sd(a,b); }
779 F64vec2_COMI(eq)
780 F64vec2_COMI(lt)
781 F64vec2_COMI(le)
782 F64vec2_COMI(gt)
783 F64vec2_COMI(ge)
784 F64vec2_COMI(neq)
785#undef F64vec2_COMI
786
787#define F64vec2_UCOMI(op) friend int ucomi##op (const F64vec2 &a,const F64vec2 &b) { return _mm_ucomi##op##_sd(a,b); }
788 F64vec2_UCOMI(eq)
789 F64vec2_UCOMI(lt)
790 F64vec2_UCOMI(le)
791 F64vec2_UCOMI(gt)
792 F64vec2_UCOMI(ge)
793 F64vec2_UCOMI(neq)
794#undef F64vec2_UCOMI
795
796#if defined(_ENABLE_VEC_DEBUG)
797
798 friend std::ostream & operator<<(std::ostream & os,const F64vec2 &a) {
799 double *dp = (double*)&a;
800 os << " [1]:" << *(dp+1)
801 << " [0]:" << *dp;
802 return os;
803 }
804#endif
805
806 const double &operator[](int i) const {
807 assert((0 <= i) && (i <= 1));
808 double *dp = (double*)&vec;
809 return *(dp+i);
810 }
811
812 double &operator[](int i) {
813 assert((0 <= i) && (i <= 1));
814 double *dp = (double*)&vec;
815 return *(dp+i);
816 }
817};
818
819inline F64vec2 unpack_low(const F64vec2 &a,const F64vec2 &b) { return _mm_unpacklo_pd(a,b); }
820inline F64vec2 unpack_high(const F64vec2 &a,const F64vec2 &b) { return _mm_unpackhi_pd(a,b); }
821inline int move_mask(const F64vec2 &a) { return _mm_movemask_pd(a); }
822inline void loadu(F64vec2 &a,double *p) { a = _mm_loadu_pd(p); }
823inline void storeu(double *p,const F64vec2 &a) { _mm_storeu_pd(p,a); }
824inline void store_nta(double *p,F64vec2 &a) { _mm_stream_pd(p,a); }
825
826#define F64vec2_SELECT(op) inline F64vec2 select_##op (const F64vec2 &a,const F64vec2 &b,const F64vec2 &c,const F64vec2 &d) { F64vec2 mask = _mm_cmp##op##_pd(a,b); return((mask & c) | F64vec2((_mm_andnot_pd(mask,d)))); }
827F64vec2_SELECT(eq)
828F64vec2_SELECT(lt)
829F64vec2_SELECT(le)
830F64vec2_SELECT(gt)
831F64vec2_SELECT(ge)
832F64vec2_SELECT(neq)
833F64vec2_SELECT(nlt)
834F64vec2_SELECT(nle)
835#undef F64vec2_SELECT
836
837inline int F64vec2ToInt(const F64vec2 &a) { return _mm_cvttsd_si32(a); }
838inline F64vec2 F32vec4ToF64vec2(const F32vec4 &a) { return _mm_cvtps_pd(a); }
839inline F32vec4 F64vec2ToF32vec4(const F64vec2 &a) { return _mm_cvtpd_ps(a); }
840inline F64vec2 IntToF64vec2(const F64vec2 &a,int b) { return _mm_cvtsi32_sd(a,b); }
841
842#pragma pack(pop)
843
844#endif /* ifdef __SSE__ */
845
846#pragma pack(pop)
847#endif
848#endif