authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-07 20:42:46-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
log6778da4516e68c271cb50fe9c252ab4084daf16b
tree46e3bb0082569ecb678039e36fa2a422ae02a994
parentf8708e2c4d93eece5b3e131fd2d1b5b210806cd6

x86_64: implement binary operations for `f16` and `f16` vectors

7 files changed, 354 insertions(+), 46 deletions(-)

src/arch/x86_64/CodeGen.zig+240-21
......@@ -4497,14 +4497,15 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
44974497 const tag = self.air.instructions.items(.tag)[inst];
44984498 try self.genBinOpMir(switch (ty_bits) {
44994499 // No point using an extra prefix byte for *pd which performs the same operation.
4500 32, 64 => switch (tag) {
4500 16, 32, 64, 128 => switch (tag) {
45014501 .neg => .xorps,
45024502 .fabs => .andnps,
45034503 else => unreachable,
45044504 },
4505 else => return self.fail("TODO implement airFloatSign for {}", .{
4505 80 => return self.fail("TODO implement airFloatSign for {}", .{
45064506 ty.fmt(self.bin_file.options.module.?),
45074507 }),
4508 else => unreachable,
45084509 }, vec_ty, dst_mcv, sign_mcv);
45094510 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
45104511}
......@@ -6112,9 +6113,53 @@ fn genBinOp(
61126113 return dst_mcv;
61136114 }
61146115
6116 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
61156117 const mir_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
61166118 else => unreachable,
61176119 .Float => switch (lhs_ty.floatBits(self.target.*)) {
6120 16 => if (self.hasFeature(.f16c)) {
6121 const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
6122 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6123 defer self.register_manager.unlockReg(tmp_lock);
6124
6125 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6126 .vpinsrw,
6127 dst_reg,
6128 dst_reg,
6129 src_mcv.mem(.word),
6130 Immediate.u(1),
6131 ) else try self.asmRegisterRegisterRegister(
6132 .vpunpcklwd,
6133 dst_reg,
6134 dst_reg,
6135 (if (src_mcv.isRegister())
6136 src_mcv.getReg().?
6137 else
6138 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6139 );
6140 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6141 try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
6142 try self.asmRegisterRegisterRegister(
6143 switch (air_tag) {
6144 .add => .vaddss,
6145 .sub => .vsubss,
6146 .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
6147 .max => .vmaxss,
6148 .min => .vmaxss,
6149 else => unreachable,
6150 },
6151 dst_reg,
6152 dst_reg,
6153 tmp_reg,
6154 );
6155 try self.asmRegisterRegisterImmediate(
6156 .vcvtps2ph,
6157 dst_reg,
6158 dst_reg,
6159 Immediate.u(0b1_00),
6160 );
6161 return dst_mcv;
6162 } else null,
61186163 32 => switch (air_tag) {
61196164 .add => if (self.hasFeature(.avx)) .vaddss else .addss,
61206165 .sub => if (self.hasFeature(.avx)) .vsubss else .subss,
......@@ -6141,12 +6186,178 @@ fn genBinOp(
61416186 .min => if (self.hasFeature(.avx)) .vminsd else .minsd,
61426187 else => unreachable,
61436188 },
6144 16, 80, 128 => null,
6189 80, 128 => null,
61456190 else => unreachable,
61466191 },
61476192 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
61486193 else => null,
61496194 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
6195 16 => if (self.hasFeature(.f16c)) switch (lhs_ty.vectorLen()) {
6196 1 => {
6197 const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
6198 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6199 defer self.register_manager.unlockReg(tmp_lock);
6200
6201 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6202 .vpinsrw,
6203 dst_reg,
6204 dst_reg,
6205 src_mcv.mem(.word),
6206 Immediate.u(1),
6207 ) else try self.asmRegisterRegisterRegister(
6208 .vpunpcklwd,
6209 dst_reg,
6210 dst_reg,
6211 (if (src_mcv.isRegister())
6212 src_mcv.getReg().?
6213 else
6214 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6215 );
6216 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6217 try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
6218 try self.asmRegisterRegisterRegister(
6219 switch (air_tag) {
6220 .add => .vaddss,
6221 .sub => .vsubss,
6222 .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
6223 .max => .vmaxss,
6224 .min => .vmaxss,
6225 else => unreachable,
6226 },
6227 dst_reg,
6228 dst_reg,
6229 tmp_reg,
6230 );
6231 try self.asmRegisterRegisterImmediate(
6232 .vcvtps2ph,
6233 dst_reg,
6234 dst_reg,
6235 Immediate.u(0b1_00),
6236 );
6237 return dst_mcv;
6238 },
6239 2 => {
6240 const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
6241 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6242 defer self.register_manager.unlockReg(tmp_lock);
6243
6244 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
6245 .vpinsrd,
6246 dst_reg,
6247 src_mcv.mem(.dword),
6248 Immediate.u(1),
6249 ) else try self.asmRegisterRegisterRegister(
6250 .vunpcklps,
6251 dst_reg,
6252 dst_reg,
6253 (if (src_mcv.isRegister())
6254 src_mcv.getReg().?
6255 else
6256 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6257 );
6258 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6259 try self.asmRegisterRegisterRegister(.vmovhlps, tmp_reg, dst_reg, dst_reg);
6260 try self.asmRegisterRegisterRegister(
6261 switch (air_tag) {
6262 .add => .vaddps,
6263 .sub => .vsubps,
6264 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6265 .max => .vmaxps,
6266 .min => .vmaxps,
6267 else => unreachable,
6268 },
6269 dst_reg,
6270 dst_reg,
6271 tmp_reg,
6272 );
6273 try self.asmRegisterRegisterImmediate(
6274 .vcvtps2ph,
6275 dst_reg,
6276 dst_reg,
6277 Immediate.u(0b1_00),
6278 );
6279 return dst_mcv;
6280 },
6281 3...4 => {
6282 const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
6283 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6284 defer self.register_manager.unlockReg(tmp_lock);
6285
6286 try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6287 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6288 .vcvtph2ps,
6289 tmp_reg,
6290 src_mcv.mem(.qword),
6291 ) else try self.asmRegisterRegister(
6292 .vcvtph2ps,
6293 tmp_reg,
6294 (if (src_mcv.isRegister())
6295 src_mcv.getReg().?
6296 else
6297 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6298 );
6299 try self.asmRegisterRegisterRegister(
6300 switch (air_tag) {
6301 .add => .vaddps,
6302 .sub => .vsubps,
6303 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6304 .max => .vmaxps,
6305 .min => .vmaxps,
6306 else => unreachable,
6307 },
6308 dst_reg,
6309 dst_reg,
6310 tmp_reg,
6311 );
6312 try self.asmRegisterRegisterImmediate(
6313 .vcvtps2ph,
6314 dst_reg,
6315 dst_reg,
6316 Immediate.u(0b1_00),
6317 );
6318 return dst_mcv;
6319 },
6320 5...8 => {
6321 const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256();
6322 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6323 defer self.register_manager.unlockReg(tmp_lock);
6324
6325 try self.asmRegisterRegister(.vcvtph2ps, dst_reg.to256(), dst_reg);
6326 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6327 .vcvtph2ps,
6328 tmp_reg,
6329 src_mcv.mem(.xword),
6330 ) else try self.asmRegisterRegister(
6331 .vcvtph2ps,
6332 tmp_reg,
6333 (if (src_mcv.isRegister())
6334 src_mcv.getReg().?
6335 else
6336 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6337 );
6338 try self.asmRegisterRegisterRegister(
6339 switch (air_tag) {
6340 .add => .vaddps,
6341 .sub => .vsubps,
6342 .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6343 .max => .vmaxps,
6344 .min => .vmaxps,
6345 else => unreachable,
6346 },
6347 dst_reg.to256(),
6348 dst_reg.to256(),
6349 tmp_reg,
6350 );
6351 try self.asmRegisterRegisterImmediate(
6352 .vcvtps2ph,
6353 dst_reg,
6354 dst_reg.to256(),
6355 Immediate.u(0b1_00),
6356 );
6357 return dst_mcv;
6358 },
6359 else => null,
6360 } else null,
61506361 32 => switch (lhs_ty.vectorLen()) {
61516362 1 => switch (air_tag) {
61526363 .add => if (self.hasFeature(.avx)) .vaddss else .addss,
......@@ -6223,14 +6434,13 @@ fn genBinOp(
62236434 } else null,
62246435 else => null,
62256436 },
6226 16, 80, 128 => null,
6437 80, 128 => null,
62276438 else => unreachable,
62286439 },
62296440 },
62306441 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
62316442 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
62326443 });
6233 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
62346444 if (self.hasFeature(.avx)) {
62356445 const src1_alias =
62366446 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
......@@ -7139,21 +7349,21 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
71397349 const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
71407350 defer self.register_manager.unlockReg(tmp2_lock);
71417351
7142 if (src_mcv.isRegister())
7143 try self.asmRegisterRegisterRegister(
7144 .vpunpcklwd,
7145 tmp1_reg,
7146 dst_reg.to128(),
7147 src_mcv.getReg().?.to128(),
7148 )
7149 else
7150 try self.asmRegisterRegisterMemoryImmediate(
7151 .vpinsrw,
7152 tmp1_reg,
7153 dst_reg.to128(),
7154 src_mcv.mem(.word),
7155 Immediate.u(1),
7156 );
7352 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
7353 .vpinsrw,
7354 tmp1_reg,
7355 dst_reg.to128(),
7356 src_mcv.mem(.word),
7357 Immediate.u(1),
7358 ) else try self.asmRegisterRegisterRegister(
7359 .vpunpcklwd,
7360 tmp1_reg,
7361 dst_reg.to128(),
7362 (if (src_mcv.isRegister())
7363 src_mcv.getReg().?
7364 else
7365 try self.copyToTmpRegister(ty, src_mcv)).to128(),
7366 );
71577367 try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);
71587368 try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);
71597369 try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);
......@@ -8139,7 +8349,16 @@ fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.Tag {
81398349 },
81408350 .Vector => switch (ty.childType().zigTypeTag()) {
81418351 .Float => switch (ty.childType().floatBits(self.target.*)) {
8142 16 => unreachable, // needs special handling
8352 16 => switch (ty.vectorLen()) {
8353 1 => unreachable, // needs special handling
8354 2 => return if (self.hasFeature(.avx)) .vmovss else .movss,
8355 3...4 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
8356 5...8 => return if (self.hasFeature(.avx))
8357 if (aligned) .vmovaps else .vmovups
8358 else if (aligned) .movaps else .movups,
8359 9...16 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
8360 else => {},
8361 },
81438362 32 => switch (ty.vectorLen()) {
81448363 1 => return if (self.hasFeature(.avx)) .vmovss else .movss,
81458364 2...4 => return if (self.hasFeature(.avx))
src/arch/x86_64/Encoding.zig+13-10
......@@ -270,7 +270,7 @@ pub const Mnemonic = enum {
270270 divps, divss,
271271 maxps, maxss,
272272 minps, minss,
273 movaps, movss, movups,
273 movaps, movhlps, movss, movups,
274274 mulps, mulss,
275275 orps,
276276 pextrw, pinsrw,
......@@ -303,6 +303,8 @@ pub const Mnemonic = enum {
303303 // SSE3
304304 movddup, movshdup, movsldup,
305305 // SSE4.1
306 pextrb, pextrd, pextrq,
307 pinsrb, pinsrd, pinsrq,
306308 roundpd, roundps, roundsd, roundss,
307309 // AVX
308310 vaddpd, vaddps, vaddsd, vaddss,
......@@ -311,13 +313,14 @@ pub const Mnemonic = enum {
311313 vmaxpd, vmaxps, vmaxsd, vmaxss,
312314 vminpd, vminps, vminsd, vminss,
313315 vmovapd, vmovaps,
314 vmovddup,
316 vmovddup, vmovhlps,
315317 vmovsd,
316318 vmovshdup, vmovsldup,
317319 vmovss,
318320 vmovupd, vmovups,
319321 vmulpd, vmulps, vmulsd, vmulss,
320 vpextrw, vpinsrw,
322 vpextrb, vpextrd, vpextrq, vpextrw,
323 vpinsrb, vpinsrd, vpinsrq, vpinsrw,
321324 vpshufhw, vpshuflw,
322325 vpsrld, vpsrlq, vpsrlw,
323326 vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
......@@ -359,7 +362,7 @@ pub const Op = enum {
359362 cl,
360363 r8, r16, r32, r64,
361364 rm8, rm16, rm32, rm64,
362 r32_m16, r64_m16,
365 r32_m8, r32_m16, r64_m16,
363366 m8, m16, m32, m64, m80, m128, m256,
364367 rel8, rel16, rel32,
365368 m,
......@@ -444,7 +447,7 @@ pub const Op = enum {
444447 pub fn immBitSize(op: Op) u64 {
445448 return switch (op) {
446449 .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
447 .al, .cl, .r8, .rm8 => unreachable,
450 .al, .cl, .r8, .rm8, .r32_m8 => unreachable,
448451 .ax, .r16, .rm16 => unreachable,
449452 .eax, .r32, .rm32, .r32_m16 => unreachable,
450453 .rax, .r64, .rm64, .r64_m16 => unreachable,
......@@ -467,7 +470,7 @@ pub const Op = enum {
467470 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
468471 .al, .cl, .r8, .rm8 => 8,
469472 .ax, .r16, .rm16 => 16,
470 .eax, .r32, .rm32, .r32_m16 => 32,
473 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
471474 .rax, .r64, .rm64, .r64_m16 => 64,
472475 .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
473476 .ymm, .ymm_m256 => 256,
......@@ -480,7 +483,7 @@ pub const Op = enum {
480483 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
481484 .rel8, .rel16, .rel32 => unreachable,
482485 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm, .ymm => unreachable,
483 .m8, .rm8 => 8,
486 .m8, .rm8, .r32_m8 => 8,
484487 .m16, .rm16, .r32_m16, .r64_m16 => 16,
485488 .m32, .rm32, .xmm_m32 => 32,
486489 .m64, .rm64, .xmm_m64 => 64,
......@@ -509,7 +512,7 @@ pub const Op = enum {
509512 .al, .ax, .eax, .rax,
510513 .r8, .r16, .r32, .r64,
511514 .rm8, .rm16, .rm32, .rm64,
512 .r32_m16, .r64_m16,
515 .r32_m8, .r32_m16, .r64_m16,
513516 .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
514517 .ymm, .ymm_m256,
515518 => true,
......@@ -535,7 +538,7 @@ pub const Op = enum {
535538 // zig fmt: off
536539 return switch (op) {
537540 .rm8, .rm16, .rm32, .rm64,
538 .r32_m16, .r64_m16,
541 .r32_m8, .r32_m16, .r64_m16,
539542 .m8, .m16, .m32, .m64, .m80, .m128, .m256,
540543 .m,
541544 .xmm_m32, .xmm_m64, .xmm_m128,
......@@ -559,7 +562,7 @@ pub const Op = enum {
559562 .al, .ax, .eax, .rax, .cl => .general_purpose,
560563 .r8, .r16, .r32, .r64 => .general_purpose,
561564 .rm8, .rm16, .rm32, .rm64 => .general_purpose,
562 .r32_m16, .r64_m16 => .general_purpose,
565 .r32_m8, .r32_m16, .r64_m16 => .general_purpose,
563566 .sreg => .segment,
564567 .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
565568 .ymm, .ymm_m256 => .floating_point,
src/arch/x86_64/Lower.zig+22
......@@ -137,6 +137,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
137137 .minps,
138138 .minss,
139139 .movaps,
140 .movhlps,
140141 .movss,
141142 .movups,
142143 .mulps,
......@@ -149,6 +150,8 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
149150 .subps,
150151 .subss,
151152 .ucomiss,
153 .unpckhps,
154 .unpcklps,
152155 .xorps,
153156
154157 .addpd,
......@@ -187,12 +190,20 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
187190 .subpd,
188191 .subsd,
189192 .ucomisd,
193 .unpckhpd,
194 .unpcklpd,
190195 .xorpd,
191196
192197 .movddup,
193198 .movshdup,
194199 .movsldup,
195200
201 .pextrb,
202 .pextrd,
203 .pextrq,
204 .pinsrb,
205 .pinsrd,
206 .pinsrq,
196207 .roundpd,
197208 .roundps,
198209 .roundsd,
......@@ -221,6 +232,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
221232 .vmovapd,
222233 .vmovaps,
223234 .vmovddup,
235 .vmovhlps,
224236 .vmovsd,
225237 .vmovshdup,
226238 .vmovsldup,
......@@ -231,7 +243,13 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
231243 .vmulps,
232244 .vmulsd,
233245 .vmulss,
246 .vpextrb,
247 .vpextrd,
248 .vpextrq,
234249 .vpextrw,
250 .vpinsrb,
251 .vpinsrd,
252 .vpinsrq,
235253 .vpinsrw,
236254 .vpshufhw,
237255 .vpshuflw,
......@@ -258,6 +276,10 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
258276 .vsubps,
259277 .vsubsd,
260278 .vsubss,
279 .vunpckhpd,
280 .vunpckhps,
281 .vunpcklpd,
282 .vunpcklps,
261283
262284 .vcvtph2ps,
263285 .vcvtps2ph,
src/arch/x86_64/Mir.zig+44
......@@ -192,6 +192,8 @@ pub const Inst = struct {
192192 minss,
193193 /// Move aligned packed single-precision floating-point values
194194 movaps,
195 /// Move packed single-precision floating-point values high to low
196 movhlps,
195197 /// Move scalar single-precision floating-point value
196198 movss,
197199 /// Move unaligned packed single-precision floating-point values
......@@ -216,6 +218,10 @@ pub const Inst = struct {
216218 subss,
217219 /// Unordered compare scalar single-precision floating-point values
218220 ucomiss,
221 /// Unpack and interleave high packed single-precision floating-point values
222 unpckhps,
223 /// Unpack and interleave low packed single-precision floating-point values
224 unpcklps,
219225 /// Bitwise logical xor of packed single precision floating-point values
220226 xorps,
221227
......@@ -291,6 +297,10 @@ pub const Inst = struct {
291297 subsd,
292298 /// Unordered compare scalar double-precision floating-point values
293299 ucomisd,
300 /// Unpack and interleave high packed double-precision floating-point values
301 unpckhpd,
302 /// Unpack and interleave low packed double-precision floating-point values
303 unpcklpd,
294304 /// Bitwise logical xor of packed double precision floating-point values
295305 xorpd,
296306
......@@ -301,6 +311,18 @@ pub const Inst = struct {
301311 /// Replicate single floating-point values
302312 movsldup,
303313
314 /// Extract Byte
315 pextrb,
316 /// Extract Doubleword
317 pextrd,
318 /// Extract Quadword
319 pextrq,
320 /// Insert Byte
321 pinsrb,
322 /// Insert Doubleword
323 pinsrd,
324 /// Insert Quadword
325 pinsrq,
304326 /// Round packed double-precision floating-point values
305327 roundpd,
306328 /// Round packed single-precision floating-point values
......@@ -354,6 +376,8 @@ pub const Inst = struct {
354376 vmovapd,
355377 /// Move aligned packed single-precision floating-point values
356378 vmovaps,
379 /// Move packed single-precision floating-point values high to low
380 vmovhlps,
357381 /// Replicate double floating-point values
358382 vmovddup,
359383 /// Move or merge scalar double-precision floating-point value
......@@ -376,8 +400,20 @@ pub const Inst = struct {
376400 vmulsd,
377401 /// Multiply scalar single-precision floating-point values
378402 vmulss,
403 /// Extract Byte
404 vpextrb,
405 /// Extract Doubleword
406 vpextrd,
407 /// Extract Quadword
408 vpextrq,
379409 /// Extract word
380410 vpextrw,
411 /// Insert Byte
412 vpinsrb,
413 /// Insert Doubleword
414 vpinsrd,
415 /// Insert Quadword
416 vpinsrq,
381417 /// Insert word
382418 vpinsrw,
383419 /// Shuffle packed high words
......@@ -430,6 +466,14 @@ pub const Inst = struct {
430466 vsubsd,
431467 /// Subtract scalar single-precision floating-point values
432468 vsubss,
469 /// Unpack and interleave high packed double-precision floating-point values
470 vunpckhpd,
471 /// Unpack and interleave high packed single-precision floating-point values
472 vunpckhps,
473 /// Unpack and interleave low packed double-precision floating-point values
474 vunpcklpd,
475 /// Unpack and interleave low packed single-precision floating-point values
476 vunpcklps,
433477
434478 /// Convert 16-bit floating-point values to single-precision floating-point values
435479 vcvtph2ps,
src/arch/x86_64/encodings.zig+20
......@@ -865,6 +865,8 @@ pub const table = [_]Entry{
865865 .{ .movaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .none, .sse },
866866 .{ .movaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .none, .sse },
867867
868 .{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
869
868870 .{ .movss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse },
869871 .{ .movss, .mr, &.{ .xmm_m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse },
870872
......@@ -988,8 +990,16 @@ pub const table = [_]Entry{
988990 .{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
989991
990992 // SSE4.1
993 .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
994 .{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },
995 .{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },
996
991997 .{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
992998
999 .{ .pinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .none, .sse4_1 },
1000 .{ .pinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 },
1001 .{ .pinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 },
1002
9931003 .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
9941004
9951005 .{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
......@@ -1062,6 +1072,8 @@ pub const table = [_]Entry{
10621072 .{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
10631073 .{ .vmovddup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
10641074
1075 .{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1076
10651077 .{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
10661078 .{ .vmovsd, .rm, &.{ .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
10671079 .{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
......@@ -1098,9 +1110,17 @@ pub const table = [_]Entry{
10981110
10991111 .{ .vmulss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
11001112
1113 .{ .vpextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx },
1114 .{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
1115 .{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
1116
11011117 .{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx },
11021118 .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
11031119
1120 .{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
1121 .{ .vpinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
1122 .{ .vpinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
1123
11041124 .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
11051125
11061126 .{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx },
test/behavior/floatop.zig+11-11
......@@ -8,6 +8,8 @@ const has_f80_rt = switch (builtin.cpu.arch) {
88 .x86_64, .x86 => true,
99 else => false,
1010};
11const no_x86_64_hardware_f16_support = builtin.zig_backend == .stage2_x86_64 and
12 !std.Target.x86.featureSetHas(builtin.cpu.features, .f16c);
1113
1214const epsilon_16 = 0.001;
1315const epsilon = 0.000001;
......@@ -52,8 +54,7 @@ fn testFloatComparisons() !void {
5254}
5355
5456test "different sized float comparisons" {
55 if (builtin.zig_backend == .stage2_x86_64 and
56 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest; // TODO
57 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
5758 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
5859 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
5960 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -152,7 +153,7 @@ fn testSqrtWithVectors() !void {
152153}
153154
154155test "more @sqrt f16 tests" {
155 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
156 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
156157 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
157158 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
158159 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -202,7 +203,7 @@ fn testSqrtLegacy(comptime T: type, x: T) !void {
202203}
203204
204205test "@sin" {
205 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
206 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
206207 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
207208 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
208209 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -241,7 +242,7 @@ fn testSinWithVectors() !void {
241242}
242243
243244test "@cos" {
244 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
245 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
245246 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
246247 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
247248 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -280,7 +281,7 @@ fn testCosWithVectors() !void {
280281}
281282
282283test "@exp" {
283 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
284 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
284285 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
285286 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
286287 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -318,7 +319,7 @@ fn testExpWithVectors() !void {
318319}
319320
320321test "@exp2" {
321 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
322 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
322323 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
323324 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
324325 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -403,7 +404,7 @@ test "@log with @vectors" {
403404}
404405
405406test "@log2" {
406 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
407 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
407408 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
408409 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
409410 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -445,7 +446,7 @@ fn testLog2WithVectors() !void {
445446}
446447
447448test "@log10" {
448 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
449 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
449450 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
450451 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
451452 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -881,7 +882,7 @@ fn testTruncLegacy(comptime T: type, x: T) !void {
881882}
882883
883884test "negation f16" {
884 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
885 if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
885886 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
886887 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
887888 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -1040,7 +1041,6 @@ test "comptime_float zero divided by zero produces zero" {
10401041}
10411042
10421043test "nan negation f16" {
1043 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
10441044 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
10451045 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10461046 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/muladd.zig+4-4
......@@ -2,11 +2,11 @@ const std = @import("std");
22const builtin = @import("builtin");
33const expect = std.testing.expect;
44
5const stage2_x86_64_without_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
5const no_x86_64_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
66 !std.Target.x86.featureSetHas(builtin.cpu.features, .fma);
77
88test "@mulAdd" {
9 if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
9 if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
1010 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1111 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1212 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -120,7 +120,7 @@ fn vector32() !void {
120120
121121test "vector f32" {
122122 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
123 if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
123 if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
124124 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
125125 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
126126 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -143,7 +143,7 @@ fn vector64() !void {
143143
144144test "vector f64" {
145145 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
146 if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
146 if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
147147 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
148148 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
149149 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO