Commit 2286c19c20897727825a6fd8dd63de1ada6724ee | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-05-15 09:51:57-07:00
committer	GitHub <noreply@github.com> 2023-05-15 09:51:57-07:00
log	2286c19c20897727825a6fd8dd63de1ada6724ee
tree	1d8a36c24bd566626af7944c535364ac18f55ab9
parent	de6cafa80fba3c1fb93bf82b76c1efffbc61bf98
parent	cea9ac772a518ff249d47fc2cb7b2776c786ac07
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15628 from jacobly0/x86_64-behavior

27 files changed, 2115 insertions(+), 719 deletions(-)

src/arch/x86_64/CodeGen.zig+1198-381

...	...	@@ -1079,9 +1079,9 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1079	1079	.fixes = Mir.Inst.Fixes.fromCondition(cc),
1080	1080	.r1 = reg,
1081	1081	} },
1082		.z_and_np, .nz_or_p => .{ .r_scratch = .{
	1082	.z_and_np, .nz_or_p => .{ .rr = .{
1083	1083	.r1 = reg,
1084		.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
	1084	.r2 = (try self.register_manager.allocReg(null, gp)).to8(),
1085	1085	} },
1086	1086	},
1087	1087	});
...	...	@@ -1120,8 +1120,8 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
1120	1120	.fixes = Mir.Inst.Fixes.fromCondition(cc),
1121	1121	.payload = payload,
1122	1122	} },
1123		.z_and_np, .nz_or_p => .{ .x_scratch = .{
1124		.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
	1123	.z_and_np, .nz_or_p => .{ .rx = .{
	1124	.r1 = (try self.register_manager.allocReg(null, gp)).to8(),
1125	1125	.payload = payload,
1126	1126	} },
1127	1127	},
...	...	@@ -1460,6 +1460,15 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist
1460	1460	}
1461	1461
1462	1462	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
	1463	const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
	1464	.signed => \|s\| @bitCast(u32, s),
	1465	.unsigned => \|u\| @intCast(u32, u),
	1466	} });
	1467	assert(payload + 1 == switch (m) {
	1468	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1469	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1470	else => unreachable,
	1471	});
1463	1472	_ = try self.addInst(.{
1464	1473	.tag = tag[1],
1465	1474	.ops = switch (m) {
...	...	@@ -1475,17 +1484,9 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
1475	1484	},
1476	1485	.data = .{ .x = .{
1477	1486	.fixes = tag[0],
1478		.payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1479		.signed => \|s\| @bitCast(u32, s),
1480		.unsigned => \|u\| @intCast(u32, u),
1481		} }),
	1487	.payload = payload,
1482	1488	} },
1483	1489	});
1484		_ = switch (m) {
1485		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1486		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1487		else => unreachable,
1488		};
1489	1490	}
1490	1491
1491	1492	fn asmMemoryRegisterRegister(
...	...	@@ -1549,7 +1550,9 @@ fn gen(self: *Self) InnerError!void {
1549	1550	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
1550	1551	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1551	1552	const backpatch_frame_align = try self.asmPlaceholder();
	1553	const backpatch_frame_align_extra = try self.asmPlaceholder();
1552	1554	const backpatch_stack_alloc = try self.asmPlaceholder();
	1555	const backpatch_stack_alloc_extra = try self.asmPlaceholder();
1553	1556
1554	1557	switch (self.ret_mcv.long) {
1555	1558	.none, .unreach => {},
...	...	@@ -1598,24 +1601,67 @@ fn gen(self: *Self) InnerError!void {
1598	1601	const need_stack_adjust = frame_layout.stack_adjust > 0;
1599	1602	const need_save_reg = frame_layout.save_reg_list.count() > 0;
1600	1603	if (need_frame_align) {
	1604	const page_align = @as(u32, math.maxInt(u32)) << 12;
1601	1605	self.mir_instructions.set(backpatch_frame_align, .{
1602	1606	.tag = .@"and",
1603	1607	.ops = .ri_s,
1604	1608	.data = .{ .ri = .{
1605	1609	.r1 = .rsp,
1606		.i = frame_layout.stack_mask,
	1610	.i = @max(frame_layout.stack_mask, page_align),
1607	1611	} },
1608	1612	});
	1613	if (frame_layout.stack_mask < page_align) {
	1614	self.mir_instructions.set(backpatch_frame_align_extra, .{
	1615	.tag = .pseudo,
	1616	.ops = .pseudo_probe_align_ri_s,
	1617	.data = .{ .ri = .{
	1618	.r1 = .rsp,
	1619	.i = ~frame_layout.stack_mask & page_align,
	1620	} },
	1621	});
	1622	}
1609	1623	}
1610	1624	if (need_stack_adjust) {
1611		self.mir_instructions.set(backpatch_stack_alloc, .{
1612		.tag = .sub,
1613		.ops = .ri_s,
1614		.data = .{ .ri = .{
1615		.r1 = .rsp,
1616		.i = frame_layout.stack_adjust,
1617		} },
1618		});
	1625	const page_size: u32 = 1 << 12;
	1626	if (frame_layout.stack_adjust <= page_size) {
	1627	self.mir_instructions.set(backpatch_stack_alloc, .{
	1628	.tag = .sub,
	1629	.ops = .ri_s,
	1630	.data = .{ .ri = .{
	1631	.r1 = .rsp,
	1632	.i = frame_layout.stack_adjust,
	1633	} },
	1634	});
	1635	} else if (frame_layout.stack_adjust <
	1636	page_size * Lower.pseudo_probe_adjust_unrolled_max_insts)
	1637	{
	1638	self.mir_instructions.set(backpatch_stack_alloc, .{
	1639	.tag = .pseudo,
	1640	.ops = .pseudo_probe_adjust_unrolled_ri_s,
	1641	.data = .{ .ri = .{
	1642	.r1 = .rsp,
	1643	.i = frame_layout.stack_adjust,
	1644	} },
	1645	});
	1646	} else {
	1647	self.mir_instructions.set(backpatch_stack_alloc, .{
	1648	.tag = .pseudo,
	1649	.ops = .pseudo_probe_adjust_setup_rri_s,
	1650	.data = .{ .rri = .{
	1651	.r1 = .rsp,
	1652	.r2 = .rax,
	1653	.i = frame_layout.stack_adjust,
	1654	} },
	1655	});
	1656	self.mir_instructions.set(backpatch_stack_alloc_extra, .{
	1657	.tag = .pseudo,
	1658	.ops = .pseudo_probe_adjust_loop_rr,
	1659	.data = .{ .rr = .{
	1660	.r1 = .rsp,
	1661	.r2 = .rax,
	1662	} },
	1663	});
	1664	}
1619	1665	}
1620	1666	if (need_frame_align or need_stack_adjust) {
1621	1667	self.mir_instructions.set(backpatch_stack_dealloc, .{
...	...	@@ -2215,11 +2261,11 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2215	2261	},
2216	2262	.Vector => switch (ty.childType().zigTypeTag()) {
2217	2263	.Float => switch (ty.childType().floatBits(self.target.*)) {
2218		16, 32, 64 => if (self.hasFeature(.avx)) 32 else 16,
2219		80, 128 => break :need_mem,
	2264	16, 32, 64, 128 => if (self.hasFeature(.avx)) 32 else 16,
	2265	80 => break :need_mem,
2220	2266	else => unreachable,
2221	2267	},
2222		else => break :need_mem,
	2268	else => if (self.hasFeature(.avx)) 32 else 16,
2223	2269	},
2224	2270	else => 8,
2225	2271	})) {
...	...	@@ -2455,12 +2501,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2455	2501	}
2456	2502	} else if (src_bits == 64 and dst_bits == 32) {
2457	2503	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2458		.{ .v_, .cvtsd2ss },
	2504	.{ .v_ss, .cvtsd2 },
2459	2505	dst_reg,
2460	2506	dst_reg,
2461	2507	src_mcv.mem(.qword),
2462	2508	) else try self.asmRegisterRegisterRegister(
2463		.{ .v_, .cvtsd2ss },
	2509	.{ .v_ss, .cvtsd2 },
2464	2510	dst_reg,
2465	2511	dst_reg,
2466	2512	(if (src_mcv.isRegister())
...	...	@@ -2468,11 +2514,11 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2468	2514	else
2469	2515	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2470	2516	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2471		.{ ._, .cvtsd2ss },
	2517	.{ ._ss, .cvtsd2 },
2472	2518	dst_reg,
2473	2519	src_mcv.mem(.qword),
2474	2520	) else try self.asmRegisterRegister(
2475		.{ ._, .cvtsd2ss },
	2521	.{ ._ss, .cvtsd2 },
2476	2522	dst_reg,
2477	2523	(if (src_mcv.isRegister())
2478	2524	src_mcv.getReg().?
...	...	@@ -2506,22 +2552,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2506	2552	src_mcv.getReg().?
2507	2553	else
2508	2554	try self.copyToTmpRegister(src_ty, src_mcv);
2509		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
	2555	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());
2510	2556	switch (dst_bits) {
2511	2557	32 => {},
2512		64 => try self.asmRegisterRegisterRegister(.{ .v_, .cvtss2sd }, dst_reg, dst_reg, dst_reg),
	2558	64 => try self.asmRegisterRegisterRegister(.{ .v_sd, .cvtss2 }, dst_reg, dst_reg, dst_reg),
2513	2559	else => return self.fail("TODO implement airFpext from {} to {}", .{
2514	2560	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2515	2561	}),
2516	2562	}
2517	2563	} else if (src_bits == 32 and dst_bits == 64) {
2518	2564	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2519		.{ .v_, .cvtss2sd },
	2565	.{ .v_sd, .cvtss2 },
2520	2566	dst_reg,
2521	2567	dst_reg,
2522	2568	src_mcv.mem(.dword),
2523	2569	) else try self.asmRegisterRegisterRegister(
2524		.{ .v_, .cvtss2sd },
	2570	.{ .v_sd, .cvtss2 },
2525	2571	dst_reg,
2526	2572	dst_reg,
2527	2573	(if (src_mcv.isRegister())
...	...	@@ -2529,11 +2575,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2529	2575	else
2530	2576	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2531	2577	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2532		.{ ._, .cvtss2sd },
	2578	.{ ._sd, .cvtss2 },
2533	2579	dst_reg,
2534	2580	src_mcv.mem(.dword),
2535	2581	) else try self.asmRegisterRegister(
2536		.{ ._, .cvtss2sd },
	2582	.{ ._sd, .cvtss2 },
2537	2583	dst_reg,
2538	2584	(if (src_mcv.isRegister())
2539	2585	src_mcv.getReg().?
...	...	@@ -2754,8 +2800,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
2754	2800	const result = result: {
2755	2801	const tag = self.air.instructions.items(.tag)[inst];
2756	2802	const dst_ty = self.air.typeOfIndex(inst);
2757		if (dst_ty.zigTypeTag() == .Float)
2758		break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
	2803	switch (dst_ty.zigTypeTag()) {
	2804	.Float, .Vector => break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs),
	2805	else => {},
	2806	}
2759	2807
2760	2808	const dst_info = dst_ty.intInfo(self.target.*);
2761	2809	var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
...	...	@@ -3421,14 +3469,17 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3421	3469	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
3422	3470	}
3423	3471
3424		const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	3472	const dst_mcv: MCValue = if (src_mcv.isRegister() and
	3473	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3425	3474	src_mcv
	3475	else if (self.liveness.isUnused(inst))
	3476	.{ .register = try self.copyToTmpRegister(dst_ty, src_mcv) }
3426	3477	else
3427	3478	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
3428	3479
3429	3480	const pl_ty = dst_ty.childType();
3430	3481	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
3431		try self.genSetMem(.{ .reg = dst_mcv.register }, pl_abi_size, Type.bool, .{ .immediate = 1 });
	3482	try self.genSetMem(.{ .reg = dst_mcv.getReg().? }, pl_abi_size, Type.bool, .{ .immediate = 1 });
3432	3483	break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
3433	3484	};
3434	3485	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -4177,9 +4228,18 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4177	4228
4178	4229	const src_bits = src_ty.bitSize(self.target.*);
4179	4230	if (self.hasFeature(.lzcnt)) {
4180		if (src_bits <= 64) {
	4231	if (src_bits <= 8) {
	4232	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
	4233	try self.truncateRegister(src_ty, wide_reg);
	4234	try self.genBinOpMir(.{ ._, .lzcnt }, Type.u32, dst_mcv, .{ .register = wide_reg });
	4235	try self.genBinOpMir(
	4236	.{ ._, .sub },
	4237	dst_ty,
	4238	dst_mcv,
	4239	.{ .immediate = 8 + self.regExtraBits(src_ty) },
	4240	);
	4241	} else if (src_bits <= 64) {
4181	4242	try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
4182
4183	4243	const extra_bits = self.regExtraBits(src_ty);
4184	4244	if (extra_bits > 0) {
4185	4245	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
...	...	@@ -4218,7 +4278,17 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4218	4278	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4219	4279	.immediate = src_bits ^ (src_bits - 1),
4220	4280	});
4221		try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
	4281	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
	4282	defer self.register_manager.unlockReg(imm_lock);
	4283
	4284	if (src_bits <= 8) {
	4285	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
	4286	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
	4287	defer self.register_manager.unlockReg(wide_lock);
	4288
	4289	try self.truncateRegister(src_ty, wide_reg);
	4290	try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });
	4291	} else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
4222	4292
4223	4293	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4224	4294	try self.asmCmovccRegisterRegister(
...	...	@@ -4232,7 +4302,20 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4232	4302	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4233	4303	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
4234	4304	});
4235		try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
	4305	const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
	4306	defer self.register_manager.unlockReg(imm_lock);
	4307
	4308	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
	4309	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
	4310	defer self.register_manager.unlockReg(wide_lock);
	4311
	4312	try self.truncateRegister(src_ty, wide_reg);
	4313	try self.genBinOpMir(
	4314	.{ ._, .bsr },
	4315	if (src_bits <= 8) Type.u16 else src_ty,
	4316	dst_mcv,
	4317	.{ .register = wide_reg },
	4318	);
4236	4319
4237	4320	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4238	4321	try self.asmCmovccRegisterRegister(
...	...	@@ -4274,24 +4357,25 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4274	4357
4275	4358	if (self.hasFeature(.bmi)) {
4276	4359	if (src_bits <= 64) {
4277		const extra_bits = self.regExtraBits(src_ty);
	4360	const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
	4361	const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;
4278	4362	const masked_mcv = if (extra_bits > 0) masked: {
4279	4363	const tmp_mcv = tmp: {
4280	4364	if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
4281	4365	break :tmp src_mcv;
4282		try self.genSetReg(dst_reg, src_ty, src_mcv);
	4366	try self.genSetReg(dst_reg, wide_ty, src_mcv);
4283	4367	break :tmp dst_mcv;
4284	4368	};
4285	4369	try self.genBinOpMir(
4286	4370	.{ ._, .@"or" },
4287		src_ty,
	4371	wide_ty,
4288	4372	tmp_mcv,
4289	4373	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
4290	4374	@intCast(u6, src_bits) },
4291	4375	);
4292	4376	break :masked tmp_mcv;
4293	4377	} else mat_src_mcv;
4294		try self.genBinOpMir(.{ ._, .tzcnt }, src_ty, dst_mcv, masked_mcv);
	4378	try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv);
4295	4379	} else if (src_bits <= 128) {
4296	4380	const tmp_reg = try self.register_manager.allocReg(null, gp);
4297	4381	const tmp_mcv = MCValue{ .register = tmp_reg };
...	...	@@ -4320,7 +4404,17 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4320	4404	return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
4321	4405
4322	4406	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
4323		try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
	4407	const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
	4408	defer self.register_manager.unlockReg(width_lock);
	4409
	4410	if (src_bits <= 8 or !math.isPowerOfTwo(src_bits)) {
	4411	const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
	4412	const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
	4413	defer self.register_manager.unlockReg(wide_lock);
	4414
	4415	try self.truncateRegister(src_ty, wide_reg);
	4416	try self.genBinOpMir(.{ ._, .bsf }, Type.u16, dst_mcv, .{ .register = wide_reg });
	4417	} else try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
4324	4418
4325	4419	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4326	4420	try self.asmCmovccRegisterRegister(
...	...	@@ -4632,61 +4726,136 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4632	4726	}
4633	4727
4634	4728	fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
	4729	const tag = self.air.instructions.items(.tag)[inst];
4635	4730	const un_op = self.air.instructions.items(.data)[inst].un_op;
4636	4731	const ty = self.air.typeOf(un_op);
4637		const ty_bits = ty.floatBits(self.target.*);
	4732	const abi_size: u32 = switch (ty.abiSize(self.target.*)) {
	4733	1...16 => 16,
	4734	17...32 => 32,
	4735	else => return self.fail("TODO implement airFloatSign for {}", .{
	4736	ty.fmt(self.bin_file.options.module.?),
	4737	}),
	4738	};
	4739	const scalar_bits = ty.scalarType().floatBits(self.target.*);
	4740
	4741	const src_mcv = try self.resolveInst(un_op);
	4742	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
	4743	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	4744
	4745	const dst_mcv: MCValue = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
	4746	src_mcv
	4747	else if (self.hasFeature(.avx))
	4748	.{ .register = try self.register_manager.allocReg(inst, sse) }
	4749	else
	4750	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
	4751	const dst_reg = dst_mcv.getReg().?;
	4752	const dst_lock = self.register_manager.lockReg(dst_reg);
	4753	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
4638	4754
4639	4755	var arena = std.heap.ArenaAllocator.init(self.gpa);
4640	4756	defer arena.deinit();
4641	4757
4642		const ExpectedContents = union {
4643		f16: Value.Payload.Float_16,
4644		f32: Value.Payload.Float_32,
4645		f64: Value.Payload.Float_64,
4646		f80: Value.Payload.Float_80,
4647		f128: Value.Payload.Float_128,
	4758	const ExpectedContents = struct {
	4759	scalar: union {
	4760	i64: Value.Payload.I64,
	4761	big: struct {
	4762	limbs: [
	4763	@max(
	4764	std.math.big.int.Managed.default_capacity,
	4765	std.math.big.int.calcTwosCompLimbCount(128),
	4766	)
	4767	]std.math.big.Limb,
	4768	pl: Value.Payload.BigInt,
	4769	},
	4770	},
	4771	repeated: Value.Payload.SubValue,
4648	4772	};
4649	4773	var stack align(@alignOf(ExpectedContents)) =
4650	4774	std.heap.stackFallback(@sizeOf(ExpectedContents), arena.allocator());
4651	4775
	4776	var int_pl = Type.Payload.Bits{
	4777	.base = .{ .tag = .int_signed },
	4778	.data = scalar_bits,
	4779	};
4652	4780	var vec_pl = Type.Payload.Array{
4653	4781	.base = .{ .tag = .vector },
4654	4782	.data = .{
4655		.len = @divExact(128, ty_bits),
4656		.elem_type = ty,
	4783	.len = @divExact(abi_size * 8, scalar_bits),
	4784	.elem_type = Type.initPayload(&int_pl.base),
4657	4785	},
4658	4786	};
4659	4787	const vec_ty = Type.initPayload(&vec_pl.base);
4660
4661		var sign_pl = Value.Payload.SubValue{
4662		.base = .{ .tag = .repeated },
4663		.data = try Value.floatToValue(-0.0, stack.get(), ty, self.target.*),
	4788	const sign_val = switch (tag) {
	4789	.neg => try vec_ty.minInt(stack.get(), self.target.*),
	4790	.fabs => try vec_ty.maxInt(stack.get(), self.target.*),
	4791	else => unreachable,
4664	4792	};
4665		const sign_val = Value.initPayload(&sign_pl.base);
4666	4793
4667	4794	const sign_mcv = try self.genTypedValue(.{ .ty = vec_ty, .val = sign_val });
4668
4669		const src_mcv = try self.resolveInst(un_op);
4670		const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
4671		src_mcv
	4795	const sign_mem = if (sign_mcv.isMemory())
	4796	sign_mcv.mem(Memory.PtrSize.fromSize(abi_size))
4672	4797	else
4673		try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
4674		const dst_lock = self.register_manager.lockReg(dst_mcv.register);
4675		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	4798	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	4799	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
	4800	});
4676	4801
4677		const tag = self.air.instructions.items(.tag)[inst];
4678		try self.genBinOpMir(switch (ty_bits) {
4679		// No point using an extra prefix byte for *pd which performs the same operation.
4680		16, 32, 64, 128 => switch (tag) {
4681		.neg => .{ ._ps, .xor },
4682		.fabs => .{ ._ps, .andn },
	4802	if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
	4803	switch (scalar_bits) {
	4804	16, 128 => if (abi_size <= 16 or self.hasFeature(.avx2)) switch (tag) {
	4805	.neg => .{ .vp_, .xor },
	4806	.fabs => .{ .vp_, .@"and" },
	4807	else => unreachable,
	4808	} else switch (tag) {
	4809	.neg => .{ .v_ps, .xor },
	4810	.fabs => .{ .v_ps, .@"and" },
	4811	else => unreachable,
	4812	},
	4813	32 => switch (tag) {
	4814	.neg => .{ .v_ps, .xor },
	4815	.fabs => .{ .v_ps, .@"and" },
	4816	else => unreachable,
	4817	},
	4818	64 => switch (tag) {
	4819	.neg => .{ .v_pd, .xor },
	4820	.fabs => .{ .v_pd, .@"and" },
	4821	else => unreachable,
	4822	},
	4823	80 => return self.fail("TODO implement airFloatSign for {}", .{
	4824	ty.fmt(self.bin_file.options.module.?),
	4825	}),
4683	4826	else => unreachable,
4684	4827	},
4685		80 => return self.fail("TODO implement airFloatSign for {}", .{
4686		ty.fmt(self.bin_file.options.module.?),
4687		}),
4688		else => unreachable,
4689		}, vec_ty, dst_mcv, sign_mcv);
	4828	registerAlias(dst_reg, abi_size),
	4829	registerAlias(if (src_mcv.isRegister())
	4830	src_mcv.getReg().?
	4831	else
	4832	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	4833	sign_mem,
	4834	) else try self.asmRegisterMemory(
	4835	switch (scalar_bits) {
	4836	16, 128 => switch (tag) {
	4837	.neg => .{ .p_, .xor },
	4838	.fabs => .{ .p_, .@"and" },
	4839	else => unreachable,
	4840	},
	4841	32 => switch (tag) {
	4842	.neg => .{ ._ps, .xor },
	4843	.fabs => .{ ._ps, .@"and" },
	4844	else => unreachable,
	4845	},
	4846	64 => switch (tag) {
	4847	.neg => .{ ._pd, .xor },
	4848	.fabs => .{ ._pd, .@"and" },
	4849	else => unreachable,
	4850	},
	4851	80 => return self.fail("TODO implement airFloatSign for {}", .{
	4852	ty.fmt(self.bin_file.options.module.?),
	4853	}),
	4854	else => unreachable,
	4855	},
	4856	registerAlias(dst_reg, abi_size),
	4857	sign_mem,
	4858	);
4690	4859	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4691	4860	}
4692	4861
...	...	@@ -4740,7 +4909,6 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
4740	4909	})) \|tag\| tag else return self.fail("TODO implement genRound for {}", .{
4741	4910	ty.fmt(self.bin_file.options.module.?),
4742	4911	});
4743
4744	4912	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4745	4913	const dst_alias = registerAlias(dst_reg, abi_size);
4746	4914	switch (mir_tag[0]) {
...	...	@@ -4799,7 +4967,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4799	4967	src_mcv.getReg().?
4800	4968	else
4801	4969	try self.copyToTmpRegister(ty, src_mcv);
4802		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
	4970	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());
4803	4971	try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg);
4804	4972	try self.asmRegisterRegisterImmediate(
4805	4973	.{ .v_, .cvtps2ph },
...	...	@@ -4819,7 +4987,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4819	4987	16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
4820	4988	1 => {
4821	4989	try self.asmRegisterRegister(
4822		.{ .v_, .cvtph2ps },
	4990	.{ .v_ps, .cvtph2 },
4823	4991	dst_reg,
4824	4992	(if (src_mcv.isRegister())
4825	4993	src_mcv.getReg().?
...	...	@@ -4843,13 +5011,13 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4843	5011	2...8 => {
4844	5012	const wide_reg = registerAlias(dst_reg, abi_size * 2);
4845	5013	if (src_mcv.isMemory()) try self.asmRegisterMemory(
4846		.{ .v_, .cvtph2ps },
	5014	.{ .v_ps, .cvtph2 },
4847	5015	wide_reg,
4848	5016	src_mcv.mem(Memory.PtrSize.fromSize(
4849	5017	@intCast(u32, @divExact(wide_reg.bitSize(), 16)),
4850	5018	)),
4851	5019	) else try self.asmRegisterRegister(
4852		.{ .v_, .cvtph2ps },
	5020	.{ .v_ps, .cvtph2 },
4853	5021	wide_reg,
4854	5022	(if (src_mcv.isRegister())
4855	5023	src_mcv.getReg().?
...	...	@@ -5256,64 +5424,24 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
5256	5424
5257	5425	fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
5258	5426	const ptr_field_ty = self.air.typeOfIndex(inst);
5259		const mcv = try self.resolveInst(operand);
5260	5427	const ptr_container_ty = self.air.typeOf(operand);
5261	5428	const container_ty = ptr_container_ty.childType();
5262		const field_offset = switch (container_ty.containerLayout()) {
5263		.Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),
	5429	const field_offset = @intCast(i32, switch (container_ty.containerLayout()) {
	5430	.Auto, .Extern => container_ty.structFieldOffset(index, self.target.*),
5264	5431	.Packed => if (container_ty.zigTypeTag() == .Struct and
5265	5432	ptr_field_ty.ptrInfo().data.host_size == 0)
5266	5433	container_ty.packedStructFieldByteOffset(index, self.target.*)
5267	5434	else
5268	5435	0,
5269		};
5270
5271		const result: MCValue = result: {
5272		switch (mcv) {
5273		.load_frame, .lea_tlv, .load_tlv => {
5274		const offset_reg = try self.copyToTmpRegister(Type.usize, .{
5275		.immediate = field_offset,
5276		});
5277		const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
5278		defer self.register_manager.unlockReg(offset_reg_lock);
5279
5280		const dst_mcv = try self.copyToRegisterWithInstTracking(inst, Type.usize, switch (mcv) {
5281		.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
5282		else => mcv,
5283		});
5284		try self.genBinOpMir(.{ ._, .add }, Type.usize, dst_mcv, .{ .register = offset_reg });
5285		break :result dst_mcv;
5286		},
5287		.indirect => \|reg_off\| break :result .{ .indirect = .{
5288		.reg = reg_off.reg,
5289		.off = reg_off.off + @intCast(i32, field_offset),
5290		} },
5291		.lea_frame => \|frame_addr\| break :result .{ .lea_frame = .{
5292		.index = frame_addr.index,
5293		.off = frame_addr.off + @intCast(i32, field_offset),
5294		} },
5295		.register, .register_offset => {
5296		const src_reg = mcv.getReg().?;
5297		const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
5298		defer self.register_manager.unlockReg(src_lock);
	5436	});
5299	5437
5300		const dst_mcv: MCValue = if (self.reuseOperand(inst, operand, 0, mcv))
5301		mcv
5302		else
5303		.{ .register = try self.copyToTmpRegister(ptr_field_ty, mcv) };
5304		break :result .{ .register_offset = .{
5305		.reg = dst_mcv.getReg().?,
5306		.off = switch (dst_mcv) {
5307		.register => 0,
5308		.register_offset => \|reg_off\| reg_off.off,
5309		else => unreachable,
5310		} + @intCast(i32, field_offset),
5311		} };
5312		},
5313		else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),
5314		}
5315		};
5316		return result;
	5438	const src_mcv = try self.resolveInst(operand);
	5439	const dst_mcv = if (switch (src_mcv) {
	5440	.immediate, .lea_frame => true,
	5441	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),
	5442	else => false,
	5443	}) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv);
	5444	return dst_mcv.offset(field_offset);
5317	5445	}
5318	5446
5319	5447	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5324,8 +5452,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5324	5452	const index = extra.field_index;
5325	5453
5326	5454	const container_ty = self.air.typeOf(operand);
	5455	const container_rc = regClassForType(container_ty);
5327	5456	const field_ty = container_ty.structFieldType(index);
5328	5457	if (!field_ty.hasRuntimeBitsIgnoreComptime()) break :result .none;
	5458	const field_rc = regClassForType(field_ty);
	5459	const field_is_gp = field_rc.supersetOf(gp);
5329	5460
5330	5461	const src_mcv = try self.resolveInst(operand);
5331	5462	const field_off = switch (container_ty.containerLayout()) {
...	...	@@ -5358,7 +5489,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5358	5489	return self.fail("TODO implement struct_field_val with large packed field", .{});
5359	5490	}
5360	5491
5361		const dst_reg = try self.register_manager.allocReg(inst, gp);
	5492	const dst_reg = try self.register_manager.allocReg(if (field_is_gp) inst else null, gp);
5362	5493	const field_extra_bits = self.regExtraBits(field_ty);
5363	5494	const load_abi_size =
5364	5495	if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
...	...	@@ -5409,30 +5540,23 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5409	5540	if (field_extra_bits > 0) try self.truncateRegister(field_ty, dst_reg);
5410	5541
5411	5542	const dst_mcv = MCValue{ .register = dst_reg };
5412		const dst_rc = regClassForType(field_ty);
5413		if (dst_rc.eql(gp)) break :result dst_mcv;
5414
5415		const result_reg = try self.register_manager.allocReg(inst, dst_rc);
5416		try self.genSetReg(result_reg, field_ty, dst_mcv);
5417		break :result .{ .register = result_reg };
	5543	break :result if (field_is_gp)
	5544	dst_mcv
	5545	else
	5546	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
5418	5547	},
5419	5548	.register => \|reg\| {
5420	5549	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5421	5550	defer self.register_manager.unlockReg(reg_lock);
5422	5551
5423		const dst_mcv = if (self.reuseOperand(inst, operand, 0, src_mcv))
5424		src_mcv
	5552	const dst_reg = if (src_mcv.isRegister() and field_rc.supersetOf(container_rc) and
	5553	self.reuseOperand(inst, operand, 0, src_mcv))
	5554	src_mcv.getReg().?
5425	5555	else
5426		try self.copyToRegisterWithInstTracking(
5427		inst,
5428		Type.usize,
5429		.{ .register = reg.to64() },
5430		);
5431		const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
5432		.register => \|a_reg\| self.register_manager.lockReg(a_reg),
5433		else => null,
5434		};
5435		defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
	5556	try self.copyToTmpRegister(Type.usize, .{ .register = reg.to64() });
	5557	const dst_mcv = MCValue{ .register = dst_reg };
	5558	const dst_lock = self.register_manager.lockReg(dst_reg);
	5559	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5436	5560
5437	5561	// Shift by struct_field_offset.
5438	5562	try self.genShiftBinOpMir(
...	...	@@ -5459,7 +5583,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5459	5583	registerAlias(dst_mcv.register, field_byte_size),
5460	5584	);
5461	5585	}
5462		break :result dst_mcv;
	5586
	5587	break :result if (field_rc.supersetOf(gp))
	5588	dst_mcv
	5589	else
	5590	try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
5463	5591	},
5464	5592	.register_overflow => \|ro\| {
5465	5593	switch (index) {
...	...	@@ -6339,12 +6467,13 @@ fn genBinOp(
6339	6467	else
6340	6468	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6341	6469	);
6342		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6470	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
6343	6471	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
6344	6472	try self.asmRegisterRegisterRegister(
6345	6473	switch (air_tag) {
6346	6474	.add => .{ .v_ss, .add },
6347	6475	.sub => .{ .v_ss, .sub },
	6476	.mul => .{ .v_ss, .mul },
6348	6477	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
6349	6478	.max => .{ .v_ss, .max },
6350	6479	.min => .{ .v_ss, .max },
...	...	@@ -6393,6 +6522,234 @@ fn genBinOp(
6393	6522	},
6394	6523	.Vector => switch (lhs_ty.childType().zigTypeTag()) {
6395	6524	else => null,
	6525	.Int => switch (lhs_ty.childType().intInfo(self.target.*).bits) {
	6526	8 => switch (lhs_ty.vectorLen()) {
	6527	1...16 => switch (air_tag) {
	6528	.add,
	6529	.addwrap,
	6530	=> if (self.hasFeature(.avx)) .{ .vp_b, .add } else .{ .p_b, .add },
	6531	.sub,
	6532	.subwrap,
	6533	=> if (self.hasFeature(.avx)) .{ .vp_b, .sub } else .{ .p_b, .sub },
	6534	.bit_and => if (self.hasFeature(.avx)) .{ .vp_, .@"and" } else .{ .p_, .@"and" },
	6535	.bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
	6536	.xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
	6537	.min => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6538	.signed => if (self.hasFeature(.avx))
	6539	.{ .vp_b, .mins }
	6540	else if (self.hasFeature(.sse4_1))
	6541	.{ .p_b, .mins }
	6542	else
	6543	null,
	6544	.unsigned => if (self.hasFeature(.avx))
	6545	.{ .vp_b, .minu }
	6546	else if (self.hasFeature(.sse4_1))
	6547	.{ .p_b, .minu }
	6548	else
	6549	null,
	6550	},
	6551	.max => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6552	.signed => if (self.hasFeature(.avx))
	6553	.{ .vp_b, .maxs }
	6554	else if (self.hasFeature(.sse4_1))
	6555	.{ .p_b, .maxs }
	6556	else
	6557	null,
	6558	.unsigned => if (self.hasFeature(.avx))
	6559	.{ .vp_b, .maxu }
	6560	else if (self.hasFeature(.sse4_1))
	6561	.{ .p_b, .maxu }
	6562	else
	6563	null,
	6564	},
	6565	else => null,
	6566	},
	6567	17...32 => switch (air_tag) {
	6568	.add,
	6569	.addwrap,
	6570	=> if (self.hasFeature(.avx2)) .{ .vp_b, .add } else null,
	6571	.sub,
	6572	.subwrap,
	6573	=> if (self.hasFeature(.avx2)) .{ .vp_b, .sub } else null,
	6574	.bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
	6575	.bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
	6576	.xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
	6577	.min => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6578	.signed => if (self.hasFeature(.avx2)) .{ .vp_b, .mins } else null,
	6579	.unsigned => if (self.hasFeature(.avx)) .{ .vp_b, .minu } else null,
	6580	},
	6581	.max => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6582	.signed => if (self.hasFeature(.avx2)) .{ .vp_b, .maxs } else null,
	6583	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_b, .maxu } else null,
	6584	},
	6585	else => null,
	6586	},
	6587	else => null,
	6588	},
	6589	16 => switch (lhs_ty.vectorLen()) {
	6590	1...8 => switch (air_tag) {
	6591	.add,
	6592	.addwrap,
	6593	=> if (self.hasFeature(.avx)) .{ .vp_w, .add } else .{ .p_w, .add },
	6594	.sub,
	6595	.subwrap,
	6596	=> if (self.hasFeature(.avx)) .{ .vp_w, .sub } else .{ .p_w, .sub },
	6597	.mul,
	6598	.mulwrap,
	6599	=> if (self.hasFeature(.avx)) .{ .vp_w, .mull } else .{ .p_d, .mull },
	6600	.bit_and => if (self.hasFeature(.avx)) .{ .vp_, .@"and" } else .{ .p_, .@"and" },
	6601	.bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
	6602	.xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
	6603	.min => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6604	.signed => if (self.hasFeature(.avx))
	6605	.{ .vp_w, .mins }
	6606	else
	6607	.{ .p_w, .mins },
	6608	.unsigned => if (self.hasFeature(.avx))
	6609	.{ .vp_w, .minu }
	6610	else
	6611	.{ .p_w, .minu },
	6612	},
	6613	.max => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6614	.signed => if (self.hasFeature(.avx))
	6615	.{ .vp_w, .maxs }
	6616	else
	6617	.{ .p_w, .maxs },
	6618	.unsigned => if (self.hasFeature(.avx))
	6619	.{ .vp_w, .maxu }
	6620	else
	6621	.{ .p_w, .maxu },
	6622	},
	6623	else => null,
	6624	},
	6625	9...16 => switch (air_tag) {
	6626	.add,
	6627	.addwrap,
	6628	=> if (self.hasFeature(.avx2)) .{ .vp_w, .add } else null,
	6629	.sub,
	6630	.subwrap,
	6631	=> if (self.hasFeature(.avx2)) .{ .vp_w, .sub } else null,
	6632	.mul,
	6633	.mulwrap,
	6634	=> if (self.hasFeature(.avx2)) .{ .vp_w, .mull } else null,
	6635	.bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
	6636	.bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
	6637	.xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
	6638	.min => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6639	.signed => if (self.hasFeature(.avx2)) .{ .vp_w, .mins } else null,
	6640	.unsigned => if (self.hasFeature(.avx)) .{ .vp_w, .minu } else null,
	6641	},
	6642	.max => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6643	.signed => if (self.hasFeature(.avx2)) .{ .vp_w, .maxs } else null,
	6644	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .maxu } else null,
	6645	},
	6646	else => null,
	6647	},
	6648	else => null,
	6649	},
	6650	32 => switch (lhs_ty.vectorLen()) {
	6651	1...4 => switch (air_tag) {
	6652	.add,
	6653	.addwrap,
	6654	=> if (self.hasFeature(.avx)) .{ .vp_d, .add } else .{ .p_d, .add },
	6655	.sub,
	6656	.subwrap,
	6657	=> if (self.hasFeature(.avx)) .{ .vp_d, .sub } else .{ .p_d, .sub },
	6658	.mul,
	6659	.mulwrap,
	6660	=> if (self.hasFeature(.avx))
	6661	.{ .vp_d, .mull }
	6662	else if (self.hasFeature(.sse4_1))
	6663	.{ .p_d, .mull }
	6664	else
	6665	null,
	6666	.bit_and => if (self.hasFeature(.avx)) .{ .vp_, .@"and" } else .{ .p_, .@"and" },
	6667	.bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
	6668	.xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
	6669	.min => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6670	.signed => if (self.hasFeature(.avx))
	6671	.{ .vp_d, .mins }
	6672	else if (self.hasFeature(.sse4_1))
	6673	.{ .p_d, .mins }
	6674	else
	6675	null,
	6676	.unsigned => if (self.hasFeature(.avx))
	6677	.{ .vp_d, .minu }
	6678	else if (self.hasFeature(.sse4_1))
	6679	.{ .p_d, .minu }
	6680	else
	6681	null,
	6682	},
	6683	.max => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6684	.signed => if (self.hasFeature(.avx))
	6685	.{ .vp_d, .maxs }
	6686	else if (self.hasFeature(.sse4_1))
	6687	.{ .p_d, .maxs }
	6688	else
	6689	null,
	6690	.unsigned => if (self.hasFeature(.avx))
	6691	.{ .vp_d, .maxu }
	6692	else if (self.hasFeature(.sse4_1))
	6693	.{ .p_d, .maxu }
	6694	else
	6695	null,
	6696	},
	6697	else => null,
	6698	},
	6699	5...8 => switch (air_tag) {
	6700	.add,
	6701	.addwrap,
	6702	=> if (self.hasFeature(.avx2)) .{ .vp_d, .add } else null,
	6703	.sub,
	6704	.subwrap,
	6705	=> if (self.hasFeature(.avx2)) .{ .vp_d, .sub } else null,
	6706	.mul,
	6707	.mulwrap,
	6708	=> if (self.hasFeature(.avx2)) .{ .vp_d, .mull } else null,
	6709	.bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
	6710	.bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
	6711	.xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
	6712	.min => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6713	.signed => if (self.hasFeature(.avx2)) .{ .vp_d, .mins } else null,
	6714	.unsigned => if (self.hasFeature(.avx)) .{ .vp_d, .minu } else null,
	6715	},
	6716	.max => switch (lhs_ty.childType().intInfo(self.target.*).signedness) {
	6717	.signed => if (self.hasFeature(.avx2)) .{ .vp_d, .maxs } else null,
	6718	.unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .maxu } else null,
	6719	},
	6720	else => null,
	6721	},
	6722	else => null,
	6723	},
	6724	64 => switch (lhs_ty.vectorLen()) {
	6725	1...2 => switch (air_tag) {
	6726	.add,
	6727	.addwrap,
	6728	=> if (self.hasFeature(.avx)) .{ .vp_q, .add } else .{ .p_q, .add },
	6729	.sub,
	6730	.subwrap,
	6731	=> if (self.hasFeature(.avx)) .{ .vp_q, .sub } else .{ .p_q, .sub },
	6732	.bit_and => if (self.hasFeature(.avx)) .{ .vp_, .@"and" } else .{ .p_, .@"and" },
	6733	.bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
	6734	.xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
	6735	else => null,
	6736	},
	6737	3...4 => switch (air_tag) {
	6738	.add,
	6739	.addwrap,
	6740	=> if (self.hasFeature(.avx2)) .{ .vp_q, .add } else null,
	6741	.sub,
	6742	.subwrap,
	6743	=> if (self.hasFeature(.avx2)) .{ .vp_q, .sub } else null,
	6744	.bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
	6745	.bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
	6746	.xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
	6747	else => null,
	6748	},
	6749	else => null,
	6750	},
	6751	else => null,
	6752	},
6396	6753	.Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
6397	6754	16 => if (self.hasFeature(.f16c)) switch (lhs_ty.vectorLen()) {
6398	6755	1 => {
...	...	@@ -6415,12 +6772,13 @@ fn genBinOp(
6415	6772	else
6416	6773	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6417	6774	);
6418		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6775	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
6419	6776	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
6420	6777	try self.asmRegisterRegisterRegister(
6421	6778	switch (air_tag) {
6422	6779	.add => .{ .v_ss, .add },
6423	6780	.sub => .{ .v_ss, .sub },
	6781	.mul => .{ .v_ss, .mul },
6424	6782	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
6425	6783	.max => .{ .v_ss, .max },
6426	6784	.min => .{ .v_ss, .max },
...	...	@@ -6457,7 +6815,7 @@ fn genBinOp(
6457	6815	else
6458	6816	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6459	6817	);
6460		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6818	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
6461	6819	try self.asmRegisterRegisterRegister(
6462	6820	.{ .v_ps, .movhl },
6463	6821	tmp_reg,
...	...	@@ -6468,6 +6826,7 @@ fn genBinOp(
6468	6826	switch (air_tag) {
6469	6827	.add => .{ .v_ps, .add },
6470	6828	.sub => .{ .v_ps, .sub },
	6829	.mul => .{ .v_ps, .mul },
6471	6830	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6472	6831	.max => .{ .v_ps, .max },
6473	6832	.min => .{ .v_ps, .max },
...	...	@@ -6490,13 +6849,13 @@ fn genBinOp(
6490	6849	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6491	6850	defer self.register_manager.unlockReg(tmp_lock);
6492	6851
6493		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6852	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
6494	6853	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6495		.{ .v_, .cvtph2ps },
	6854	.{ .v_ps, .cvtph2 },
6496	6855	tmp_reg,
6497	6856	src_mcv.mem(.qword),
6498	6857	) else try self.asmRegisterRegister(
6499		.{ .v_, .cvtph2ps },
	6858	.{ .v_ps, .cvtph2 },
6500	6859	tmp_reg,
6501	6860	(if (src_mcv.isRegister())
6502	6861	src_mcv.getReg().?
...	...	@@ -6507,6 +6866,7 @@ fn genBinOp(
6507	6866	switch (air_tag) {
6508	6867	.add => .{ .v_ps, .add },
6509	6868	.sub => .{ .v_ps, .sub },
	6869	.mul => .{ .v_ps, .mul },
6510	6870	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6511	6871	.max => .{ .v_ps, .max },
6512	6872	.min => .{ .v_ps, .max },
...	...	@@ -6529,13 +6889,13 @@ fn genBinOp(
6529	6889	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6530	6890	defer self.register_manager.unlockReg(tmp_lock);
6531	6891
6532		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg.to256(), dst_reg);
	6892	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg);
6533	6893	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6534		.{ .v_, .cvtph2ps },
	6894	.{ .v_ps, .cvtph2 },
6535	6895	tmp_reg,
6536	6896	src_mcv.mem(.xword),
6537	6897	) else try self.asmRegisterRegister(
6538		.{ .v_, .cvtph2ps },
	6898	.{ .v_ps, .cvtph2 },
6539	6899	tmp_reg,
6540	6900	(if (src_mcv.isRegister())
6541	6901	src_mcv.getReg().?
...	...	@@ -6546,6 +6906,7 @@ fn genBinOp(
6546	6906	switch (air_tag) {
6547	6907	.add => .{ .v_ps, .add },
6548	6908	.sub => .{ .v_ps, .sub },
	6909	.mul => .{ .v_ps, .mul },
6549	6910	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
6550	6911	.max => .{ .v_ps, .max },
6551	6912	.min => .{ .v_ps, .max },
...	...	@@ -6681,7 +7042,7 @@ fn genBinOp(
6681	7042	);
6682	7043	}
6683	7044	switch (air_tag) {
6684		.add, .sub, .mul, .div_float, .div_exact => {},
	7045	.add, .addwrap, .sub, .subwrap, .mul, .mulwrap, .div_float, .div_exact => {},
6685	7046	.div_trunc, .div_floor => try self.genRound(
6686	7047	lhs_ty,
6687	7048	dst_reg,
...	...	@@ -6692,6 +7053,7 @@ fn genBinOp(
6692	7053	else => unreachable,
6693	7054	},
6694	7055	),
	7056	.bit_and, .bit_or, .xor => {},
6695	7057	.max, .min => {}, // TODO: unordered select
6696	7058	else => unreachable,
6697	7059	}
...	...	@@ -6712,7 +7074,6 @@ fn genBinOpMir(
6712	7074	.dead,
6713	7075	.undef,
6714	7076	.immediate,
6715		.register_offset,
6716	7077	.eflags,
6717	7078	.register_overflow,
6718	7079	.lea_direct,
...	...	@@ -6721,7 +7082,9 @@ fn genBinOpMir(
6721	7082	.lea_frame,
6722	7083	.reserved_frame,
6723	7084	=> unreachable, // unmodifiable destination
6724		.register => \|dst_reg\| {
	7085	.register, .register_offset => {
	7086	assert(dst_mcv.isRegister());
	7087	const dst_reg = dst_mcv.getReg().?;
6725	7088	const dst_alias = registerAlias(dst_reg, abi_size);
6726	7089	switch (src_mcv) {
6727	7090	.none,
...	...	@@ -7185,13 +7548,13 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
7185	7548	switch (self.debug_output) {
7186	7549	.dwarf => \|dw\| {
7187	7550	const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
7188		.register => \|reg\| .{ .register = reg.dwarfLocOp() },
	7551	.register => \|reg\| .{ .register = reg.dwarfNum() },
7189	7552	// TODO use a frame index
7190	7553	.load_frame => return,
7191	7554	//.stack_offset => \|off\| .{
7192	7555	// .stack = .{
7193	7556	// // TODO handle -fomit-frame-pointer
7194		// .fp_register = Register.rbp.dwarfLocOpDeref(),
	7557	// .fp_register = Register.rbp.dwarfNum(),
7195	7558	// .offset = -off,
7196	7559	// },
7197	7560	//},
...	...	@@ -7223,11 +7586,11 @@ fn genVarDbgInfo(
7223	7586	switch (self.debug_output) {
7224	7587	.dwarf => \|dw\| {
7225	7588	const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
7226		.register => \|reg\| .{ .register = reg.dwarfLocOp() },
	7589	.register => \|reg\| .{ .register = reg.dwarfNum() },
7227	7590	// TODO use a frame index
7228	7591	.load_frame, .lea_frame => return,
7229	7592	//=> \|off\| .{ .stack = .{
7230		// .fp_register = Register.rbp.dwarfLocOpDeref(),
	7593	// .fp_register = Register.rbp.dwarfNum(),
7231	7594	// .offset = -off,
7232	7595	//} },
7233	7596	.memory => \|address\| .{ .memory = address },
...	...	@@ -7364,11 +7727,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7364	7727	// on linking.
7365	7728	const mod = self.bin_file.options.module.?;
7366	7729	if (self.air.value(callee)) \|func_value\| {
7367		if (func_value.castTag(.function)) \|func_payload\| {
7368		const func = func_payload.data;
7369
	7730	if (if (func_value.castTag(.function)) \|func_payload\|
	7731	func_payload.data.owner_decl
	7732	else if (func_value.castTag(.decl_ref)) \|decl_ref_payload\|
	7733	decl_ref_payload.data
	7734	else
	7735	null) \|owner_decl\|
	7736	{
7370	7737	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
7371		const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
	7738	const atom_index = try elf_file.getOrCreateAtomForDecl(owner_decl);
7372	7739	const atom = elf_file.getAtom(atom_index);
7373	7740	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
7374	7741	const got_addr = atom.getOffsetTableAddress(elf_file);
...	...	@@ -7377,17 +7744,17 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7377	7744	.disp = @intCast(i32, got_addr),
7378	7745	}));
7379	7746	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
7380		const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
	7747	const atom = try coff_file.getOrCreateAtomForDecl(owner_decl);
7381	7748	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
7382	7749	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
7383	7750	try self.asmRegister(.{ ._, .call }, .rax);
7384	7751	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
7385		const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
	7752	const atom = try macho_file.getOrCreateAtomForDecl(owner_decl);
7386	7753	const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
7387	7754	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
7388	7755	try self.asmRegister(.{ ._, .call }, .rax);
7389	7756	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
7390		const decl_block_index = try p9.seeDecl(func.owner_decl);
	7757	const decl_block_index = try p9.seeDecl(owner_decl);
7391	7758	const decl_block = p9.getDeclBlock(decl_block_index);
7392	7759	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
7393	7760	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
...	...	@@ -7577,7 +7944,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7577	7944	else
7578	7945	try self.copyToTmpRegister(ty, src_mcv)).to128(),
7579	7946	);
7580		try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, tmp1_reg, tmp1_reg);
	7947	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, tmp1_reg, tmp1_reg);
7581	7948	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg);
7582	7949	try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv);
7583	7950	} else return self.fail("TODO implement airCmp for {}", .{
...	...	@@ -8568,47 +8935,190 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8568	8935	return self.finishAirResult(inst, result);
8569	8936	}
8570	8937
8571		fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
	8938	const MoveStrategy = union(enum) {
	8939	move: Mir.Inst.FixedTag,
	8940	insert_extract: InsertExtract,
	8941	vex_insert_extract: InsertExtract,
	8942
	8943	const InsertExtract = struct {
	8944	insert: Mir.Inst.FixedTag,
	8945	extract: Mir.Inst.FixedTag,
	8946	};
	8947	};
	8948	fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy {
8572	8949	switch (ty.zigTypeTag()) {
8573		else => return .{ ._, .mov },
	8950	else => return .{ .move = .{ ._, .mov } },
8574	8951	.Float => switch (ty.floatBits(self.target.*)) {
8575		16 => unreachable, // needs special handling
8576		32 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
8577		64 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
8578		128 => return if (self.hasFeature(.avx))
8579		if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8580		else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	8952	16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	8953	.insert = .{ .vp_w, .insr },
	8954	.extract = .{ .vp_w, .extr },
	8955	} } else .{ .insert_extract = .{
	8956	.insert = .{ .p_w, .insr },
	8957	.extract = .{ .p_w, .extr },
	8958	} },
	8959	32 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov } },
	8960	64 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov } },
	8961	128 => return .{ .move = if (self.hasFeature(.avx))
	8962	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	8963	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
8581	8964	else => {},
8582	8965	},
8583	8966	.Vector => switch (ty.childType().zigTypeTag()) {
	8967	.Int => switch (ty.childType().intInfo(self.target.*).bits) {
	8968	8 => switch (ty.vectorLen()) {
	8969	1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
	8970	.insert = .{ .vp_b, .insr },
	8971	.extract = .{ .vp_b, .extr },
	8972	} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
	8973	.insert = .{ .p_b, .insr },
	8974	.extract = .{ .p_b, .extr },
	8975	} },
	8976	2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	8977	.insert = .{ .vp_w, .insr },
	8978	.extract = .{ .vp_w, .extr },
	8979	} } else .{ .insert_extract = .{
	8980	.insert = .{ .p_w, .insr },
	8981	.extract = .{ .p_w, .extr },
	8982	} },
	8983	3...4 => return .{ .move = if (self.hasFeature(.avx))
	8984	.{ .v_d, .mov }
	8985	else
	8986	.{ ._d, .mov } },
	8987	5...8 => return .{ .move = if (self.hasFeature(.avx))
	8988	.{ .v_q, .mov }
	8989	else
	8990	.{ ._q, .mov } },
	8991	9...16 => return .{ .move = if (self.hasFeature(.avx))
	8992	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	8993	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	8994	17...32 => if (self.hasFeature(.avx))
	8995	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	8996	else => {},
	8997	},
	8998	16 => switch (ty.vectorLen()) {
	8999	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	9000	.insert = .{ .vp_w, .insr },
	9001	.extract = .{ .vp_w, .extr },
	9002	} } else .{ .insert_extract = .{
	9003	.insert = .{ .p_w, .insr },
	9004	.extract = .{ .p_w, .extr },
	9005	} },
	9006	2 => return .{ .move = if (self.hasFeature(.avx))
	9007	.{ .v_d, .mov }
	9008	else
	9009	.{ ._d, .mov } },
	9010	3...4 => return .{ .move = if (self.hasFeature(.avx))
	9011	.{ .v_q, .mov }
	9012	else
	9013	.{ ._q, .mov } },
	9014	5...8 => return .{ .move = if (self.hasFeature(.avx))
	9015	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	9016	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	9017	9...16 => if (self.hasFeature(.avx))
	9018	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	9019	else => {},
	9020	},
	9021	32 => switch (ty.vectorLen()) {
	9022	1 => return .{ .move = if (self.hasFeature(.avx))
	9023	.{ .v_d, .mov }
	9024	else
	9025	.{ ._d, .mov } },
	9026	2 => return .{ .move = if (self.hasFeature(.avx))
	9027	.{ .v_q, .mov }
	9028	else
	9029	.{ ._q, .mov } },
	9030	3...4 => return .{ .move = if (self.hasFeature(.avx))
	9031	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	9032	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	9033	5...8 => if (self.hasFeature(.avx))
	9034	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	9035	else => {},
	9036	},
	9037	64 => switch (ty.vectorLen()) {
	9038	1 => return .{ .move = if (self.hasFeature(.avx))
	9039	.{ .v_q, .mov }
	9040	else
	9041	.{ ._q, .mov } },
	9042	2 => return .{ .move = if (self.hasFeature(.avx))
	9043	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	9044	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	9045	3...4 => if (self.hasFeature(.avx))
	9046	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	9047	else => {},
	9048	},
	9049	128 => switch (ty.vectorLen()) {
	9050	1 => return .{ .move = if (self.hasFeature(.avx))
	9051	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	9052	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	9053	2 => if (self.hasFeature(.avx))
	9054	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	9055	else => {},
	9056	},
	9057	256 => switch (ty.vectorLen()) {
	9058	1 => if (self.hasFeature(.avx))
	9059	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
	9060	else => {},
	9061	},
	9062	else => {},
	9063	},
8584	9064	.Float => switch (ty.childType().floatBits(self.target.*)) {
8585	9065	16 => switch (ty.vectorLen()) {
8586		1 => unreachable, // needs special handling
8587		2 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
8588		3...4 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
8589		5...8 => return if (self.hasFeature(.avx))
8590		if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8591		else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	9066	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
	9067	.insert = .{ .vp_w, .insr },
	9068	.extract = .{ .vp_w, .extr },
	9069	} } else .{ .insert_extract = .{
	9070	.insert = .{ .p_w, .insr },
	9071	.extract = .{ .p_w, .extr },
	9072	} },
	9073	2 => return .{ .move = if (self.hasFeature(.avx))
	9074	.{ .v_d, .mov }
	9075	else
	9076	.{ ._d, .mov } },
	9077	3...4 => return .{ .move = if (self.hasFeature(.avx))
	9078	.{ .v_q, .mov }
	9079	else
	9080	.{ ._q, .mov } },
	9081	5...8 => return .{ .move = if (self.hasFeature(.avx))
	9082	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	9083	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
8592	9084	9...16 => if (self.hasFeature(.avx))
8593		return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
	9085	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
8594	9086	else => {},
8595	9087	},
8596	9088	32 => switch (ty.vectorLen()) {
8597		1 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
8598		2...4 => return if (self.hasFeature(.avx))
	9089	1 => return .{ .move = if (self.hasFeature(.avx))
	9090	.{ .v_ss, .mov }
	9091	else
	9092	.{ ._ss, .mov } },
	9093	2 => return .{ .move = if (self.hasFeature(.avx))
	9094	.{ .v_sd, .mov }
	9095	else
	9096	.{ ._sd, .mov } },
	9097	3...4 => return .{ .move = if (self.hasFeature(.avx))
8599	9098	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8600		else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	9099	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
8601	9100	5...8 => if (self.hasFeature(.avx))
8602		return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
	9101	return .{ .move = if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu } },
8603	9102	else => {},
8604	9103	},
8605	9104	64 => switch (ty.vectorLen()) {
8606		1 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
8607		2 => return if (self.hasFeature(.avx))
8608		if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
8609		else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	9105	1 => return .{ .move = if (self.hasFeature(.avx))
	9106	.{ .v_sd, .mov }
	9107	else
	9108	.{ ._sd, .mov } },
	9109	2 => return .{ .move = if (self.hasFeature(.avx))
	9110	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
	9111	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
8610	9112	3...4 => if (self.hasFeature(.avx))
8611		return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
	9113	return .{ .move = if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } },
	9114	else => {},
	9115	},
	9116	128 => switch (ty.vectorLen()) {
	9117	1 => return .{ .move = if (self.hasFeature(.avx))
	9118	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
	9119	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
	9120	2 => if (self.hasFeature(.avx))
	9121	return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
8612	9122	else => {},
8613	9123	},
8614	9124	else => {},
...	...	@@ -8616,15 +9126,11 @@ fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
8616	9126	else => {},
8617	9127	},
8618	9128	}
8619		return self.fail("TODO movMirTag for {}", .{ty.fmt(self.bin_file.options.module.?)});
	9129	return self.fail("TODO moveStrategy for {}", .{ty.fmt(self.bin_file.options.module.?)});
8620	9130	}
8621	9131
8622	9132	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
8623		const src_lock = switch (src_mcv) {
8624		.register => \|reg\| self.register_manager.lockReg(reg),
8625		.register_overflow => \|ro\| self.register_manager.lockReg(ro.reg),
8626		else => null,
8627		};
	9133	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
8628	9134	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
8629	9135
8630	9136	switch (dst_mcv) {
...	...	@@ -8720,34 +9226,94 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8720	9226	);
8721	9227	}
8722	9228	},
8723		.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
8724		if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
8725		switch (ty.zigTypeTag()) {
8726		else => .{ ._, .mov },
8727		.Float, .Vector => .{ ._ps, .mova },
8728		}
8729		else switch (abi_size) {
8730		2 => return try self.asmRegisterRegisterImmediate(
8731		if (dst_reg.class() == .floating_point) .{ .p_w, .insr } else .{ .p_w, .extr },
8732		registerAlias(dst_reg, 4),
8733		registerAlias(src_reg, 4),
8734		Immediate.u(0),
	9229	.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) switch (dst_reg.class()) {
	9230	.general_purpose => switch (src_reg.class()) {
	9231	.general_purpose => try self.asmRegisterRegister(
	9232	.{ ._, .mov },
	9233	registerAlias(dst_reg, abi_size),
	9234	registerAlias(src_reg, abi_size),
8735	9235	),
8736		4 => .{ ._d, .mov },
8737		8 => .{ ._q, .mov },
8738		else => return self.fail(
8739		"unsupported register copy from {s} to {s}",
8740		.{ @tagName(src_reg), @tagName(dst_reg) },
	9236	.segment => try self.asmRegisterRegister(
	9237	.{ ._, .mov },
	9238	registerAlias(dst_reg, abi_size),
	9239	src_reg,
8741	9240	),
	9241	.sse => try self.asmRegisterRegister(
	9242	switch (abi_size) {
	9243	1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
	9244	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
	9245	else => unreachable,
	9246	},
	9247	registerAlias(dst_reg, @max(abi_size, 4)),
	9248	src_reg.to128(),
	9249	),
	9250	.x87, .mmx => unreachable,
8742	9251	},
8743		registerAlias(dst_reg, abi_size),
8744		registerAlias(src_reg, abi_size),
8745		),
	9252	.segment => try self.asmRegisterRegister(
	9253	.{ ._, .mov },
	9254	dst_reg,
	9255	switch (src_reg.class()) {
	9256	.general_purpose, .segment => registerAlias(src_reg, abi_size),
	9257	.sse => try self.copyToTmpRegister(ty, src_mcv),
	9258	.x87, .mmx => unreachable,
	9259	},
	9260	),
	9261	.sse => switch (src_reg.class()) {
	9262	.general_purpose => try self.asmRegisterRegister(
	9263	switch (abi_size) {
	9264	1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
	9265	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
	9266	else => unreachable,
	9267	},
	9268	dst_reg.to128(),
	9269	registerAlias(src_reg, @max(abi_size, 4)),
	9270	),
	9271	.segment => try self.genSetReg(
	9272	dst_reg,
	9273	ty,
	9274	.{ .register = try self.copyToTmpRegister(ty, src_mcv) },
	9275	),
	9276	.sse => try self.asmRegisterRegister(
	9277	if (@as(?Mir.Inst.FixedTag, switch (ty.scalarType().zigTypeTag()) {
	9278	else => switch (abi_size) {
	9279	1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
	9280	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
	9281	9...16 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else .{ ._, .movdqa },
	9282	17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
	9283	else => null,
	9284	},
	9285	.Float => switch (ty.floatBits(self.target.*)) {
	9286	16, 128 => switch (abi_size) {
	9287	2...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
	9288	5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
	9289	9...16 => if (self.hasFeature(.avx))
	9290	.{ .v_, .movdqa }
	9291	else
	9292	.{ ._, .movdqa },
	9293	17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
	9294	else => null,
	9295	},
	9296	32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova },
	9297	64 => if (self.hasFeature(.avx)) .{ .v_pd, .mova } else .{ ._pd, .mova },
	9298	80 => null,
	9299	else => unreachable,
	9300	},
	9301	})) \|tag\| tag else return self.fail("TODO implement genSetReg for {}", .{
	9302	ty.fmt(self.bin_file.options.module.?),
	9303	}),
	9304	registerAlias(dst_reg, abi_size),
	9305	registerAlias(src_reg, abi_size),
	9306	),
	9307	.x87, .mmx => unreachable,
	9308	},
	9309	.x87, .mmx => unreachable,
	9310	},
8746	9311	.register_offset,
8747	9312	.indirect,
8748	9313	.load_frame,
8749	9314	.lea_frame,
8750	9315	=> {
	9316	const dst_alias = registerAlias(dst_reg, abi_size);
8751	9317	const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
8752	9318	.register_offset, .indirect => \|reg_off\| .{
8753	9319	.base = .{ .reg = reg_off.reg },
...	...	@@ -8759,71 +9325,81 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8759	9325	},
8760	9326	else => unreachable,
8761	9327	});
8762		if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8763		try self.asmRegisterMemoryImmediate(
8764		.{ .p_w, .insr },
8765		registerAlias(dst_reg, abi_size),
	9328	switch (@as(MoveStrategy, switch (src_mcv) {
	9329	.register_offset => \|reg_off\| switch (reg_off.off) {
	9330	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
	9331	else => .{ .move = .{ ._, .lea } },
	9332	},
	9333	.indirect => try self.moveStrategy(ty, false),
	9334	.load_frame => \|frame_addr\| try self.moveStrategy(
	9335	ty,
	9336	self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
	9337	),
	9338	.lea_frame => .{ .move = .{ ._, .lea } },
	9339	else => unreachable,
	9340	})) {
	9341	.move => \|tag\| try self.asmRegisterMemory(tag, dst_alias, src_mem),
	9342	.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
	9343	ie.insert,
	9344	dst_alias,
8766	9345	src_mem,
8767	9346	Immediate.u(0),
8768		)
8769		else
8770		try self.asmRegisterMemory(
8771		switch (src_mcv) {
8772		.register_offset => \|reg_off\| switch (reg_off.off) {
8773		0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
8774		else => .{ ._, .lea },
8775		},
8776		.indirect => try self.movMirTag(ty, false),
8777		.load_frame => \|frame_addr\| try self.movMirTag(
8778		ty,
8779		self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
8780		),
8781		.lea_frame => .{ ._, .lea },
8782		else => unreachable,
8783		},
8784		registerAlias(dst_reg, abi_size),
	9347	),
	9348	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
	9349	ie.insert,
	9350	dst_alias,
	9351	dst_alias,
8785	9352	src_mem,
8786		);
	9353	Immediate.u(0),
	9354	),
	9355	}
8787	9356	},
8788	9357	.memory, .load_direct, .load_got, .load_tlv => {
8789	9358	switch (src_mcv) {
8790	9359	.memory => \|addr\| if (math.cast(i32, @bitCast(i64, addr))) \|small_addr\| {
	9360	const dst_alias = registerAlias(dst_reg, abi_size);
8791	9361	const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
8792	9362	.base = .{ .reg = .ds },
8793	9363	.disp = small_addr,
8794	9364	});
8795		return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8796		self.asmRegisterMemoryImmediate(
8797		.{ .p_w, .insr },
8798		registerAlias(dst_reg, abi_size),
	9365	switch (try self.moveStrategy(ty, mem.isAlignedGeneric(
	9366	u32,
	9367	@bitCast(u32, small_addr),
	9368	ty.abiAlignment(self.target.*),
	9369	))) {
	9370	.move => \|tag\| try self.asmRegisterMemory(tag, dst_alias, src_mem),
	9371	.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
	9372	ie.insert,
	9373	dst_alias,
8799	9374	src_mem,
8800	9375	Immediate.u(0),
8801		)
8802		else
8803		self.asmRegisterMemory(
8804		try self.movMirTag(ty, mem.isAlignedGeneric(
8805		u32,
8806		@bitCast(u32, small_addr),
8807		ty.abiAlignment(self.target.*),
8808		)),
8809		registerAlias(dst_reg, abi_size),
	9376	),
	9377	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
	9378	ie.insert,
	9379	dst_alias,
	9380	dst_alias,
8810	9381	src_mem,
8811		);
	9382	Immediate.u(0),
	9383	),
	9384	}
8812	9385	},
8813		.load_direct => \|sym_index\| if (!ty.isRuntimeFloat()) {
8814		const atom_index = try self.owner.getSymbolIndex(self);
8815		_ = try self.addInst(.{
8816		.tag = .mov,
8817		.ops = .direct_reloc,
8818		.data = .{ .rx = .{
8819		.r1 = dst_reg.to64(),
8820		.payload = try self.addExtra(Mir.Reloc{
8821		.atom_index = atom_index,
8822		.sym_index = sym_index,
8823		}),
8824		} },
8825		});
8826		return;
	9386	.load_direct => \|sym_index\| switch (ty.zigTypeTag()) {
	9387	else => {
	9388	const atom_index = try self.owner.getSymbolIndex(self);
	9389	_ = try self.addInst(.{
	9390	.tag = .mov,
	9391	.ops = .direct_reloc,
	9392	.data = .{ .rx = .{
	9393	.r1 = dst_reg.to64(),
	9394	.payload = try self.addExtra(Mir.Reloc{
	9395	.atom_index = atom_index,
	9396	.sym_index = sym_index,
	9397	}),
	9398	} },
	9399	});
	9400	return;
	9401	},
	9402	.Float, .Vector => {},
8827	9403	},
8828	9404	.load_got, .load_tlv => {},
8829	9405	else => unreachable,
...	...	@@ -8833,22 +9409,26 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8833	9409	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
8834	9410	defer self.register_manager.unlockReg(addr_lock);
8835	9411
	9412	const dst_alias = registerAlias(dst_reg, abi_size);
8836	9413	const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
8837	9414	.base = .{ .reg = addr_reg },
8838	9415	});
8839		if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8840		try self.asmRegisterMemoryImmediate(
8841		.{ .p_w, .insr },
8842		registerAlias(dst_reg, abi_size),
	9416	switch (try self.moveStrategy(ty, false)) {
	9417	.move => \|tag\| try self.asmRegisterMemory(tag, dst_alias, src_mem),
	9418	.insert_extract => \|ie\| try self.asmRegisterMemoryImmediate(
	9419	ie.insert,
	9420	dst_alias,
8843	9421	src_mem,
8844	9422	Immediate.u(0),
8845		)
8846		else
8847		try self.asmRegisterMemory(
8848		try self.movMirTag(ty, false),
8849		registerAlias(dst_reg, abi_size),
	9423	),
	9424	.vex_insert_extract => \|ie\| try self.asmRegisterRegisterMemoryImmediate(
	9425	ie.insert,
	9426	dst_alias,
	9427	dst_alias,
8850	9428	src_mem,
8851		);
	9429	Immediate.u(0),
	9430	),
	9431	}
8852	9432	},
8853	9433	.lea_direct, .lea_got => \|sym_index\| {
8854	9434	const atom_index = try self.owner.getSymbolIndex(self);
...	...	@@ -8950,36 +9530,33 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8950	9530	Memory.PtrSize.fromSize(abi_size),
8951	9531	.{ .base = base, .disp = disp },
8952	9532	);
8953		if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8954		try self.asmMemoryRegisterImmediate(
8955		.{ .p_w, .extr },
	9533	const src_alias = registerAlias(src_reg, abi_size);
	9534	switch (try self.moveStrategy(ty, switch (base) {
	9535	.none => mem.isAlignedGeneric(
	9536	u32,
	9537	@bitCast(u32, disp),
	9538	ty.abiAlignment(self.target.*),
	9539	),
	9540	.reg => \|reg\| switch (reg) {
	9541	.es, .cs, .ss, .ds => mem.isAlignedGeneric(
	9542	u32,
	9543	@bitCast(u32, disp),
	9544	ty.abiAlignment(self.target.*),
	9545	),
	9546	else => false,
	9547	},
	9548	.frame => \|frame_index\| self.getFrameAddrAlignment(
	9549	.{ .index = frame_index, .off = disp },
	9550	) >= ty.abiAlignment(self.target.*),
	9551	})) {
	9552	.move => \|tag\| try self.asmMemoryRegister(tag, dst_mem, src_alias),
	9553	.insert_extract, .vex_insert_extract => \|ie\| try self.asmMemoryRegisterImmediate(
	9554	ie.extract,
8956	9555	dst_mem,
8957		src_reg.to128(),
	9556	src_alias,
8958	9557	Immediate.u(0),
8959		)
8960		else
8961		try self.asmMemoryRegister(
8962		try self.movMirTag(ty, switch (base) {
8963		.none => mem.isAlignedGeneric(
8964		u32,
8965		@bitCast(u32, disp),
8966		ty.abiAlignment(self.target.*),
8967		),
8968		.reg => \|reg\| switch (reg) {
8969		.es, .cs, .ss, .ds => mem.isAlignedGeneric(
8970		u32,
8971		@bitCast(u32, disp),
8972		ty.abiAlignment(self.target.*),
8973		),
8974		else => false,
8975		},
8976		.frame => \|frame_index\| self.getFrameAddrAlignment(
8977		.{ .index = frame_index, .off = disp },
8978		) >= ty.abiAlignment(self.target.*),
8979		}),
8980		dst_mem,
8981		registerAlias(src_reg, abi_size),
8982		);
	9558	),
	9559	}
8983	9560	},
8984	9561	.register_overflow => \|ro\| {
8985	9562	try self.genSetMem(
...	...	@@ -9069,7 +9646,7 @@ fn genInlineMemcpyRegisterRegister(
9069	9646	try self.asmMemoryRegister(
9070	9647	switch (src_reg.class()) {
9071	9648	.general_purpose, .segment => .{ ._, .mov },
9072		.floating_point => .{ ._ss, .mov },
	9649	.sse => .{ ._ss, .mov },
9073	9650	},
9074	9651	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
9075	9652	registerAlias(src_reg, abi_size),
...	...	@@ -9235,10 +9812,10 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
9235	9812	});
9236	9813
9237	9814	const src_mcv = try self.resolveInst(ty_op.operand);
9238		const src_reg = switch (src_mcv) {
9239		.register => \|reg\| reg,
9240		else => try self.copyToTmpRegister(src_ty, src_mcv),
9241		};
	9815	const src_reg = if (src_mcv.isRegister())
	9816	src_mcv.getReg().?
	9817	else
	9818	try self.copyToTmpRegister(src_ty, src_mcv);
9242	9819	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
9243	9820	defer self.register_manager.unlockReg(src_lock);
9244	9821
...	...	@@ -9249,23 +9826,23 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
9249	9826	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
9250	9827	defer self.register_manager.unlockReg(dst_lock);
9251	9828
9252		try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
9253		32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
9254		.{ ._, .cvtsi2ss }
9255		else
9256		return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
9257		src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
9258		}),
9259		64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
9260		.{ ._, .cvtsi2sd }
9261		else
9262		return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
9263		src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
9264		}),
9265		else => return self.fail("TODO implement airIntToFloat from {} to {}", .{
9266		src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
9267		}),
9268		}, dst_reg.to128(), registerAlias(src_reg, src_size));
	9829	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (dst_ty.zigTypeTag()) {
	9830	.Float => switch (dst_ty.floatBits(self.target.*)) {
	9831	32 => if (self.hasFeature(.avx)) .{ .v_ss, .cvtsi2 } else .{ ._ss, .cvtsi2 },
	9832	64 => if (self.hasFeature(.avx)) .{ .v_sd, .cvtsi2 } else .{ ._sd, .cvtsi2 },
	9833	16, 80, 128 => null,
	9834	else => unreachable,
	9835	},
	9836	else => null,
	9837	})) \|tag\| tag else return self.fail("TODO implement airIntToFloat from {} to {}", .{
	9838	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	9839	});
	9840	const dst_alias = dst_reg.to128();
	9841	const src_alias = registerAlias(src_reg, src_size);
	9842	switch (mir_tag[0]) {
	9843	.v_ss, .v_sd => try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, src_alias),
	9844	else => try self.asmRegisterRegister(mir_tag, dst_alias, src_alias),
	9845	}
9269	9846
9270	9847	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
9271	9848	}
...	...	@@ -9275,46 +9852,50 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
9275	9852
9276	9853	const src_ty = self.air.typeOf(ty_op.operand);
9277	9854	const dst_ty = self.air.typeOfIndex(inst);
9278		const operand = try self.resolveInst(ty_op.operand);
9279		const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
9280		const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	9855	const dst_bits = @intCast(u32, dst_ty.bitSize(self.target.*));
	9856	const dst_signedness =
	9857	if (dst_ty.isAbiInt()) dst_ty.intInfo(self.target.*).signedness else .unsigned;
9281	9858
9282		switch (src_abi_size) {
9283		4, 8 => {},
9284		else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
9285		}
9286		if (dst_abi_size > 8) {
9287		return self.fail("TODO convert float with abiSize={}", .{dst_abi_size});
9288		}
	9859	const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {
	9860	.signed => dst_bits,
	9861	.unsigned => dst_bits + 1,
	9862	}, 32), 8) catch unreachable;
	9863	if (dst_size > 8) return self.fail("TODO implement airFloatToInt from {} to {}", .{
	9864	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	9865	});
9289	9866
9290		// move float src to ST(0)
9291		const frame_addr: FrameAddr = switch (operand) {
9292		.load_frame => \|frame_addr\| frame_addr,
9293		else => frame_addr: {
9294		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(src_ty, self.target.*));
9295		try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, operand);
9296		break :frame_addr .{ .index = frame_index };
9297		},
9298		};
9299		try self.asmMemory(
9300		.{ .f_, .ld },
9301		Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
9302		.base = .{ .frame = frame_addr.index },
9303		.disp = frame_addr.off,
9304		}),
9305		);
	9867	const src_mcv = try self.resolveInst(ty_op.operand);
	9868	const src_reg = if (src_mcv.isRegister())
	9869	src_mcv.getReg().?
	9870	else
	9871	try self.copyToTmpRegister(src_ty, src_mcv);
	9872	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	9873	defer self.register_manager.unlockReg(src_lock);
	9874
	9875	const dst_reg = try self.register_manager.allocReg(inst, regClassForType(dst_ty));
	9876	const dst_mcv = MCValue{ .register = dst_reg };
	9877	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	9878	defer self.register_manager.unlockReg(dst_lock);
9306	9879
9307		// convert
9308		const stack_dst = try self.allocRegOrMem(inst, false);
9309		try self.asmMemory(
9310		.{ .f_p, .istt },
9311		Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
9312		.base = .{ .frame = stack_dst.load_frame.index },
9313		.disp = stack_dst.load_frame.off,
	9880	try self.asmRegisterRegister(
	9881	if (@as(?Mir.Inst.FixedTag, switch (src_ty.zigTypeTag()) {
	9882	.Float => switch (src_ty.floatBits(self.target.*)) {
	9883	32 => if (self.hasFeature(.avx)) .{ .v_, .cvttss2si } else .{ ._, .cvttss2si },
	9884	64 => if (self.hasFeature(.avx)) .{ .v_, .cvttsd2si } else .{ ._, .cvttsd2si },
	9885	16, 80, 128 => null,
	9886	else => unreachable,
	9887	},
	9888	else => null,
	9889	})) \|tag\| tag else return self.fail("TODO implement airFloatToInt from {} to {}", .{
	9890	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
9314	9891	}),
	9892	registerAlias(dst_reg, dst_size),
	9893	src_reg.to128(),
9315	9894	);
9316	9895
9317		return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
	9896	if (dst_bits < dst_size * 8) try self.truncateRegister(dst_ty, dst_reg);
	9897
	9898	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
9318	9899	}
9319	9900
9320	9901	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -9977,9 +10558,200 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9977	10558
9978	10559	fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
9979	10560	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
9980		_ = ty_op;
9981		return self.fail("TODO implement airSplat for x86_64", .{});
9982		//return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	10561	const vector_ty = self.air.typeOfIndex(inst);
	10562	const dst_rc = regClassForType(vector_ty);
	10563	const scalar_ty = vector_ty.scalarType();
	10564
	10565	const src_mcv = try self.resolveInst(ty_op.operand);
	10566	const result: MCValue = result: {
	10567	switch (scalar_ty.zigTypeTag()) {
	10568	else => {},
	10569	.Float => switch (scalar_ty.floatBits(self.target.*)) {
	10570	32 => switch (vector_ty.vectorLen()) {
	10571	1 => {
	10572	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
	10573	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10574	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
	10575	break :result .{ .register = dst_reg };
	10576	},
	10577	2...4 => {
	10578	if (self.hasFeature(.avx)) {
	10579	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10580	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	10581	.{ .v_ss, .broadcast },
	10582	dst_reg.to128(),
	10583	src_mcv.mem(.dword),
	10584	) else {
	10585	const src_reg = if (src_mcv.isRegister())
	10586	src_mcv.getReg().?
	10587	else
	10588	try self.copyToTmpRegister(scalar_ty, src_mcv);
	10589	try self.asmRegisterRegisterRegisterImmediate(
	10590	.{ .v_ps, .shuf },
	10591	dst_reg.to128(),
	10592	src_reg.to128(),
	10593	src_reg.to128(),
	10594	Immediate.u(0),
	10595	);
	10596	}
	10597	break :result .{ .register = dst_reg };
	10598	} else {
	10599	const dst_mcv = if (src_mcv.isRegister() and
	10600	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	10601	src_mcv
	10602	else
	10603	try self.copyToRegisterWithInstTracking(inst, scalar_ty, src_mcv);
	10604	const dst_reg = dst_mcv.getReg().?;
	10605	try self.asmRegisterRegisterImmediate(
	10606	.{ ._ps, .shuf },
	10607	dst_reg.to128(),
	10608	dst_reg.to128(),
	10609	Immediate.u(0),
	10610	);
	10611	break :result dst_mcv;
	10612	}
	10613	},
	10614	5...8 => if (self.hasFeature(.avx)) {
	10615	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10616	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	10617	.{ .v_ss, .broadcast },
	10618	dst_reg.to256(),
	10619	src_mcv.mem(.dword),
	10620	) else {
	10621	const src_reg = if (src_mcv.isRegister())
	10622	src_mcv.getReg().?
	10623	else
	10624	try self.copyToTmpRegister(scalar_ty, src_mcv);
	10625	if (self.hasFeature(.avx2)) try self.asmRegisterRegister(
	10626	.{ .v_ss, .broadcast },
	10627	dst_reg.to256(),
	10628	src_reg.to128(),
	10629	) else {
	10630	try self.asmRegisterRegisterRegisterImmediate(
	10631	.{ .v_ps, .shuf },
	10632	dst_reg.to128(),
	10633	src_reg.to128(),
	10634	src_reg.to128(),
	10635	Immediate.u(0),
	10636	);
	10637	try self.asmRegisterRegisterRegisterImmediate(
	10638	.{ .v_f128, .insert },
	10639	dst_reg.to256(),
	10640	dst_reg.to256(),
	10641	dst_reg.to128(),
	10642	Immediate.u(1),
	10643	);
	10644	}
	10645	}
	10646	break :result .{ .register = dst_reg };
	10647	},
	10648	else => {},
	10649	},
	10650	64 => switch (vector_ty.vectorLen()) {
	10651	1 => {
	10652	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
	10653	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10654	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
	10655	break :result .{ .register = dst_reg };
	10656	},
	10657	2 => {
	10658	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10659	if (self.hasFeature(.sse3)) {
	10660	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	10661	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
	10662	dst_reg.to128(),
	10663	src_mcv.mem(.qword),
	10664	) else try self.asmRegisterRegister(
	10665	if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
	10666	dst_reg.to128(),
	10667	(if (src_mcv.isRegister())
	10668	src_mcv.getReg().?
	10669	else
	10670	try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(),
	10671	);
	10672	break :result .{ .register = dst_reg };
	10673	} else try self.asmRegisterRegister(
	10674	.{ ._ps, .movlh },
	10675	dst_reg.to128(),
	10676	(if (src_mcv.isRegister())
	10677	src_mcv.getReg().?
	10678	else
	10679	try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(),
	10680	);
	10681	},
	10682	3...4 => if (self.hasFeature(.avx)) {
	10683	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10684	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	10685	.{ .v_sd, .broadcast },
	10686	dst_reg.to256(),
	10687	src_mcv.mem(.qword),
	10688	) else {
	10689	const src_reg = if (src_mcv.isRegister())
	10690	src_mcv.getReg().?
	10691	else
	10692	try self.copyToTmpRegister(scalar_ty, src_mcv);
	10693	if (self.hasFeature(.avx2)) try self.asmRegisterRegister(
	10694	.{ .v_sd, .broadcast },
	10695	dst_reg.to256(),
	10696	src_reg.to128(),
	10697	) else {
	10698	try self.asmRegisterRegister(
	10699	.{ .v_, .movddup },
	10700	dst_reg.to128(),
	10701	src_reg.to128(),
	10702	);
	10703	try self.asmRegisterRegisterRegisterImmediate(
	10704	.{ .v_f128, .insert },
	10705	dst_reg.to256(),
	10706	dst_reg.to256(),
	10707	dst_reg.to128(),
	10708	Immediate.u(1),
	10709	);
	10710	}
	10711	}
	10712	break :result .{ .register = dst_reg };
	10713	},
	10714	else => {},
	10715	},
	10716	128 => switch (vector_ty.vectorLen()) {
	10717	1 => {
	10718	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
	10719	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10720	try self.genSetReg(dst_reg, scalar_ty, src_mcv);
	10721	break :result .{ .register = dst_reg };
	10722	},
	10723	2 => if (self.hasFeature(.avx)) {
	10724	const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
	10725	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	10726	.{ .v_f128, .broadcast },
	10727	dst_reg.to256(),
	10728	src_mcv.mem(.xword),
	10729	) else {
	10730	const src_reg = if (src_mcv.isRegister())
	10731	src_mcv.getReg().?
	10732	else
	10733	try self.copyToTmpRegister(scalar_ty, src_mcv);
	10734	try self.asmRegisterRegisterRegisterImmediate(
	10735	.{ .v_f128, .insert },
	10736	dst_reg.to256(),
	10737	src_reg.to256(),
	10738	src_reg.to128(),
	10739	Immediate.u(1),
	10740	);
	10741	}
	10742	break :result .{ .register = dst_reg };
	10743	},
	10744	else => {},
	10745	},
	10746	16, 80 => {},
	10747	else => unreachable,
	10748	},
	10749	}
	10750	return self.fail("TODO implement airSplat for {}", .{
	10751	vector_ty.fmt(self.bin_file.options.module.?),
	10752	});
	10753	};
	10754	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
9983	10755	}
9984	10756
9985	10757	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -10142,9 +10914,46 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
10142	10914	fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
10143	10915	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
10144	10916	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
10145		_ = extra;
10146		return self.fail("TODO implement airUnionInit for x86_64", .{});
10147		//return self.finishAir(inst, result, .{ extra.init, .none, .none });
	10917	const result: MCValue = result: {
	10918	const union_ty = self.air.typeOfIndex(inst);
	10919	const layout = union_ty.unionGetLayout(self.target.*);
	10920
	10921	const src_ty = self.air.typeOf(extra.init);
	10922	const src_mcv = try self.resolveInst(extra.init);
	10923	if (layout.tag_size == 0) {
	10924	if (self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
	10925
	10926	const dst_mcv = try self.allocRegOrMem(inst, true);
	10927	try self.genCopy(union_ty, dst_mcv, src_mcv);
	10928	break :result dst_mcv;
	10929	}
	10930
	10931	const dst_mcv = try self.allocRegOrMem(inst, false);
	10932
	10933	const union_obj = union_ty.cast(Type.Payload.Union).?.data;
	10934	const field_name = union_obj.fields.keys()[extra.field_index];
	10935	const tag_ty = union_ty.unionTagTypeSafety().?;
	10936	const field_index = @intCast(u32, tag_ty.enumFieldIndex(field_name).?);
	10937	var tag_pl = Value.Payload.U32{ .base = .{ .tag = .enum_field_index }, .data = field_index };
	10938	const tag_val = Value.initPayload(&tag_pl.base);
	10939	var tag_int_pl: Value.Payload.U64 = undefined;
	10940	const tag_int_val = tag_val.enumToInt(tag_ty, &tag_int_pl);
	10941	const tag_int = tag_int_val.toUnsignedInt(self.target.*);
	10942	const tag_off = if (layout.tag_align < layout.payload_align)
	10943	@intCast(i32, layout.payload_size)
	10944	else
	10945	0;
	10946	try self.genCopy(tag_ty, dst_mcv.address().offset(tag_off).deref(), .{ .immediate = tag_int });
	10947
	10948	const pl_off = if (layout.tag_align < layout.payload_align)
	10949	0
	10950	else
	10951	@intCast(i32, layout.tag_size);
	10952	try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv);
	10953
	10954	break :result dst_mcv;
	10955	};
	10956	return self.finishAir(inst, result, .{ extra.init, .none, .none });
10148	10957	}
10149	10958
10150	10959	fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -10616,13 +11425,21 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
10616	11425	reg.to64()
10617	11426	else
10618	11427	unreachable,
10619		.floating_point => if (size_bytes <= 16)
	11428	.segment => if (size_bytes <= 2)
	11429	reg
	11430	else
	11431	unreachable,
	11432	.x87 => unreachable,
	11433	.mmx => if (size_bytes <= 8)
	11434	reg
	11435	else
	11436	unreachable,
	11437	.sse => if (size_bytes <= 16)
10620	11438	reg.to128()
10621	11439	else if (size_bytes <= 32)
10622	11440	reg.to256()
10623	11441	else
10624	11442	unreachable,
10625		.segment => unreachable,
10626	11443	};
10627	11444	}
10628	11445

src/arch/x86_64/Encoding.zig+127-75

...	...	@@ -233,7 +233,6 @@ pub const Mnemonic = enum {
233	233	cmpxchg, cmpxchg8b, cmpxchg16b,
234	234	cqo, cwd, cwde,
235	235	div,
236		fisttp, fld,
237	236	idiv, imul, int3,
238	237	ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
239	238	jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
...	...	@@ -259,21 +258,30 @@ pub const Mnemonic = enum {
259	258	@"test", tzcnt,
260	259	ud2,
261	260	xadd, xchg, xor,
	261	// X87
	262	fisttp, fld,
262	263	// MMX
263		movd,
	264	movd, movq,
	265	paddb, paddd, paddq, paddsb, paddsw, paddusb, paddusw, paddw,
	266	pand, pandn, por, pxor,
	267	pmulhw, pmullw,
	268	psubb, psubd, psubq, psubsb, psubsw, psubusb, psubusw, psubw,
264	269	// SSE
265	270	addps, addss,
266	271	andps,
267	272	andnps,
268	273	cmpss,
269		cvtsi2ss,
	274	cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si,
270	275	divps, divss,
271	276	maxps, maxss,
272	277	minps, minss,
273		movaps, movhlps, movss, movups,
	278	movaps, movhlps, movlhps,
	279	movss, movups,
274	280	mulps, mulss,
275	281	orps,
276	282	pextrw, pinsrw,
	283	pmaxsw, pmaxub, pminsw, pminub,
	284	shufps,
277	285	sqrtps, sqrtss,
278	286	subps, subss,
279	287	ucomiss,
...	...	@@ -283,12 +291,15 @@ pub const Mnemonic = enum {
283	291	andpd,
284	292	andnpd,
285	293	//cmpsd,
286		cvtsd2ss, cvtsi2sd, cvtss2sd,
	294	cvtdq2pd, cvtdq2ps, cvtpd2dq, cvtpd2pi, cvtpd2ps, cvtpi2pd,
	295	cvtps2dq, cvtps2pd, cvtsd2si, cvtsd2ss, cvtsi2sd, cvtss2sd,
	296	cvttpd2dq, cvttpd2pi, cvttps2dq, cvttsd2si,
287	297	divpd, divsd,
288	298	maxpd, maxsd,
289	299	minpd, minsd,
290	300	movapd,
291		movq, //movd, movsd,
	301	movdqa, movdqu,
	302	//movsd,
292	303	movupd,
293	304	mulpd, mulsd,
294	305	orpd,
...	...	@@ -296,6 +307,7 @@ pub const Mnemonic = enum {
296	307	psrld, psrlq, psrlw,
297	308	punpckhbw, punpckhdq, punpckhqdq, punpckhwd,
298	309	punpcklbw, punpckldq, punpcklqdq, punpcklwd,
	310	shufpd,
299	311	sqrtpd, sqrtsd,
300	312	subpd, subsd,
301	313	ucomisd,
...	...	@@ -303,31 +315,57 @@ pub const Mnemonic = enum {
303	315	// SSE3
304	316	movddup, movshdup, movsldup,
305	317	// SSE4.1
	318	extractps,
	319	insertps,
306	320	pextrb, pextrd, pextrq,
307	321	pinsrb, pinsrd, pinsrq,
	322	pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
	323	pmulld,
308	324	roundpd, roundps, roundsd, roundss,
309	325	// AVX
310	326	vaddpd, vaddps, vaddsd, vaddss,
311		vcvtsd2ss, vcvtsi2sd, vcvtsi2ss, vcvtss2sd,
	327	vandnpd, vandnps, vandpd, vandps,
	328	vbroadcastf128, vbroadcastsd, vbroadcastss,
	329	vcvtdq2pd, vcvtdq2ps, vcvtpd2dq, vcvtpd2ps,
	330	vcvtps2dq, vcvtps2pd, vcvtsd2si, vcvtsd2ss,
	331	vcvtsi2sd, vcvtsi2ss, vcvtss2sd, vcvtss2si,
	332	vcvttpd2dq, vcvttps2dq, vcvttsd2si, vcvttss2si,
312	333	vdivpd, vdivps, vdivsd, vdivss,
	334	vextractf128, vextractps,
	335	vinsertf128, vinsertps,
313	336	vmaxpd, vmaxps, vmaxsd, vmaxss,
314	337	vminpd, vminps, vminsd, vminss,
315	338	vmovapd, vmovaps,
316		vmovddup, vmovhlps,
	339	vmovd,
	340	vmovddup,
	341	vmovdqa, vmovdqu,
	342	vmovhlps, vmovlhps,
	343	vmovq,
317	344	vmovsd,
318	345	vmovshdup, vmovsldup,
319	346	vmovss,
320	347	vmovupd, vmovups,
321	348	vmulpd, vmulps, vmulsd, vmulss,
	349	vorpd, vorps,
	350	vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
	351	vpand, vpandn,
322	352	vpextrb, vpextrd, vpextrq, vpextrw,
323	353	vpinsrb, vpinsrd, vpinsrq, vpinsrw,
	354	vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,
	355	vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,
	356	vpmulhw, vpmulld, vpmullw,
	357	vpor,
324	358	vpshufhw, vpshuflw,
325	359	vpsrld, vpsrlq, vpsrlw,
	360	vpsubb, vpsubd, vpsubq, vpsubsb, vpsubsw, vpsubusb, vpsubusw, vpsubw,
326	361	vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
327	362	vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
	363	vpxor,
328	364	vroundpd, vroundps, vroundsd, vroundss,
	365	vshufpd, vshufps,
329	366	vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
330	367	vsubpd, vsubps, vsubsd, vsubss,
	368	vxorpd, vxorps,
331	369	// F16C
332	370	vcvtph2ps, vcvtps2ph,
333	371	// FMA
...	...	@@ -368,80 +406,84 @@ pub const Op = enum {
368	406	m,
369	407	moffs,
370	408	sreg,
	409	st, mm, mm_m64,
371	410	xmm, xmm_m32, xmm_m64, xmm_m128,
372	411	ymm, ymm_m256,
373	412	// zig fmt: on
374	413
375	414	pub fn fromOperand(operand: Instruction.Operand) Op {
376		switch (operand) {
377		.none => return .none,
378
379		.reg => \|reg\| {
380		switch (reg.class()) {
381		.segment => return .sreg,
382		.floating_point => return switch (reg.bitSize()) {
383		128 => .xmm,
384		256 => .ymm,
	415	return switch (operand) {
	416	.none => .none,
	417
	418	.reg => \|reg\| switch (reg.class()) {
	419	.general_purpose => if (reg.to64() == .rax)
	420	switch (reg) {
	421	.al => .al,
	422	.ax => .ax,
	423	.eax => .eax,
	424	.rax => .rax,
385	425	else => unreachable,
386		},
387		.general_purpose => {
388		if (reg.to64() == .rax) return switch (reg) {
389		.al => .al,
390		.ax => .ax,
391		.eax => .eax,
392		.rax => .rax,
393		else => unreachable,
394		};
395		if (reg == .cl) return .cl;
396		return switch (reg.bitSize()) {
397		8 => .r8,
398		16 => .r16,
399		32 => .r32,
400		64 => .r64,
401		else => unreachable,
402		};
403		},
404		}
	426	}
	427	else if (reg == .cl)
	428	.cl
	429	else switch (reg.bitSize()) {
	430	8 => .r8,
	431	16 => .r16,
	432	32 => .r32,
	433	64 => .r64,
	434	else => unreachable,
	435	},
	436	.segment => .sreg,
	437	.x87 => .st,
	438	.mmx => .mm,
	439	.sse => switch (reg.bitSize()) {
	440	128 => .xmm,
	441	256 => .ymm,
	442	else => unreachable,
	443	},
405	444	},
406	445
407	446	.mem => \|mem\| switch (mem) {
408		.moffs => return .moffs,
409		.sib, .rip => {
410		const bit_size = mem.bitSize();
411		return switch (bit_size) {
412		8 => .m8,
413		16 => .m16,
414		32 => .m32,
415		64 => .m64,
416		80 => .m80,
417		128 => .m128,
418		256 => .m256,
419		else => unreachable,
420		};
	447	.moffs => .moffs,
	448	.sib, .rip => switch (mem.bitSize()) {
	449	8 => .m8,
	450	16 => .m16,
	451	32 => .m32,
	452	64 => .m64,
	453	80 => .m80,
	454	128 => .m128,
	455	256 => .m256,
	456	else => unreachable,
421	457	},
422	458	},
423	459
424		.imm => \|imm\| {
425		switch (imm) {
426		.signed => \|x\| {
427		if (x == 1) return .unity;
428		if (math.cast(i8, x)) \|_\| return .imm8s;
429		if (math.cast(i16, x)) \|_\| return .imm16s;
430		return .imm32s;
431		},
432		.unsigned => \|x\| {
433		if (x == 1) return .unity;
434		if (math.cast(i8, x)) \|_\| return .imm8s;
435		if (math.cast(u8, x)) \|_\| return .imm8;
436		if (math.cast(i16, x)) \|_\| return .imm16s;
437		if (math.cast(u16, x)) \|_\| return .imm16;
438		if (math.cast(i32, x)) \|_\| return .imm32s;
439		if (math.cast(u32, x)) \|_\| return .imm32;
440		return .imm64;
441		},
442		}
	460	.imm => \|imm\| switch (imm) {
	461	.signed => \|x\| if (x == 1)
	462	.unity
	463	else if (math.cast(i8, x)) \|_\|
	464	.imm8s
	465	else if (math.cast(i16, x)) \|_\|
	466	.imm16s
	467	else
	468	.imm32s,
	469	.unsigned => \|x\| if (x == 1)
	470	.unity
	471	else if (math.cast(i8, x)) \|_\|
	472	.imm8s
	473	else if (math.cast(u8, x)) \|_\|
	474	.imm8
	475	else if (math.cast(i16, x)) \|_\|
	476	.imm16s
	477	else if (math.cast(u16, x)) \|_\|
	478	.imm16
	479	else if (math.cast(i32, x)) \|_\|
	480	.imm32s
	481	else if (math.cast(u32, x)) \|_\|
	482	.imm32
	483	else
	484	.imm64,
443	485	},
444		}
	486	};
445	487	}
446	488
447	489	pub fn immBitSize(op: Op) u64 {
...	...	@@ -451,6 +493,7 @@ pub const Op = enum {
451	493	.ax, .r16, .rm16 => unreachable,
452	494	.eax, .r32, .rm32, .r32_m16 => unreachable,
453	495	.rax, .r64, .rm64, .r64_m16 => unreachable,
	496	.st, .mm, .mm_m64 => unreachable,
454	497	.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
455	498	.ymm, .ymm_m256 => unreachable,
456	499	.m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
...	...	@@ -471,7 +514,8 @@ pub const Op = enum {
471	514	.al, .cl, .r8, .rm8 => 8,
472	515	.ax, .r16, .rm16 => 16,
473	516	.eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
474		.rax, .r64, .rm64, .r64_m16 => 64,
	517	.rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,
	518	.st => 80,
475	519	.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
476	520	.ymm, .ymm_m256 => 256,
477	521	};
...	...	@@ -482,11 +526,11 @@ pub const Op = enum {
482	526	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
483	527	.unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
484	528	.rel8, .rel16, .rel32 => unreachable,
485		.al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm, .ymm => unreachable,
	529	.al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .st, .mm, .xmm, .ymm => unreachable,
486	530	.m8, .rm8, .r32_m8 => 8,
487	531	.m16, .rm16, .r32_m16, .r64_m16 => 16,
488	532	.m32, .rm32, .xmm_m32 => 32,
489		.m64, .rm64, .xmm_m64 => 64,
	533	.m64, .rm64, .mm_m64, .xmm_m64 => 64,
490	534	.m80 => 80,
491	535	.m128, .xmm_m128 => 128,
492	536	.m256, .ymm_m256 => 256,
...	...	@@ -513,6 +557,7 @@ pub const Op = enum {
513	557	.r8, .r16, .r32, .r64,
514	558	.rm8, .rm16, .rm32, .rm64,
515	559	.r32_m8, .r32_m16, .r64_m16,
	560	.st, .mm, .mm_m64,
516	561	.xmm, .xmm_m32, .xmm_m64, .xmm_m128,
517	562	.ymm, .ymm_m256,
518	563	=> true,
...	...	@@ -541,6 +586,7 @@ pub const Op = enum {
541	586	.r32_m8, .r32_m16, .r64_m16,
542	587	.m8, .m16, .m32, .m64, .m80, .m128, .m256,
543	588	.m,
	589	.mm_m64,
544	590	.xmm_m32, .xmm_m64, .xmm_m128,
545	591	.ymm_m256,
546	592	=> true,
...	...	@@ -564,8 +610,10 @@ pub const Op = enum {
564	610	.rm8, .rm16, .rm32, .rm64 => .general_purpose,
565	611	.r32_m8, .r32_m16, .r64_m16 => .general_purpose,
566	612	.sreg => .segment,
567		.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
568		.ymm, .ymm_m256 => .floating_point,
	613	.st => .x87,
	614	.mm, .mm_m64 => .mmx,
	615	.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
	616	.ymm, .ymm_m256 => .sse,
569	617	};
570	618	}
571	619
...	...	@@ -682,8 +730,12 @@ pub const Feature = enum {
682	730	none,
683	731	avx,
684	732	avx2,
	733	bmi,
685	734	f16c,
686	735	fma,
	736	lzcnt,
	737	movbe,
	738	popcnt,
687	739	sse,
688	740	sse2,
689	741	sse3,
...	...	@@ -705,7 +757,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
705	757	}
706	758
707	759	const mnemonic_to_encodings_map = init: {
708		@setEvalBranchQuota(20_000);
	760	@setEvalBranchQuota(30_000);
709	761	const encodings = @import("encodings.zig");
710	762	var entries = encodings.table;
711	763	std.sort.sort(encodings.Entry, &entries, {}, struct {

src/arch/x86_64/Lower.zig+115-16

...	...	@@ -9,19 +9,33 @@ result_insts_len: u8 = undefined,
9	9	result_relocs_len: u8 = undefined,
10	10	result_insts: [
11	11	std.mem.max(usize, &.{
	12	1, // non-pseudo instructions
12	13	2, // cmovcc: cmovcc \ cmovcc
13	14	3, // setcc: setcc \ setcc \ logicop
14	15	2, // jcc: jcc \ jcc
	16	pseudo_probe_align_insts,
	17	pseudo_probe_adjust_unrolled_max_insts,
	18	pseudo_probe_adjust_setup_insts,
	19	pseudo_probe_adjust_loop_insts,
15	20	abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
16	21	abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
17	22	})
18	23	]Instruction = undefined,
19	24	result_relocs: [
20	25	std.mem.max(usize, &.{
	26	1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea
21	27	2, // jcc: jcc \ jcc
	28	2, // test \ jcc \ probe \ sub \ jmp
	29	1, // probe \ sub \ jcc
22	30	})
23	31	]Reloc = undefined,
24	32
	33	pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp
	34	pub const pseudo_probe_adjust_unrolled_max_insts =
	35	pseudo_probe_adjust_setup_insts + pseudo_probe_adjust_loop_insts;
	36	pub const pseudo_probe_adjust_setup_insts = 2; // mov \ sub
	37	pub const pseudo_probe_adjust_loop_insts = 3; // probe \ sub \ jcc
	38
25	39	pub const Error = error{
26	40	OutOfMemory,
27	41	LowerFail,
...	...	@@ -62,6 +76,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
62	76	else => try lower.generic(inst),
63	77	.pseudo => switch (inst.ops) {
64	78	.pseudo_cmov_z_and_np_rr => {
	79	assert(inst.data.rr.fixes == ._);
65	80	try lower.emit(.none, .cmovnz, &.{
66	81	.{ .reg = inst.data.rr.r2 },
67	82	.{ .reg = inst.data.rr.r1 },
...	...	@@ -72,6 +87,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
72	87	});
73	88	},
74	89	.pseudo_cmov_nz_or_p_rr => {
	90	assert(inst.data.rr.fixes == ._);
75	91	try lower.emit(.none, .cmovnz, &.{
76	92	.{ .reg = inst.data.rr.r1 },
77	93	.{ .reg = inst.data.rr.r2 },
...	...	@@ -84,6 +100,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
84	100	.pseudo_cmov_nz_or_p_rm_sib,
85	101	.pseudo_cmov_nz_or_p_rm_rip,
86	102	=> {
	103	assert(inst.data.rx.fixes == ._);
87	104	try lower.emit(.none, .cmovnz, &.{
88	105	.{ .reg = inst.data.rx.r1 },
89	106	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
...	...	@@ -94,58 +111,63 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
94	111	});
95	112	},
96	113	.pseudo_set_z_and_np_r => {
	114	assert(inst.data.rr.fixes == ._);
97	115	try lower.emit(.none, .setz, &.{
98		.{ .reg = inst.data.r_scratch.r1 },
	116	.{ .reg = inst.data.rr.r1 },
99	117	});
100	118	try lower.emit(.none, .setnp, &.{
101		.{ .reg = inst.data.r_scratch.scratch_reg },
	119	.{ .reg = inst.data.rr.r2 },
102	120	});
103	121	try lower.emit(.none, .@"and", &.{
104		.{ .reg = inst.data.r_scratch.r1 },
105		.{ .reg = inst.data.r_scratch.scratch_reg },
	122	.{ .reg = inst.data.rr.r1 },
	123	.{ .reg = inst.data.rr.r2 },
106	124	});
107	125	},
108	126	.pseudo_set_z_and_np_m_sib,
109	127	.pseudo_set_z_and_np_m_rip,
110	128	=> {
	129	assert(inst.data.rx.fixes == ._);
111	130	try lower.emit(.none, .setz, &.{
112		.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	131	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
113	132	});
114	133	try lower.emit(.none, .setnp, &.{
115		.{ .reg = inst.data.x_scratch.scratch_reg },
	134	.{ .reg = inst.data.rx.r1 },
116	135	});
117	136	try lower.emit(.none, .@"and", &.{
118		.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
119		.{ .reg = inst.data.x_scratch.scratch_reg },
	137	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	138	.{ .reg = inst.data.rx.r1 },
120	139	});
121	140	},
122	141	.pseudo_set_nz_or_p_r => {
	142	assert(inst.data.rr.fixes == ._);
123	143	try lower.emit(.none, .setnz, &.{
124		.{ .reg = inst.data.r_scratch.r1 },
	144	.{ .reg = inst.data.rr.r1 },
125	145	});
126	146	try lower.emit(.none, .setp, &.{
127		.{ .reg = inst.data.r_scratch.scratch_reg },
	147	.{ .reg = inst.data.rr.r2 },
128	148	});
129	149	try lower.emit(.none, .@"or", &.{
130		.{ .reg = inst.data.r_scratch.r1 },
131		.{ .reg = inst.data.r_scratch.scratch_reg },
	150	.{ .reg = inst.data.rr.r1 },
	151	.{ .reg = inst.data.rr.r2 },
132	152	});
133	153	},
134	154	.pseudo_set_nz_or_p_m_sib,
135	155	.pseudo_set_nz_or_p_m_rip,
136	156	=> {
	157	assert(inst.data.rx.fixes == ._);
137	158	try lower.emit(.none, .setnz, &.{
138		.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	159	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
139	160	});
140	161	try lower.emit(.none, .setp, &.{
141		.{ .reg = inst.data.x_scratch.scratch_reg },
	162	.{ .reg = inst.data.rx.r1 },
142	163	});
143	164	try lower.emit(.none, .@"or", &.{
144		.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
145		.{ .reg = inst.data.x_scratch.scratch_reg },
	165	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	166	.{ .reg = inst.data.rx.r1 },
146	167	});
147	168	},
148	169	.pseudo_j_z_and_np_inst => {
	170	assert(inst.data.inst.fixes == ._);
149	171	try lower.emit(.none, .jnz, &.{
150	172	.{ .imm = lower.reloc(.{ .inst = index + 1 }) },
151	173	});
...	...	@@ -154,6 +176,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
154	176	});
155	177	},
156	178	.pseudo_j_nz_or_p_inst => {
	179	assert(inst.data.inst.fixes == ._);
157	180	try lower.emit(.none, .jnz, &.{
158	181	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
159	182	});
...	...	@@ -162,6 +185,78 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
162	185	});
163	186	},
164	187
	188	.pseudo_probe_align_ri_s => {
	189	try lower.emit(.none, .@"test", &.{
	190	.{ .reg = inst.data.ri.r1 },
	191	.{ .imm = Immediate.s(@bitCast(i32, inst.data.ri.i)) },
	192	});
	193	try lower.emit(.none, .jz, &.{
	194	.{ .imm = lower.reloc(.{ .inst = index + 1 }) },
	195	});
	196	try lower.emit(.none, .lea, &.{
	197	.{ .reg = inst.data.ri.r1 },
	198	.{ .mem = Memory.sib(.qword, .{
	199	.base = .{ .reg = inst.data.ri.r1 },
	200	.disp = -page_size,
	201	}) },
	202	});
	203	try lower.emit(.none, .@"test", &.{
	204	.{ .mem = Memory.sib(.dword, .{
	205	.base = .{ .reg = inst.data.ri.r1 },
	206	}) },
	207	.{ .reg = inst.data.ri.r1.to32() },
	208	});
	209	try lower.emit(.none, .jmp, &.{
	210	.{ .imm = lower.reloc(.{ .inst = index }) },
	211	});
	212	assert(lower.result_insts_len == pseudo_probe_align_insts);
	213	},
	214	.pseudo_probe_adjust_unrolled_ri_s => {
	215	var offset = page_size;
	216	while (offset < @bitCast(i32, inst.data.ri.i)) : (offset += page_size) {
	217	try lower.emit(.none, .@"test", &.{
	218	.{ .mem = Memory.sib(.dword, .{
	219	.base = .{ .reg = inst.data.ri.r1 },
	220	.disp = -offset,
	221	}) },
	222	.{ .reg = inst.data.ri.r1.to32() },
	223	});
	224	}
	225	try lower.emit(.none, .sub, &.{
	226	.{ .reg = inst.data.ri.r1 },
	227	.{ .imm = Immediate.s(@bitCast(i32, inst.data.ri.i)) },
	228	});
	229	assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
	230	},
	231	.pseudo_probe_adjust_setup_rri_s => {
	232	try lower.emit(.none, .mov, &.{
	233	.{ .reg = inst.data.rri.r2.to32() },
	234	.{ .imm = Immediate.s(@bitCast(i32, inst.data.rri.i)) },
	235	});
	236	try lower.emit(.none, .sub, &.{
	237	.{ .reg = inst.data.rri.r1 },
	238	.{ .reg = inst.data.rri.r2 },
	239	});
	240	assert(lower.result_insts_len == pseudo_probe_adjust_setup_insts);
	241	},
	242	.pseudo_probe_adjust_loop_rr => {
	243	try lower.emit(.none, .@"test", &.{
	244	.{ .mem = Memory.sib(.dword, .{
	245	.base = .{ .reg = inst.data.rr.r1 },
	246	.scale_index = .{ .scale = 1, .index = inst.data.rr.r2 },
	247	.disp = -page_size,
	248	}) },
	249	.{ .reg = inst.data.rr.r1.to32() },
	250	});
	251	try lower.emit(.none, .sub, &.{
	252	.{ .reg = inst.data.rr.r2 },
	253	.{ .imm = Immediate.s(page_size) },
	254	});
	255	try lower.emit(.none, .jae, &.{
	256	.{ .imm = lower.reloc(.{ .inst = index }) },
	257	});
	258	assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts);
	259	},
165	260	.pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
166	261	.pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
167	262
...	...	@@ -300,6 +395,8 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
300	395	else
301	396	.none,
302	397	}, mnemonic: {
	398	@setEvalBranchQuota(2_000);
	399
303	400	comptime var max_len = 0;
304	401	inline for (@typeInfo(Mnemonic).Enum.fields) \|field\| max_len = @max(field.name.len, max_len);
305	402	var buf: [max_len]u8 = undefined;
...	...	@@ -438,6 +535,8 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er
438	535	}});
439	536	}
440	537
	538	const page_size: i32 = 1 << 12;
	539
441	540	const abi = @import("abi.zig");
442	541	const assert = std.debug.assert;
443	542	const bits = @import("bits.zig");

src/arch/x86_64/Mir.zig+108-24

...	...	@@ -236,6 +236,14 @@ pub const Inst = struct {
236	236
237	237	/// VEX-Encoded ___
238	238	v_,
	239	/// VEX-Encoded ___ Byte
	240	v_b,
	241	/// VEX-Encoded ___ Word
	242	v_w,
	243	/// VEX-Encoded ___ Doubleword
	244	v_d,
	245	/// VEX-Encoded ___ QuadWord
	246	v_q,
239	247	/// VEX-Encoded Packed ___
240	248	vp_,
241	249	/// VEX-Encoded Packed ___ Byte
...	...	@@ -256,6 +264,8 @@ pub const Inst = struct {
256	264	v_sd,
257	265	/// VEX-Encoded ___ Packed Double-Precision Values
258	266	v_pd,
	267	/// VEX-Encoded ___ 128-Bits Of Floating-Point Data
	268	v_f128,
259	269
260	270	/// Mask ___ Byte
261	271	k_b,
...	...	@@ -278,6 +288,7 @@ pub const Inst = struct {
278	288	/// Add with carry
279	289	adc,
280	290	/// Add
	291	/// Add packed integers
281	292	/// Add packed single-precision floating-point values
282	293	/// Add scalar single-precision floating-point values
283	294	/// Add packed double-precision floating-point values
...	...	@@ -410,6 +421,7 @@ pub const Inst = struct {
410	421	/// Double precision shift right
411	422	sh,
412	423	/// Subtract
	424	/// Subtract packed integers
413	425	/// Subtract packed single-precision floating-point values
414	426	/// Subtract scalar single-precision floating-point values
415	427	/// Subtract packed double-precision floating-point values
...	...	@@ -434,11 +446,45 @@ pub const Inst = struct {
434	446	/// Bitwise logical xor of packed double-precision floating-point values
435	447	xor,
436	448
	449	/// Add packed signed integers with signed saturation
	450	adds,
	451	/// Add packed unsigned integers with unsigned saturation
	452	addus,
437	453	/// Bitwise logical and not of packed single-precision floating-point values
438	454	/// Bitwise logical and not of packed double-precision floating-point values
439	455	andn,
	456	/// Maximum of packed signed integers
	457	maxs,
	458	/// Maximum of packed unsigned integers
	459	maxu,
	460	/// Minimum of packed signed integers
	461	mins,
	462	/// Minimum of packed unsigned integers
	463	minu,
	464	/// Multiply packed signed integers and store low result
	465	mull,
	466	/// Multiply packed signed integers and store high result
	467	mulh,
	468	/// Subtract packed signed integers with signed saturation
	469	subs,
	470	/// Subtract packed unsigned integers with unsigned saturation
	471	subus,
	472
	473	/// Convert packed doubleword integers to packed single-precision floating-point values
	474	/// Convert packed doubleword integers to packed double-precision floating-point values
	475	cvtpi2,
	476	/// Convert packed single-precision floating-point values to packed doubleword integers
	477	cvtps2pi,
440	478	/// Convert doubleword integer to scalar single-precision floating-point value
441		cvtsi2ss,
	479	/// Convert doubleword integer to scalar double-precision floating-point value
	480	cvtsi2,
	481	/// Convert scalar single-precision floating-point value to doubleword integer
	482	cvtss2si,
	483	/// Convert with truncation packed single-precision floating-point values to packed doubleword integers
	484	cvttps2pi,
	485	/// Convert with truncation scalar single-precision floating-point value to doubleword integer
	486	cvttss2si,
	487
442	488	/// Maximum of packed single-precision floating-point values
443	489	/// Maximum of scalar single-precision floating-point values
444	490	/// Maximum of packed double-precision floating-point values
...	...	@@ -454,6 +500,8 @@ pub const Inst = struct {
454	500	mova,
455	501	/// Move packed single-precision floating-point values high to low
456	502	movhl,
	503	/// Move packed single-precision floating-point values low to high
	504	movlh,
457	505	/// Move unaligned packed single-precision floating-point values
458	506	/// Move unaligned packed double-precision floating-point values
459	507	movu,
...	...	@@ -482,12 +530,40 @@ pub const Inst = struct {
482	530	/// Unpack and interleave low packed double-precision floating-point values
483	531	unpckl,
484	532
	533	/// Convert packed doubleword integers to packed single-precision floating-point values
	534	/// Convert packed doubleword integers to packed double-precision floating-point values
	535	cvtdq2,
	536	/// Convert packed double-precision floating-point values to packed doubleword integers
	537	cvtpd2dq,
	538	/// Convert packed double-precision floating-point values to packed doubleword integers
	539	cvtpd2pi,
	540	/// Convert packed double-precision floating-point values to packed single-precision floating-point values
	541	cvtpd2,
	542	/// Convert packed single-precision floating-point values to packed doubleword integers
	543	cvtps2dq,
	544	/// Convert packed single-precision floating-point values to packed double-precision floating-point values
	545	cvtps2,
	546	/// Convert scalar double-precision floating-point value to doubleword integer
	547	cvtsd2si,
485	548	/// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
486		cvtsd2ss,
487		/// Convert doubleword integer to scalar double-precision floating-point value
488		cvtsi2sd,
	549	cvtsd2,
489	550	/// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
490		cvtss2sd,
	551	cvtss2,
	552	/// Convert with truncation packed double-precision floating-point values to packed doubleword integers
	553	cvttpd2dq,
	554	/// Convert with truncation packed double-precision floating-point values to packed doubleword integers
	555	cvttpd2pi,
	556	/// Convert with truncation packed single-precision floating-point values to packed doubleword integers
	557	cvttps2dq,
	558	/// Convert with truncation scalar double-precision floating-point value to doubleword integer
	559	cvttsd2si,
	560	/// Move aligned packed integer values
	561	movdqa,
	562	/// Move unaligned packed integer values
	563	movdqu,
	564	/// Packed interleave shuffle of quadruplets of single-precision floating-point values
	565	/// Packed interleave shuffle of pairs of double-precision floating-point values
	566	shuf,
491	567	/// Shuffle packed high words
492	568	shufh,
493	569	/// Shuffle packed low words
...	...	@@ -520,14 +596,22 @@ pub const Inst = struct {
520	596	/// Replicate single floating-point values
521	597	movsldup,
522	598
	599	/// Extract packed floating-point values
	600	extract,
	601	/// Insert scalar single-precision floating-point value
	602	/// Insert packed floating-point values
	603	insert,
523	604	/// Round packed single-precision floating-point values
524	605	/// Round scalar single-precision floating-point value
525	606	/// Round packed double-precision floating-point values
526	607	/// Round scalar double-precision floating-point value
527	608	round,
528	609
	610	/// Load with broadcast floating-point data
	611	broadcast,
	612
529	613	/// Convert 16-bit floating-point values to single-precision floating-point values
530		cvtph2ps,
	614	cvtph2,
531	615	/// Convert single-precision floating-point values to 16-bit floating-point values
532	616	cvtps2ph,
533	617
...	...	@@ -696,27 +780,27 @@ pub const Inst = struct {
696	780	pseudo_cmov_nz_or_p_rm_rip,
697	781	/// Set byte if zero flag set and parity flag not set
698	782	/// Requires a scratch register!
699		/// Uses `r_scratch` payload.
	783	/// Uses `rr` payload.
700	784	pseudo_set_z_and_np_r,
701	785	/// Set byte if zero flag set and parity flag not set
702	786	/// Requires a scratch register!
703		/// Uses `x_scratch` payload.
	787	/// Uses `rx` payload.
704	788	pseudo_set_z_and_np_m_sib,
705	789	/// Set byte if zero flag set and parity flag not set
706	790	/// Requires a scratch register!
707		/// Uses `x_scratch` payload.
	791	/// Uses `rx` payload.
708	792	pseudo_set_z_and_np_m_rip,
709	793	/// Set byte if zero flag not set or parity flag set
710	794	/// Requires a scratch register!
711		/// Uses `r_scratch` payload.
	795	/// Uses `rr` payload.
712	796	pseudo_set_nz_or_p_r,
713	797	/// Set byte if zero flag not set or parity flag set
714	798	/// Requires a scratch register!
715		/// Uses `x_scratch` payload.
	799	/// Uses `rx` payload.
716	800	pseudo_set_nz_or_p_m_sib,
717	801	/// Set byte if zero flag not set or parity flag set
718	802	/// Requires a scratch register!
719		/// Uses `x_scratch` payload.
	803	/// Uses `rx` payload.
720	804	pseudo_set_nz_or_p_m_rip,
721	805	/// Jump if zero flag set and parity flag not set
722	806	/// Uses `inst` payload.
...	...	@@ -725,6 +809,18 @@ pub const Inst = struct {
725	809	/// Uses `inst` payload.
726	810	pseudo_j_nz_or_p_inst,
727	811
	812	/// Probe alignment
	813	/// Uses `ri` payload
	814	pseudo_probe_align_ri_s,
	815	/// Probe adjust unrolled
	816	/// Uses `ri` payload
	817	pseudo_probe_adjust_unrolled_ri_s,
	818	/// Probe adjust setup
	819	/// Uses `rri` payload
	820	pseudo_probe_adjust_setup_rri_s,
	821	/// Probe adjust loop
	822	/// Uses `rr` payload
	823	pseudo_probe_adjust_loop_rr,
728	824	/// Push registers
729	825	/// Uses `reg_list` payload.
730	826	pseudo_push_reg_list,
...	...	@@ -821,18 +917,6 @@ pub const Inst = struct {
821	917	i: u8,
822	918	payload: u32,
823	919	},
824		/// Register, scratch register
825		r_scratch: struct {
826		fixes: Fixes = ._,
827		r1: Register,
828		scratch_reg: Register,
829		},
830		/// Scratch register, followed by Custom payload found in extra.
831		x_scratch: struct {
832		fixes: Fixes = ._,
833		scratch_reg: Register,
834		payload: u32,
835		},
836	920	/// Custom payload found in extra.
837	921	x: struct {
838	922	fixes: Fixes = ._,

src/arch/x86_64/abi.zig+2-28

...	...	@@ -165,34 +165,6 @@ pub fn classifySystemV(ty: Type, target: Target, ctx: Context) [8]Class {
165	165	},
166	166	.Vector => {
167	167	const elem_ty = ty.childType();
168		if (ctx == .arg) {
169		const bit_size = ty.bitSize(target);
170		if (bit_size > 128) {
171		const has_avx512 = target.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.avx512f));
172		if (has_avx512 and bit_size <= 512) return .{
173		.integer, .integer, .integer, .integer,
174		.integer, .integer, .integer, .integer,
175		};
176		const has_avx = target.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.avx));
177		if (has_avx and bit_size <= 256) return .{
178		.integer, .integer, .integer, .integer,
179		.none, .none, .none, .none,
180		};
181		return memory_class;
182		}
183		if (bit_size > 80) return .{
184		.integer, .integer, .none, .none,
185		.none, .none, .none, .none,
186		};
187		if (bit_size > 64) return .{
188		.x87, .none, .none, .none,
189		.none, .none, .none, .none,
190		};
191		return .{
192		.integer, .none, .none, .none,
193		.none, .none, .none, .none,
194		};
195		}
196	168	const bits = elem_ty.bitSize(target) * ty.arrayLen();
197	169	if (bits <= 64) return .{
198	170	.sse, .none, .none, .none,
...	...	@@ -202,6 +174,7 @@ pub fn classifySystemV(ty: Type, target: Target, ctx: Context) [8]Class {
202	174	.sse, .sseup, .none, .none,
203	175	.none, .none, .none, .none,
204	176	};
	177	if (ctx == .arg and !std.Target.x86.featureSetHas(target.cpu.features, .avx)) return memory_class;
205	178	if (bits <= 192) return .{
206	179	.sse, .sseup, .sseup, .none,
207	180	.none, .none, .none, .none,
...	...	@@ -210,6 +183,7 @@ pub fn classifySystemV(ty: Type, target: Target, ctx: Context) [8]Class {
210	183	.sse, .sseup, .sseup, .sseup,
211	184	.none, .none, .none, .none,
212	185	};
	186	if (ctx == .arg and !std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return memory_class;
213	187	if (bits <= 320) return .{
214	188	.sse, .sseup, .sseup, .sseup,
215	189	.sseup, .none, .none, .none,

src/arch/x86_64/bits.zig+38-45

...	...	@@ -175,15 +175,21 @@ pub const Register = enum(u7) {
175	175	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
176	176	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
177	177
	178	mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7,
	179
	180	st0, st1, st2, st3, st4, st5, st6, st7,
	181
178	182	es, cs, ss, ds, fs, gs,
179	183
180	184	none,
181	185	// zig fmt: on
182	186
183		pub const Class = enum(u2) {
	187	pub const Class = enum {
184	188	general_purpose,
185		floating_point,
186	189	segment,
	190	x87,
	191	mmx,
	192	sse,
187	193	};
188	194
189	195	pub fn class(reg: Register) Class {
...	...	@@ -195,8 +201,10 @@ pub const Register = enum(u7) {
195	201	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => .general_purpose,
196	202	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => .general_purpose,
197	203
198		@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => .floating_point,
199		@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => .floating_point,
	204	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => .sse,
	205	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => .sse,
	206	@enumToInt(Register.mm0) ... @enumToInt(Register.mm7) => .mmx,
	207	@enumToInt(Register.st0) ... @enumToInt(Register.st7) => .x87,
200	208
201	209	@enumToInt(Register.es) ... @enumToInt(Register.gs) => .segment,
202	210
...	...	@@ -216,8 +224,10 @@ pub const Register = enum(u7) {
216	224
217	225	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0) - 16,
218	226	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0) - 16,
	227	@enumToInt(Register.mm0) ... @enumToInt(Register.mm7) => @enumToInt(Register.mm0) - 32,
	228	@enumToInt(Register.st0) ... @enumToInt(Register.st7) => @enumToInt(Register.st0) - 40,
219	229
220		@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es) - 32,
	230	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es) - 48,
221	231
222	232	else => unreachable,
223	233	// zig fmt: on
...	...	@@ -236,6 +246,8 @@ pub const Register = enum(u7) {
236	246
237	247	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => 256,
238	248	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => 128,
	249	@enumToInt(Register.mm0) ... @enumToInt(Register.mm7) => 64,
	250	@enumToInt(Register.st0) ... @enumToInt(Register.st7) => 80,
239	251
240	252	@enumToInt(Register.es) ... @enumToInt(Register.gs) => 16,
241	253
...	...	@@ -271,6 +283,8 @@ pub const Register = enum(u7) {
271	283
272	284	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
273	285	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
	286	@enumToInt(Register.mm0) ... @enumToInt(Register.mm7) => @enumToInt(Register.mm0),
	287	@enumToInt(Register.st0) ... @enumToInt(Register.st7) => @enumToInt(Register.st0),
274	288
275	289	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es),
276	290
...	...	@@ -326,8 +340,8 @@ pub const Register = enum(u7) {
326	340	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.al));
327	341	}
328	342
329		fn fpBase(reg: Register) u7 {
330		assert(reg.class() == .floating_point);
	343	fn sseBase(reg: Register) u7 {
	344	assert(reg.class() == .sse);
331	345	return switch (@enumToInt(reg)) {
332	346	@enumToInt(Register.ymm0)...@enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
333	347	@enumToInt(Register.xmm0)...@enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
...	...	@@ -336,49 +350,24 @@ pub const Register = enum(u7) {
336	350	}
337	351
338	352	pub fn to256(reg: Register) Register {
339		return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.ymm0));
	353	return @intToEnum(Register, @enumToInt(reg) - reg.sseBase() + @enumToInt(Register.ymm0));
340	354	}
341	355
342	356	pub fn to128(reg: Register) Register {
343		return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.xmm0));
344		}
345
346		pub fn dwarfLocOp(reg: Register) u8 {
347		return switch (reg.class()) {
348		.general_purpose => switch (reg.to64()) {
349		.rax => DW.OP.reg0,
350		.rdx => DW.OP.reg1,
351		.rcx => DW.OP.reg2,
352		.rbx => DW.OP.reg3,
353		.rsi => DW.OP.reg4,
354		.rdi => DW.OP.reg5,
355		.rbp => DW.OP.reg6,
356		.rsp => DW.OP.reg7,
357		else => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.reg0,
358		},
359		.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.reg17,
360		else => unreachable,
361		};
	357	return @intToEnum(Register, @enumToInt(reg) - reg.sseBase() + @enumToInt(Register.xmm0));
362	358	}
363	359
364		/// DWARF encodings that push a value onto the DWARF stack that is either
365		/// the contents of a register or the result of adding the contents a given
366		/// register to a given signed offset.
367		pub fn dwarfLocOpDeref(reg: Register) u8 {
	360	/// DWARF register encoding
	361	pub fn dwarfNum(reg: Register) u6 {
368	362	return switch (reg.class()) {
369		.general_purpose => switch (reg.to64()) {
370		.rax => DW.OP.breg0,
371		.rdx => DW.OP.breg1,
372		.rcx => DW.OP.breg2,
373		.rbx => DW.OP.breg3,
374		.rsi => DW.OP.breg4,
375		.rdi => DW.OP.breg5,
376		.rbp => DW.OP.breg6,
377		.rsp => DW.OP.breg7,
378		else => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.breg0,
379		},
380		.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.breg17,
381		else => unreachable,
	363	.general_purpose => if (reg.isExtended())
	364	reg.enc()
	365	else
	366	@truncate(u3, @as(u24, 0o54673120) >> @as(u5, reg.enc()) * 3),
	367	.sse => 17 + @as(u6, reg.enc()),
	368	.x87 => 33 + @as(u6, reg.enc()),
	369	.mmx => 41 + @as(u6, reg.enc()),
	370	.segment => 50 + @as(u6, reg.enc()),
382	371	};
383	372	}
384	373	};
...	...	@@ -392,6 +381,8 @@ test "Register id - different classes" {
392	381	try expect(Register.ymm0.id() == 0b10000);
393	382	try expect(Register.ymm0.id() != Register.rax.id());
394	383	try expect(Register.xmm0.id() == Register.ymm0.id());
	384	try expect(Register.xmm0.id() != Register.mm0.id());
	385	try expect(Register.mm0.id() != Register.st0.id());
395	386
396	387	try expect(Register.es.id() == 0b100000);
397	388	}
...	...	@@ -407,7 +398,9 @@ test "Register enc - different classes" {
407	398
408	399	test "Register classes" {
409	400	try expect(Register.r11.class() == .general_purpose);
410		try expect(Register.ymm11.class() == .floating_point);
	401	try expect(Register.ymm11.class() == .sse);
	402	try expect(Register.mm3.class() == .mmx);
	403	try expect(Register.st3.class() == .x87);
411	404	try expect(Register.fs.class() == .segment);
412	405	}
413	406

src/arch/x86_64/encodings.zig+359-31

...	...	@@ -272,14 +272,6 @@ pub const table = [_]Entry{
272	272	.{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none, .none },
273	273	.{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long, .none },
274	274
275		.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
276		.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
277		.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
278
279		.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
280		.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
281		.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
282
283	275	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .none, .none },
284	276	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .rex, .none },
285	277	.{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .short, .none },
...	...	@@ -354,9 +346,9 @@ pub const table = [_]Entry{
354	346	.{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none, .none },
355	347	.{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long, .none },
356	348
357		.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .none },
358		.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
359		.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .none },
	349	.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt },
	350	.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt },
	351	.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt },
360	352
361	353	.{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
362	354
...	...	@@ -395,12 +387,12 @@ pub const table = [_]Entry{
395	387	.{ .mov, .mi, &.{ .rm32, .imm32 }, &.{ 0xc7 }, 0, .none, .none },
396	388	.{ .mov, .mi, &.{ .rm64, .imm32s }, &.{ 0xc7 }, 0, .long, .none },
397	389
398		.{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .short, .none },
399		.{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
400		.{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long, .none },
401		.{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .short, .none },
402		.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
403		.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .none },
	390	.{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .short, .movbe },
	391	.{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .movbe },
	392	.{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long, .movbe },
	393	.{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .short, .movbe },
	394	.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe },
	395	.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe },
404	396
405	397	.{ .movs, .np, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
406	398	.{ .movs, .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
...	...	@@ -482,9 +474,9 @@ pub const table = [_]Entry{
482	474	.{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .short, .none },
483	475	.{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none, .none },
484	476
485		.{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .none },
486		.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
487		.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .none },
	477	.{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .popcnt },
	478	.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt },
	479	.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt },
488	480
489	481	.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },
490	482	.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },
...	...	@@ -784,9 +776,9 @@ pub const table = [_]Entry{
784	776	.{ .@"test", .mr, &.{ .rm32, .r32 }, &.{ 0x85 }, 0, .none, .none },
785	777	.{ .@"test", .mr, &.{ .rm64, .r64 }, &.{ 0x85 }, 0, .long, .none },
786	778
787		.{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .none },
788		.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
789		.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .none },
	779	.{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .bmi },
	780	.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi },
	781	.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi },
790	782
791	783	.{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
792	784
...	...	@@ -836,6 +828,15 @@ pub const table = [_]Entry{
836	828	.{ .xor, .rm, &.{ .r32, .rm32 }, &.{ 0x33 }, 0, .none, .none },
837	829	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
838	830
	831	// X87
	832	.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
	833	.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
	834	.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
	835
	836	.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
	837	.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
	838	.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
	839
839	840	// SSE
840	841	.{ .addps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .none, .sse },
841	842
...	...	@@ -847,9 +848,21 @@ pub const table = [_]Entry{
847	848
848	849	.{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse },
849	850
	851	.{ .cvtpi2ps, .rm, &.{ .xmm, .mm_m64 }, &.{ 0x0f, 0x2a }, 0, .none, .sse },
	852
	853	.{ .cvtps2pi, .rm, &.{ .mm, .xmm_m64 }, &.{ 0x0f, 0x2d }, 0, .none, .sse },
	854
850	855	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse },
851	856	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse },
852	857
	858	.{ .cvtss2si, .rm, &.{ .r32, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2d }, 0, .none, .sse },
	859	.{ .cvtss2si, .rm, &.{ .r64, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2d }, 0, .long, .sse },
	860
	861	.{ .cvttps2pi, .rm, &.{ .mm, .xmm_m64 }, &.{ 0x0f, 0x2c }, 0, .none, .sse },
	862
	863	.{ .cvttss2si, .rm, &.{ .r32, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2c }, 0, .none, .sse },
	864	.{ .cvttss2si, .rm, &.{ .r64, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2c }, 0, .long, .sse },
	865
853	866	.{ .divps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .none, .sse },
854	867
855	868	.{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse },
...	...	@@ -867,6 +880,8 @@ pub const table = [_]Entry{
867	880
868	881	.{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
869	882
	883	.{ .movlhps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .none, .sse },
	884
870	885	.{ .movss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse },
871	886	.{ .movss, .mr, &.{ .xmm_m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse },
872	887
...	...	@@ -879,14 +894,16 @@ pub const table = [_]Entry{
879	894
880	895	.{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .none, .sse },
881	896
882		.{ .subps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .none, .sse },
883
884		.{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
	897	.{ .shufps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x0f, 0xc6 }, 0, .none, .sse },
885	898
886	899	.{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },
887	900
888	901	.{ .sqrtss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse },
889	902
	903	.{ .subps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .none, .sse },
	904
	905	.{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
	906
890	907	.{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .none, .sse },
891	908
892	909	.{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .none, .sse },
...	...	@@ -902,6 +919,25 @@ pub const table = [_]Entry{
902	919
903	920	.{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 },
904	921
	922	.{ .cvtdq2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .none, .sse2 },
	923
	924	.{ .cvtdq2ps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5b }, 0, .none, .sse2 },
	925
	926	.{ .cvtpd2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf2, 0x0f, 0xe6 }, 0, .none, .sse2 },
	927
	928	.{ .cvtpd2pi, .rm, &.{ .mm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x2d }, 0, .none, .sse2 },
	929
	930	.{ .cvtpd2ps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5a }, 0, .none, .sse2 },
	931
	932	.{ .cvtpi2pd, .rm, &.{ .xmm, .mm_m64 }, &.{ 0x66, 0x0f, 0x2a }, 0, .none, .sse2 },
	933
	934	.{ .cvtps2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5b }, 0, .none, .sse2 },
	935
	936	.{ .cvtps2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x0f, 0x5a }, 0, .none, .sse2 },
	937
	938	.{ .cvtsd2si, .rm, &.{ .r32, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2d }, 0, .none, .sse2 },
	939	.{ .cvtsd2si, .rm, &.{ .r64, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2d }, 0, .long, .sse2 },
	940
905	941	.{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .none, .sse2 },
906	942
907	943	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .none, .sse2 },
...	...	@@ -909,6 +945,15 @@ pub const table = [_]Entry{
909	945
910	946	.{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 },
911	947
	948	.{ .cvttpd2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe6 }, 0, .none, .sse2 },
	949
	950	.{ .cvttpd2pi, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x2c }, 0, .none, .sse2 },
	951
	952	.{ .cvttps2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x5b }, 0, .none, .sse2 },
	953
	954	.{ .cvttsd2si, .rm, &.{ .r32, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2c }, 0, .none, .sse2 },
	955	.{ .cvttsd2si, .rm, &.{ .r64, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2c }, 0, .long, .sse2 },
	956
912	957	.{ .divpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .none, .sse2 },
913	958
914	959	.{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 },
...	...	@@ -925,11 +970,16 @@ pub const table = [_]Entry{
925	970	.{ .movapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .none, .sse2 },
926	971
927	972	.{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .none, .sse2 },
928		.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 },
929
930	973	.{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .long, .sse2 },
	974	.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 },
931	975	.{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .long, .sse2 },
932	976
	977	.{ .movdqa, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6f }, 0, .none, .sse2 },
	978	.{ .movdqa, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x7f }, 0, .none, .sse2 },
	979
	980	.{ .movdqu, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 },
	981	.{ .movdqu, .mr, &.{ .xmm_m128, .xmm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 },
	982
933	983	.{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .none, .sse2 },
934	984	.{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .none, .sse2 },
935	985
...	...	@@ -942,10 +992,39 @@ pub const table = [_]Entry{
942	992
943	993	.{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
944	994
	995	.{ .paddb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .none, .sse2 },
	996	.{ .paddw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .none, .sse2 },
	997	.{ .paddd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .none, .sse2 },
	998	.{ .paddq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd4 }, 0, .none, .sse2 },
	999
	1000	.{ .paddsb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xec }, 0, .none, .sse2 },
	1001	.{ .paddsw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xed }, 0, .none, .sse2 },
	1002
	1003	.{ .paddusb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdc }, 0, .none, .sse2 },
	1004	.{ .paddusw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdd }, 0, .none, .sse2 },
	1005
	1006	.{ .pand, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdb }, 0, .none, .sse2 },
	1007
	1008	.{ .pandn, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .none, .sse2 },
	1009
945	1010	.{ .pextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
946	1011
947	1012	.{ .pinsrw, .rmi, &.{ .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
948	1013
	1014	.{ .pmaxsw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xee }, 0, .none, .sse2 },
	1015
	1016	.{ .pmaxub, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xde }, 0, .none, .sse2 },
	1017
	1018	.{ .pminsw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xea }, 0, .none, .sse2 },
	1019
	1020	.{ .pminub, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xda }, 0, .none, .sse2 },
	1021
	1022	.{ .pmulhw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .none, .sse2 },
	1023
	1024	.{ .pmullw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd5 }, 0, .none, .sse2 },
	1025
	1026	.{ .por, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .none, .sse2 },
	1027
949	1028	.{ .pshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .none, .sse2 },
950	1029
951	1030	.{ .pshuflw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 },
...	...	@@ -957,6 +1036,18 @@ pub const table = [_]Entry{
957	1036	.{ .psrlq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 },
958	1037	.{ .psrlq, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 },
959	1038
	1039	.{ .psubb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xf8 }, 0, .none, .sse2 },
	1040	.{ .psubw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xf9 }, 0, .none, .sse2 },
	1041	.{ .psubd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfa }, 0, .none, .sse2 },
	1042
	1043	.{ .psubsb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe8 }, 0, .none, .sse2 },
	1044	.{ .psubsw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe9 }, 0, .none, .sse2 },
	1045
	1046	.{ .psubq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfb }, 0, .none, .sse2 },
	1047
	1048	.{ .psubusb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd8 }, 0, .none, .sse2 },
	1049	.{ .psubusw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd9 }, 0, .none, .sse2 },
	1050
960	1051	.{ .punpckhbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 },
961	1052	.{ .punpckhwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 },
962	1053	.{ .punpckhdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 },
...	...	@@ -967,6 +1058,10 @@ pub const table = [_]Entry{
967	1058	.{ .punpckldq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 },
968	1059	.{ .punpcklqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 },
969	1060
	1061	.{ .pxor, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xef }, 0, .none, .sse2 },
	1062
	1063	.{ .shufpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0xc6 }, 0, .none, .sse2 },
	1064
970	1065	.{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
971	1066
972	1067	.{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
...	...	@@ -990,6 +1085,10 @@ pub const table = [_]Entry{
990	1085	.{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
991	1086
992	1087	// SSE4.1
	1088	.{ .extractps, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .none, .sse4_1 },
	1089
	1090	.{ .insertps, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .none, .sse4_1 },
	1091
993	1092	.{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
994	1093	.{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },
995	1094	.{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },
...	...	@@ -1000,6 +1099,22 @@ pub const table = [_]Entry{
1000	1099	.{ .pinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 },
1001	1100	.{ .pinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 },
1002	1101
	1102	.{ .pmaxsb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .none, .sse4_1 },
	1103	.{ .pmaxsd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3d }, 0, .none, .sse4_1 },
	1104
	1105	.{ .pmaxuw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3e }, 0, .none, .sse4_1 },
	1106
	1107	.{ .pmaxud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3f }, 0, .none, .sse4_1 },
	1108
	1109	.{ .pminsb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x38 }, 0, .none, .sse4_1 },
	1110	.{ .pminsd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x39 }, 0, .none, .sse4_1 },
	1111
	1112	.{ .pminuw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3a }, 0, .none, .sse4_1 },
	1113
	1114	.{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },
	1115
	1116	.{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
	1117
1003	1118	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
1004	1119
1005	1120	.{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
...	...	@@ -1019,15 +1134,68 @@ pub const table = [_]Entry{
1019	1134
1020	1135	.{ .vaddss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
1021	1136
	1137	.{ .vandnpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .vex_128_wig, .avx },
	1138	.{ .vandnpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x55 }, 0, .vex_256_wig, .avx },
	1139
	1140	.{ .vandnps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .vex_128_wig, .avx },
	1141	.{ .vandnps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x55 }, 0, .vex_256_wig, .avx },
	1142
	1143	.{ .vandpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .vex_128_wig, .avx },
	1144	.{ .vandpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x54 }, 0, .vex_256_wig, .avx },
	1145
	1146	.{ .vandps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .vex_128_wig, .avx },
	1147	.{ .vandps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x54 }, 0, .vex_256_wig, .avx },
	1148
	1149	.{ .vbroadcastss, .rm, &.{ .xmm, .m32 }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx },
	1150	.{ .vbroadcastss, .rm, &.{ .ymm, .m32 }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx },
	1151	.{ .vbroadcastsd, .rm, &.{ .ymm, .m64 }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx },
	1152	.{ .vbroadcastf128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x1a }, 0, .vex_256_w0, .avx },
	1153
	1154	.{ .vcvtdq2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx },
	1155	.{ .vcvtdq2pd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx },
	1156
	1157	.{ .vcvtdq2ps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5b }, 0, .vex_128_wig, .avx },
	1158	.{ .vcvtdq2ps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x5b }, 0, .vex_256_wig, .avx },
	1159
	1160	.{ .vcvtpd2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf2, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx },
	1161	.{ .vcvtpd2dq, .rm, &.{ .xmm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx },
	1162
	1163	.{ .vcvtpd2ps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5a }, 0, .vex_128_wig, .avx },
	1164	.{ .vcvtpd2ps, .rm, &.{ .xmm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5a }, 0, .vex_256_wig, .avx },
	1165
	1166	.{ .vcvtps2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5b }, 0, .vex_128_wig, .avx },
	1167	.{ .vcvtps2dq, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5b }, 0, .vex_256_wig, .avx },
	1168
	1169	.{ .vcvtps2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x0f, 0x5a }, 0, .vex_128_wig, .avx },
	1170	.{ .vcvtps2pd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x0f, 0x5a }, 0, .vex_256_wig, .avx },
	1171
	1172	.{ .vcvtsd2si, .rm, &.{ .r32, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2d }, 0, .vex_lig_w0, .sse2 },
	1173	.{ .vcvtsd2si, .rm, &.{ .r64, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2d }, 0, .vex_lig_w1, .sse2 },
	1174
1022	1175	.{ .vcvtsd2ss, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
1023	1176
1024	1177	.{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
1025	1178	.{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
1026	1179
1027		.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
1028		.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
	1180	.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
	1181	.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
	1182
	1183	.{ .vcvtss2sd, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
	1184
	1185	.{ .vcvtss2si, .rm, &.{ .r32, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2d }, 0, .vex_lig_w0, .avx },
	1186	.{ .vcvtss2si, .rm, &.{ .r64, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2d }, 0, .vex_lig_w1, .avx },
	1187
	1188	.{ .vcvttpd2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx },
	1189	.{ .vcvttpd2dq, .rm, &.{ .xmm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx },
1029	1190
1030		.{ .vcvtss2sd, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
	1191	.{ .vcvttps2dq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x5b }, 0, .vex_128_wig, .avx },
	1192	.{ .vcvttps2dq, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x5b }, 0, .vex_256_wig, .avx },
	1193
	1194	.{ .vcvttsd2si, .rm, &.{ .r32, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2c }, 0, .vex_lig_w0, .sse2 },
	1195	.{ .vcvttsd2si, .rm, &.{ .r64, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x2c }, 0, .vex_lig_w1, .sse2 },
	1196
	1197	.{ .vcvttss2si, .rm, &.{ .r32, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2c }, 0, .vex_lig_w0, .avx },
	1198	.{ .vcvttss2si, .rm, &.{ .r64, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x2c }, 0, .vex_lig_w1, .avx },
1031	1199
1032	1200	.{ .vdivpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
1033	1201	.{ .vdivpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
...	...	@@ -1039,6 +1207,14 @@ pub const table = [_]Entry{
1039	1207
1040	1208	.{ .vdivss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
1041	1209
	1210	.{ .vextractf128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x19 }, 0, .vex_256_w0, .avx },
	1211
	1212	.{ .vextractps, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .vex_128_wig, .avx },
	1213
	1214	.{ .vinsertf128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x18 }, 0, .vex_256_w0, .avx },
	1215
	1216	.{ .vinsertps, .rvmi, &.{ .xmm, .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .vex_128_wig, .avx },
	1217
1042	1218	.{ .vmaxpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
1043	1219	.{ .vmaxpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
1044	1220
...	...	@@ -1069,11 +1245,31 @@ pub const table = [_]Entry{
1069	1245	.{ .vmovaps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
1070	1246	.{ .vmovaps, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
1071	1247
	1248	.{ .vmovd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .vex_128_w0, .avx },
	1249	.{ .vmovq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .vex_128_w1, .avx },
	1250	.{ .vmovd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .vex_128_w0, .avx },
	1251	.{ .vmovq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .vex_128_w1, .avx },
	1252
1072	1253	.{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1073	1254	.{ .vmovddup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
1074	1255
	1256	.{ .vmovdqa, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6f }, 0, .vex_128_wig, .avx },
	1257	.{ .vmovdqa, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x7f }, 0, .vex_128_wig, .avx },
	1258	.{ .vmovdqa, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6f }, 0, .vex_256_wig, .avx },
	1259	.{ .vmovdqa, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x7f }, 0, .vex_256_wig, .avx },
	1260
	1261	.{ .vmovdqu, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .vex_128_wig, .avx },
	1262	.{ .vmovdqu, .mr, &.{ .xmm_m128, .xmm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .vex_128_wig, .avx },
	1263	.{ .vmovdqu, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x6f }, 0, .vex_256_wig, .avx },
	1264	.{ .vmovdqu, .mr, &.{ .ymm_m256, .ymm }, &.{ 0xf3, 0x0f, 0x7f }, 0, .vex_256_wig, .avx },
	1265
1075	1266	.{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
1076	1267
	1268	.{ .vmovlhps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
	1269
	1270	.{ .vmovq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx },
	1271	.{ .vmovq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx },
	1272
1077	1273	.{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
1078	1274	.{ .vmovsd, .rm, &.{ .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
1079	1275	.{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
...	...	@@ -1110,6 +1306,27 @@ pub const table = [_]Entry{
1110	1306
1111	1307	.{ .vmulss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
1112	1308
	1309	.{ .vorpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .vex_128_wig, .avx },
	1310	.{ .vorpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x56 }, 0, .vex_256_wig, .avx },
	1311
	1312	.{ .vorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx },
	1313	.{ .vorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx },
	1314
	1315	.{ .vpaddb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_128_wig, .avx },
	1316	.{ .vpaddw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_128_wig, .avx },
	1317	.{ .vpaddd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_128_wig, .avx },
	1318	.{ .vpaddq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd4 }, 0, .vex_128_wig, .avx },
	1319
	1320	.{ .vpaddsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xec }, 0, .vex_128_wig, .avx },
	1321	.{ .vpaddsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xed }, 0, .vex_128_wig, .avx },
	1322
	1323	.{ .vpaddusb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdc }, 0, .vex_128_wig, .avx },
	1324	.{ .vpaddusw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdd }, 0, .vex_128_wig, .avx },
	1325
	1326	.{ .vpand, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdb }, 0, .vex_128_wig, .avx },
	1327
	1328	.{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },
	1329
1113	1330	.{ .vpextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx },
1114	1331	.{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
1115	1332	.{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
...	...	@@ -1123,6 +1340,32 @@ pub const table = [_]Entry{
1123	1340
1124	1341	.{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
1125	1342
	1343	.{ .vpmaxsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx },
	1344	.{ .vpmaxsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx },
	1345	.{ .vpmaxsd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3d }, 0, .vex_128_wig, .avx },
	1346
	1347	.{ .vpmaxub, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xde }, 0, .vex_128_wig, .avx },
	1348	.{ .vpmaxuw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3e }, 0, .vex_128_wig, .avx },
	1349
	1350	.{ .vpmaxud, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3f }, 0, .vex_128_wig, .avx },
	1351
	1352	.{ .vpminsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x38 }, 0, .vex_128_wig, .avx },
	1353	.{ .vpminsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xea }, 0, .vex_128_wig, .avx },
	1354	.{ .vpminsd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x39 }, 0, .vex_128_wig, .avx },
	1355
	1356	.{ .vpminub, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xda }, 0, .vex_128_wig, .avx },
	1357	.{ .vpminuw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3a }, 0, .vex_128_wig, .avx },
	1358
	1359	.{ .vpminud, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .vex_128_wig, .avx },
	1360
	1361	.{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },
	1362
	1363	.{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },
	1364
	1365	.{ .vpmullw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd5 }, 0, .vex_128_wig, .avx },
	1366
	1367	.{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
	1368
1126	1369	.{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx },
1127	1370	.{ .vpsrlw, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_128_wig, .avx },
1128	1371	.{ .vpsrld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_128_wig, .avx },
...	...	@@ -1130,6 +1373,18 @@ pub const table = [_]Entry{
1130	1373	.{ .vpsrlq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_128_wig, .avx },
1131	1374	.{ .vpsrlq, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_128_wig, .avx },
1132	1375
	1376	.{ .vpsubb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xf8 }, 0, .vex_128_wig, .avx },
	1377	.{ .vpsubw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xf9 }, 0, .vex_128_wig, .avx },
	1378	.{ .vpsubd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfa }, 0, .vex_128_wig, .avx },
	1379
	1380	.{ .vpsubsb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe8 }, 0, .vex_128_wig, .avx },
	1381	.{ .vpsubsw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe9 }, 0, .vex_128_wig, .avx },
	1382
	1383	.{ .vpsubq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfb }, 0, .vex_128_wig, .avx },
	1384
	1385	.{ .vpsubusb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd8 }, 0, .vex_128_wig, .avx },
	1386	.{ .vpsubusw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd9 }, 0, .vex_128_wig, .avx },
	1387
1133	1388	.{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },
1134	1389	.{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },
1135	1390	.{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },
...	...	@@ -1140,6 +1395,8 @@ pub const table = [_]Entry{
1140	1395	.{ .vpunpckldq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx },
1141	1396	.{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx },
1142	1397
	1398	.{ .vpxor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xef }, 0, .vex_128_wig, .avx },
	1399
1143	1400	.{ .vroundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_128_wig, .avx },
1144	1401	.{ .vroundpd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_256_wig, .avx },
1145	1402
...	...	@@ -1150,6 +1407,12 @@ pub const table = [_]Entry{
1150	1407
1151	1408	.{ .vroundss, .rvmi, &.{ .xmm, .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .vex_lig_wig, .avx },
1152	1409
	1410	.{ .vshufpd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0xc6 }, 0, .vex_128_wig, .avx },
	1411	.{ .vshufpd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0xc6 }, 0, .vex_256_wig, .avx },
	1412
	1413	.{ .vshufps, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x0f, 0xc6 }, 0, .vex_128_wig, .avx },
	1414	.{ .vshufps, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x0f, 0xc6 }, 0, .vex_256_wig, .avx },
	1415
1153	1416	.{ .vsqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
1154	1417	.{ .vsqrtpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
1155	1418
...	...	@@ -1170,6 +1433,12 @@ pub const table = [_]Entry{
1170	1433
1171	1434	.{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
1172	1435
	1436	.{ .vxorpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx },
	1437	.{ .vxorpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx },
	1438
	1439	.{ .vxorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .vex_128_wig, .avx },
	1440	.{ .vxorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x57 }, 0, .vex_256_wig, .avx },
	1441
1173	1442	// F16C
1174	1443	.{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
1175	1444	.{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c },
...	...	@@ -1201,6 +1470,51 @@ pub const table = [_]Entry{
1201	1470	.{ .vfmadd231ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w0, .fma },
1202	1471
1203	1472	// AVX2
	1473	.{ .vbroadcastss, .rm, &.{ .xmm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx2 },
	1474	.{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
	1475	.{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
	1476
	1477	.{ .vpaddb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_256_wig, .avx2 },
	1478	.{ .vpaddw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_256_wig, .avx2 },
	1479	.{ .vpaddd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_256_wig, .avx2 },
	1480	.{ .vpaddq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xd4 }, 0, .vex_256_wig, .avx2 },
	1481
	1482	.{ .vpaddsb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xec }, 0, .vex_256_wig, .avx2 },
	1483	.{ .vpaddsw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xed }, 0, .vex_256_wig, .avx2 },
	1484
	1485	.{ .vpaddusb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdc }, 0, .vex_256_wig, .avx2 },
	1486	.{ .vpaddusw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdd }, 0, .vex_256_wig, .avx2 },
	1487
	1488	.{ .vpand, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdb }, 0, .vex_256_wig, .avx2 },
	1489
	1490	.{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
	1491
	1492	.{ .vpmaxsb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_256_wig, .avx },
	1493	.{ .vpmaxsw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xee }, 0, .vex_256_wig, .avx },
	1494	.{ .vpmaxsd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x3d }, 0, .vex_256_wig, .avx },
	1495
	1496	.{ .vpmaxub, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xde }, 0, .vex_256_wig, .avx },
	1497	.{ .vpmaxuw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x3e }, 0, .vex_256_wig, .avx },
	1498
	1499	.{ .vpmaxud, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x3f }, 0, .vex_256_wig, .avx },
	1500
	1501	.{ .vpminsb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x38 }, 0, .vex_256_wig, .avx },
	1502	.{ .vpminsw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xea }, 0, .vex_256_wig, .avx },
	1503	.{ .vpminsd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x39 }, 0, .vex_256_wig, .avx },
	1504
	1505	.{ .vpminub, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xda }, 0, .vex_256_wig, .avx },
	1506	.{ .vpminuw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x3a }, 0, .vex_256_wig, .avx },
	1507
	1508	.{ .vpminud, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .vex_256_wig, .avx },
	1509
	1510	.{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx },
	1511
	1512	.{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx },
	1513
	1514	.{ .vpmullw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xd5 }, 0, .vex_256_wig, .avx },
	1515
	1516	.{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
	1517
1204	1518	.{ .vpsrlw, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_256_wig, .avx2 },
1205	1519	.{ .vpsrlw, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_256_wig, .avx2 },
1206	1520	.{ .vpsrld, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_256_wig, .avx2 },
...	...	@@ -1208,6 +1522,18 @@ pub const table = [_]Entry{
1208	1522	.{ .vpsrlq, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_256_wig, .avx2 },
1209	1523	.{ .vpsrlq, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_256_wig, .avx2 },
1210	1524
	1525	.{ .vpsubb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xf8 }, 0, .vex_256_wig, .avx2 },
	1526	.{ .vpsubw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xf9 }, 0, .vex_256_wig, .avx2 },
	1527	.{ .vpsubd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfa }, 0, .vex_256_wig, .avx2 },
	1528
	1529	.{ .vpsubsb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe8 }, 0, .vex_256_wig, .avx2 },
	1530	.{ .vpsubsw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe9 }, 0, .vex_256_wig, .avx2 },
	1531
	1532	.{ .vpsubq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfb }, 0, .vex_256_wig, .avx2 },
	1533
	1534	.{ .vpsubusb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xd8 }, 0, .vex_256_wig, .avx2 },
	1535	.{ .vpsubusw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xd9 }, 0, .vex_256_wig, .avx2 },
	1536
1211	1537	.{ .vpunpckhbw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_256_wig, .avx2 },
1212	1538	.{ .vpunpckhwd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_256_wig, .avx2 },
1213	1539	.{ .vpunpckhdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_256_wig, .avx2 },
...	...	@@ -1217,5 +1543,7 @@ pub const table = [_]Entry{
1217	1543	.{ .vpunpcklwd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_256_wig, .avx2 },
1218	1544	.{ .vpunpckldq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_256_wig, .avx2 },
1219	1545	.{ .vpunpcklqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_256_wig, .avx2 },
	1546
	1547	.{ .vpxor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xef }, 0, .vex_256_wig, .avx2 },
1220	1548	};
1221	1549	// zig fmt: on

src/codegen.zig+39-9

...	...	@@ -380,7 +380,7 @@ pub fn generateSymbol(
380	380
381	381	return Result.ok;
382	382	},
383		.field_ptr, .elem_ptr => return lowerParentPtr(
	383	.field_ptr, .elem_ptr, .opt_payload_ptr => return lowerParentPtr(
384	384	bin_file,
385	385	src_loc,
386	386	typed_value,
...	...	@@ -812,7 +812,6 @@ fn lowerParentPtr(
812	812	reloc_info: RelocInfo,
813	813	) CodeGenError!Result {
814	814	const target = bin_file.options.target;
815
816	815	switch (parent_ptr.tag()) {
817	816	.field_ptr => {
818	817	const field_ptr = parent_ptr.castTag(.field_ptr).?.data;
...	...	@@ -858,6 +857,31 @@ fn lowerParentPtr(
858	857	reloc_info.offset(@intCast(u32, elem_ptr.index * elem_ptr.elem_ty.abiSize(target))),
859	858	);
860	859	},
	860	.opt_payload_ptr => {
	861	const opt_payload_ptr = parent_ptr.castTag(.opt_payload_ptr).?.data;
	862	return lowerParentPtr(
	863	bin_file,
	864	src_loc,
	865	typed_value,
	866	opt_payload_ptr.container_ptr,
	867	code,
	868	debug_output,
	869	reloc_info,
	870	);
	871	},
	872	.eu_payload_ptr => {
	873	const eu_payload_ptr = parent_ptr.castTag(.eu_payload_ptr).?.data;
	874	const pl_ty = eu_payload_ptr.container_ty.errorUnionPayload();
	875	return lowerParentPtr(
	876	bin_file,
	877	src_loc,
	878	typed_value,
	879	eu_payload_ptr.container_ptr,
	880	code,
	881	debug_output,
	882	reloc_info.offset(@intCast(u32, errUnionPayloadOffset(pl_ty, target))),
	883	);
	884	},
861	885	.variable, .decl_ref, .decl_ref_mut => \|tag\| return lowerDeclRef(
862	886	bin_file,
863	887	src_loc,
...	...	@@ -1189,12 +1213,16 @@ pub fn genTypedValue(
1189	1213	.enum_simple => {
1190	1214	return GenResult.mcv(.{ .immediate = field_index.data });
1191	1215	},
1192		.enum_full, .enum_nonexhaustive => {
1193		const enum_full = typed_value.ty.cast(Type.Payload.EnumFull).?.data;
1194		if (enum_full.values.count() != 0) {
1195		const tag_val = enum_full.values.keys()[field_index.data];
	1216	.enum_numbered, .enum_full, .enum_nonexhaustive => {
	1217	const enum_values = if (typed_value.ty.castTag(.enum_numbered)) \|pl\|
	1218	pl.data.values
	1219	else
	1220	typed_value.ty.cast(Type.Payload.EnumFull).?.data.values;
	1221	if (enum_values.count() != 0) {
	1222	const tag_val = enum_values.keys()[field_index.data];
	1223	var buf: Type.Payload.Bits = undefined;
1196	1224	return genTypedValue(bin_file, src_loc, .{
1197		.ty = enum_full.tag_ty,
	1225	.ty = typed_value.ty.intTagType(&buf),
1198	1226	.val = tag_val,
1199	1227	}, owner_decl_index);
1200	1228	} else {
...	...	@@ -1258,9 +1286,10 @@ pub fn genTypedValue(
1258	1286	}
1259	1287
1260	1288	pub fn errUnionPayloadOffset(payload_ty: Type, target: std.Target) u64 {
	1289	if (!payload_ty.hasRuntimeBitsIgnoreComptime()) return 0;
1261	1290	const payload_align = payload_ty.abiAlignment(target);
1262	1291	const error_align = Type.anyerror.abiAlignment(target);
1263		if (payload_align >= error_align) {
	1292	if (payload_align >= error_align or !payload_ty.hasRuntimeBitsIgnoreComptime()) {
1264	1293	return 0;
1265	1294	} else {
1266	1295	return mem.alignForwardGeneric(u64, Type.anyerror.abiSize(target), payload_align);
...	...	@@ -1268,9 +1297,10 @@ pub fn errUnionPayloadOffset(payload_ty: Type, target: std.Target) u64 {
1268	1297	}
1269	1298
1270	1299	pub fn errUnionErrorOffset(payload_ty: Type, target: std.Target) u64 {
	1300	if (!payload_ty.hasRuntimeBitsIgnoreComptime()) return 0;
1271	1301	const payload_align = payload_ty.abiAlignment(target);
1272	1302	const error_align = Type.anyerror.abiAlignment(target);
1273		if (payload_align >= error_align) {
	1303	if (payload_align >= error_align and payload_ty.hasRuntimeBitsIgnoreComptime()) {
1274	1304	return mem.alignForwardGeneric(u64, payload_ty.abiSize(target), error_align);
1275	1305	} else {
1276	1306	return 0;

src/link/Dwarf.zig+115-71

...	...	@@ -184,12 +184,14 @@ pub const DeclState = struct {
184	184	try dbg_info_buffer.append(@enumToInt(AbbrevKind.pad1));
185	185	},
186	186	.Bool => {
187		try dbg_info_buffer.appendSlice(&[_]u8{
188		@enumToInt(AbbrevKind.base_type),
189		DW.ATE.boolean, // DW.AT.encoding , DW.FORM.data1
190		1, // DW.AT.byte_size, DW.FORM.data1
191		'b', 'o', 'o', 'l', 0, // DW.AT.name, DW.FORM.string
192		});
	187	try dbg_info_buffer.ensureUnusedCapacity(12);
	188	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.base_type));
	189	// DW.AT.encoding, DW.FORM.data1
	190	dbg_info_buffer.appendAssumeCapacity(DW.ATE.boolean);
	191	// DW.AT.byte_size, DW.FORM.udata
	192	try leb128.writeULEB128(dbg_info_buffer.writer(), ty.abiSize(target));
	193	// DW.AT.name, DW.FORM.string
	194	try dbg_info_buffer.writer().print("{}\x00", .{ty.fmt(module)});
193	195	},
194	196	.Int => {
195	197	const info = ty.intInfo(target);
...	...	@@ -200,9 +202,9 @@ pub const DeclState = struct {
200	202	.signed => DW.ATE.signed,
201	203	.unsigned => DW.ATE.unsigned,
202	204	});
203		// DW.AT.byte_size, DW.FORM.data1
204		dbg_info_buffer.appendAssumeCapacity(@intCast(u8, ty.abiSize(target)));
205		// DW.AT.name, DW.FORM.string
	205	// DW.AT.byte_size, DW.FORM.udata
	206	try leb128.writeULEB128(dbg_info_buffer.writer(), ty.abiSize(target));
	207	// DW.AT.name, DW.FORM.string
206	208	try dbg_info_buffer.writer().print("{}\x00", .{ty.fmt(module)});
207	209	},
208	210	.Optional => {
...	...	@@ -211,9 +213,9 @@ pub const DeclState = struct {
211	213	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.base_type));
212	214	// DW.AT.encoding, DW.FORM.data1
213	215	dbg_info_buffer.appendAssumeCapacity(DW.ATE.address);
214		// DW.AT.byte_size, DW.FORM.data1
215		dbg_info_buffer.appendAssumeCapacity(@intCast(u8, ty.abiSize(target)));
216		// DW.AT.name, DW.FORM.string
	216	// DW.AT.byte_size, DW.FORM.udata
	217	try leb128.writeULEB128(dbg_info_buffer.writer(), ty.abiSize(target));
	218	// DW.AT.name, DW.FORM.string
217	219	try dbg_info_buffer.writer().print("{}\x00", .{ty.fmt(module)});
218	220	} else {
219	221	// Non-pointer optionals are structs: struct { .maybe = , .val = }
...	...	@@ -221,7 +223,7 @@ pub const DeclState = struct {
221	223	const payload_ty = ty.optionalChild(buf);
222	224	// DW.AT.structure_type
223	225	try dbg_info_buffer.append(@enumToInt(AbbrevKind.struct_type));
224		// DW.AT.byte_size, DW.FORM.sdata
	226	// DW.AT.byte_size, DW.FORM.udata
225	227	const abi_size = ty.abiSize(target);
226	228	try leb128.writeULEB128(dbg_info_buffer.writer(), abi_size);
227	229	// DW.AT.name, DW.FORM.string
...	...	@@ -236,7 +238,7 @@ pub const DeclState = struct {
236	238	var index = dbg_info_buffer.items.len;
237	239	try dbg_info_buffer.resize(index + 4);
238	240	try self.addTypeRelocGlobal(atom_index, Type.bool, @intCast(u32, index));
239		// DW.AT.data_member_location, DW.FORM.sdata
	241	// DW.AT.data_member_location, DW.FORM.udata
240	242	try dbg_info_buffer.ensureUnusedCapacity(6);
241	243	dbg_info_buffer.appendAssumeCapacity(0);
242	244	// DW.AT.member
...	...	@@ -248,7 +250,7 @@ pub const DeclState = struct {
248	250	index = dbg_info_buffer.items.len;
249	251	try dbg_info_buffer.resize(index + 4);
250	252	try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
251		// DW.AT.data_member_location, DW.FORM.sdata
	253	// DW.AT.data_member_location, DW.FORM.udata
252	254	const offset = abi_size - payload_ty.abiSize(target);
253	255	try leb128.writeULEB128(dbg_info_buffer.writer(), offset);
254	256	// DW.AT.structure_type delimit children
...	...	@@ -263,8 +265,8 @@ pub const DeclState = struct {
263	265	// DW.AT.structure_type
264	266	try dbg_info_buffer.ensureUnusedCapacity(2);
265	267	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_type));
266		// DW.AT.byte_size, DW.FORM.sdata
267		dbg_info_buffer.appendAssumeCapacity(ptr_bytes * 2);
	268	// DW.AT.byte_size, DW.FORM.udata
	269	try leb128.writeULEB128(dbg_info_buffer.writer(), ty.abiSize(target));
268	270	// DW.AT.name, DW.FORM.string
269	271	try dbg_info_buffer.writer().print("{}\x00", .{ty.fmt(module)});
270	272	// DW.AT.member
...	...	@@ -279,7 +281,7 @@ pub const DeclState = struct {
279	281	var buf = try arena.create(Type.SlicePtrFieldTypeBuffer);
280	282	const ptr_ty = ty.slicePtrFieldType(buf);
281	283	try self.addTypeRelocGlobal(atom_index, ptr_ty, @intCast(u32, index));
282		// DW.AT.data_member_location, DW.FORM.sdata
	284	// DW.AT.data_member_location, DW.FORM.udata
283	285	try dbg_info_buffer.ensureUnusedCapacity(6);
284	286	dbg_info_buffer.appendAssumeCapacity(0);
285	287	// DW.AT.member
...	...	@@ -291,7 +293,7 @@ pub const DeclState = struct {
291	293	index = dbg_info_buffer.items.len;
292	294	try dbg_info_buffer.resize(index + 4);
293	295	try self.addTypeRelocGlobal(atom_index, Type.usize, @intCast(u32, index));
294		// DW.AT.data_member_location, DW.FORM.sdata
	296	// DW.AT.data_member_location, DW.FORM.udata
295	297	try dbg_info_buffer.ensureUnusedCapacity(2);
296	298	dbg_info_buffer.appendAssumeCapacity(ptr_bytes);
297	299	// DW.AT.structure_type delimit children
...	...	@@ -329,9 +331,8 @@ pub const DeclState = struct {
329	331	.Struct => blk: {
330	332	// DW.AT.structure_type
331	333	try dbg_info_buffer.append(@enumToInt(AbbrevKind.struct_type));
332		// DW.AT.byte_size, DW.FORM.sdata
333		const abi_size = ty.abiSize(target);
334		try leb128.writeULEB128(dbg_info_buffer.writer(), abi_size);
	334	// DW.AT.byte_size, DW.FORM.udata
	335	try leb128.writeULEB128(dbg_info_buffer.writer(), ty.abiSize(target));
335	336
336	337	switch (ty.tag()) {
337	338	.tuple, .anon_struct => {
...	...	@@ -348,7 +349,7 @@ pub const DeclState = struct {
348	349	var index = dbg_info_buffer.items.len;
349	350	try dbg_info_buffer.resize(index + 4);
350	351	try self.addTypeRelocGlobal(atom_index, field, @intCast(u32, index));
351		// DW.AT.data_member_location, DW.FORM.sdata
	352	// DW.AT.data_member_location, DW.FORM.udata
352	353	const field_off = ty.structFieldOffset(field_index, target);
353	354	try leb128.writeULEB128(dbg_info_buffer.writer(), field_off);
354	355	}
...	...	@@ -380,7 +381,7 @@ pub const DeclState = struct {
380	381	var index = dbg_info_buffer.items.len;
381	382	try dbg_info_buffer.resize(index + 4);
382	383	try self.addTypeRelocGlobal(atom_index, field.ty, @intCast(u32, index));
383		// DW.AT.data_member_location, DW.FORM.sdata
	384	// DW.AT.data_member_location, DW.FORM.udata
384	385	const field_off = ty.structFieldOffset(field_index, target);
385	386	try leb128.writeULEB128(dbg_info_buffer.writer(), field_off);
386	387	}
...	...	@@ -393,9 +394,8 @@ pub const DeclState = struct {
393	394	.Enum => {
394	395	// DW.AT.enumeration_type
395	396	try dbg_info_buffer.append(@enumToInt(AbbrevKind.enum_type));
396		// DW.AT.byte_size, DW.FORM.sdata
397		const abi_size = ty.abiSize(target);
398		try leb128.writeULEB128(dbg_info_buffer.writer(), abi_size);
	397	// DW.AT.byte_size, DW.FORM.udata
	398	try leb128.writeULEB128(dbg_info_buffer.writer(), ty.abiSize(target));
399	399	// DW.AT.name, DW.FORM.string
400	400	const enum_name = try ty.nameAllocArena(arena, module);
401	401	try dbg_info_buffer.ensureUnusedCapacity(enum_name.len + 1);
...	...	@@ -446,7 +446,7 @@ pub const DeclState = struct {
446	446	if (is_tagged) {
447	447	// DW.AT.structure_type
448	448	try dbg_info_buffer.append(@enumToInt(AbbrevKind.struct_type));
449		// DW.AT.byte_size, DW.FORM.sdata
	449	// DW.AT.byte_size, DW.FORM.udata
450	450	try leb128.writeULEB128(dbg_info_buffer.writer(), layout.abi_size);
451	451	// DW.AT.name, DW.FORM.string
452	452	try dbg_info_buffer.ensureUnusedCapacity(union_name.len + 1);
...	...	@@ -463,13 +463,13 @@ pub const DeclState = struct {
463	463	const inner_union_index = dbg_info_buffer.items.len;
464	464	try dbg_info_buffer.resize(inner_union_index + 4);
465	465	try self.addTypeRelocLocal(atom_index, @intCast(u32, inner_union_index), 5);
466		// DW.AT.data_member_location, DW.FORM.sdata
	466	// DW.AT.data_member_location, DW.FORM.udata
467	467	try leb128.writeULEB128(dbg_info_buffer.writer(), payload_offset);
468	468	}
469	469
470	470	// DW.AT.union_type
471	471	try dbg_info_buffer.append(@enumToInt(AbbrevKind.union_type));
472		// DW.AT.byte_size, DW.FORM.sdata,
	472	// DW.AT.byte_size, DW.FORM.udata,
473	473	try leb128.writeULEB128(dbg_info_buffer.writer(), layout.payload_size);
474	474	// DW.AT.name, DW.FORM.string
475	475	if (is_tagged) {
...	...	@@ -490,7 +490,7 @@ pub const DeclState = struct {
490	490	const index = dbg_info_buffer.items.len;
491	491	try dbg_info_buffer.resize(index + 4);
492	492	try self.addTypeRelocGlobal(atom_index, field.ty, @intCast(u32, index));
493		// DW.AT.data_member_location, DW.FORM.sdata
	493	// DW.AT.data_member_location, DW.FORM.udata
494	494	try dbg_info_buffer.append(0);
495	495	}
496	496	// DW.AT.union_type delimit children
...	...	@@ -507,7 +507,7 @@ pub const DeclState = struct {
507	507	const index = dbg_info_buffer.items.len;
508	508	try dbg_info_buffer.resize(index + 4);
509	509	try self.addTypeRelocGlobal(atom_index, union_obj.tag_ty, @intCast(u32, index));
510		// DW.AT.data_member_location, DW.FORM.sdata
	510	// DW.AT.data_member_location, DW.FORM.udata
511	511	try leb128.writeULEB128(dbg_info_buffer.writer(), tag_offset);
512	512
513	513	// DW.AT.structure_type delimit children
...	...	@@ -534,7 +534,7 @@ pub const DeclState = struct {
534	534
535	535	// DW.AT.structure_type
536	536	try dbg_info_buffer.append(@enumToInt(AbbrevKind.struct_type));
537		// DW.AT.byte_size, DW.FORM.sdata
	537	// DW.AT.byte_size, DW.FORM.udata
538	538	try leb128.writeULEB128(dbg_info_buffer.writer(), abi_size);
539	539	// DW.AT.name, DW.FORM.string
540	540	const name = try ty.nameAllocArena(arena, module);
...	...	@@ -551,7 +551,7 @@ pub const DeclState = struct {
551	551	const index = dbg_info_buffer.items.len;
552	552	try dbg_info_buffer.resize(index + 4);
553	553	try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
554		// DW.AT.data_member_location, DW.FORM.sdata
	554	// DW.AT.data_member_location, DW.FORM.udata
555	555	try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);
556	556	}
557	557
...	...	@@ -566,7 +566,7 @@ pub const DeclState = struct {
566	566	const index = dbg_info_buffer.items.len;
567	567	try dbg_info_buffer.resize(index + 4);
568	568	try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));
569		// DW.AT.data_member_location, DW.FORM.sdata
	569	// DW.AT.data_member_location, DW.FORM.udata
570	570	try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);
571	571	}
572	572
...	...	@@ -608,23 +608,44 @@ pub const DeclState = struct {
608	608
609	609	switch (loc) {
610	610	.register => \|reg\| {
611		try dbg_info.ensureUnusedCapacity(3);
	611	try dbg_info.ensureUnusedCapacity(4);
612	612	dbg_info.appendAssumeCapacity(@enumToInt(AbbrevKind.parameter));
613		dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
614		1, // ULEB128 dwarf expression length
615		reg,
616		});
	613	// DW.AT.location, DW.FORM.exprloc
	614	var expr_len = std.io.countingWriter(std.io.null_writer);
	615	if (reg < 32) {
	616	expr_len.writer().writeByte(DW.OP.reg0 + reg) catch unreachable;
	617	} else {
	618	expr_len.writer().writeByte(DW.OP.regx) catch unreachable;
	619	leb128.writeULEB128(expr_len.writer(), reg) catch unreachable;
	620	}
	621	leb128.writeULEB128(dbg_info.writer(), expr_len.bytes_written) catch unreachable;
	622	if (reg < 32) {
	623	dbg_info.appendAssumeCapacity(DW.OP.reg0 + reg);
	624	} else {
	625	dbg_info.appendAssumeCapacity(DW.OP.regx);
	626	leb128.writeULEB128(dbg_info.writer(), reg) catch unreachable;
	627	}
617	628	},
618	629	.stack => \|info\| {
619		try dbg_info.ensureUnusedCapacity(8);
	630	try dbg_info.ensureUnusedCapacity(9);
620	631	dbg_info.appendAssumeCapacity(@enumToInt(AbbrevKind.parameter));
621		const fixup = dbg_info.items.len;
622		dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
623		1, // we will backpatch it after we encode the displacement in LEB128
624		info.fp_register, // frame pointer
625		});
	632	// DW.AT.location, DW.FORM.exprloc
	633	var expr_len = std.io.countingWriter(std.io.null_writer);
	634	if (info.fp_register < 32) {
	635	expr_len.writer().writeByte(DW.OP.breg0 + info.fp_register) catch unreachable;
	636	} else {
	637	expr_len.writer().writeByte(DW.OP.bregx) catch unreachable;
	638	leb128.writeULEB128(expr_len.writer(), info.fp_register) catch unreachable;
	639	}
	640	leb128.writeILEB128(expr_len.writer(), info.offset) catch unreachable;
	641	leb128.writeULEB128(dbg_info.writer(), expr_len.bytes_written) catch unreachable;
	642	if (info.fp_register < 32) {
	643	dbg_info.appendAssumeCapacity(DW.OP.breg0 + info.fp_register);
	644	} else {
	645	dbg_info.appendAssumeCapacity(DW.OP.bregx);
	646	leb128.writeULEB128(dbg_info.writer(), info.fp_register) catch unreachable;
	647	}
626	648	leb128.writeILEB128(dbg_info.writer(), info.offset) catch unreachable;
627		dbg_info.items[fixup] += @intCast(u8, dbg_info.items.len - fixup - 2);
628	649	},
629	650	.wasm_local => \|value\| {
630	651	const leb_size = link.File.Wasm.getULEB128Size(value);
...	...	@@ -647,8 +668,8 @@ pub const DeclState = struct {
647	668
648	669	try dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
649	670	const index = dbg_info.items.len;
650		try dbg_info.resize(index + 4); // dw.at.type, dw.form.ref4
651		try self.addTypeRelocGlobal(atom_index, ty, @intCast(u32, index)); // DW.AT.type, DW.FORM.ref4
	671	try dbg_info.resize(index + 4); // dw.at.type, dw.form.ref4
	672	try self.addTypeRelocGlobal(atom_index, ty, @intCast(u32, index)); // DW.AT.type, DW.FORM.ref4
652	673	dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
653	674	}
654	675
...	...	@@ -670,22 +691,45 @@ pub const DeclState = struct {
670	691
671	692	switch (loc) {
672	693	.register => \|reg\| {
673		try dbg_info.ensureUnusedCapacity(2);
674		dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
675		1, // ULEB128 dwarf expression length
676		reg,
677		});
	694	try dbg_info.ensureUnusedCapacity(4);
	695	dbg_info.appendAssumeCapacity(@enumToInt(AbbrevKind.parameter));
	696	// DW.AT.location, DW.FORM.exprloc
	697	var expr_len = std.io.countingWriter(std.io.null_writer);
	698	if (reg < 32) {
	699	expr_len.writer().writeByte(DW.OP.reg0 + reg) catch unreachable;
	700	} else {
	701	expr_len.writer().writeByte(DW.OP.regx) catch unreachable;
	702	leb128.writeULEB128(expr_len.writer(), reg) catch unreachable;
	703	}
	704	leb128.writeULEB128(dbg_info.writer(), expr_len.bytes_written) catch unreachable;
	705	if (reg < 32) {
	706	dbg_info.appendAssumeCapacity(DW.OP.reg0 + reg);
	707	} else {
	708	dbg_info.appendAssumeCapacity(DW.OP.regx);
	709	leb128.writeULEB128(dbg_info.writer(), reg) catch unreachable;
	710	}
678	711	},
679	712
680	713	.stack => \|info\| {
681		try dbg_info.ensureUnusedCapacity(7);
682		const fixup = dbg_info.items.len;
683		dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
684		1, // we will backpatch it after we encode the displacement in LEB128
685		info.fp_register,
686		});
	714	try dbg_info.ensureUnusedCapacity(9);
	715	dbg_info.appendAssumeCapacity(@enumToInt(AbbrevKind.parameter));
	716	// DW.AT.location, DW.FORM.exprloc
	717	var expr_len = std.io.countingWriter(std.io.null_writer);
	718	if (info.fp_register < 32) {
	719	expr_len.writer().writeByte(DW.OP.breg0 + info.fp_register) catch unreachable;
	720	} else {
	721	expr_len.writer().writeByte(DW.OP.bregx) catch unreachable;
	722	leb128.writeULEB128(expr_len.writer(), info.fp_register) catch unreachable;
	723	}
	724	leb128.writeILEB128(expr_len.writer(), info.offset) catch unreachable;
	725	leb128.writeULEB128(dbg_info.writer(), expr_len.bytes_written) catch unreachable;
	726	if (info.fp_register < 32) {
	727	dbg_info.appendAssumeCapacity(DW.OP.breg0 + info.fp_register);
	728	} else {
	729	dbg_info.appendAssumeCapacity(DW.OP.bregx);
	730	leb128.writeULEB128(dbg_info.writer(), info.fp_register) catch unreachable;
	731	}
687	732	leb128.writeILEB128(dbg_info.writer(), info.offset) catch unreachable;
688		dbg_info.items[fixup] += @intCast(u8, dbg_info.items.len - fixup - 2);
689	733	},
690	734
691	735	.wasm_local => \|value\| {
...	...	@@ -790,7 +834,7 @@ pub const DeclState = struct {
790	834
791	835	try dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
792	836	const index = dbg_info.items.len;
793		try dbg_info.resize(index + 4); // dw.at.type, dw.form.ref4
	837	try dbg_info.resize(index + 4); // dw.at.type, dw.form.ref4
794	838	try self.addTypeRelocGlobal(atom_index, child_ty, @intCast(u32, index));
795	839	dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
796	840	}
...	...	@@ -993,13 +1037,13 @@ pub fn initDeclState(self: Dwarf, mod: Module, decl_index: Module.Decl.Index)
993	1037	// "relocations" and have to be in this fixed place so that functions can be
994	1038	// moved in virtual address space.
995	1039	assert(dbg_info_low_pc_reloc_index == dbg_info_buffer.items.len);
996		dbg_info_buffer.items.len += ptr_width_bytes; // DW.AT.low_pc, DW.FORM.addr
	1040	dbg_info_buffer.items.len += ptr_width_bytes; // DW.AT.low_pc, DW.FORM.addr
997	1041	assert(self.getRelocDbgInfoSubprogramHighPC() == dbg_info_buffer.items.len);
998		dbg_info_buffer.items.len += 4; // DW.AT.high_pc, DW.FORM.data4
	1042	dbg_info_buffer.items.len += 4; // DW.AT.high_pc, DW.FORM.data4
999	1043	//
1000	1044	if (fn_ret_has_bits) {
1001	1045	try decl_state.addTypeRelocGlobal(di_atom_index, fn_ret_type, @intCast(u32, dbg_info_buffer.items.len));
1002		dbg_info_buffer.items.len += 4; // DW.AT.type, DW.FORM.ref4
	1046	dbg_info_buffer.items.len += 4; // DW.AT.type, DW.FORM.ref4
1003	1047	}
1004	1048
1005	1049	dbg_info_buffer.appendSliceAssumeCapacity(decl_name_with_null); // DW.AT.name, DW.FORM.string
...	...	@@ -1619,7 +1663,7 @@ pub fn writeDbgAbbrev(self: *Dwarf) !void {
1619	1663	DW.AT.encoding,
1620	1664	DW.FORM.data1,
1621	1665	DW.AT.byte_size,
1622		DW.FORM.data1,
	1666	DW.FORM.udata,
1623	1667	DW.AT.name,
1624	1668	DW.FORM.string,
1625	1669	0,
...	...	@@ -1635,7 +1679,7 @@ pub fn writeDbgAbbrev(self: *Dwarf) !void {
1635	1679	DW.TAG.structure_type,
1636	1680	DW.CHILDREN.yes, // header
1637	1681	DW.AT.byte_size,
1638		DW.FORM.sdata,
	1682	DW.FORM.udata,
1639	1683	DW.AT.name,
1640	1684	DW.FORM.string,
1641	1685	0,
...	...	@@ -1648,14 +1692,14 @@ pub fn writeDbgAbbrev(self: *Dwarf) !void {
1648	1692	DW.AT.type,
1649	1693	DW.FORM.ref4,
1650	1694	DW.AT.data_member_location,
1651		DW.FORM.sdata,
	1695	DW.FORM.udata,
1652	1696	0,
1653	1697	0, // table sentinel
1654	1698	@enumToInt(AbbrevKind.enum_type),
1655	1699	DW.TAG.enumeration_type,
1656	1700	DW.CHILDREN.yes, // header
1657	1701	DW.AT.byte_size,
1658		DW.FORM.sdata,
	1702	DW.FORM.udata,
1659	1703	DW.AT.name,
1660	1704	DW.FORM.string,
1661	1705	0,
...	...	@@ -1673,7 +1717,7 @@ pub fn writeDbgAbbrev(self: *Dwarf) !void {
1673	1717	DW.TAG.union_type,
1674	1718	DW.CHILDREN.yes, // header
1675	1719	DW.AT.byte_size,
1676		DW.FORM.sdata,
	1720	DW.FORM.udata,
1677	1721	DW.AT.name,
1678	1722	DW.FORM.string,
1679	1723	0,
...	...	@@ -2628,7 +2672,7 @@ fn addDbgInfoErrorSet(
2628	2672
2629	2673	// DW.AT.enumeration_type
2630	2674	try dbg_info_buffer.append(@enumToInt(AbbrevKind.enum_type));
2631		// DW.AT.byte_size, DW.FORM.sdata
	2675	// DW.AT.byte_size, DW.FORM.udata
2632	2676	const abi_size = Type.anyerror.abiSize(target);
2633	2677	try leb128.writeULEB128(dbg_info_buffer.writer(), abi_size);
2634	2678	// DW.AT.name, DW.FORM.string

src/type.zig+11-1

...	...	@@ -5433,8 +5433,18 @@ pub const Type = extern union {
5433	5433	}
5434	5434	}
5435	5435
	5436	// Works for vectors and vectors of integers.
	5437	pub fn maxInt(ty: Type, arena: Allocator, target: Target) !Value {
	5438	const scalar = try maxIntScalar(ty.scalarType(), arena, target);
	5439	if (ty.zigTypeTag() == .Vector and scalar.tag() != .the_only_possible_value) {
	5440	return Value.Tag.repeated.create(arena, scalar);
	5441	} else {
	5442	return scalar;
	5443	}
	5444	}
	5445
5436	5446	/// Asserts that self.zigTypeTag() == .Int.
5437		pub fn maxInt(self: Type, arena: Allocator, target: Target) !Value {
	5447	pub fn maxIntScalar(self: Type, arena: Allocator, target: Target) !Value {
5438	5448	assert(self.zigTypeTag() == .Int);
5439	5449	const info = self.intInfo(target);
5440	5450

test/behavior/bugs/1277.zig-1

...	...	@@ -14,7 +14,6 @@ fn f() i32 {
14	14	test "don't emit an LLVM global for a const function when it's in an optional in a struct" {
15	15	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
16	16	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
17		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
18	17	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
19	18	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
20	19

test/behavior/bugs/12801-2.zig-1

...	...	@@ -16,7 +16,6 @@ const Auto = struct {
16	16	test {
17	17	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
18	18	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
19		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
20	19	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
21	20	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
22	21

test/behavior/bugs/13366.zig-1

...	...	@@ -14,7 +14,6 @@ const Block = struct {
14	14
15	15	test {
16	16	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
17		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
18	17	if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
19	18	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
20	19	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO

test/behavior/cast.zig-1

...	...	@@ -153,7 +153,6 @@ test "@intToFloat(f80)" {
153	153	test "@floatToInt" {
154	154	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
155	155	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
156		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
157	156	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
158	157	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
159	158	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;

test/behavior/field_parent_ptr.zig-4

...	...	@@ -11,7 +11,6 @@ test "@fieldParentPtr non-first field" {
11	11	}
12	12
13	13	test "@fieldParentPtr first field" {
14		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
15	14	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
16	15	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
17	16	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
...	...	@@ -51,7 +50,6 @@ fn testParentFieldPtrFirst(a: *const bool) !void {
51	50	}
52	51
53	52	test "@fieldParentPtr untagged union" {
54		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
55	53	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
56	54	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
57	55	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
...	...	@@ -79,7 +77,6 @@ fn testFieldParentPtrUnion(c: *const i32) !void {
79	77	}
80	78
81	79	test "@fieldParentPtr tagged union" {
82		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
83	80	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
84	81	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
85	82	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
...	...	@@ -107,7 +104,6 @@ fn testFieldParentPtrTaggedUnion(c: *const i32) !void {
107	104	}
108	105
109	106	test "@fieldParentPtr extern union" {
110		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
111	107	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
112	108	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
113	109	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO

test/behavior/floatop.zig+1-2

...	...	@@ -184,7 +184,7 @@ test "more @sqrt f16 tests" {
184	184	test "another, possibly redundant @sqrt test" {
185	185	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
186	186	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
187		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	187	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
188	188	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
189	189
190	190	try testSqrtLegacy(f64, 12.0);
...	...	@@ -532,7 +532,6 @@ fn testFabs() !void {
532	532
533	533	test "@fabs with vectors" {
534	534	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
535		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
536	535	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
537	536	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
538	537	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;

test/behavior/fn.zig-1

...	...	@@ -502,7 +502,6 @@ test "method call with optional pointer first param" {
502	502	}
503	503
504	504	test "using @ptrCast on function pointers" {
505		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
506	505	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
507	506	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
508	507	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/int_comparison_elision.zig-1

...	...	@@ -15,7 +15,6 @@ test "int comparison elision" {
15	15
16	16	// TODO: support int types > 128 bits wide in other backends
17	17	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
18		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
19	18	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
20	19	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
21	20

test/behavior/math.zig-3

...	...	@@ -61,7 +61,6 @@ fn assertFalse(b: bool) !void {
61	61	}
62	62
63	63	test "@clz" {
64		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
65	64	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
66	65	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
67	66	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -142,7 +141,6 @@ fn expectVectorsEqual(a: anytype, b: anytype) !void {
142	141	}
143	142
144	143	test "@ctz" {
145		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
146	144	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
147	145	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
148	146	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1612,7 +1610,6 @@ test "absFloat" {
1612	1610	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1613	1611	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1614	1612	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1615		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1616	1613	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1617	1614	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1618	1615

test/behavior/maximum_minimum.zig-1

...	...	@@ -146,7 +146,6 @@ test "@min/@max more than two arguments" {
146	146
147	147	test "@min/@max more than two vector arguments" {
148	148	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
149		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
150	149	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
151	150	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
152	151	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/memset.zig-2

...	...	@@ -120,7 +120,6 @@ test "memset with large array element, runtime known" {
120	120	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
121	121	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
122	122	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
123		if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
124	123
125	124	const A = [128]u64;
126	125	var buf: [5]A = undefined;
...	...	@@ -139,7 +138,6 @@ test "memset with large array element, comptime known" {
139	138	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
140	139	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
141	140	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
142		if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
143	141
144	142	const A = [128]u64;
145	143	var buf: [5]A = undefined;

test/behavior/optional.zig-2

...	...	@@ -74,7 +74,6 @@ test "optional with void type" {
74	74	test "address of unwrap optional" {
75	75	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
76	76	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
77		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
78	77	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
79	78	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
80	79
...	...	@@ -365,7 +364,6 @@ test "optional pointer to zero bit optional payload" {
365	364	}
366	365
367	366	test "optional pointer to zero bit error union payload" {
368		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
369	367	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
370	368	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
371	369	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO

test/behavior/struct.zig-1

...	...	@@ -1352,7 +1352,6 @@ test "struct field init value is size of the struct" {
1352	1352	}
1353	1353
1354	1354	test "under-aligned struct field" {
1355		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1356	1355	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1357	1356	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1358	1357	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/translate_c_macros.zig-1

...	...	@@ -65,7 +65,6 @@ test "cast negative integer to pointer" {
65	65
66	66	test "casting to union with a macro" {
67	67	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
68		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
69	68	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
70	69	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
71	70	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/tuple.zig-1

...	...	@@ -367,7 +367,6 @@ test "branching inside tuple literal" {
367	367	test "tuple initialized with a runtime known value" {
368	368	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
369	369	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
370		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
371	370	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
372	371	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
373	372

test/behavior/union.zig-10

...	...	@@ -362,7 +362,6 @@ const MultipleChoice = union(enum(u32)) {
362	362	D = 1000,
363	363	};
364	364	test "simple union(enum(u32))" {
365		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
366	365	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
367	366	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
368	367	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -641,7 +640,6 @@ test "tagged union with all void fields but a meaningful tag" {
641	640	}
642	641
643	642	test "union(enum(u32)) with specified and unspecified tag values" {
644		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
645	643	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
646	644	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
647	645	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -821,7 +819,6 @@ test "return union init with void payload" {
821	819	}
822	820
823	821	test "@unionInit stored to a const" {
824		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
825	822	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
826	823	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
827	824	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
...	...	@@ -994,7 +991,6 @@ test "function call result coerces from tagged union to the tag" {
994	991	}
995	992
996	993	test "cast from anonymous struct to union" {
997		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
998	994	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
999	995	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1000	996	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
...	...	@@ -1028,7 +1024,6 @@ test "cast from anonymous struct to union" {
1028	1024	}
1029	1025
1030	1026	test "cast from pointer to anonymous struct to pointer to union" {
1031		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1032	1027	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1033	1028	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1034	1029	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
...	...	@@ -1169,7 +1164,6 @@ test "union enum type gets a separate scope" {
1169	1164	}
1170	1165
1171	1166	test "global variable struct contains union initialized to non-most-aligned field" {
1172		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1173	1167	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1174	1168	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1175	1169	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1199,7 +1193,6 @@ test "global variable struct contains union initialized to non-most-aligned fiel
1199	1193	test "union with no result loc initiated with a runtime value" {
1200	1194	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1201	1195	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1202		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1203	1196	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1204	1197	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1205	1198
...	...	@@ -1217,7 +1210,6 @@ test "union with no result loc initiated with a runtime value" {
1217	1210	test "union with a large struct field" {
1218	1211	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1219	1212	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1220		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1221	1213	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1222	1214	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1223	1215
...	...	@@ -1288,7 +1280,6 @@ test "extern union most-aligned field is smaller" {
1288	1280	}
1289	1281
1290	1282	test "return an extern union from C calling convention" {
1291		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1292	1283	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1293	1284	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1294	1285	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1397,7 +1388,6 @@ test "union and enum field order doesn't match" {
1397	1388	}
1398	1389
1399	1390	test "@unionInit uses tag value instead of field index" {
1400		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1401	1391	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1402	1392	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1403	1393	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO

test/behavior/vector.zig+2-5

...	...	@@ -26,7 +26,8 @@ test "implicit cast vector to array - bool" {
26	26
27	27	test "vector wrap operators" {
28	28	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
29		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	29	if (builtin.zig_backend == .stage2_x86_64 and
	30	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
30	31	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
31	32	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
32	33	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -119,7 +120,6 @@ test "vector float operators" {
119	120
120	121	test "vector bit operators" {
121	122	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
122		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
123	123	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
124	124	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
125	125	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1129,7 +1129,6 @@ test "loading the second vector from a slice of vectors" {
1129	1129	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1130	1130	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1131	1131	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1132		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1133	1132
1134	1133	@setRuntimeSafety(false);
1135	1134	var small_bases = [2]@Vector(2, u8){
...	...	@@ -1219,7 +1218,6 @@ test "zero multiplicand" {
1219	1218	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1220	1219	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1221	1220	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1222		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1223	1221
1224	1222	const zeros = @Vector(2, u32){ 0.0, 0.0 };
1225	1223	var ones = @Vector(2, u32){ 1.0, 1.0 };
...	...	@@ -1324,7 +1322,6 @@ test "store to vector in slice" {
1324	1322	test "addition of vectors represented as strings" {
1325	1323	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1326	1324	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1327		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1328	1325	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1329	1326
1330	1327	const V = @Vector(3, u8);