Commit ecb5feaf94bf49dc4c180f09c170223d6c1898b3 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-05-08 06:51:05-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
log	ecb5feaf94bf49dc4c180f09c170223d6c1898b3
tree	9b168cfd880fb316f86fba14b3773297745e3ba3
parent	6c14eb2863c7c00f809c5e447ceb8186b55f2eef

x86_64: continue to optimize mir tag usage

Migrate mnemonic literals to tuples that represent the compressed
storage.  225 tags left in use, many tags left to compress.

3 files changed, 870 insertions(+), 708 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Lower.zig
    Mir.zig

src/arch/x86_64/CodeGen.zig+802-667

...	...	@@ -1167,11 +1167,13 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1167	1167	});
1168	1168	}
1169	1169
1170		fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
	1170	fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1171	1171	_ = try self.addInst(.{
1172		.tag = tag,
	1172	.tag = tag[1],
1173	1173	.ops = .none,
1174		.data = undefined,
	1174	.data = .{ .none = .{
	1175	.fixes = tag[0],
	1176	} },
1175	1177	});
1176	1178	}
1177	1179
...	...	@@ -1183,22 +1185,26 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1183	1185	});
1184	1186	}
1185	1187
1186		fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
	1188	fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1187	1189	_ = try self.addInst(.{
1188		.tag = tag,
	1190	.tag = tag[1],
1189	1191	.ops = .r,
1190		.data = .{ .r = .{ .r1 = reg } },
	1192	.data = .{ .r = .{
	1193	.fixes = tag[0],
	1194	.r1 = reg,
	1195	} },
1191	1196	});
1192	1197	}
1193	1198
1194		fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
	1199	fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1195	1200	_ = try self.addInst(.{
1196		.tag = tag,
	1201	.tag = tag[1],
1197	1202	.ops = switch (imm) {
1198	1203	.signed => .i_s,
1199	1204	.unsigned => .i_u,
1200	1205	},
1201	1206	.data = .{ .i = .{
	1207	.fixes = tag[0],
1202	1208	.i = switch (imm) {
1203	1209	.signed => \|s\| @bitCast(u32, s),
1204	1210	.unsigned => \|u\| @intCast(u32, u),
...	...	@@ -1207,24 +1213,29 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
1207	1213	});
1208	1214	}
1209	1215
1210		fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {
	1216	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1211	1217	_ = try self.addInst(.{
1212		.tag = tag,
	1218	.tag = tag[1],
1213	1219	.ops = .rr,
1214		.data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },
	1220	.data = .{ .rr = .{
	1221	.fixes = tag[0],
	1222	.r1 = reg1,
	1223	.r2 = reg2,
	1224	} },
1215	1225	});
1216	1226	}
1217	1227
1218		fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
	1228	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1219	1229	const ops: Mir.Inst.Ops = switch (imm) {
1220	1230	.signed => .ri_s,
1221	1231	.unsigned => \|u\| if (math.cast(u32, u)) \|_\| .ri_u else .ri64,
1222	1232	};
1223	1233	_ = try self.addInst(.{
1224		.tag = tag,
	1234	.tag = tag[1],
1225	1235	.ops = ops,
1226	1236	.data = switch (ops) {
1227	1237	.ri_s, .ri_u => .{ .ri = .{
	1238	.fixes = tag[0],
1228	1239	.r1 = reg,
1229	1240	.i = switch (imm) {
1230	1241	.signed => \|s\| @bitCast(u32, s),
...	...	@@ -1232,6 +1243,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1232	1243	},
1233	1244	} },
1234	1245	.ri64 => .{ .rx = .{
	1246	.fixes = tag[0],
1235	1247	.r1 = reg,
1236	1248	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
1237	1249	} },
...	...	@@ -1242,47 +1254,59 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1242	1254
1243	1255	fn asmRegisterRegisterRegister(
1244	1256	self: *Self,
1245		tag: Mir.Inst.Tag,
	1257	tag: Mir.Inst.FixedTag,
1246	1258	reg1: Register,
1247	1259	reg2: Register,
1248	1260	reg3: Register,
1249	1261	) !void {
1250	1262	_ = try self.addInst(.{
1251		.tag = tag,
	1263	.tag = tag[1],
1252	1264	.ops = .rrr,
1253		.data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },
	1265	.data = .{ .rrr = .{
	1266	.fixes = tag[0],
	1267	.r1 = reg1,
	1268	.r2 = reg2,
	1269	.r3 = reg3,
	1270	} },
1254	1271	});
1255	1272	}
1256	1273
1257	1274	fn asmRegisterRegisterRegisterImmediate(
1258	1275	self: *Self,
1259		tag: Mir.Inst.Tag,
	1276	tag: Mir.Inst.FixedTag,
1260	1277	reg1: Register,
1261	1278	reg2: Register,
1262	1279	reg3: Register,
1263	1280	imm: Immediate,
1264	1281	) !void {
1265	1282	_ = try self.addInst(.{
1266		.tag = tag,
	1283	.tag = tag[1],
1267	1284	.ops = .rrri,
1268		.data = .{ .rrri = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3, .i = @intCast(u8, imm.unsigned) } },
	1285	.data = .{ .rrri = .{
	1286	.fixes = tag[0],
	1287	.r1 = reg1,
	1288	.r2 = reg2,
	1289	.r3 = reg3,
	1290	.i = @intCast(u8, imm.unsigned),
	1291	} },
1269	1292	});
1270	1293	}
1271	1294
1272	1295	fn asmRegisterRegisterImmediate(
1273	1296	self: *Self,
1274		tag: Mir.Inst.Tag,
	1297	tag: Mir.Inst.FixedTag,
1275	1298	reg1: Register,
1276	1299	reg2: Register,
1277	1300	imm: Immediate,
1278	1301	) !void {
1279	1302	_ = try self.addInst(.{
1280		.tag = tag,
	1303	.tag = tag[1],
1281	1304	.ops = switch (imm) {
1282	1305	.signed => .rri_s,
1283	1306	.unsigned => .rri_u,
1284	1307	},
1285	1308	.data = .{ .rri = .{
	1309	.fixes = tag[0],
1286	1310	.r1 = reg1,
1287	1311	.r2 = reg2,
1288	1312	.i = switch (imm) {
...	...	@@ -1295,19 +1319,20 @@ fn asmRegisterRegisterImmediate(
1295	1319
1296	1320	fn asmRegisterRegisterMemory(
1297	1321	self: *Self,
1298		tag: Mir.Inst.Tag,
	1322	tag: Mir.Inst.FixedTag,
1299	1323	reg1: Register,
1300	1324	reg2: Register,
1301	1325	m: Memory,
1302	1326	) !void {
1303	1327	_ = try self.addInst(.{
1304		.tag = tag,
	1328	.tag = tag[1],
1305	1329	.ops = switch (m) {
1306	1330	.sib => .rrm_sib,
1307	1331	.rip => .rrm_rip,
1308	1332	else => unreachable,
1309	1333	},
1310	1334	.data = .{ .rrx = .{
	1335	.fixes = tag[0],
1311	1336	.r1 = reg1,
1312	1337	.r2 = reg2,
1313	1338	.payload = switch (m) {
...	...	@@ -1319,15 +1344,16 @@ fn asmRegisterRegisterMemory(
1319	1344	});
1320	1345	}
1321	1346
1322		fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
	1347	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1323	1348	_ = try self.addInst(.{
1324		.tag = tag,
	1349	.tag = tag[1],
1325	1350	.ops = switch (m) {
1326	1351	.sib => .m_sib,
1327	1352	.rip => .m_rip,
1328	1353	else => unreachable,
1329	1354	},
1330	1355	.data = .{ .x = .{
	1356	.fixes = tag[0],
1331	1357	.payload = switch (m) {
1332	1358	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1333	1359	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...	...	@@ -1337,15 +1363,16 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
1337	1363	});
1338	1364	}
1339	1365
1340		fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
	1366	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1341	1367	_ = try self.addInst(.{
1342		.tag = tag,
	1368	.tag = tag[1],
1343	1369	.ops = switch (m) {
1344	1370	.sib => .rm_sib,
1345	1371	.rip => .rm_rip,
1346	1372	else => unreachable,
1347	1373	},
1348	1374	.data = .{ .rx = .{
	1375	.fixes = tag[0],
1349	1376	.r1 = reg,
1350	1377	.payload = switch (m) {
1351	1378	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
...	...	@@ -1358,19 +1385,20 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
1358	1385
1359	1386	fn asmRegisterMemoryImmediate(
1360	1387	self: *Self,
1361		tag: Mir.Inst.Tag,
	1388	tag: Mir.Inst.FixedTag,
1362	1389	reg: Register,
1363	1390	m: Memory,
1364	1391	imm: Immediate,
1365	1392	) !void {
1366	1393	_ = try self.addInst(.{
1367		.tag = tag,
	1394	.tag = tag[1],
1368	1395	.ops = switch (m) {
1369	1396	.sib => .rmi_sib,
1370	1397	.rip => .rmi_rip,
1371	1398	else => unreachable,
1372	1399	},
1373	1400	.data = .{ .rix = .{
	1401	.fixes = tag[0],
1374	1402	.r1 = reg,
1375	1403	.i = @intCast(u8, imm.unsigned),
1376	1404	.payload = switch (m) {
...	...	@@ -1384,20 +1412,21 @@ fn asmRegisterMemoryImmediate(
1384	1412
1385	1413	fn asmRegisterRegisterMemoryImmediate(
1386	1414	self: *Self,
1387		tag: Mir.Inst.Tag,
	1415	tag: Mir.Inst.FixedTag,
1388	1416	reg1: Register,
1389	1417	reg2: Register,
1390	1418	m: Memory,
1391	1419	imm: Immediate,
1392	1420	) !void {
1393	1421	_ = try self.addInst(.{
1394		.tag = tag,
	1422	.tag = tag[1],
1395	1423	.ops = switch (m) {
1396	1424	.sib => .rrmi_sib,
1397	1425	.rip => .rrmi_rip,
1398	1426	else => unreachable,
1399	1427	},
1400	1428	.data = .{ .rrix = .{
	1429	.fixes = tag[0],
1401	1430	.r1 = reg1,
1402	1431	.r2 = reg2,
1403	1432	.i = @intCast(u8, imm.unsigned),
...	...	@@ -1410,15 +1439,16 @@ fn asmRegisterRegisterMemoryImmediate(
1410	1439	});
1411	1440	}
1412	1441
1413		fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
	1442	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1414	1443	_ = try self.addInst(.{
1415		.tag = tag,
	1444	.tag = tag[1],
1416	1445	.ops = switch (m) {
1417	1446	.sib => .mr_sib,
1418	1447	.rip => .mr_rip,
1419	1448	else => unreachable,
1420	1449	},
1421	1450	.data = .{ .rx = .{
	1451	.fixes = tag[0],
1422	1452	.r1 = reg,
1423	1453	.payload = switch (m) {
1424	1454	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
...	...	@@ -1429,9 +1459,9 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
1429	1459	});
1430	1460	}
1431	1461
1432		fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
	1462	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1433	1463	_ = try self.addInst(.{
1434		.tag = tag,
	1464	.tag = tag[1],
1435	1465	.ops = switch (m) {
1436	1466	.sib => switch (imm) {
1437	1467	.signed => .mi_sib_s,
...	...	@@ -1443,57 +1473,64 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
1443	1473	},
1444	1474	else => unreachable,
1445	1475	},
1446		.data = .{ .ix = .{
1447		.i = switch (imm) {
	1476	.data = .{ .x = .{
	1477	.fixes = tag[0],
	1478	.payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1448	1479	.signed => \|s\| @bitCast(u32, s),
1449	1480	.unsigned => \|u\| @intCast(u32, u),
1450		},
1451		.payload = switch (m) {
1452		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1453		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1454		else => unreachable,
1455		},
	1481	} }),
1456	1482	} },
1457	1483	});
	1484	_ = switch (m) {
	1485	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1486	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1487	else => unreachable,
	1488	};
1458	1489	}
1459	1490
1460	1491	fn asmMemoryRegisterRegister(
1461	1492	self: *Self,
1462		tag: Mir.Inst.Tag,
	1493	tag: Mir.Inst.FixedTag,
1463	1494	m: Memory,
1464	1495	reg1: Register,
1465	1496	reg2: Register,
1466	1497	) !void {
1467	1498	_ = try self.addInst(.{
1468		.tag = tag,
	1499	.tag = tag[1],
1469	1500	.ops = switch (m) {
1470	1501	.sib => .mrr_sib,
1471	1502	.rip => .mrr_rip,
1472	1503	else => unreachable,
1473	1504	},
1474		.data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {
1475		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1476		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1477		else => unreachable,
1478		} } },
	1505	.data = .{ .rrx = .{
	1506	.fixes = tag[0],
	1507	.r1 = reg1,
	1508	.r2 = reg2,
	1509	.payload = switch (m) {
	1510	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1511	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1512	else => unreachable,
	1513	},
	1514	} },
1479	1515	});
1480	1516	}
1481	1517
1482	1518	fn asmMemoryRegisterImmediate(
1483	1519	self: *Self,
1484		tag: Mir.Inst.Tag,
	1520	tag: Mir.Inst.FixedTag,
1485	1521	m: Memory,
1486	1522	reg: Register,
1487	1523	imm: Immediate,
1488	1524	) !void {
1489	1525	_ = try self.addInst(.{
1490		.tag = tag,
	1526	.tag = tag[1],
1491	1527	.ops = switch (m) {
1492	1528	.sib => .mri_sib,
1493	1529	.rip => .mri_rip,
1494	1530	else => unreachable,
1495	1531	},
1496	1532	.data = .{ .rix = .{
	1533	.fixes = tag[0],
1497	1534	.r1 = reg,
1498	1535	.i = @intCast(u8, imm.unsigned),
1499	1536	.payload = switch (m) {
...	...	@@ -1508,9 +1545,9 @@ fn asmMemoryRegisterImmediate(
1508	1545	fn gen(self: *Self) InnerError!void {
1509	1546	const cc = self.fn_type.fnCallingConvention();
1510	1547	if (cc != .Naked) {
1511		try self.asmRegister(.push, .rbp);
	1548	try self.asmRegister(.{ ._, .push }, .rbp);
1512	1549	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
1513		try self.asmRegisterRegister(.mov, .rbp, .rsp);
	1550	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1514	1551	const backpatch_frame_align = try self.asmPlaceholder();
1515	1552	const backpatch_stack_alloc = try self.asmPlaceholder();
1516	1553
...	...	@@ -1553,8 +1590,8 @@ fn gen(self: *Self) InnerError!void {
1553	1590	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1554	1591	const backpatch_stack_dealloc = try self.asmPlaceholder();
1555	1592	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
1556		try self.asmRegister(.pop, .rbp);
1557		try self.asmOpOnly(.ret);
	1593	try self.asmRegister(.{ ._, .pop }, .rbp);
	1594	try self.asmOpOnly(.{ ._, .ret });
1558	1595
1559	1596	const frame_layout = try self.computeFrameLayout();
1560	1597	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
...	...	@@ -1927,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1927	1964	};
1928	1965	const tag_val = Value.initPayload(&tag_pl.base);
1929	1966	const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
1930		try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);
	1967	try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
1931	1968	const skip_reloc = try self.asmJccReloc(undefined, .ne);
1932	1969
1933	1970	try self.genSetMem(
...	...	@@ -1947,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1947	1984	try self.airTrap();
1948	1985
1949	1986	for (exitlude_jump_relocs) \|reloc\| try self.performReloc(reloc);
1950		try self.asmOpOnly(.ret);
	1987	try self.asmOpOnly(.{ ._, .ret });
1951	1988	},
1952	1989	else => return self.fail(
1953	1990	"TODO implement {s} for {}",
...	...	@@ -2406,7 +2443,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2406	2443	else
2407	2444	try self.copyToTmpRegister(src_ty, src_mcv);
2408	2445	try self.asmRegisterRegisterImmediate(
2409		.vcvtps2ph,
	2446	.{ ._, .vcvtps2ph },
2410	2447	dst_reg,
2411	2448	mat_src_reg.to128(),
2412	2449	Immediate.u(0b1_00),
...	...	@@ -2418,12 +2455,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2418	2455	}
2419	2456	} else if (src_bits == 64 and dst_bits == 32) {
2420	2457	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2421		.vcvtsd2ss,
	2458	.{ ._, .vcvtsd2ss },
2422	2459	dst_reg,
2423	2460	dst_reg,
2424	2461	src_mcv.mem(.qword),
2425	2462	) else try self.asmRegisterRegisterRegister(
2426		.vcvtsd2ss,
	2463	.{ ._, .vcvtsd2ss },
2427	2464	dst_reg,
2428	2465	dst_reg,
2429	2466	(if (src_mcv.isRegister())
...	...	@@ -2431,11 +2468,11 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2431	2468	else
2432	2469	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2433	2470	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2434		.cvtsd2ss,
	2471	.{ ._, .cvtsd2ss },
2435	2472	dst_reg,
2436	2473	src_mcv.mem(.qword),
2437	2474	) else try self.asmRegisterRegister(
2438		.cvtsd2ss,
	2475	.{ ._, .cvtsd2ss },
2439	2476	dst_reg,
2440	2477	(if (src_mcv.isRegister())
2441	2478	src_mcv.getReg().?
...	...	@@ -2469,22 +2506,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2469	2506	src_mcv.getReg().?
2470	2507	else
2471	2508	try self.copyToTmpRegister(src_ty, src_mcv);
2472		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
	2509	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
2473	2510	switch (dst_bits) {
2474	2511	32 => {},
2475		64 => try self.asmRegisterRegisterRegister(.vcvtss2sd, dst_reg, dst_reg, dst_reg),
	2512	64 => try self.asmRegisterRegisterRegister(.{ ._, .vcvtss2sd }, dst_reg, dst_reg, dst_reg),
2476	2513	else => return self.fail("TODO implement airFpext from {} to {}", .{
2477	2514	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2478	2515	}),
2479	2516	}
2480	2517	} else if (src_bits == 32 and dst_bits == 64) {
2481	2518	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2482		.vcvtss2sd,
	2519	.{ ._, .vcvtss2sd },
2483	2520	dst_reg,
2484	2521	dst_reg,
2485	2522	src_mcv.mem(.dword),
2486	2523	) else try self.asmRegisterRegisterRegister(
2487		.vcvtss2sd,
	2524	.{ ._, .vcvtss2sd },
2488	2525	dst_reg,
2489	2526	dst_reg,
2490	2527	(if (src_mcv.isRegister())
...	...	@@ -2492,11 +2529,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2492	2529	else
2493	2530	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2494	2531	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2495		.cvtss2sd,
	2532	.{ ._, .cvtss2sd },
2496	2533	dst_reg,
2497	2534	src_mcv.mem(.dword),
2498	2535	) else try self.asmRegisterRegister(
2499		.cvtss2sd,
	2536	.{ ._, .cvtss2sd },
2500	2537	dst_reg,
2501	2538	(if (src_mcv.isRegister())
2502	2539	src_mcv.getReg().?
...	...	@@ -2537,12 +2574,12 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2537	2574	switch (dst_mcv) {
2538	2575	.register => \|dst_reg\| {
2539	2576	const min_abi_size = @min(dst_abi_size, src_abi_size);
2540		const tag: Mir.Inst.Tag = switch (signedness) {
2541		.signed => if (min_abi_size >= 4) .movsxd else .movsx,
2542		.unsigned => if (min_abi_size >= 4) .mov else .movzx,
	2577	const tag: Mir.Inst.FixedTag = switch (signedness) {
	2578	.signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
	2579	.unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx },
2543	2580	};
2544		const dst_alias = switch (tag) {
2545		.movsx, .movsxd => dst_reg.to64(),
	2581	const dst_alias = switch (tag[1]) {
	2582	.movsx => dst_reg.to64(),
2546	2583	.mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
2547	2584	else => unreachable,
2548	2585	};
...	...	@@ -2570,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2570	2607	try self.genCopy(min_ty, dst_mcv, src_mcv);
2571	2608	const extra = dst_abi_size * 8 - dst_int_info.bits;
2572	2609	if (extra > 0) {
2573		try self.genShiftBinOpMir(switch (signedness) {
2574		.signed => .sal,
2575		.unsigned => .shl,
2576		}, dst_ty, dst_mcv, .{ .immediate = extra });
2577		try self.genShiftBinOpMir(switch (signedness) {
2578		.signed => .sar,
2579		.unsigned => .shr,
2580		}, dst_ty, dst_mcv, .{ .immediate = extra });
	2610	try self.genShiftBinOpMir(
	2611	switch (signedness) {
	2612	.signed => .{ ._l, .sa },
	2613	.unsigned => .{ ._l, .sh },
	2614	},
	2615	dst_ty,
	2616	dst_mcv,
	2617	.{ .immediate = extra },
	2618	);
	2619	try self.genShiftBinOpMir(
	2620	switch (signedness) {
	2621	.signed => .{ ._r, .sa },
	2622	.unsigned => .{ ._r, .sh },
	2623	},
	2624	dst_ty,
	2625	dst_mcv,
	2626	.{ .immediate = extra },
	2627	);
2581	2628	}
2582	2629	},
2583	2630	}
...	...	@@ -2762,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2762	2809	const reg_bits = self.regBitSize(ty);
2763	2810	const cc: Condition = if (ty.isSignedInt()) cc: {
2764	2811	try self.genSetReg(limit_reg, ty, dst_mcv);
2765		try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2766		try self.genBinOpMir(.xor, ty, limit_mcv, .{
	2812	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	2813	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2767	2814	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2768	2815	});
2769	2816	break :cc .o;
...	...	@@ -2773,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2773	2820	});
2774	2821	break :cc .c;
2775	2822	};
2776		try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
	2823	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
2777	2824
2778	2825	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2779	2826	try self.asmCmovccRegisterRegister(
...	...	@@ -2813,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2813	2860	const reg_bits = self.regBitSize(ty);
2814	2861	const cc: Condition = if (ty.isSignedInt()) cc: {
2815	2862	try self.genSetReg(limit_reg, ty, dst_mcv);
2816		try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2817		try self.genBinOpMir(.xor, ty, limit_mcv, .{
	2863	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	2864	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2818	2865	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2819	2866	});
2820	2867	break :cc .o;
...	...	@@ -2822,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2822	2869	try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
2823	2870	break :cc .c;
2824	2871	};
2825		try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
	2872	try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
2826	2873
2827	2874	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2828	2875	try self.asmCmovccRegisterRegister(
...	...	@@ -2864,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
2864	2911	const reg_bits = self.regBitSize(ty);
2865	2912	const cc: Condition = if (ty.isSignedInt()) cc: {
2866	2913	try self.genSetReg(limit_reg, ty, lhs_mcv);
2867		try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
2868		try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2869		try self.genBinOpMir(.xor, ty, limit_mcv, .{
	2914	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
	2915	try self.genShiftBinOpMir(.{ ._, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	2916	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2870	2917	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2871	2918	});
2872	2919	break :cc .o;
...	...	@@ -2979,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2979	3026	};
2980	3027	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
2981	3028
2982		try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
	3029	try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs);
2983	3030	const cc = Condition.ne;
2984	3031
2985	3032	const tuple_ty = self.air.typeOfIndex(inst);
...	...	@@ -3066,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(
3066	3113	src_mcv;
3067	3114	try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
3068	3115	try self.truncateRegister(hi_limb_ty, scratch_reg);
3069		try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
	3116	try self.genBinOpMir(.{ ._, .cmp }, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
3070	3117
3071	3118	const eq_reg = temp_regs[2];
3072	3119	if (overflow_cc) \|_\| {
3073	3120	try self.asmSetccRegister(eq_reg.to8(), .ne);
3074		try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
	3121	try self.genBinOpMir(
	3122	.{ ._, .@"or" },
	3123	Type.u8,
	3124	.{ .register = overflow_reg },
	3125	.{ .register = eq_reg },
	3126	);
3075	3127	}
3076	3128
3077	3129	const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
...	...	@@ -3200,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3200	3252	/// Generates signed or unsigned integer multiplication/division.
3201	3253	/// Clobbers .rax and .rdx registers.
3202	3254	/// Quotient is saved in .rax and remainder in .rdx.
3203		fn genIntMulDivOpMir(
3204		self: *Self,
3205		tag: Mir.Inst.Tag,
3206		ty: Type,
3207		lhs: MCValue,
3208		rhs: MCValue,
3209		) !void {
	3255	fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
3210	3256	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
3211	3257	if (abi_size > 8) {
3212	3258	return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
3213	3259	}
3214	3260
3215	3261	try self.genSetReg(.rax, ty, lhs);
3216		switch (tag) {
	3262	switch (tag[1]) {
3217	3263	else => unreachable,
3218		.mul, .imul => {},
3219		.div => try self.asmRegisterRegister(.xor, .edx, .edx),
3220		.idiv => switch (self.regBitSize(ty)) {
3221		8 => try self.asmOpOnly(.cbw),
3222		16 => try self.asmOpOnly(.cwd),
3223		32 => try self.asmOpOnly(.cdq),
3224		64 => try self.asmOpOnly(.cqo),
	3264	.mul => {},
	3265	.div => switch (tag[0]) {
	3266	._ => try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx),
	3267	.i_ => switch (self.regBitSize(ty)) {
	3268	8 => try self.asmOpOnly(.{ ._, .cbw }),
	3269	16 => try self.asmOpOnly(.{ ._, .cwd }),
	3270	32 => try self.asmOpOnly(.{ ._, .cdq }),
	3271	64 => try self.asmOpOnly(.{ ._, .cqo }),
	3272	else => unreachable,
	3273	},
3225	3274	else => unreachable,
3226	3275	},
3227	3276	}
...	...	@@ -3259,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
3259	3308	const divisor_lock = self.register_manager.lockReg(divisor);
3260	3309	defer if (divisor_lock) \|lock\| self.register_manager.unlockReg(lock);
3261	3310
3262		try self.genIntMulDivOpMir(switch (int_info.signedness) {
3263		.signed => .idiv,
3264		.unsigned => .div,
3265		}, ty, .{ .register = dividend }, .{ .register = divisor });
	3311	try self.genIntMulDivOpMir(
	3312	switch (int_info.signedness) {
	3313	.signed => .{ .i_, .div },
	3314	.unsigned => .{ ._, .div },
	3315	},
	3316	ty,
	3317	.{ .register = dividend },
	3318	.{ .register = divisor },
	3319	);
3266	3320
3267	3321	try self.asmRegisterRegister(
3268		.xor,
	3322	.{ ._, .xor },
3269	3323	registerAlias(divisor, abi_size),
3270	3324	registerAlias(dividend, abi_size),
3271	3325	);
3272	3326	try self.asmRegisterImmediate(
3273		.sar,
	3327	.{ ._r, .sa },
3274	3328	registerAlias(divisor, abi_size),
3275	3329	Immediate.u(int_info.bits - 1),
3276	3330	);
3277	3331	try self.asmRegisterRegister(
3278		.@"test",
	3332	.{ ._, .@"test" },
3279	3333	registerAlias(.rdx, abi_size),
3280	3334	registerAlias(.rdx, abi_size),
3281	3335	);
...	...	@@ -3284,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
3284	3338	registerAlias(.rdx, abi_size),
3285	3339	.z,
3286	3340	);
3287		try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });
	3341	try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
3288	3342	return MCValue{ .register = divisor };
3289	3343	}
3290	3344
...	...	@@ -3406,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
3406	3460	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
3407	3461	if (err_off > 0) {
3408	3462	const shift = @intCast(u6, err_off * 8);
3409		try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });
	3463	try self.genShiftBinOpMir(
	3464	.{ ._r, .sh },
	3465	err_union_ty,
	3466	result,
	3467	.{ .immediate = shift },
	3468	);
3410	3469	} else {
3411	3470	try self.truncateRegister(Type.anyerror, result.register);
3412	3471	}
...	...	@@ -3458,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(
3458	3517	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
3459	3518	if (payload_off > 0) {
3460	3519	const shift = @intCast(u6, payload_off * 8);
3461		try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });
	3520	try self.genShiftBinOpMir(
	3521	.{ ._r, .sh },
	3522	err_union_ty,
	3523	result_mcv,
	3524	.{ .immediate = shift },
	3525	);
3462	3526	} else {
3463	3527	try self.truncateRegister(payload_ty, result_mcv.register);
3464	3528	}
...	...	@@ -3495,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3495	3559	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3496	3560	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3497	3561	try self.asmRegisterMemory(
3498		.mov,
	3562	.{ ._, .mov },
3499	3563	registerAlias(dst_reg, err_abi_size),
3500	3564	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3501	3565	.base = .{ .reg = src_reg },
...	...	@@ -3533,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
3533	3597	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3534	3598	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3535	3599	try self.asmRegisterMemory(
3536		.lea,
	3600	.{ ._, .lea },
3537	3601	registerAlias(dst_reg, dst_abi_size),
3538	3602	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3539	3603	);
...	...	@@ -3559,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3559	3623	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3560	3624	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3561	3625	try self.asmMemoryImmediate(
3562		.mov,
	3626	.{ ._, .mov },
3563	3627	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3564	3628	.base = .{ .reg = src_reg },
3565	3629	.disp = err_off,
...	...	@@ -3580,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3580	3644	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3581	3645	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3582	3646	try self.asmRegisterMemory(
3583		.lea,
	3647	.{ ._, .lea },
3584	3648	registerAlias(dst_reg, dst_abi_size),
3585	3649	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3586	3650	);
...	...	@@ -3631,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
3631	3695	else => unreachable,
3632	3696
3633	3697	.register => \|opt_reg\| try self.asmRegisterImmediate(
3634		.bts,
	3698	.{ ._s, .bt },
3635	3699	opt_reg,
3636	3700	Immediate.u(@intCast(u6, pl_abi_size * 8)),
3637	3701	),
3638	3702
3639	3703	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
3640		.mov,
	3704	.{ ._, .mov },
3641	3705	Memory.sib(.byte, .{
3642	3706	.base = .{ .frame = frame_addr.index },
3643	3707	.disp = frame_addr.off + pl_abi_size,
...	...	@@ -3749,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
3749	3813
3750	3814	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3751	3815	try self.asmRegisterMemory(
3752		.lea,
	3816	.{ ._, .lea },
3753	3817	registerAlias(dst_reg, dst_abi_size),
3754	3818	Memory.sib(.qword, .{
3755	3819	.base = .{ .reg = src_reg },
...	...	@@ -3823,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
3823	3887	try self.genSetReg(addr_reg, Type.usize, slice_mcv);
3824	3888	// TODO we could allocate register here, but need to expect addr register and potentially
3825	3889	// offset register.
3826		try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{
	3890	try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
3827	3891	.register = offset_reg,
3828	3892	});
3829	3893	return MCValue{ .register = addr_reg.to64() };
...	...	@@ -3881,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3881	3945	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
3882	3946	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
3883	3947	try self.asmRegisterMemory(
3884		.lea,
	3948	.{ ._, .lea },
3885	3949	addr_reg,
3886	3950	Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
3887	3951	);
3888	3952	},
3889	3953	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
3890		.lea,
	3954	.{ ._, .lea },
3891	3955	addr_reg,
3892	3956	Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
3893	3957	),
...	...	@@ -3903,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3903	3967	// TODO we could allocate register here, but need to expect addr register and potentially
3904	3968	// offset register.
3905	3969	const dst_mcv = try self.allocRegOrMem(inst, false);
3906		try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
	3970	try self.genBinOpMir(
	3971	.{ ._, .add },
	3972	Type.usize,
	3973	.{ .register = addr_reg },
	3974	.{ .register = offset_reg },
	3975	);
3907	3976	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3908	3977
3909	3978	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -3937,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
3937	4006	try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3938	4007	const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);
3939	4008	defer self.register_manager.unlockReg(elem_ptr_lock);
3940		try self.asmRegisterRegister(.add, elem_ptr_reg, offset_reg);
	4009	try self.asmRegisterRegister(
	4010	.{ ._, .add },
	4011	elem_ptr_reg,
	4012	offset_reg,
	4013	);
3941	4014
3942	4015	const dst_mcv = try self.allocRegOrMem(inst, true);
3943	4016	const dst_lock = switch (dst_mcv) {
...	...	@@ -3977,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
3977	4050	defer self.register_manager.unlockReg(offset_reg_lock);
3978	4051
3979	4052	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
3980		try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
	4053	try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg });
3981	4054
3982	4055	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
3983	4056	}
...	...	@@ -4010,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
4010	4083	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
4011	4084	// TODO reusing the operand
4012	4085	const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
4013		try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
	4086	try self.genBinOpMir(
	4087	.{ ._, .add },
	4088	ptr_union_ty,
	4089	.{ .register = reg },
	4090	.{ .immediate = layout.payload_size },
	4091	);
4014	4092	break :blk MCValue{ .register = reg };
4015	4093	} else ptr;
4016	4094
...	...	@@ -4063,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
4063	4141	else
4064	4142	0;
4065	4143	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
4066		try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });
	4144	try self.genShiftBinOpMir(.{ ._r, .sh }, Type.usize, result, .{ .immediate = shift });
4067	4145	break :blk MCValue{
4068	4146	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
4069	4147	};
...	...	@@ -4100,11 +4178,11 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4100	4178	const src_bits = src_ty.bitSize(self.target.*);
4101	4179	if (self.hasFeature(.lzcnt)) {
4102	4180	if (src_bits <= 64) {
4103		try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
	4181	try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
4104	4182
4105	4183	const extra_bits = self.regExtraBits(src_ty);
4106	4184	if (extra_bits > 0) {
4107		try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
	4185	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
4108	4186	}
4109	4187	} else if (src_bits <= 128) {
4110	4188	const tmp_reg = try self.register_manager.allocReg(null, gp);
...	...	@@ -4112,13 +4190,23 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4112	4190	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4113	4191	defer self.register_manager.unlockReg(tmp_lock);
4114	4192
4115		try self.genBinOpMir(.lzcnt, Type.u64, dst_mcv, mat_src_mcv);
4116		try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
4117		try self.genBinOpMir(.lzcnt, Type.u64, tmp_mcv, mat_src_mcv.address().offset(8).deref());
	4193	try self.genBinOpMir(.{ ._, .lzcnt }, Type.u64, dst_mcv, mat_src_mcv);
	4194	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
	4195	try self.genBinOpMir(
	4196	.{ ._, .lzcnt },
	4197	Type.u64,
	4198	tmp_mcv,
	4199	mat_src_mcv.address().offset(8).deref(),
	4200	);
4118	4201	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
4119	4202
4120	4203	if (src_bits < 128) {
4121		try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = 128 - src_bits });
	4204	try self.genBinOpMir(
	4205	.{ ._, .sub },
	4206	dst_ty,
	4207	dst_mcv,
	4208	.{ .immediate = 128 - src_bits },
	4209	);
4122	4210	}
4123	4211	} else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
4124	4212	break :result dst_mcv;
...	...	@@ -4130,7 +4218,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4130	4218	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4131	4219	.immediate = src_bits ^ (src_bits - 1),
4132	4220	});
4133		try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
	4221	try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
4134	4222
4135	4223	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4136	4224	try self.asmCmovccRegisterRegister(
...	...	@@ -4139,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4139	4227	.z,
4140	4228	);
4141	4229
4142		try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
	4230	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
4143	4231	} else {
4144	4232	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
4145	4233	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
4146	4234	});
4147		try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
	4235	try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
4148	4236
4149	4237	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4150	4238	try self.asmCmovccRegisterRegister(
...	...	@@ -4154,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
4154	4242	);
4155	4243
4156	4244	try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
4157		try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
	4245	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
4158	4246	}
4159	4247	break :result dst_mcv;
4160	4248	};
...	...	@@ -4195,7 +4283,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4195	4283	break :tmp dst_mcv;
4196	4284	};
4197	4285	try self.genBinOpMir(
4198		.@"or",
	4286	.{ ._, .@"or" },
4199	4287	src_ty,
4200	4288	tmp_mcv,
4201	4289	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
...	...	@@ -4203,7 +4291,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4203	4291	);
4204	4292	break :masked tmp_mcv;
4205	4293	} else mat_src_mcv;
4206		try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
	4294	try self.genBinOpMir(.{ ._, .tzcnt }, src_ty, dst_mcv, masked_mcv);
4207	4295	} else if (src_bits <= 128) {
4208	4296	const tmp_reg = try self.register_manager.allocReg(null, gp);
4209	4297	const tmp_mcv = MCValue{ .register = tmp_reg };
...	...	@@ -4213,16 +4301,16 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4213	4301	const masked_mcv = if (src_bits < 128) masked: {
4214	4302	try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
4215	4303	try self.genBinOpMir(
4216		.@"or",
	4304	.{ ._, .@"or" },
4217	4305	Type.u64,
4218	4306	dst_mcv,
4219	4307	.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
4220	4308	);
4221	4309	break :masked dst_mcv;
4222	4310	} else mat_src_mcv.address().offset(8).deref();
4223		try self.genBinOpMir(.tzcnt, Type.u64, dst_mcv, masked_mcv);
4224		try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
4225		try self.genBinOpMir(.tzcnt, Type.u64, tmp_mcv, mat_src_mcv);
	4311	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
	4312	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
	4313	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
4226	4314	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
4227	4315	} else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
4228	4316	break :result dst_mcv;
...	...	@@ -4232,7 +4320,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
4232	4320	return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
4233	4321
4234	4322	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
4235		try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
	4323	try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
4236	4324
4237	4325	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
4238	4326	try self.asmCmovccRegisterRegister(
...	...	@@ -4270,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4270	4358	.{ .register = try self.register_manager.allocReg(inst, gp) };
4271	4359
4272	4360	const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
4273		try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);
	4361	try self.genBinOpMir(.{ ._, .popcnt }, popcnt_ty, dst_mcv, mat_src_mcv);
4274	4362	break :result dst_mcv;
4275	4363	}
4276	4364
...	...	@@ -4301,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
4301	4389	undefined;
4302	4390
4303	4391	// dst = operand
4304		try self.asmRegisterRegister(.mov, tmp, dst);
	4392	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4305	4393	// tmp = operand
4306		try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
	4394	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
4307	4395	// tmp = operand >> 1
4308	4396	if (src_abi_size > 4) {
4309		try self.asmRegisterImmediate(.mov, imm, imm_0_1);
4310		try self.asmRegisterRegister(.@"and", tmp, imm);
4311		} else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
	4397	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
	4398	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4399	} else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
4312	4400	// tmp = (operand >> 1) & 0x55...55
4313		try self.asmRegisterRegister(.sub, dst, tmp);
	4401	try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp);
4314	4402	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
4315		try self.asmRegisterRegister(.mov, tmp, dst);
	4403	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4316	4404	// tmp = temp1
4317		try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
	4405	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
4318	4406	// dst = temp1 >> 2
4319	4407	if (src_abi_size > 4) {
4320		try self.asmRegisterImmediate(.mov, imm, imm_00_11);
4321		try self.asmRegisterRegister(.@"and", tmp, imm);
4322		try self.asmRegisterRegister(.@"and", dst, imm);
	4408	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
	4409	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4410	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4323	4411	} else {
4324		try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
4325		try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
	4412	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
	4413	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
4326	4414	}
4327	4415	// tmp = temp1 & 0x33...33
4328	4416	// dst = (temp1 >> 2) & 0x33...33
4329		try self.asmRegisterRegister(.add, tmp, dst);
	4417	try self.asmRegisterRegister(.{ ._, .add }, tmp, dst);
4330	4418	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
4331		try self.asmRegisterRegister(.mov, dst, tmp);
	4419	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
4332	4420	// dst = temp2
4333		try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));
	4421	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
4334	4422	// tmp = temp2 >> 4
4335		try self.asmRegisterRegister(.add, dst, tmp);
	4423	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
4336	4424	// dst = temp2 + (temp2 >> 4)
4337	4425	if (src_abi_size > 4) {
4338		try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
4339		try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);
4340		try self.asmRegisterRegister(.@"and", dst, imm);
4341		try self.asmRegisterRegister(.imul, dst, tmp);
	4426	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
	4427	try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001);
	4428	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
	4429	try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp);
4342	4430	} else {
4343		try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
	4431	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4344	4432	if (src_abi_size > 1) {
4345		try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);
	4433	try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001);
4346	4434	}
4347	4435	}
4348	4436	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
4349	4437	// dst = temp3 * 0x01...01
4350	4438	if (src_abi_size > 1) {
4351		try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));
	4439	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
4352	4440	}
4353	4441	// dst = (temp3 * 0x01...01) >> (bits - 8)
4354	4442	}
...	...	@@ -4377,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4377	4465	16 => if ((mem_ok or src_mcv.isRegister()) and
4378	4466	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
4379	4467	{
4380		try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });
	4468	try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 });
4381	4469	return src_mcv;
4382	4470	},
4383	4471	32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
4384		try self.genUnOpMir(.bswap, src_ty, src_mcv);
	4472	try self.genUnOpMir(.{ ._, .bswap }, src_ty, src_mcv);
4385	4473	return src_mcv;
4386	4474	},
4387	4475	}
...	...	@@ -4398,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4398	4486	try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
4399	4487	switch (src_bits) {
4400	4488	else => unreachable,
4401		16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
4402		32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),
	4489	16 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
	4490	32, 64 => try self.genUnOpMir(.{ ._, .bswap }, src_ty, dst_mcv),
4403	4491	}
4404		} else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
	4492	} else try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4405	4493	return dst_mcv;
4406	4494	}
4407	4495
...	...	@@ -4410,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4410	4498	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4411	4499	defer self.register_manager.unlockReg(dst_lock);
4412	4500
4413		try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
	4501	try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4414	4502	return dst_mcv;
4415	4503	}
4416	4504
...	...	@@ -4424,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
4424	4512	switch (self.regExtraBits(src_ty)) {
4425	4513	0 => {},
4426	4514	else => \|extra\| try self.genBinOpMir(
4427		if (src_ty.isSignedInt()) .sar else .shr,
	4515	if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4428	4516	src_ty,
4429	4517	dst_mcv,
4430	4518	.{ .immediate = extra },
...	...	@@ -4464,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4464	4552	const imm_0_1 = Immediate.u(mask / 0b1_1);
4465	4553
4466	4554	// dst = temp1 = bswap(operand)
4467		try self.asmRegisterRegister(.mov, tmp, dst);
	4555	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4468	4556	// tmp = temp1
4469		try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));
	4557	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
4470	4558	// dst = temp1 >> 4
4471	4559	if (src_abi_size > 4) {
4472		try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
4473		try self.asmRegisterRegister(.@"and", tmp, imm);
4474		try self.asmRegisterRegister(.@"and", dst, imm);
	4560	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
	4561	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4562	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4475	4563	} else {
4476		try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);
4477		try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
	4564	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
	4565	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4478	4566	}
4479	4567	// tmp = temp1 & 0x0F...0F
4480	4568	// dst = (temp1 >> 4) & 0x0F...0F
4481		try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));
	4569	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
4482	4570	// tmp = (temp1 & 0x0F...0F) << 4
4483		try self.asmRegisterRegister(.@"or", dst, tmp);
	4571	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
4484	4572	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
4485		try self.asmRegisterRegister(.mov, tmp, dst);
	4573	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4486	4574	// tmp = temp2
4487		try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
	4575	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
4488	4576	// dst = temp2 >> 2
4489	4577	if (src_abi_size > 4) {
4490		try self.asmRegisterImmediate(.mov, imm, imm_00_11);
4491		try self.asmRegisterRegister(.@"and", tmp, imm);
4492		try self.asmRegisterRegister(.@"and", dst, imm);
	4578	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
	4579	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4580	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4493	4581	} else {
4494		try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
4495		try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
	4582	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
	4583	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
4496	4584	}
4497	4585	// tmp = temp2 & 0x33...33
4498	4586	// dst = (temp2 >> 2) & 0x33...33
4499	4587	try self.asmRegisterMemory(
4500		.lea,
	4588	.{ ._, .lea },
4501	4589	if (src_abi_size > 4) tmp.to64() else tmp.to32(),
4502	4590	Memory.sib(.qword, .{
4503	4591	.base = .{ .reg = dst.to64() },
...	...	@@ -4505,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4505	4593	}),
4506	4594	);
4507	4595	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
4508		try self.asmRegisterRegister(.mov, dst, tmp);
	4596	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
4509	4597	// dst = temp3
4510		try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
	4598	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
4511	4599	// tmp = temp3 >> 1
4512	4600	if (src_abi_size > 4) {
4513		try self.asmRegisterImmediate(.mov, imm, imm_0_1);
4514		try self.asmRegisterRegister(.@"and", dst, imm);
4515		try self.asmRegisterRegister(.@"and", tmp, imm);
	4601	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
	4602	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
	4603	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4516	4604	} else {
4517		try self.asmRegisterImmediate(.@"and", dst, imm_0_1);
4518		try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
	4605	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1);
	4606	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
4519	4607	}
4520	4608	// dst = temp3 & 0x55...55
4521	4609	// tmp = (temp3 >> 1) & 0x55...55
4522	4610	try self.asmRegisterMemory(
4523		.lea,
	4611	.{ ._, .lea },
4524	4612	if (src_abi_size > 4) dst.to64() else dst.to32(),
4525	4613	Memory.sib(.qword, .{
4526	4614	.base = .{ .reg = tmp.to64() },
...	...	@@ -4533,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4533	4621	switch (self.regExtraBits(src_ty)) {
4534	4622	0 => {},
4535	4623	else => \|extra\| try self.genBinOpMir(
4536		if (src_ty.isSignedInt()) .sar else .shr,
	4624	if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4537	4625	src_ty,
4538	4626	dst_mcv,
4539	4627	.{ .immediate = extra },
...	...	@@ -4590,8 +4678,8 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
4590	4678	try self.genBinOpMir(switch (ty_bits) {
4591	4679	// No point using an extra prefix byte for *pd which performs the same operation.
4592	4680	16, 32, 64, 128 => switch (tag) {
4593		.neg => .xorps,
4594		.fabs => .andnps,
	4681	.neg => .{ ._, .xorps },
	4682	.fabs => .{ ._, .andnps },
4595	4683	else => unreachable,
4596	4684	},
4597	4685	80 => return self.fail("TODO implement airFloatSign for {}", .{
...	...	@@ -4622,25 +4710,25 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
4622	4710	if (!self.hasFeature(.sse4_1))
4623	4711	return self.fail("TODO implement genRound without sse4_1 feature", .{});
4624	4712
4625		const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
	4713	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
4626	4714	.Float => switch (ty.floatBits(self.target.*)) {
4627		32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
4628		64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
	4715	32 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
	4716	64 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
4629	4717	16, 80, 128 => null,
4630	4718	else => unreachable,
4631	4719	},
4632	4720	.Vector => switch (ty.childType().zigTypeTag()) {
4633	4721	.Float => switch (ty.childType().floatBits(self.target.*)) {
4634	4722	32 => switch (ty.vectorLen()) {
4635		1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
4636		2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
4637		5...8 => if (self.hasFeature(.avx)) .vroundps else null,
	4723	1 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
	4724	2...4 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else .{ ._, .roundps },
	4725	5...8 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else null,
4638	4726	else => null,
4639	4727	},
4640	4728	64 => switch (ty.vectorLen()) {
4641		1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
4642		2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
4643		3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
	4729	1 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
	4730	2 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else .{ ._, .roundpd },
	4731	3...4 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else null,
4644	4732	else => null,
4645	4733	},
4646	4734	16, 80, 128 => null,
...	...	@@ -4655,7 +4743,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
4655	4743
4656	4744	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4657	4745	const dst_alias = registerAlias(dst_reg, abi_size);
4658		switch (mir_tag) {
	4746	switch (mir_tag[1]) {
4659	4747	.vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
4660	4748	mir_tag,
4661	4749	dst_alias,
...	...	@@ -4704,25 +4792,25 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4704	4792	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
4705	4793
4706	4794	const result: MCValue = result: {
4707		const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
	4795	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
4708	4796	.Float => switch (ty.floatBits(self.target.*)) {
4709	4797	16 => if (self.hasFeature(.f16c)) {
4710	4798	const mat_src_reg = if (src_mcv.isRegister())
4711	4799	src_mcv.getReg().?
4712	4800	else
4713	4801	try self.copyToTmpRegister(ty, src_mcv);
4714		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
4715		try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
	4802	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
	4803	try self.asmRegisterRegisterRegister(.{ ._, .vsqrtss }, dst_reg, dst_reg, dst_reg);
4716	4804	try self.asmRegisterRegisterImmediate(
4717		.vcvtps2ph,
	4805	.{ ._, .vcvtps2ph },
4718	4806	dst_reg,
4719	4807	dst_reg,
4720	4808	Immediate.u(0b1_00),
4721	4809	);
4722	4810	break :result dst_mcv;
4723	4811	} else null,
4724		32 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
4725		64 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
	4812	32 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
	4813	64 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
4726	4814	80, 128 => null,
4727	4815	else => unreachable,
4728	4816	},
...	...	@@ -4731,16 +4819,21 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4731	4819	16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
4732	4820	1 => {
4733	4821	try self.asmRegisterRegister(
4734		.vcvtph2ps,
	4822	.{ ._, .vcvtph2ps },
4735	4823	dst_reg,
4736	4824	(if (src_mcv.isRegister())
4737	4825	src_mcv.getReg().?
4738	4826	else
4739	4827	try self.copyToTmpRegister(ty, src_mcv)).to128(),
4740	4828	);
4741		try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
	4829	try self.asmRegisterRegisterRegister(
	4830	.{ ._, .vsqrtss },
	4831	dst_reg,
	4832	dst_reg,
	4833	dst_reg,
	4834	);
4742	4835	try self.asmRegisterRegisterImmediate(
4743		.vcvtps2ph,
	4836	.{ ._, .vcvtps2ph },
4744	4837	dst_reg,
4745	4838	dst_reg,
4746	4839	Immediate.u(0b1_00),
...	...	@@ -4750,22 +4843,22 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4750	4843	2...8 => {
4751	4844	const wide_reg = registerAlias(dst_reg, abi_size * 2);
4752	4845	if (src_mcv.isMemory()) try self.asmRegisterMemory(
4753		.vcvtph2ps,
	4846	.{ ._, .vcvtph2ps },
4754	4847	wide_reg,
4755	4848	src_mcv.mem(Memory.PtrSize.fromSize(
4756	4849	@intCast(u32, @divExact(wide_reg.bitSize(), 16)),
4757	4850	)),
4758	4851	) else try self.asmRegisterRegister(
4759		.vcvtph2ps,
	4852	.{ ._, .vcvtph2ps },
4760	4853	wide_reg,
4761	4854	(if (src_mcv.isRegister())
4762	4855	src_mcv.getReg().?
4763	4856	else
4764	4857	try self.copyToTmpRegister(ty, src_mcv)).to128(),
4765	4858	);
4766		try self.asmRegisterRegister(.vsqrtps, wide_reg, wide_reg);
	4859	try self.asmRegisterRegister(.{ ._, .vsqrtps }, wide_reg, wide_reg);
4767	4860	try self.asmRegisterRegisterImmediate(
4768		.vcvtps2ph,
	4861	.{ ._, .vcvtps2ph },
4769	4862	dst_reg,
4770	4863	wide_reg,
4771	4864	Immediate.u(0b1_00),
...	...	@@ -4775,15 +4868,15 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4775	4868	else => null,
4776	4869	} else null,
4777	4870	32 => switch (ty.vectorLen()) {
4778		1 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
4779		2...4 => if (self.hasFeature(.avx)) .vsqrtps else .sqrtps,
4780		5...8 => if (self.hasFeature(.avx)) .vsqrtps else null,
	4871	1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
	4872	2...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else .{ ._, .sqrtps },
	4873	5...8 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else null,
4781	4874	else => null,
4782	4875	},
4783	4876	64 => switch (ty.vectorLen()) {
4784		1 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
4785		2 => if (self.hasFeature(.avx)) .vsqrtpd else .sqrtpd,
4786		3...4 => if (self.hasFeature(.avx)) .vsqrtpd else null,
	4877	1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
	4878	2 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else .{ ._, .sqrtpd },
	4879	3...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else null,
4787	4880	else => null,
4788	4881	},
4789	4882	80, 128 => null,
...	...	@@ -4795,7 +4888,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4795	4888	})) \|tag\| tag else return self.fail("TODO implement airSqrt for {}", .{
4796	4889	ty.fmt(self.bin_file.options.module.?),
4797	4890	});
4798		switch (mir_tag) {
	4891	switch (mir_tag[1]) {
4799	4892	.vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
4800	4893	mir_tag,
4801	4894	dst_reg,
...	...	@@ -4911,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4911	5004	if (load_abi_size <= 8) {
4912	5005	const load_reg = registerAlias(dst_reg, load_abi_size);
4913	5006	try self.asmRegisterMemory(
4914		.mov,
	5007	.{ ._, .mov },
4915	5008	load_reg,
4916	5009	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4917	5010	.base = .{ .reg = ptr_reg },
4918	5011	.disp = val_byte_off,
4919	5012	}),
4920	5013	);
4921		try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
	5014	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
4922	5015	} else {
4923	5016	const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
4924	5017	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	...	@@ -4926,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4926	5019
4927	5020	const dst_alias = registerAlias(dst_reg, val_abi_size);
4928	5021	try self.asmRegisterMemory(
4929		.mov,
	5022	.{ ._, .mov },
4930	5023	dst_alias,
4931	5024	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4932	5025	.base = .{ .reg = ptr_reg },
...	...	@@ -4934,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4934	5027	}),
4935	5028	);
4936	5029	try self.asmRegisterMemory(
4937		.mov,
	5030	.{ ._, .mov },
4938	5031	tmp_reg,
4939	5032	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4940	5033	.base = .{ .reg = ptr_reg },
4941	5034	.disp = val_byte_off + 1,
4942	5035	}),
4943	5036	);
4944		try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
	5037	try self.asmRegisterRegisterImmediate(
	5038	.{ ._rd, .sh },
	5039	dst_alias,
	5040	tmp_reg,
	5041	Immediate.u(val_bit_off),
	5042	);
4945	5043	}
4946	5044
4947	5045	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
...	...	@@ -5047,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5047	5145	const part_mask_not = part_mask ^
5048	5146	(@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));
5049	5147	if (limb_abi_size <= 4) {
5050		try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not));
	5148	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
5051	5149	} else if (math.cast(i32, @bitCast(i64, part_mask_not))) \|small\| {
5052		try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small));
	5150	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
5053	5151	} else {
5054	5152	const part_mask_reg = try self.register_manager.allocReg(null, gp);
5055		try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not));
5056		try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
	5153	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
	5154	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
5057	5155	}
5058	5156
5059	5157	if (src_bit_size <= 64) {
...	...	@@ -5064,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
5064	5162
5065	5163	try self.genSetReg(tmp_reg, src_ty, src_mcv);
5066	5164	switch (limb_i) {
5067		0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
5068		1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
5069		.immediate = limb_abi_bits - src_bit_off,
5070		}),
	5165	0 => try self.genShiftBinOpMir(
	5166	.{ ._l, .sh },
	5167	src_ty,
	5168	tmp_mcv,
	5169	.{ .immediate = src_bit_off },
	5170	),
	5171	1 => try self.genShiftBinOpMir(
	5172	.{ ._r, .sh },
	5173	src_ty,
	5174	tmp_mcv,
	5175	.{ .immediate = limb_abi_bits - src_bit_off },
	5176	),
5071	5177	else => unreachable,
5072	5178	}
5073		try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
5074		try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
	5179	try self.genBinOpMir(.{ ._, .@"and" }, src_ty, tmp_mcv, .{ .immediate = part_mask });
	5180	try self.asmMemoryRegister(
	5181	.{ ._, .@"or" },
	5182	limb_mem,
	5183	registerAlias(tmp_reg, limb_abi_size),
	5184	);
5075	5185	} else return self.fail("TODO: implement packed store of {}", .{
5076	5186	src_ty.fmt(self.bin_file.options.module.?),
5077	5187	});
...	...	@@ -5171,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
5171	5281	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
5172	5282	else => mcv,
5173	5283	});
5174		try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
	5284	try self.genBinOpMir(.{ ._, .add }, Type.usize, dst_mcv, .{ .register = offset_reg });
5175	5285	break :result dst_mcv;
5176	5286	},
5177	5287	.indirect => \|reg_off\| break :result .{ .indirect = .{
...	...	@@ -5255,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5255	5365	if (load_abi_size <= 8) {
5256	5366	const load_reg = registerAlias(dst_reg, load_abi_size);
5257	5367	try self.asmRegisterMemory(
5258		.mov,
	5368	.{ ._, .mov },
5259	5369	load_reg,
5260	5370	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
5261	5371	.base = .{ .frame = frame_addr.index },
5262	5372	.disp = frame_addr.off + field_byte_off,
5263	5373	}),
5264	5374	);
5265		try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
	5375	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
5266	5376	} else {
5267	5377	const tmp_reg = registerAlias(
5268	5378	try self.register_manager.allocReg(null, gp),
...	...	@@ -5273,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5273	5383
5274	5384	const dst_alias = registerAlias(dst_reg, field_abi_size);
5275	5385	try self.asmRegisterMemory(
5276		.mov,
	5386	.{ ._, .mov },
5277	5387	dst_alias,
5278	5388	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
5279	5389	.base = .{ .frame = frame_addr.index },
...	...	@@ -5281,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5281	5391	}),
5282	5392	);
5283	5393	try self.asmRegisterMemory(
5284		.mov,
	5394	.{ ._, .mov },
5285	5395	tmp_reg,
5286	5396	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
5287	5397	.base = .{ .frame = frame_addr.index },
...	...	@@ -5289,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5289	5399	}),
5290	5400	);
5291	5401	try self.asmRegisterRegisterImmediate(
5292		.shrd,
	5402	.{ ._rd, .sh },
5293	5403	dst_alias,
5294	5404	tmp_reg,
5295	5405	Immediate.u(field_bit_off),
...	...	@@ -5325,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5325	5435	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
5326	5436
5327	5437	// Shift by struct_field_offset.
5328		try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_off });
	5438	try self.genShiftBinOpMir(
	5439	.{ ._r, .sh },
	5440	Type.usize,
	5441	dst_mcv,
	5442	.{ .immediate = field_off },
	5443	);
5329	5444
5330	5445	// Mask to field_bit_size bits
5331	5446	const field_bit_size = field_ty.bitSize(self.target.*);
5332	5447	const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);
5333	5448
5334	5449	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
5335		try self.genBinOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg });
	5450	try self.genBinOpMir(.{ ._, .@"and" }, Type.usize, dst_mcv, .{ .register = tmp_reg });
5336	5451
5337	5452	const signedness =
5338	5453	if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;
5339	5454	const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
5340	5455	if (signedness == .signed and field_byte_size < 8) {
5341	5456	try self.asmRegisterRegister(
5342		if (field_byte_size >= 4) .movsxd else .movsx,
	5457	if (field_byte_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
5343	5458	dst_mcv.register,
5344	5459	registerAlias(dst_mcv.register, field_byte_size),
5345	5460	);
...	...	@@ -5451,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
5451	5566
5452	5567	if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {
5453	5568	const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);
5454		try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask });
5455		} else try self.genUnOpMir(.not, limb_ty, limb_mcv);
	5569	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
	5570	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
5456	5571	}
5457	5572	},
5458		.neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),
	5573	.neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
5459	5574	else => unreachable,
5460	5575	}
5461	5576	return dst_mcv;
5462	5577	}
5463	5578
5464		fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
	5579	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
5465	5580	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
5466	5581	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
5467	5582	mir_tag,
...	...	@@ -5504,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
5504	5619	/// Clobbers .rcx for non-immediate shift value.
5505	5620	fn genShiftBinOpMir(
5506	5621	self: *Self,
5507		tag: Mir.Inst.Tag,
	5622	tag: Mir.Inst.FixedTag,
5508	5623	ty: Type,
5509	5624	lhs_mcv: MCValue,
5510	5625	shift_mcv: MCValue,
...	...	@@ -5589,16 +5704,16 @@ fn genShiftBinOpMir(
5589	5704	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5590	5705	defer self.register_manager.unlockReg(tmp_lock);
5591	5706
5592		const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {
5593		.shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },
5594		.shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },
	5707	const info: struct { offsets: [2]i32, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) {
	5708	._l => .{ .offsets = .{ 0, 8 }, .double_tag = .{ ._ld, .sh } },
	5709	._r => .{ .offsets = .{ 8, 0 }, .double_tag = .{ ._rd, .sh } },
5595	5710	else => unreachable,
5596	5711	};
5597	5712	switch (lhs_mcv) {
5598	5713	.load_frame => \|dst_frame_addr\| switch (rhs_mcv) {
5599	5714	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
5600	5715	try self.asmRegisterMemory(
5601		.mov,
	5716	.{ ._, .mov },
5602	5717	tmp_reg,
5603	5718	Memory.sib(.qword, .{
5604	5719	.base = .{ .frame = dst_frame_addr.index },
...	...	@@ -5625,7 +5740,7 @@ fn genShiftBinOpMir(
5625	5740	} else {
5626	5741	assert(rhs_imm < 128);
5627	5742	try self.asmRegisterMemory(
5628		.mov,
	5743	.{ ._, .mov },
5629	5744	tmp_reg,
5630	5745	Memory.sib(.qword, .{
5631	5746	.base = .{ .frame = dst_frame_addr.index },
...	...	@@ -5636,34 +5751,30 @@ fn genShiftBinOpMir(
5636	5751	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
5637	5752	}
5638	5753	try self.asmMemoryRegister(
5639		.mov,
	5754	.{ ._, .mov },
5640	5755	Memory.sib(.qword, .{
5641	5756	.base = .{ .frame = dst_frame_addr.index },
5642	5757	.disp = dst_frame_addr.off + info.offsets[1],
5643	5758	}),
5644	5759	tmp_reg,
5645	5760	);
5646		switch (tag) {
5647		.shl, .sal, .shr => {
5648		try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
5649		try self.asmMemoryRegister(
5650		.mov,
5651		Memory.sib(.qword, .{
5652		.base = .{ .frame = dst_frame_addr.index },
5653		.disp = dst_frame_addr.off + info.offsets[0],
5654		}),
5655		tmp_reg,
5656		);
5657		},
5658		.sar => try self.asmMemoryImmediate(
5659		tag,
	5761	if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
	5762	tag,
	5763	Memory.sib(.qword, .{
	5764	.base = .{ .frame = dst_frame_addr.index },
	5765	.disp = dst_frame_addr.off + info.offsets[0],
	5766	}),
	5767	Immediate.u(63),
	5768	) else {
	5769	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
	5770	try self.asmMemoryRegister(
	5771	.{ ._, .mov },
5660	5772	Memory.sib(.qword, .{
5661	5773	.base = .{ .frame = dst_frame_addr.index },
5662	5774	.disp = dst_frame_addr.off + info.offsets[0],
5663	5775	}),
5664		Immediate.u(63),
5665		),
5666		else => unreachable,
	5776	tmp_reg,
	5777	);
5667	5778	}
5668	5779	},
5669	5780	else => {
...	...	@@ -5677,7 +5788,7 @@ fn genShiftBinOpMir(
5677	5788
5678	5789	try self.genSetReg(.cl, Type.u8, rhs_mcv);
5679	5790	try self.asmRegisterMemory(
5680		.mov,
	5791	.{ ._, .mov },
5681	5792	first_reg,
5682	5793	Memory.sib(.qword, .{
5683	5794	.base = .{ .frame = dst_frame_addr.index },
...	...	@@ -5685,32 +5796,28 @@ fn genShiftBinOpMir(
5685	5796	}),
5686	5797	);
5687	5798	try self.asmRegisterMemory(
5688		.mov,
	5799	.{ ._, .mov },
5689	5800	second_reg,
5690	5801	Memory.sib(.qword, .{
5691	5802	.base = .{ .frame = dst_frame_addr.index },
5692	5803	.disp = dst_frame_addr.off + info.offsets[1],
5693	5804	}),
5694	5805	);
5695		switch (tag) {
5696		.shl, .sal, .shr => try self.asmRegisterRegister(
5697		.xor,
5698		tmp_reg.to32(),
5699		tmp_reg.to32(),
5700		),
5701		.sar => {
5702		try self.asmRegisterRegister(.mov, tmp_reg, first_reg);
5703		try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
5704		},
5705		else => unreachable,
5706		}
	5806	if (tag[0] == ._r and tag[1] == .sa) {
	5807	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
	5808	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
	5809	} else try self.asmRegisterRegister(
	5810	.{ ._, .xor },
	5811	tmp_reg.to32(),
	5812	tmp_reg.to32(),
	5813	);
5707	5814	try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
5708	5815	try self.asmRegisterRegister(tag, first_reg, .cl);
5709		try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));
	5816	try self.asmRegisterImmediate(.{ ._, .cmp }, .cl, Immediate.u(64));
5710	5817	try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
5711	5818	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
5712	5819	try self.asmMemoryRegister(
5713		.mov,
	5820	.{ ._, .mov },
5714	5821	Memory.sib(.qword, .{
5715	5822	.base = .{ .frame = dst_frame_addr.index },
5716	5823	.disp = dst_frame_addr.off + info.offsets[1],
...	...	@@ -5718,7 +5825,7 @@ fn genShiftBinOpMir(
5718	5825	second_reg,
5719	5826	);
5720	5827	try self.asmMemoryRegister(
5721		.mov,
	5828	.{ ._, .mov },
5722	5829	Memory.sib(.qword, .{
5723	5830	.base = .{ .frame = dst_frame_addr.index },
5724	5831	.disp = dst_frame_addr.off + info.offsets[0],
...	...	@@ -5743,7 +5850,7 @@ fn genShiftBinOpMir(
5743	5850	/// Asserts .rcx is free.
5744	5851	fn genShiftBinOp(
5745	5852	self: *Self,
5746		tag: Air.Inst.Tag,
	5853	air_tag: Air.Inst.Tag,
5747	5854	maybe_inst: ?Air.Inst.Index,
5748	5855	lhs_mcv: MCValue,
5749	5856	rhs_mcv: MCValue,
...	...	@@ -5788,14 +5895,14 @@ fn genShiftBinOp(
5788	5895	};
5789	5896
5790	5897	const signedness = lhs_ty.intInfo(self.target.*).signedness;
5791		try self.genShiftBinOpMir(switch (tag) {
	5898	try self.genShiftBinOpMir(switch (air_tag) {
5792	5899	.shl, .shl_exact => switch (signedness) {
5793		.signed => .sal,
5794		.unsigned => .shl,
	5900	.signed => .{ ._l, .sa },
	5901	.unsigned => .{ ._l, .sh },
5795	5902	},
5796	5903	.shr, .shr_exact => switch (signedness) {
5797		.signed => .sar,
5798		.unsigned => .shr,
	5904	.signed => .{ ._r, .sa },
	5905	.unsigned => .{ ._r, .sh },
5799	5906	},
5800	5907	else => unreachable,
5801	5908	}, lhs_ty, dst_mcv, rhs_mcv);
...	...	@@ -5855,20 +5962,18 @@ fn genMulDivBinOp(
5855	5962	try self.register_manager.getReg(.rax, track_inst_rax);
5856	5963	try self.register_manager.getReg(.rdx, track_inst_rdx);
5857	5964
5858		const mir_tag: Mir.Inst.Tag = switch (signedness) {
	5965	try self.genIntMulDivOpMir(switch (signedness) {
5859	5966	.signed => switch (tag) {
5860		.mul, .mulwrap => .imul,
5861		.div_trunc, .div_exact, .rem => .idiv,
	5967	.mul, .mulwrap => .{ .i_, .mul },
	5968	.div_trunc, .div_exact, .rem => .{ .i_, .div },
5862	5969	else => unreachable,
5863	5970	},
5864	5971	.unsigned => switch (tag) {
5865		.mul, .mulwrap => .mul,
5866		.div_trunc, .div_exact, .rem => .div,
	5972	.mul, .mulwrap => .{ ._, .mul },
	5973	.div_trunc, .div_exact, .rem => .{ ._, .div },
5867	5974	else => unreachable,
5868	5975	},
5869		};
5870
5871		try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs);
	5976	}, ty, lhs, rhs);
5872	5977
5873	5978	if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
5874	5979	.mul, .mulwrap, .div_trunc, .div_exact => .rax,
...	...	@@ -5878,7 +5983,7 @@ fn genMulDivBinOp(
5878	5983
5879	5984	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
5880	5985	try self.asmMemoryRegister(
5881		.mov,
	5986	.{ ._, .mov },
5882	5987	Memory.sib(.qword, .{
5883	5988	.base = .{ .frame = dst_mcv.load_frame.index },
5884	5989	.disp = dst_mcv.load_frame.off,
...	...	@@ -5886,7 +5991,7 @@ fn genMulDivBinOp(
5886	5991	.rax,
5887	5992	);
5888	5993	try self.asmMemoryRegister(
5889		.mov,
	5994	.{ ._, .mov },
5890	5995	Memory.sib(.qword, .{
5891	5996	.base = .{ .frame = dst_mcv.load_frame.index },
5892	5997	.disp = dst_mcv.load_frame.off + 8,
...	...	@@ -5927,12 +6032,12 @@ fn genMulDivBinOp(
5927	6032	try self.copyToRegisterWithInstTracking(inst, ty, lhs)
5928	6033	else
5929	6034	.{ .register = try self.copyToTmpRegister(ty, lhs) };
5930		try self.genBinOpMir(.sub, ty, result, div_floor);
	6035	try self.genBinOpMir(.{ ._, .sub }, ty, result, div_floor);
5931	6036
5932	6037	return result;
5933	6038	},
5934	6039	.unsigned => {
5935		try self.genIntMulDivOpMir(.div, ty, lhs, rhs);
	6040	try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, rhs);
5936	6041	return .{ .register = registerAlias(.rdx, abi_size) };
5937	6042	},
5938	6043	}
...	...	@@ -5974,7 +6079,7 @@ fn genMulDivBinOp(
5974	6079	switch (signedness) {
5975	6080	.signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),
5976	6081	.unsigned => {
5977		try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs);
	6082	try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, actual_rhs);
5978	6083	return .{ .register = registerAlias(.rax, abi_size) };
5979	6084	},
5980	6085	}
...	...	@@ -6072,11 +6177,11 @@ fn genBinOp(
6072	6177	switch (air_tag) {
6073	6178	.add,
6074	6179	.addwrap,
6075		=> try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv),
	6180	=> try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv),
6076	6181
6077	6182	.sub,
6078	6183	.subwrap,
6079		=> try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv),
	6184	=> try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv),
6080	6185
6081	6186	.ptr_add,
6082	6187	.ptr_sub,
...	...	@@ -6088,22 +6193,27 @@ fn genBinOp(
6088	6193
6089	6194	const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
6090	6195	try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
6091		try self.genBinOpMir(switch (air_tag) {
6092		.ptr_add => .add,
6093		.ptr_sub => .sub,
6094		else => unreachable,
6095		}, lhs_ty, dst_mcv, tmp_mcv);
	6196	try self.genBinOpMir(
	6197	switch (air_tag) {
	6198	.ptr_add => .{ ._, .add },
	6199	.ptr_sub => .{ ._, .sub },
	6200	else => unreachable,
	6201	},
	6202	lhs_ty,
	6203	dst_mcv,
	6204	tmp_mcv,
	6205	);
6096	6206	},
6097	6207
6098	6208	.bool_or,
6099	6209	.bit_or,
6100		=> try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),
	6210	=> try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv),
6101	6211
6102	6212	.bool_and,
6103	6213	.bit_and,
6104		=> try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),
	6214	=> try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv),
6105	6215
6106		.xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
	6216	.xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv),
6107	6217
6108	6218	.min,
6109	6219	.max,
...	...	@@ -6129,7 +6239,7 @@ fn genBinOp(
6129	6239	};
6130	6240	defer if (mat_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
6131	6241
6132		try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);
	6242	try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
6133	6243
6134	6244	const int_info = lhs_ty.intInfo(self.target.*);
6135	6245	const cc: Condition = switch (int_info.signedness) {
...	...	@@ -6206,7 +6316,7 @@ fn genBinOp(
6206	6316	}
6207	6317
6208	6318	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
6209		const mir_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
	6319	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
6210	6320	else => unreachable,
6211	6321	.Float => switch (lhs_ty.floatBits(self.target.*)) {
6212	6322	16 => if (self.hasFeature(.f16c)) {
...	...	@@ -6215,13 +6325,13 @@ fn genBinOp(
6215	6325	defer self.register_manager.unlockReg(tmp_lock);
6216	6326
6217	6327	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6218		.vpinsrw,
	6328	.{ ._, .vpinsrw },
6219	6329	dst_reg,
6220	6330	dst_reg,
6221	6331	src_mcv.mem(.word),
6222	6332	Immediate.u(1),
6223	6333	) else try self.asmRegisterRegisterRegister(
6224		.vpunpcklwd,
	6334	.{ ._, .vpunpcklwd },
6225	6335	dst_reg,
6226	6336	dst_reg,
6227	6337	(if (src_mcv.isRegister())
...	...	@@ -6229,15 +6339,15 @@ fn genBinOp(
6229	6339	else
6230	6340	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6231	6341	);
6232		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6233		try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
	6342	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
	6343	try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
6234	6344	try self.asmRegisterRegisterRegister(
6235	6345	switch (air_tag) {
6236		.add => .vaddss,
6237		.sub => .vsubss,
6238		.div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
6239		.max => .vmaxss,
6240		.min => .vmaxss,
	6346	.add => .{ ._, .vaddss },
	6347	.sub => .{ ._, .vsubss },
	6348	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
	6349	.max => .{ ._, .vmaxss },
	6350	.min => .{ ._, .vmaxss },
6241	6351	else => unreachable,
6242	6352	},
6243	6353	dst_reg,
...	...	@@ -6245,7 +6355,7 @@ fn genBinOp(
6245	6355	tmp_reg,
6246	6356	);
6247	6357	try self.asmRegisterRegisterImmediate(
6248		.vcvtps2ph,
	6358	.{ ._, .vcvtps2ph },
6249	6359	dst_reg,
6250	6360	dst_reg,
6251	6361	Immediate.u(0b1_00),
...	...	@@ -6253,29 +6363,29 @@ fn genBinOp(
6253	6363	return dst_mcv;
6254	6364	} else null,
6255	6365	32 => switch (air_tag) {
6256		.add => if (self.hasFeature(.avx)) .vaddss else .addss,
6257		.sub => if (self.hasFeature(.avx)) .vsubss else .subss,
6258		.mul => if (self.hasFeature(.avx)) .vmulss else .mulss,
	6366	.add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
	6367	.sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
	6368	.mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
6259	6369	.div_float,
6260	6370	.div_trunc,
6261	6371	.div_floor,
6262	6372	.div_exact,
6263		=> if (self.hasFeature(.avx)) .vdivss else .divss,
6264		.max => if (self.hasFeature(.avx)) .vmaxss else .maxss,
6265		.min => if (self.hasFeature(.avx)) .vminss else .minss,
	6373	=> if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
	6374	.max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
	6375	.min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
6266	6376	else => unreachable,
6267	6377	},
6268	6378	64 => switch (air_tag) {
6269		.add => if (self.hasFeature(.avx)) .vaddsd else .addsd,
6270		.sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,
6271		.mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,
	6379	.add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
	6380	.sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
	6381	.mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
6272	6382	.div_float,
6273	6383	.div_trunc,
6274	6384	.div_floor,
6275	6385	.div_exact,
6276		=> if (self.hasFeature(.avx)) .vdivsd else .divsd,
6277		.max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,
6278		.min => if (self.hasFeature(.avx)) .vminsd else .minsd,
	6386	=> if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
	6387	.max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
	6388	.min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
6279	6389	else => unreachable,
6280	6390	},
6281	6391	80, 128 => null,
...	...	@@ -6291,13 +6401,13 @@ fn genBinOp(
6291	6401	defer self.register_manager.unlockReg(tmp_lock);
6292	6402
6293	6403	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
6294		.vpinsrw,
	6404	.{ ._, .vpinsrw },
6295	6405	dst_reg,
6296	6406	dst_reg,
6297	6407	src_mcv.mem(.word),
6298	6408	Immediate.u(1),
6299	6409	) else try self.asmRegisterRegisterRegister(
6300		.vpunpcklwd,
	6410	.{ ._, .vpunpcklwd },
6301	6411	dst_reg,
6302	6412	dst_reg,
6303	6413	(if (src_mcv.isRegister())
...	...	@@ -6305,15 +6415,15 @@ fn genBinOp(
6305	6415	else
6306	6416	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6307	6417	);
6308		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6309		try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
	6418	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
	6419	try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
6310	6420	try self.asmRegisterRegisterRegister(
6311	6421	switch (air_tag) {
6312		.add => .vaddss,
6313		.sub => .vsubss,
6314		.div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
6315		.max => .vmaxss,
6316		.min => .vmaxss,
	6422	.add => .{ ._, .vaddss },
	6423	.sub => .{ ._, .vsubss },
	6424	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
	6425	.max => .{ ._, .vmaxss },
	6426	.min => .{ ._, .vmaxss },
6317	6427	else => unreachable,
6318	6428	},
6319	6429	dst_reg,
...	...	@@ -6321,7 +6431,7 @@ fn genBinOp(
6321	6431	tmp_reg,
6322	6432	);
6323	6433	try self.asmRegisterRegisterImmediate(
6324		.vcvtps2ph,
	6434	.{ ._, .vcvtps2ph },
6325	6435	dst_reg,
6326	6436	dst_reg,
6327	6437	Immediate.u(0b1_00),
...	...	@@ -6334,12 +6444,12 @@ fn genBinOp(
6334	6444	defer self.register_manager.unlockReg(tmp_lock);
6335	6445
6336	6446	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
6337		.vpinsrd,
	6447	.{ ._, .vpinsrd },
6338	6448	dst_reg,
6339	6449	src_mcv.mem(.dword),
6340	6450	Immediate.u(1),
6341	6451	) else try self.asmRegisterRegisterRegister(
6342		.vunpcklps,
	6452	.{ ._, .vunpcklps },
6343	6453	dst_reg,
6344	6454	dst_reg,
6345	6455	(if (src_mcv.isRegister())
...	...	@@ -6347,15 +6457,20 @@ fn genBinOp(
6347	6457	else
6348	6458	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
6349	6459	);
6350		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6351		try self.asmRegisterRegisterRegister(.vmovhlps, tmp_reg, dst_reg, dst_reg);
	6460	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
	6461	try self.asmRegisterRegisterRegister(
	6462	.{ ._, .vmovhlps },
	6463	tmp_reg,
	6464	dst_reg,
	6465	dst_reg,
	6466	);
6352	6467	try self.asmRegisterRegisterRegister(
6353	6468	switch (air_tag) {
6354		.add => .vaddps,
6355		.sub => .vsubps,
6356		.div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6357		.max => .vmaxps,
6358		.min => .vmaxps,
	6469	.add => .{ ._, .vaddps },
	6470	.sub => .{ ._, .vsubps },
	6471	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
	6472	.max => .{ ._, .vmaxps },
	6473	.min => .{ ._, .vmaxps },
6359	6474	else => unreachable,
6360	6475	},
6361	6476	dst_reg,
...	...	@@ -6363,7 +6478,7 @@ fn genBinOp(
6363	6478	tmp_reg,
6364	6479	);
6365	6480	try self.asmRegisterRegisterImmediate(
6366		.vcvtps2ph,
	6481	.{ ._, .vcvtps2ph },
6367	6482	dst_reg,
6368	6483	dst_reg,
6369	6484	Immediate.u(0b1_00),
...	...	@@ -6375,13 +6490,13 @@ fn genBinOp(
6375	6490	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6376	6491	defer self.register_manager.unlockReg(tmp_lock);
6377	6492
6378		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
	6493	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
6379	6494	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6380		.vcvtph2ps,
	6495	.{ ._, .vcvtph2ps },
6381	6496	tmp_reg,
6382	6497	src_mcv.mem(.qword),
6383	6498	) else try self.asmRegisterRegister(
6384		.vcvtph2ps,
	6499	.{ ._, .vcvtph2ps },
6385	6500	tmp_reg,
6386	6501	(if (src_mcv.isRegister())
6387	6502	src_mcv.getReg().?
...	...	@@ -6390,11 +6505,11 @@ fn genBinOp(
6390	6505	);
6391	6506	try self.asmRegisterRegisterRegister(
6392	6507	switch (air_tag) {
6393		.add => .vaddps,
6394		.sub => .vsubps,
6395		.div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6396		.max => .vmaxps,
6397		.min => .vmaxps,
	6508	.add => .{ ._, .vaddps },
	6509	.sub => .{ ._, .vsubps },
	6510	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
	6511	.max => .{ ._, .vmaxps },
	6512	.min => .{ ._, .vmaxps },
6398	6513	else => unreachable,
6399	6514	},
6400	6515	dst_reg,
...	...	@@ -6402,7 +6517,7 @@ fn genBinOp(
6402	6517	tmp_reg,
6403	6518	);
6404	6519	try self.asmRegisterRegisterImmediate(
6405		.vcvtps2ph,
	6520	.{ ._, .vcvtps2ph },
6406	6521	dst_reg,
6407	6522	dst_reg,
6408	6523	Immediate.u(0b1_00),
...	...	@@ -6414,13 +6529,13 @@ fn genBinOp(
6414	6529	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6415	6530	defer self.register_manager.unlockReg(tmp_lock);
6416	6531
6417		try self.asmRegisterRegister(.vcvtph2ps, dst_reg.to256(), dst_reg);
	6532	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg.to256(), dst_reg);
6418	6533	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6419		.vcvtph2ps,
	6534	.{ ._, .vcvtph2ps },
6420	6535	tmp_reg,
6421	6536	src_mcv.mem(.xword),
6422	6537	) else try self.asmRegisterRegister(
6423		.vcvtph2ps,
	6538	.{ ._, .vcvtph2ps },
6424	6539	tmp_reg,
6425	6540	(if (src_mcv.isRegister())
6426	6541	src_mcv.getReg().?
...	...	@@ -6429,11 +6544,11 @@ fn genBinOp(
6429	6544	);
6430	6545	try self.asmRegisterRegisterRegister(
6431	6546	switch (air_tag) {
6432		.add => .vaddps,
6433		.sub => .vsubps,
6434		.div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6435		.max => .vmaxps,
6436		.min => .vmaxps,
	6547	.add => .{ ._, .vaddps },
	6548	.sub => .{ ._, .vsubps },
	6549	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
	6550	.max => .{ ._, .vmaxps },
	6551	.min => .{ ._, .vmaxps },
6437	6552	else => unreachable,
6438	6553	},
6439	6554	dst_reg.to256(),
...	...	@@ -6441,7 +6556,7 @@ fn genBinOp(
6441	6556	tmp_reg,
6442	6557	);
6443	6558	try self.asmRegisterRegisterImmediate(
6444		.vcvtps2ph,
	6559	.{ ._, .vcvtps2ph },
6445	6560	dst_reg,
6446	6561	dst_reg.to256(),
6447	6562	Immediate.u(0b1_00),
...	...	@@ -6452,76 +6567,76 @@ fn genBinOp(
6452	6567	} else null,
6453	6568	32 => switch (lhs_ty.vectorLen()) {
6454	6569	1 => switch (air_tag) {
6455		.add => if (self.hasFeature(.avx)) .vaddss else .addss,
6456		.sub => if (self.hasFeature(.avx)) .vsubss else .subss,
6457		.mul => if (self.hasFeature(.avx)) .vmulss else .mulss,
	6570	.add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
	6571	.sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
	6572	.mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
6458	6573	.div_float,
6459	6574	.div_trunc,
6460	6575	.div_floor,
6461	6576	.div_exact,
6462		=> if (self.hasFeature(.avx)) .vdivss else .divss,
6463		.max => if (self.hasFeature(.avx)) .vmaxss else .maxss,
6464		.min => if (self.hasFeature(.avx)) .vminss else .minss,
	6577	=> if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
	6578	.max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
	6579	.min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
6465	6580	else => unreachable,
6466	6581	},
6467	6582	2...4 => switch (air_tag) {
6468		.add => if (self.hasFeature(.avx)) .vaddps else .addps,
6469		.sub => if (self.hasFeature(.avx)) .vsubps else .subps,
6470		.mul => if (self.hasFeature(.avx)) .vmulps else .mulps,
	6583	.add => if (self.hasFeature(.avx)) .{ ._, .vaddps } else .{ ._, .addps },
	6584	.sub => if (self.hasFeature(.avx)) .{ ._, .vsubps } else .{ ._, .subps },
	6585	.mul => if (self.hasFeature(.avx)) .{ ._, .vmulps } else .{ ._, .mulps },
6471	6586	.div_float,
6472	6587	.div_trunc,
6473	6588	.div_floor,
6474	6589	.div_exact,
6475		=> if (self.hasFeature(.avx)) .vdivps else .divps,
6476		.max => if (self.hasFeature(.avx)) .vmaxps else .maxps,
6477		.min => if (self.hasFeature(.avx)) .vminps else .minps,
	6590	=> if (self.hasFeature(.avx)) .{ ._, .vdivps } else .{ ._, .divps },
	6591	.max => if (self.hasFeature(.avx)) .{ ._, .vmaxps } else .{ ._, .maxps },
	6592	.min => if (self.hasFeature(.avx)) .{ ._, .vminps } else .{ ._, .minps },
6478	6593	else => unreachable,
6479	6594	},
6480	6595	5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
6481		.add => .vaddps,
6482		.sub => .vsubps,
6483		.mul => .vmulps,
6484		.div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
6485		.max => .vmaxps,
6486		.min => .vminps,
	6596	.add => .{ ._, .vaddps },
	6597	.sub => .{ ._, .vsubps },
	6598	.mul => .{ ._, .vmulps },
	6599	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
	6600	.max => .{ ._, .vmaxps },
	6601	.min => .{ ._, .vminps },
6487	6602	else => unreachable,
6488	6603	} else null,
6489	6604	else => null,
6490	6605	},
6491	6606	64 => switch (lhs_ty.vectorLen()) {
6492	6607	1 => switch (air_tag) {
6493		.add => if (self.hasFeature(.avx)) .vaddsd else .addsd,
6494		.sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,
6495		.mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,
	6608	.add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
	6609	.sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
	6610	.mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
6496	6611	.div_float,
6497	6612	.div_trunc,
6498	6613	.div_floor,
6499	6614	.div_exact,
6500		=> if (self.hasFeature(.avx)) .vdivsd else .divsd,
6501		.max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,
6502		.min => if (self.hasFeature(.avx)) .vminsd else .minsd,
	6615	=> if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
	6616	.max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
	6617	.min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
6503	6618	else => unreachable,
6504	6619	},
6505	6620	2 => switch (air_tag) {
6506		.add => if (self.hasFeature(.avx)) .vaddpd else .addpd,
6507		.sub => if (self.hasFeature(.avx)) .vsubpd else .subpd,
6508		.mul => if (self.hasFeature(.avx)) .vmulpd else .mulpd,
	6621	.add => if (self.hasFeature(.avx)) .{ ._, .vaddpd } else .{ ._, .addpd },
	6622	.sub => if (self.hasFeature(.avx)) .{ ._, .vsubpd } else .{ ._, .subpd },
	6623	.mul => if (self.hasFeature(.avx)) .{ ._, .vmulpd } else .{ ._, .mulpd },
6509	6624	.div_float,
6510	6625	.div_trunc,
6511	6626	.div_floor,
6512	6627	.div_exact,
6513		=> if (self.hasFeature(.avx)) .vdivpd else .divpd,
6514		.max => if (self.hasFeature(.avx)) .vmaxpd else .maxpd,
6515		.min => if (self.hasFeature(.avx)) .vminpd else .minpd,
	6628	=> if (self.hasFeature(.avx)) .{ ._, .vdivpd } else .{ ._, .divpd },
	6629	.max => if (self.hasFeature(.avx)) .{ ._, .vmaxpd } else .{ ._, .maxpd },
	6630	.min => if (self.hasFeature(.avx)) .{ ._, .vminpd } else .{ ._, .minpd },
6516	6631	else => unreachable,
6517	6632	},
6518	6633	3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
6519		.add => .vaddpd,
6520		.sub => .vsubpd,
6521		.mul => .vmulpd,
6522		.div_float, .div_trunc, .div_floor, .div_exact => .vdivpd,
6523		.max => .vmaxpd,
6524		.min => .vminpd,
	6634	.add => .{ ._, .vaddpd },
	6635	.sub => .{ ._, .vsubpd },
	6636	.mul => .{ ._, .vmulpd },
	6637	.div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivpd },
	6638	.max => .{ ._, .vmaxpd },
	6639	.min => .{ ._, .vminpd },
6525	6640	else => unreachable,
6526	6641	} else null,
6527	6642	else => null,
...	...	@@ -6583,7 +6698,13 @@ fn genBinOp(
6583	6698	return dst_mcv;
6584	6699	}
6585	6700
6586		fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
	6701	fn genBinOpMir(
	6702	self: *Self,
	6703	mir_tag: Mir.Inst.FixedTag,
	6704	ty: Type,
	6705	dst_mcv: MCValue,
	6706	src_mcv: MCValue,
	6707	) !void {
6587	6708	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6588	6709	switch (dst_mcv) {
6589	6710	.none,
...	...	@@ -6788,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
6788	6909	};
6789	6910	var off: i32 = 0;
6790	6911	while (off < abi_size) : (off += 8) {
6791		const mir_limb_tag = switch (off) {
	6912	const mir_limb_tag: Mir.Inst.FixedTag = switch (off) {
6792	6913	0 => mir_tag,
6793		else => switch (mir_tag) {
6794		.add => .adc,
6795		.sub, .cmp => .sbb,
	6914	else => switch (mir_tag[1]) {
	6915	.add => .{ ._, .adc },
	6916	.sub, .cmp => .{ ._, .sbb },
6796	6917	.@"or", .@"and", .xor => mir_tag,
6797	6918	else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{
6798		@tagName(mir_tag),
	6919	@tagName(mir_tag[1]),
6799	6920	}),
6800	6921	},
6801	6922	};
...	...	@@ -6967,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6967	7088	.reserved_frame,
6968	7089	=> unreachable,
6969	7090	.register => \|src_reg\| try self.asmRegisterRegister(
6970		.imul,
	7091	.{ .i_, .mul },
6971	7092	dst_alias,
6972	7093	registerAlias(src_reg, abi_size),
6973	7094	),
6974	7095	.immediate => \|imm\| {
6975	7096	if (math.cast(i32, imm)) \|small\| {
6976	7097	try self.asmRegisterRegisterImmediate(
6977		.imul,
	7098	.{ .i_, .mul },
6978	7099	dst_alias,
6979	7100	dst_alias,
6980	7101	Immediate.s(small),
...	...	@@ -6994,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6994	7115	.lea_tlv,
6995	7116	.lea_frame,
6996	7117	=> try self.asmRegisterRegister(
6997		.imul,
	7118	.{ .i_, .mul },
6998	7119	dst_alias,
6999	7120	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
7000	7121	),
7001	7122	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
7002		.imul,
	7123	.{ .i_, .mul },
7003	7124	dst_alias,
7004	7125	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
7005	7126	.memory => \|addr\| .{
7006	7127	.base = .{ .reg = .ds },
7007	7128	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
7008	7129	return self.asmRegisterRegister(
7009		.imul,
	7130	.{ .i_, .mul },
7010	7131	dst_alias,
7011	7132	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
7012	7133	),
...	...	@@ -7131,12 +7252,12 @@ fn genVarDbgInfo(
7131	7252	}
7132	7253
7133	7254	fn airTrap(self: *Self) !void {
7134		try self.asmOpOnly(.ud2);
	7255	try self.asmOpOnly(.{ ._, .ud2 });
7135	7256	return self.finishAirBookkeeping();
7136	7257	}
7137	7258
7138	7259	fn airBreakpoint(self: *Self) !void {
7139		try self.asmOpOnly(.int3);
	7260	try self.asmOpOnly(.{ ._, .int3 });
7140	7261	return self.finishAirBookkeeping();
7141	7262	}
7142	7263
...	...	@@ -7157,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {
7157	7278	switch (order) {
7158	7279	.Unordered, .Monotonic => unreachable,
7159	7280	.Acquire, .Release, .AcqRel => {},
7160		.SeqCst => try self.asmOpOnly(.mfence),
	7281	.SeqCst => try self.asmOpOnly(.{ ._, .mfence }),
7161	7282	}
7162	7283	return self.finishAirBookkeeping();
7163	7284	}
...	...	@@ -7251,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7251	7372	const atom = elf_file.getAtom(atom_index);
7252	7373	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
7253	7374	const got_addr = atom.getOffsetTableAddress(elf_file);
7254		try self.asmMemory(.call, Memory.sib(.qword, .{
	7375	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
7255	7376	.base = .{ .reg = .ds },
7256	7377	.disp = @intCast(i32, got_addr),
7257	7378	}));
...	...	@@ -7259,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7259	7380	const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
7260	7381	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
7261	7382	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
7262		try self.asmRegister(.call, .rax);
	7383	try self.asmRegister(.{ ._, .call }, .rax);
7263	7384	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
7264	7385	const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
7265	7386	const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
7266	7387	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
7267		try self.asmRegister(.call, .rax);
	7388	try self.asmRegister(.{ ._, .call }, .rax);
7268	7389	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
7269	7390	const decl_block_index = try p9.seeDecl(func.owner_decl);
7270	7391	const decl_block = p9.getDeclBlock(decl_block_index);
...	...	@@ -7273,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7273	7394	const got_addr = p9.bases.data;
7274	7395	const got_index = decl_block.got_index.?;
7275	7396	const fn_got_addr = got_addr + got_index * ptr_bytes;
7276		try self.asmMemory(.call, Memory.sib(.qword, .{
	7397	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
7277	7398	.base = .{ .reg = .ds },
7278	7399	.disp = @intCast(i32, fn_got_addr),
7279	7400	}));
...	...	@@ -7296,7 +7417,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7296	7417	}),
7297	7418	} },
7298	7419	});
7299		try self.asmRegister(.call, .rax);
	7420	try self.asmRegister(.{ ._, .call }, .rax);
7300	7421	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
7301	7422	const atom_index = try self.owner.getSymbolIndex(self);
7302	7423	const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
...	...	@@ -7318,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7318	7439	assert(ty.zigTypeTag() == .Pointer);
7319	7440	const mcv = try self.resolveInst(callee);
7320	7441	try self.genSetReg(.rax, Type.usize, mcv);
7321		try self.asmRegister(.call, .rax);
	7442	try self.asmRegister(.{ ._, .call }, .rax);
7322	7443	}
7323	7444
7324	7445	var bt = self.liveness.iterateBigTomb(inst);
...	...	@@ -7408,7 +7529,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7408	7529	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
7409	7530	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
7410	7531
7411		try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
	7532	try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
7412	7533	break :result Condition.fromCompareOperator(
7413	7534	if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
7414	7535	if (flipped) op.reverse() else op,
...	...	@@ -7442,13 +7563,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7442	7563	defer self.register_manager.unlockReg(tmp2_lock);
7443	7564
7444	7565	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
7445		.vpinsrw,
	7566	.{ ._, .vpinsrw },
7446	7567	tmp1_reg,
7447	7568	dst_reg.to128(),
7448	7569	src_mcv.mem(.word),
7449	7570	Immediate.u(1),
7450	7571	) else try self.asmRegisterRegisterRegister(
7451		.vpunpcklwd,
	7572	.{ ._, .vpunpcklwd },
7452	7573	tmp1_reg,
7453	7574	dst_reg.to128(),
7454	7575	(if (src_mcv.isRegister())
...	...	@@ -7456,14 +7577,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7456	7577	else
7457	7578	try self.copyToTmpRegister(ty, src_mcv)).to128(),
7458	7579	);
7459		try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);
7460		try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);
7461		try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);
	7580	try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, tmp1_reg, tmp1_reg);
	7581	try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp2_reg, tmp1_reg);
	7582	try self.genBinOpMir(.{ ._, .ucomiss }, ty, tmp1_mcv, tmp2_mcv);
7462	7583	} else return self.fail("TODO implement airCmp for {}", .{
7463	7584	ty.fmt(self.bin_file.options.module.?),
7464	7585	}),
7465		32 => try self.genBinOpMir(.ucomiss, ty, .{ .register = dst_reg }, src_mcv),
7466		64 => try self.genBinOpMir(.ucomisd, ty, .{ .register = dst_reg }, src_mcv),
	7586	32 => try self.genBinOpMir(
	7587	.{ ._, .ucomiss },
	7588	ty,
	7589	.{ .register = dst_reg },
	7590	src_mcv,
	7591	),
	7592	64 => try self.genBinOpMir(
	7593	.{ ._, .ucomisd },
	7594	ty,
	7595	.{ .register = dst_reg },
	7596	src_mcv,
	7597	),
7467	7598	else => return self.fail("TODO implement airCmp for {}", .{
7468	7599	ty.fmt(self.bin_file.options.module.?),
7469	7600	}),
...	...	@@ -7507,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
7507	7638	else => try self.copyToTmpRegister(op_ty, op_mcv),
7508	7639	};
7509	7640	try self.asmRegisterMemory(
7510		.cmp,
	7641	.{ ._, .cmp },
7511	7642	registerAlias(dst_reg, op_abi_size),
7512	7643	Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
7513	7644	);
...	...	@@ -7627,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
7627	7758	},
7628	7759	.register => \|reg\| {
7629	7760	try self.spillEflagsIfOccupied();
7630		try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));
	7761	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
7631	7762	return self.asmJccReloc(undefined, .e);
7632	7763	},
7633	7764	.immediate,
...	...	@@ -7730,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
7730	7861	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7731	7862	const alias_reg = registerAlias(opt_reg, some_abi_size);
7732	7863	assert(some_abi_size * 8 == alias_reg.bitSize());
7733		try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);
	7864	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
7734	7865	return .{ .eflags = .z };
7735	7866	}
7736	7867	assert(some_info.ty.tag() == .bool);
7737	7868	const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
7738	7869	try self.asmRegisterImmediate(
7739		.bt,
	7870	.{ ._, .bt },
7740	7871	registerAlias(opt_reg, opt_abi_size),
7741	7872	Immediate.u(@intCast(u6, some_info.off * 8)),
7742	7873	);
...	...	@@ -7755,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
7755	7886	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
7756	7887	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7757	7888	try self.asmMemoryImmediate(
7758		.cmp,
	7889	.{ ._, .cmp },
7759	7890	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
7760	7891	.base = .{ .reg = addr_reg },
7761	7892	.disp = some_info.off,
...	...	@@ -7768,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
7768	7899	.indirect, .load_frame => {
7769	7900	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7770	7901	try self.asmMemoryImmediate(
7771		.cmp,
	7902	.{ ._, .cmp },
7772	7903	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
7773	7904	.indirect => \|reg_off\| .{
7774	7905	.base = .{ .reg = reg_off.reg },
...	...	@@ -7810,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
7810	7941
7811	7942	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
7812	7943	try self.asmMemoryImmediate(
7813		.cmp,
	7944	.{ ._, .cmp },
7814	7945	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
7815	7946	.base = .{ .reg = ptr_reg },
7816	7947	.disp = some_info.off,
...	...	@@ -7841,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
7841	7972	const tmp_reg = try self.copyToTmpRegister(ty, operand);
7842	7973	if (err_off > 0) {
7843	7974	const shift = @intCast(u6, err_off * 8);
7844		try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });
	7975	try self.genShiftBinOpMir(
	7976	.{ ._r, .sh },
	7977	ty,
	7978	.{ .register = tmp_reg },
	7979	.{ .immediate = shift },
	7980	);
7845	7981	} else {
7846	7982	try self.truncateRegister(Type.anyerror, tmp_reg);
7847	7983	}
7848		try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
	7984	try self.genBinOpMir(
	7985	.{ ._, .cmp },
	7986	Type.anyerror,
	7987	.{ .register = tmp_reg },
	7988	.{ .immediate = 0 },
	7989	);
7849	7990	},
7850	7991	.load_frame => \|frame_addr\| try self.genBinOpMir(
7851		.cmp,
	7992	.{ ._, .cmp },
7852	7993	Type.anyerror,
7853	7994	.{ .load_frame = .{
7854	7995	.index = frame_addr.index,
...	...	@@ -8073,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
8073	8214	try self.spillEflagsIfOccupied();
8074	8215	for (items, relocs, 0..) \|item, *reloc, i\| {
8075	8216	const item_mcv = try self.resolveInst(item);
8076		try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
	8217	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
8077	8218	reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
8078	8219	}
8079	8220
...	...	@@ -8284,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8284	8425	.qword
8285	8426	else
8286	8427	null;
8287		const mnem = mnem: {
	8428	const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
8288	8429	if (mnem_size) \|_\| {
8289	8430	if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) \|mnem\| {
8290	8431	break :mnem mnem;
...	...	@@ -8292,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8292	8433	}
8293	8434	break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
8294	8435	return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
8295		};
	8436	} };
8296	8437
8297	8438	var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
8298	8439	var ops = [1]encoder.Instruction.Operand{.none} ** 4;
...	...	@@ -8343,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8343	8484	} else if (op_it.next()) \|op_str\| return self.fail("Extra operand: '{s}'", .{op_str});
8344	8485
8345	8486	(switch (ops[0]) {
8346		.none => self.asmOpOnly(mnem),
	8487	.none => self.asmOpOnly(mnem_tag),
8347	8488	.reg => \|reg0\| switch (ops[1]) {
8348		.none => self.asmRegister(mnem, reg0),
	8489	.none => self.asmRegister(mnem_tag, reg0),
8349	8490	.reg => \|reg1\| switch (ops[2]) {
8350		.none => self.asmRegisterRegister(mnem, reg1, reg0),
	8491	.none => self.asmRegisterRegister(mnem_tag, reg1, reg0),
8351	8492	.reg => \|reg2\| switch (ops[3]) {
8352		.none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),
	8493	.none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),
8353	8494	else => error.InvalidInstruction,
8354	8495	},
8355	8496	.mem => \|mem2\| switch (ops[3]) {
8356		.none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),
	8497	.none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),
8357	8498	else => error.InvalidInstruction,
8358	8499	},
8359	8500	else => error.InvalidInstruction,
8360	8501	},
8361	8502	.mem => \|mem1\| switch (ops[2]) {
8362		.none => self.asmMemoryRegister(mnem, mem1, reg0),
	8503	.none => self.asmMemoryRegister(mnem_tag, mem1, reg0),
8363	8504	else => error.InvalidInstruction,
8364	8505	},
8365	8506	else => error.InvalidInstruction,
8366	8507	},
8367	8508	.mem => \|mem0\| switch (ops[1]) {
8368		.none => self.asmMemory(mnem, mem0),
	8509	.none => self.asmMemory(mnem_tag, mem0),
8369	8510	.reg => \|reg1\| switch (ops[2]) {
8370		.none => self.asmRegisterMemory(mnem, reg1, mem0),
	8511	.none => self.asmRegisterMemory(mnem_tag, reg1, mem0),
8371	8512	else => error.InvalidInstruction,
8372	8513	},
8373	8514	else => error.InvalidInstruction,
8374	8515	},
8375	8516	.imm => \|imm0\| switch (ops[1]) {
8376		.none => self.asmImmediate(mnem, imm0),
	8517	.none => self.asmImmediate(mnem_tag, imm0),
8377	8518	.reg => \|reg1\| switch (ops[2]) {
8378		.none => self.asmRegisterImmediate(mnem, reg1, imm0),
	8519	.none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),
8379	8520	.reg => \|reg2\| switch (ops[3]) {
8380		.none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),
	8521	.none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),
8381	8522	else => error.InvalidInstruction,
8382	8523	},
8383	8524	.mem => \|mem2\| switch (ops[3]) {
8384		.none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),
	8525	.none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),
8385	8526	else => error.InvalidInstruction,
8386	8527	},
8387	8528	else => error.InvalidInstruction,
8388	8529	},
8389	8530	.mem => \|mem1\| switch (ops[2]) {
8390		.none => self.asmMemoryImmediate(mnem, mem1, imm0),
	8531	.none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),
8391	8532	else => error.InvalidInstruction,
8392	8533	},
8393	8534	else => error.InvalidInstruction,
...	...	@@ -8396,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8396	8537	error.InvalidInstruction => return self.fail(
8397	8538	"Invalid instruction: '{s} {s} {s} {s} {s}'",
8398	8539	.{
8399		@tagName(mnem),
	8540	@tagName(mnem_tag[1]),
8400	8541	@tagName(ops[0]),
8401	8542	@tagName(ops[1]),
8402	8543	@tagName(ops[2]),
...	...	@@ -8427,44 +8568,47 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
8427	8568	return self.finishAirResult(inst, result);
8428	8569	}
8429	8570
8430		fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.Tag {
	8571	fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
8431	8572	switch (ty.zigTypeTag()) {
8432		else => return .mov,
	8573	else => return .{ ._, .mov },
8433	8574	.Float => switch (ty.floatBits(self.target.*)) {
8434	8575	16 => unreachable, // needs special handling
8435		32 => return if (self.hasFeature(.avx)) .vmovss else .movss,
8436		64 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
	8576	32 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
	8577	64 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8437	8578	128 => return if (self.hasFeature(.avx))
8438		if (aligned) .vmovaps else .vmovups
8439		else if (aligned) .movaps else .movups,
	8579	if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
	8580	else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
8440	8581	else => {},
8441	8582	},
8442	8583	.Vector => switch (ty.childType().zigTypeTag()) {
8443	8584	.Float => switch (ty.childType().floatBits(self.target.*)) {
8444	8585	16 => switch (ty.vectorLen()) {
8445	8586	1 => unreachable, // needs special handling
8446		2 => return if (self.hasFeature(.avx)) .vmovss else .movss,
8447		3...4 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
	8587	2 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
	8588	3...4 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8448	8589	5...8 => return if (self.hasFeature(.avx))
8449		if (aligned) .vmovaps else .vmovups
8450		else if (aligned) .movaps else .movups,
8451		9...16 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
	8590	if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
	8591	else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
	8592	9...16 => if (self.hasFeature(.avx))
	8593	return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8452	8594	else => {},
8453	8595	},
8454	8596	32 => switch (ty.vectorLen()) {
8455		1 => return if (self.hasFeature(.avx)) .vmovss else .movss,
	8597	1 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
8456	8598	2...4 => return if (self.hasFeature(.avx))
8457		if (aligned) .vmovaps else .vmovups
8458		else if (aligned) .movaps else .movups,
8459		5...8 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
	8599	if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
	8600	else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
	8601	5...8 => if (self.hasFeature(.avx))
	8602	return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8460	8603	else => {},
8461	8604	},
8462	8605	64 => switch (ty.vectorLen()) {
8463		1 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
	8606	1 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
8464	8607	2 => return if (self.hasFeature(.avx))
8465		if (aligned) .vmovaps else .vmovups
8466		else if (aligned) .movaps else .movups,
8467		3...4 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
	8608	if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
	8609	else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
	8610	3...4 => if (self.hasFeature(.avx))
	8611	return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
8468	8612	else => {},
8469	8613	},
8470	8614	else => {},
...	...	@@ -8558,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8558	8702	if (imm == 0) {
8559	8703	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
8560	8704	// register is the fastest way to zero a register.
8561		try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
	8705	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
8562	8706	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
8563	8707	// 32-bit moves zero-extend to 64-bit.
8564		try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
	8708	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
8565	8709	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
8566	8710	try self.asmRegisterImmediate(
8567		.mov,
	8711	.{ ._, .mov },
8568	8712	registerAlias(dst_reg, abi_size),
8569	8713	Immediate.s(@intCast(i32, @bitCast(i64, imm))),
8570	8714	);
8571	8715	} else {
8572	8716	try self.asmRegisterImmediate(
8573		.mov,
	8717	.{ ._, .mov },
8574	8718	registerAlias(dst_reg, abi_size),
8575	8719	Immediate.u(imm),
8576	8720	);
...	...	@@ -8579,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8579	8723	.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
8580	8724	if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
8581	8725	switch (ty.zigTypeTag()) {
8582		else => .mov,
8583		.Float, .Vector => .movaps,
	8726	else => .{ ._, .mov },
	8727	.Float, .Vector => .{ ._, .movaps },
8584	8728	}
8585	8729	else switch (abi_size) {
8586	8730	2 => return try self.asmRegisterRegisterImmediate(
8587		if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,
	8731	if (dst_reg.class() == .floating_point) .{ ._, .pinsrw } else .{ ._, .pextrw },
8588	8732	registerAlias(dst_reg, 4),
8589	8733	registerAlias(src_reg, 4),
8590	8734	Immediate.u(0),
8591	8735	),
8592		4 => .movd,
8593		8 => .movq,
	8736	4 => .{ ._d, .mov },
	8737	8 => .{ ._q, .mov },
8594	8738	else => return self.fail(
8595	8739	"unsupported register copy from {s} to {s}",
8596	8740	.{ @tagName(src_reg), @tagName(dst_reg) },
...	...	@@ -8617,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8617	8761	});
8618	8762	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8619	8763	try self.asmRegisterMemoryImmediate(
8620		.pinsrw,
	8764	.{ ._, .pinsrw },
8621	8765	registerAlias(dst_reg, abi_size),
8622	8766	src_mem,
8623	8767	Immediate.u(0),
...	...	@@ -8627,14 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8627	8771	switch (src_mcv) {
8628	8772	.register_offset => \|reg_off\| switch (reg_off.off) {
8629	8773	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
8630		else => .lea,
	8774	else => .{ ._, .lea },
8631	8775	},
8632	8776	.indirect => try self.movMirTag(ty, false),
8633	8777	.load_frame => \|frame_addr\| try self.movMirTag(
8634	8778	ty,
8635	8779	self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
8636	8780	),
8637		.lea_frame => .lea,
	8781	.lea_frame => .{ ._, .lea },
8638	8782	else => unreachable,
8639	8783	},
8640	8784	registerAlias(dst_reg, abi_size),
...	...	@@ -8650,7 +8794,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8650	8794	});
8651	8795	return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8652	8796	self.asmRegisterMemoryImmediate(
8653		.pinsrw,
	8797	.{ ._, .pinsrw },
8654	8798	registerAlias(dst_reg, abi_size),
8655	8799	src_mem,
8656	8800	Immediate.u(0),
...	...	@@ -8694,7 +8838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8694	8838	});
8695	8839	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8696	8840	try self.asmRegisterMemoryImmediate(
8697		.pinsrw,
	8841	.{ ._, .pinsrw },
8698	8842	registerAlias(dst_reg, abi_size),
8699	8843	src_mem,
8700	8844	Immediate.u(0),
...	...	@@ -8743,7 +8887,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8743	8887	} },
8744	8888	});
8745	8889	// TODO: spill registers before calling
8746		try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
	8890	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
8747	8891	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
8748	8892	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
8749	8893	@tagName(self.bin_file.tag),
...	...	@@ -8770,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8770	8914	else
8771	8915	Immediate.u(@intCast(u32, imm));
8772	8916	try self.asmMemoryImmediate(
8773		.mov,
	8917	.{ ._, .mov },
8774	8918	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
8775	8919	immediate,
8776	8920	);
...	...	@@ -8778,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8778	8922	3, 5...7 => unreachable,
8779	8923	else => if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
8780	8924	try self.asmMemoryImmediate(
8781		.mov,
	8925	.{ ._, .mov },
8782	8926	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
8783	8927	Immediate.s(small),
8784	8928	);
8785	8929	} else {
8786	8930	var offset: i32 = 0;
8787	8931	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
8788		.mov,
	8932	.{ ._, .mov },
8789	8933	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
8790	8934	if (ty.isSignedInt())
8791	8935	Immediate.s(@truncate(
...	...	@@ -8808,7 +8952,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
8808	8952	);
8809	8953	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
8810	8954	try self.asmMemoryRegisterImmediate(
8811		.pextrw,
	8955	.{ ._, .pextrw },
8812	8956	dst_mem,
8813	8957	src_reg.to128(),
8814	8958	Immediate.u(0),
...	...	@@ -8904,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(
8904	9048	while (remainder > 0) {
8905	9049	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
8906	9050	try self.asmMemoryRegister(
8907		.mov,
	9051	.{ ._, .mov },
8908	9052	Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
8909	9053	.base = dst_reg,
8910	9054	.disp = -next_offset,
...	...	@@ -8913,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(
8913	9057	);
8914	9058
8915	9059	if (nearest_power_of_two > 1) {
8916		try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
	9060	try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{
8917	9061	.immediate = nearest_power_of_two * 8,
8918	9062	});
8919	9063	}
...	...	@@ -8924,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(
8924	9068	} else {
8925	9069	try self.asmMemoryRegister(
8926	9070	switch (src_reg.class()) {
8927		.general_purpose, .segment => .mov,
8928		.floating_point => .movss,
	9071	.general_purpose, .segment => .{ ._, .mov },
	9072	.floating_point => .{ ._, .movss },
8929	9073	},
8930	9074	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
8931	9075	registerAlias(src_reg, abi_size),
...	...	@@ -8938,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
8938	9082	try self.genSetReg(.rdi, Type.usize, dst_ptr);
8939	9083	try self.genSetReg(.rsi, Type.usize, src_ptr);
8940	9084	try self.genSetReg(.rcx, Type.usize, len);
8941		_ = try self.addInst(.{
8942		.tag = .mov,
8943		.ops = .none,
8944		.data = .{ .none = .{ .fixes = .@"rep _sb" } },
8945		});
	9085	try self.asmOpOnly(.{ .@"rep _sb", .mov });
8946	9086	}
8947	9087
8948	9088	fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
...	...	@@ -8950,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
8950	9090	try self.genSetReg(.rdi, Type.usize, dst_ptr);
8951	9091	try self.genSetReg(.al, Type.u8, value);
8952	9092	try self.genSetReg(.rcx, Type.usize, len);
8953		_ = try self.addInst(.{
8954		.tag = .sto,
8955		.ops = .none,
8956		.data = .{ .none = .{ .fixes = .@"rep _sb" } },
8957		});
	9093	try self.asmOpOnly(.{ .@"rep _sb", .sto });
8958	9094	}
8959	9095
8960	9096	fn genLazySymbolRef(
...	...	@@ -8972,14 +9108,14 @@ fn genLazySymbolRef(
8972	9108	const got_mem =
8973	9109	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });
8974	9110	switch (tag) {
8975		.lea, .mov => try self.asmRegisterMemory(.mov, reg.to64(), got_mem),
8976		.call => try self.asmMemory(.call, got_mem),
	9111	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
	9112	.call => try self.asmMemory(.{ ._, .call }, got_mem),
8977	9113	else => unreachable,
8978	9114	}
8979	9115	switch (tag) {
8980	9116	.lea, .call => {},
8981	9117	.mov => try self.asmRegisterMemory(
8982		tag,
	9118	.{ ._, tag },
8983	9119	reg.to64(),
8984	9120	Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),
8985	9121	),
...	...	@@ -8996,7 +9132,7 @@ fn genLazySymbolRef(
8996	9132	}
8997	9133	switch (tag) {
8998	9134	.lea, .mov => {},
8999		.call => try self.asmRegister(.call, reg),
	9135	.call => try self.asmRegister(.{ ._, .call }, reg),
9000	9136	else => unreachable,
9001	9137	}
9002	9138	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
...	...	@@ -9010,7 +9146,7 @@ fn genLazySymbolRef(
9010	9146	}
9011	9147	switch (tag) {
9012	9148	.lea, .mov => {},
9013		.call => try self.asmRegister(.call, reg),
	9149	.call => try self.asmRegister(.{ ._, .call }, reg),
9014	9150	else => unreachable,
9015	9151	}
9016	9152	} else {
...	...	@@ -9115,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
9115	9251
9116	9252	try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
9117	9253	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
9118		.cvtsi2ss
	9254	.{ ._, .cvtsi2ss }
9119	9255	else
9120	9256	return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
9121	9257	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
9122	9258	}),
9123	9259	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
9124		.cvtsi2sd
	9260	.{ ._, .cvtsi2sd }
9125	9261	else
9126	9262	return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
9127	9263	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
...	...	@@ -9161,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
9161	9297	},
9162	9298	};
9163	9299	try self.asmMemory(
9164		.fld,
	9300	.{ .f_, .ld },
9165	9301	Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
9166	9302	.base = .{ .frame = frame_addr.index },
9167	9303	.disp = frame_addr.off,
...	...	@@ -9171,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
9171	9307	// convert
9172	9308	const stack_dst = try self.allocRegOrMem(inst, false);
9173	9309	try self.asmMemory(
9174		.fisttp,
	9310	.{ .f_p, .istt },
9175	9311	Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
9176	9312	.base = .{ .frame = stack_dst.load_frame.index },
9177	9313	.disp = stack_dst.load_frame.off,
...	...	@@ -9227,22 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
9227	9363	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
9228	9364
9229	9365	try self.spillEflagsIfOccupied();
9230		_ = try self.addInst(if (val_abi_size <= 8) .{
9231		.tag = .cmpxchg,
9232		.ops = .mr_sib,
9233		.data = .{ .rx = .{
9234		.fixes = .@"lock _",
9235		.r1 = registerAlias(new_reg.?, val_abi_size),
9236		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9237		} },
9238		} else .{
9239		.tag = .cmpxchg,
9240		.ops = .m_sib,
9241		.data = .{ .x = .{
9242		.fixes = .@"lock _16b",
9243		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9244		} },
9245		});
	9366	if (val_abi_size <= 8) try self.asmMemoryRegister(
	9367	.{ .@"lock _", .cmpxchg },
	9368	ptr_mem,
	9369	registerAlias(new_reg.?, val_abi_size),
	9370	) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
9246	9371
9247	9372	const result: MCValue = result: {
9248	9373	if (self.liveness.isUnused(inst)) break :result .unreach;
...	...	@@ -9340,21 +9465,17 @@ fn atomicOp(
9340	9465
9341	9466	try self.genSetReg(dst_reg, val_ty, val_mcv);
9342	9467	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
9343		try self.genUnOpMir(.neg, val_ty, dst_mcv);
	9468	try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv);
9344	9469	}
9345		_ = try self.addInst(.{
9346		.tag = tag,
9347		.ops = .mr_sib,
9348		.data = .{ .rx = .{
9349		.fixes = switch (tag) {
9350		.mov, .xchg => ._,
9351		.xadd, .add, .sub, .@"and", .@"or", .xor => .@"lock _",
9352		else => unreachable,
9353		},
9354		.r1 = registerAlias(dst_reg, val_abi_size),
9355		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9356		} },
9357		});
	9470	try self.asmMemoryRegister(
	9471	switch (tag) {
	9472	.mov, .xchg => .{ ._, tag },
	9473	.xadd, .add, .sub, .@"and", .@"or", .xor => .{ .@"lock _", tag },
	9474	else => unreachable,
	9475	},
	9476	ptr_mem,
	9477	registerAlias(dst_reg, val_abi_size),
	9478	);
9358	9479
9359	9480	return if (unused) .unreach else dst_mcv;
9360	9481	},
...	...	@@ -9364,22 +9485,22 @@ fn atomicOp(
9364	9485	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
9365	9486	defer self.register_manager.unlockReg(tmp_lock);
9366	9487
9367		try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
	9488	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
9368	9489	const loop = @intCast(u32, self.mir_instructions.len);
9369	9490	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
9370	9491	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
9371	9492	}
9372	9493	if (rmw_op) \|op\| switch (op) {
9373	9494	.Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
9374		.Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
9375		.Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
9376		.And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
	9495	.Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
	9496	.Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
	9497	.And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv),
9377	9498	.Nand => {
9378		try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);
9379		try self.genUnOpMir(.not, val_ty, tmp_mcv);
	9499	try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv);
	9500	try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv);
9380	9501	},
9381		.Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),
9382		.Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),
	9502	.Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv),
	9503	.Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv),
9383	9504	.Min, .Max => {
9384	9505	const cc: Condition = switch (if (val_ty.isAbiInt())
9385	9506	val_ty.intInfo(self.target.*).signedness
...	...	@@ -9397,7 +9518,7 @@ fn atomicOp(
9397	9518	},
9398	9519	};
9399	9520
9400		try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);
	9521	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
9401	9522	const cmov_abi_size = @max(val_abi_size, 2);
9402	9523	switch (val_mcv) {
9403	9524	.register => \|val_reg\| try self.asmCmovccRegisterRegister(
...	...	@@ -9421,24 +9542,20 @@ fn atomicOp(
9421	9542	}
9422	9543	},
9423	9544	};
9424		_ = try self.addInst(.{
9425		.tag = .cmpxchg,
9426		.ops = .mr_sib,
9427		.data = .{ .rx = .{
9428		.fixes = .@"lock _",
9429		.r1 = registerAlias(tmp_reg, val_abi_size),
9430		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9431		} },
9432		});
	9545	try self.asmMemoryRegister(
	9546	.{ .@"lock _", .cmpxchg },
	9547	ptr_mem,
	9548	registerAlias(tmp_reg, val_abi_size),
	9549	);
9433	9550	_ = try self.asmJccReloc(loop, .ne);
9434	9551	return if (unused) .unreach else .{ .register = .rax };
9435	9552	} else {
9436		try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
	9553	try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
9437	9554	.base = ptr_mem.sib.base,
9438	9555	.scale_index = ptr_mem.scaleIndex(),
9439	9556	.disp = ptr_mem.sib.disp + 0,
9440	9557	}));
9441		try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{
	9558	try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
9442	9559	.base = ptr_mem.sib.base,
9443	9560	.scale_index = ptr_mem.scaleIndex(),
9444	9561	.disp = ptr_mem.sib.disp + 8,
...	...	@@ -9453,58 +9570,51 @@ fn atomicOp(
9453	9570	const val_lo_mem = val_mem_mcv.mem(.qword);
9454	9571	const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
9455	9572	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
9456		try self.asmRegisterRegister(.mov, .rbx, .rax);
9457		try self.asmRegisterRegister(.mov, .rcx, .rdx);
	9573	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
	9574	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
9458	9575	}
9459	9576	if (rmw_op) \|op\| switch (op) {
9460	9577	.Xchg => {
9461		try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);
9462		try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);
	9578	try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem);
	9579	try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem);
9463	9580	},
9464	9581	.Add => {
9465		try self.asmRegisterMemory(.add, .rbx, val_lo_mem);
9466		try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);
	9582	try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem);
	9583	try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem);
9467	9584	},
9468	9585	.Sub => {
9469		try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);
9470		try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);
	9586	try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem);
	9587	try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem);
9471	9588	},
9472	9589	.And => {
9473		try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
9474		try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
	9590	try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
	9591	try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
9475	9592	},
9476	9593	.Nand => {
9477		try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
9478		try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
9479		try self.asmRegister(.not, .rbx);
9480		try self.asmRegister(.not, .rcx);
	9594	try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
	9595	try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
	9596	try self.asmRegister(.{ ._, .not }, .rbx);
	9597	try self.asmRegister(.{ ._, .not }, .rcx);
9481	9598	},
9482	9599	.Or => {
9483		try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);
9484		try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);
	9600	try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem);
	9601	try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem);
9485	9602	},
9486	9603	.Xor => {
9487		try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);
9488		try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);
	9604	try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
	9605	try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
9489	9606	},
9490	9607	else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
9491	9608	val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
9492	9609	}),
9493	9610	};
9494		_ = try self.addInst(.{
9495		.tag = .cmpxchg,
9496		.ops = .m_sib,
9497		.data = .{ .x = .{
9498		.fixes = .@"lock _16b",
9499		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9500		} },
9501		});
	9611	try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
9502	9612	_ = try self.asmJccReloc(loop, .ne);
9503	9613
9504	9614	if (unused) return .unreach;
9505	9615	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
9506	9616	try self.asmMemoryRegister(
9507		.mov,
	9617	.{ ._, .mov },
9508	9618	Memory.sib(.qword, .{
9509	9619	.base = .{ .frame = dst_mcv.load_frame.index },
9510	9620	.disp = dst_mcv.load_frame.off + 0,
...	...	@@ -9512,7 +9622,7 @@ fn atomicOp(
9512	9622	.rax,
9513	9623	);
9514	9624	try self.asmMemoryRegister(
9515		.mov,
	9625	.{ ._, .mov },
9516	9626	Memory.sib(.qword, .{
9517	9627	.base = .{ .frame = dst_mcv.load_frame.index },
9518	9628	.disp = dst_mcv.load_frame.off + 8,
...	...	@@ -9664,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
9664	9774	.off = elem_abi_size,
9665	9775	} });
9666	9776
9667		try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
9668		try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
	9777	try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
	9778	try self.asmRegisterRegisterImmediate(
	9779	.{ .i_, .mul },
	9780	len_reg,
	9781	len_reg,
	9782	Immediate.u(elem_abi_size),
	9783	);
9669	9784	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
9670	9785
9671	9786	try self.performReloc(skip_reloc);
...	...	@@ -9803,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9803	9918	try self.truncateRegister(err_ty, err_reg.to32());
9804	9919
9805	9920	try self.asmRegisterMemory(
9806		.mov,
	9921	.{ ._, .mov },
9807	9922	start_reg.to32(),
9808	9923	Memory.sib(.dword, .{
9809	9924	.base = .{ .reg = addr_reg.to64() },
...	...	@@ -9812,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9812	9927	}),
9813	9928	);
9814	9929	try self.asmRegisterMemory(
9815		.mov,
	9930	.{ ._, .mov },
9816	9931	end_reg.to32(),
9817	9932	Memory.sib(.dword, .{
9818	9933	.base = .{ .reg = addr_reg.to64() },
...	...	@@ -9820,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9820	9935	.disp = 8,
9821	9936	}),
9822	9937	);
9823		try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
	9938	try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
9824	9939	try self.asmRegisterMemory(
9825		.lea,
	9940	.{ ._, .lea },
9826	9941	start_reg.to64(),
9827	9942	Memory.sib(.byte, .{
9828	9943	.base = .{ .reg = addr_reg.to64() },
...	...	@@ -9831,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9831	9946	}),
9832	9947	);
9833	9948	try self.asmRegisterMemory(
9834		.lea,
	9949	.{ ._, .lea },
9835	9950	end_reg.to32(),
9836	9951	Memory.sib(.byte, .{
9837	9952	.base = .{ .reg = end_reg.to64() },
...	...	@@ -9841,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9841	9956
9842	9957	const dst_mcv = try self.allocRegOrMem(inst, false);
9843	9958	try self.asmMemoryRegister(
9844		.mov,
	9959	.{ ._, .mov },
9845	9960	Memory.sib(.qword, .{
9846	9961	.base = .{ .frame = dst_mcv.load_frame.index },
9847	9962	.disp = dst_mcv.load_frame.off,
...	...	@@ -9849,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
9849	9964	start_reg.to64(),
9850	9965	);
9851	9966	try self.asmMemoryRegister(
9852		.mov,
	9967	.{ ._, .mov },
9853	9968	Memory.sib(.qword, .{
9854	9969	.base = .{ .frame = dst_mcv.load_frame.index },
9855	9970	.disp = dst_mcv.load_frame.off + 8,
...	...	@@ -9945,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9945	10060	try self.truncateRegister(elem_ty, elem_reg);
9946	10061	}
9947	10062	if (elem_bit_off > 0) try self.genShiftBinOpMir(
9948		.shl,
	10063	.{ ._l, .sh },
9949	10064	elem_ty,
9950	10065	.{ .register = elem_reg },
9951	10066	.{ .immediate = elem_bit_off },
9952	10067	);
9953	10068	try self.genBinOpMir(
9954		.@"or",
	10069	.{ ._, .@"or" },
9955	10070	elem_ty,
9956	10071	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
9957	10072	.{ .register = elem_reg },
...	...	@@ -9962,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9962	10077	try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));
9963	10078	}
9964	10079	try self.genShiftBinOpMir(
9965		.shr,
	10080	.{ ._r, .sh },
9966	10081	elem_ty,
9967	10082	.{ .register = reg },
9968	10083	.{ .immediate = elem_abi_bits - elem_bit_off },
9969	10084	);
9970	10085	try self.genBinOpMir(
9971		.@"or",
	10086	.{ ._, .@"or" },
9972	10087	elem_ty,
9973	10088	.{ .load_frame = .{
9974	10089	.index = frame_index,
...	...	@@ -10078,25 +10193,25 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
10078	10193	}
10079	10194
10080	10195	const mir_tag = if (@as(
10081		?Mir.Inst.Tag,
	10196	?Mir.Inst.FixedTag,
10082	10197	if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
10083	10198	switch (ty.zigTypeTag()) {
10084	10199	.Float => switch (ty.floatBits(self.target.*)) {
10085		32 => .vfmadd132ss,
10086		64 => .vfmadd132sd,
	10200	32 => .{ ._, .vfmadd132ss },
	10201	64 => .{ ._, .vfmadd132sd },
10087	10202	16, 80, 128 => null,
10088	10203	else => unreachable,
10089	10204	},
10090	10205	.Vector => switch (ty.childType().zigTypeTag()) {
10091	10206	.Float => switch (ty.childType().floatBits(self.target.*)) {
10092	10207	32 => switch (ty.vectorLen()) {
10093		1 => .vfmadd132ss,
10094		2...8 => .vfmadd132ps,
	10208	1 => .{ ._, .vfmadd132ss },
	10209	2...8 => .{ ._, .vfmadd132ps },
10095	10210	else => null,
10096	10211	},
10097	10212	64 => switch (ty.vectorLen()) {
10098		1 => .vfmadd132sd,
10099		2...4 => .vfmadd132pd,
	10213	1 => .{ ._, .vfmadd132sd },
	10214	2...4 => .{ ._, .vfmadd132pd },
10100	10215	else => null,
10101	10216	},
10102	10217	16, 80, 128 => null,
...	...	@@ -10109,21 +10224,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
10109	10224	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
10110	10225	switch (ty.zigTypeTag()) {
10111	10226	.Float => switch (ty.floatBits(self.target.*)) {
10112		32 => .vfmadd213ss,
10113		64 => .vfmadd213sd,
	10227	32 => .{ ._, .vfmadd213ss },
	10228	64 => .{ ._, .vfmadd213sd },
10114	10229	16, 80, 128 => null,
10115	10230	else => unreachable,
10116	10231	},
10117	10232	.Vector => switch (ty.childType().zigTypeTag()) {
10118	10233	.Float => switch (ty.childType().floatBits(self.target.*)) {
10119	10234	32 => switch (ty.vectorLen()) {
10120		1 => .vfmadd213ss,
10121		2...8 => .vfmadd213ps,
	10235	1 => .{ ._, .vfmadd213ss },
	10236	2...8 => .{ ._, .vfmadd213ps },
10122	10237	else => null,
10123	10238	},
10124	10239	64 => switch (ty.vectorLen()) {
10125		1 => .vfmadd213sd,
10126		2...4 => .vfmadd213pd,
	10240	1 => .{ ._, .vfmadd213sd },
	10241	2...4 => .{ ._, .vfmadd213pd },
10127	10242	else => null,
10128	10243	},
10129	10244	16, 80, 128 => null,
...	...	@@ -10136,21 +10251,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
10136	10251	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
10137	10252	switch (ty.zigTypeTag()) {
10138	10253	.Float => switch (ty.floatBits(self.target.*)) {
10139		32 => .vfmadd231ss,
10140		64 => .vfmadd231sd,
	10254	32 => .{ ._, .vfmadd231ss },
	10255	64 => .{ ._, .vfmadd231sd },
10141	10256	16, 80, 128 => null,
10142	10257	else => unreachable,
10143	10258	},
10144	10259	.Vector => switch (ty.childType().zigTypeTag()) {
10145	10260	.Float => switch (ty.childType().floatBits(self.target.*)) {
10146	10261	32 => switch (ty.vectorLen()) {
10147		1 => .vfmadd231ss,
10148		2...8 => .vfmadd231ps,
	10262	1 => .{ ._, .vfmadd231ss },
	10263	2...8 => .{ ._, .vfmadd231ps },
10149	10264	else => null,
10150	10265	},
10151	10266	64 => switch (ty.vectorLen()) {
10152		1 => .vfmadd231sd,
10153		2...4 => .vfmadd231pd,
	10267	1 => .{ ._, .vfmadd231sd },
	10268	2...4 => .{ ._, .vfmadd231pd },
10154	10269	else => null,
10155	10270	},
10156	10271	16, 80, 128 => null,
...	...	@@ -10522,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
10522	10637	switch (int_info.signedness) {
10523	10638	.signed => {
10524	10639	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
10525		try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });
10526		try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });
	10640	try self.genShiftBinOpMir(
	10641	.{ ._l, .sa },
	10642	Type.isize,
	10643	.{ .register = reg },
	10644	.{ .immediate = shift },
	10645	);
	10646	try self.genShiftBinOpMir(
	10647	.{ ._r, .sa },
	10648	Type.isize,
	10649	.{ .register = reg },
	10650	.{ .immediate = shift },
	10651	);
10527	10652	},
10528	10653	.unsigned => {
10529	10654	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
10530	10655	const mask = (~@as(u64, 0)) >> shift;
10531	10656	if (int_info.bits <= 32) {
10532		try self.genBinOpMir(.@"and", Type.u32, .{ .register = reg }, .{ .immediate = mask });
	10657	try self.genBinOpMir(
	10658	.{ ._, .@"and" },
	10659	Type.u32,
	10660	.{ .register = reg },
	10661	.{ .immediate = mask },
	10662	);
10533	10663	} else {
10534	10664	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
10535		try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg });
	10665	try self.genBinOpMir(
	10666	.{ ._, .@"and" },
	10667	Type.usize,
	10668	.{ .register = reg },
	10669	.{ .register = tmp_reg },
	10670	);
10536	10671	}
10537	10672	},
10538	10673	}

src/arch/x86_64/Lower.zig+6-3

...	...	@@ -286,10 +286,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
286	286	.rri_s, .rri_u => inst.data.rri.fixes,
287	287	.ri_s, .ri_u => inst.data.ri.fixes,
288	288	.ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
289		.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => ._,
290	289	.mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
291	290	.rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
292	291	.rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
	292	.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
293	293	.m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
294	294	.extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
295	295	else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
...	...	@@ -356,8 +356,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
356	356	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
357	357	},
358	358	.mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
359		.{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
360		.{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
	359	.{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
	360	.{ .imm = lower.imm(
	361	inst.ops,
	362	lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
	363	) },
361	364	},
362	365	.rm_sib, .rm_rip => &.{
363	366	.{ .reg = inst.data.rx.r1 },

src/arch/x86_64/Mir.zig+62-38

...	...	@@ -36,6 +36,18 @@ pub const Inst = struct {
36	36	/// ___
37	37	@"_",
38	38
	39	/// Integer __
	40	i_,
	41
	42	/// ___ Left
	43	_l,
	44	/// ___ Left Double
	45	_ld,
	46	/// ___ Right
	47	_r,
	48	/// ___ Right Double
	49	_rd,
	50
39	51	/// ___ Above
40	52	_a,
41	53	/// ___ Above Or Equal
...	...	@@ -53,7 +65,7 @@ pub const Inst = struct {
53	65	/// ___ Greater Or Equal
54	66	_ge,
55	67	/// ___ Less
56		_l,
	68	//_l,
57	69	/// ___ Less Or Equal
58	70	_le,
59	71	/// ___ Not Above
...	...	@@ -97,6 +109,15 @@ pub const Inst = struct {
97	109	/// ___ Zero
98	110	_z,
99	111
	112	/// ___ Byte
	113	//_b,
	114	/// ___ Word
	115	_w,
	116	/// ___ Doubleword
	117	_d,
	118	/// ___ QuadWord
	119	_q,
	120
100	121	/// ___ String
101	122	//_s,
102	123	/// ___ String Byte
...	...	@@ -165,6 +186,18 @@ pub const Inst = struct {
165	186
166	187	/// Locked ___
167	188	@"lock _",
	189	/// ___ And Complement
	190	//_c,
	191	/// Locked ___ And Complement
	192	@"lock _c",
	193	/// ___ And Reset
	194	//_r,
	195	/// Locked ___ And Reset
	196	@"lock _r",
	197	/// ___ And Set
	198	//_s,
	199	/// Locked ___ And Set
	200	@"lock _s",
168	201	/// ___ 8 Bytes
169	202	_8b,
170	203	/// Locked ___ 8 Bytes
...	...	@@ -174,6 +207,11 @@ pub const Inst = struct {
174	207	/// Locked ___ 16 Bytes
175	208	@"lock _16b",
176	209
	210	/// Float ___
	211	f_,
	212	/// Float ___ Pop
	213	f_p,
	214
177	215	/// Packed ___
178	216	p_,
179	217	/// Packed ___ Byte
...	...	@@ -250,13 +288,10 @@ pub const Inst = struct {
250	288	/// Byte swap
251	289	bswap,
252	290	/// Bit test
253		bt,
254	291	/// Bit test and complement
255		btc,
256	292	/// Bit test and reset
257		btr,
258	293	/// Bit test and set
259		bts,
	294	bt,
260	295	/// Call
261	296	call,
262	297	/// Convert byte to word
...	...	@@ -280,21 +315,18 @@ pub const Inst = struct {
280	315	/// Convert word to doubleword
281	316	cwde,
282	317	/// Unsigned division
283		div,
284		/// Store integer with truncation
285		fisttp,
286		/// Load floating-point value
287		fld,
288	318	/// Signed division
289		idiv,
290		/// Signed multiplication
291		imul,
	319	div,
292	320	///
293	321	int3,
	322	/// Store integer with truncation
	323	istt,
294	324	/// Conditional jump
295	325	j,
296	326	/// Jump
297	327	jmp,
	328	/// Load floating-point value
	329	ld,
298	330	/// Load effective address
299	331	lea,
300	332	/// Load string
...	...	@@ -307,20 +339,17 @@ pub const Inst = struct {
307	339	mfence,
308	340	/// Move
309	341	/// Move data from string to string
	342	/// Move doubleword
	343	/// Move quadword
310	344	mov,
311	345	/// Move data after swapping bytes
312	346	movbe,
313		/// Move doubleword
314		movd,
315		/// Move quadword
316		movq,
317	347	/// Move with sign extension
318	348	movsx,
319		/// Move with sign extension
320		movsxd,
321	349	/// Move with zero extension
322	350	movzx,
323	351	/// Multiply
	352	/// Signed multiplication
324	353	mul,
325	354	/// Two's complement negation
326	355	neg,
...	...	@@ -337,19 +366,16 @@ pub const Inst = struct {
337	366	/// Push
338	367	push,
339	368	/// Rotate left through carry
340		rcl,
341	369	/// Rotate right through carry
342		rcr,
	370	rc,
343	371	/// Return
344	372	ret,
345	373	/// Rotate left
346		rol,
347	374	/// Rotate right
348		ror,
	375	ro,
349	376	/// Arithmetic shift left
350		sal,
351	377	/// Arithmetic shift right
352		sar,
	378	sa,
353	379	/// Integer subtraction with borrow
354	380	sbb,
355	381	/// Scan string
...	...	@@ -359,13 +385,10 @@ pub const Inst = struct {
359	385	/// Store fence
360	386	sfence,
361	387	/// Logical shift left
362		shl,
363	388	/// Double precision shift left
364		shld,
365	389	/// Logical shift right
366		shr,
367	390	/// Double precision shift right
368		shrd,
	391	sh,
369	392	/// Subtract
370	393	sub,
371	394	/// Store string
...	...	@@ -730,6 +753,8 @@ pub const Inst = struct {
730	753	pseudo,
731	754	};
732	755
	756	pub const FixedTag = struct { Fixes, Tag };
	757
733	758	pub const Ops = enum(u8) {
734	759	/// No data associated with this instruction (only mnemonic is used).
735	760	none,
...	...	@@ -800,16 +825,16 @@ pub const Inst = struct {
800	825	/// Uses `x` with extra data of type `MemoryRip`.
801	826	m_rip,
802	827	/// Memory (SIB), immediate (unsigned) operands.
803		/// Uses `ix` payload with extra data of type `MemorySib`.
	828	/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
804	829	mi_sib_u,
805	830	/// Memory (RIP), immediate (unsigned) operands.
806		/// Uses `ix` payload with extra data of type `MemoryRip`.
	831	/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
807	832	mi_rip_u,
808	833	/// Memory (SIB), immediate (sign-extend) operands.
809		/// Uses `ix` payload with extra data of type `MemorySib`.
	834	/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
810	835	mi_sib_s,
811	836	/// Memory (RIP), immediate (sign-extend) operands.
812		/// Uses `ix` payload with extra data of type `MemoryRip`.
	837	/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
813	838	mi_rip_s,
814	839	/// Memory (SIB), register operands.
815	840	/// Uses `rx` payload with extra data of type `MemorySib`.
...	...	@@ -974,11 +999,6 @@ pub const Inst = struct {
974	999	r1: Register,
975	1000	payload: u32,
976	1001	},
977		/// Immediate, followed by Custom payload found in extra.
978		ix: struct {
979		i: u32,
980		payload: u32,
981		},
982	1002	/// Register, register, followed by Custom payload found in extra.
983	1003	rrx: struct {
984	1004	fixes: Fixes = ._,
...	...	@@ -1081,6 +1101,10 @@ pub const RegisterList = struct {
1081	1101	}
1082	1102	};
1083	1103
	1104	pub const Imm32 = struct {
	1105	imm: u32,
	1106	};
	1107
1084	1108	pub const Imm64 = struct {
1085	1109	msb: u32,
1086	1110	lsb: u32,