Commit 6c14eb2863c7c00f809c5e447ceb8186b55f2eef | zig

author	Jacob Young <jacobly@ziglang.org> 2023-05-08 06:50:18-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
log	6c14eb2863c7c00f809c5e447ceb8186b55f2eef
tree	14d9948d53fb661206d2a8b71ba4af73e9431de1
parent	6778da4516e68c271cb50fe9c252ab4084daf16b

x86_64: optimize mir tag usage

This moves all pseudo-instructions to a single `Mir.Inst.Tag` tag and
prepares to start coalescing similar mnemonics. 239 tags left in use.

6 files changed, 839 insertions(+), 864 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig

src/arch/x86_64/CodeGen.zig+251-152

...	...	@@ -973,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
973	973	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
974	974	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
975	975	self.mir_instructions.appendAssumeCapacity(inst);
976		switch (inst.tag) {
977		else => wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)}),
978		.dbg_line,
979		.dbg_prologue_end,
980		.dbg_epilogue_begin,
981		.dead,
982		=> {},
983		}
	976	if (inst.tag != .pseudo or switch (inst.ops) {
	977	else => true,
	978	.pseudo_dbg_prologue_end_none,
	979	.pseudo_dbg_line_line_column,
	980	.pseudo_dbg_epilogue_begin_none,
	981	.pseudo_dead_none,
	982	=> false,
	983	}) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
984	984	return result_index;
985	985	}
986	986
...	...	@@ -1003,35 +1003,57 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1003	1003	return result;
1004	1004	}
1005	1005
1006		fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
	1006	/// A `cc` of `.z_and_np` clobbers `reg2`!
	1007	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1007	1008	_ = try self.addInst(.{
1008		.tag = .setcc,
1009		.ops = .r_cc,
1010		.data = .{ .r_cc = .{
1011		.r = reg,
1012		.scratch = if (cc == .z_and_np or cc == .nz_or_p)
1013		(try self.register_manager.allocReg(null, gp)).to8()
1014		else
1015		.none,
1016		.cc = cc,
	1009	.tag = switch (cc) {
	1010	else => .cmov,
	1011	.z_and_np, .nz_or_p => .pseudo,
	1012	},
	1013	.ops = switch (cc) {
	1014	else => .rr,
	1015	.z_and_np => .pseudo_cmov_z_and_np_rr,
	1016	.nz_or_p => .pseudo_cmov_nz_or_p_rr,
	1017	},
	1018	.data = .{ .rr = .{
	1019	.fixes = switch (cc) {
	1020	else => Mir.Inst.Fixes.fromCondition(cc),
	1021	.z_and_np, .nz_or_p => ._,
	1022	},
	1023	.r1 = reg1,
	1024	.r2 = reg2,
1017	1025	} },
1018	1026	});
1019	1027	}
1020	1028
1021		fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
	1029	/// A `cc` of `.z_and_np` is not supported by this encoding!
	1030	fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
1022	1031	_ = try self.addInst(.{
1023		.tag = .setcc,
1024		.ops = switch (m) {
1025		.sib => .m_sib_cc,
1026		.rip => .m_rip_cc,
1027		else => unreachable,
	1032	.tag = switch (cc) {
	1033	else => .cmov,
	1034	.z_and_np => unreachable,
	1035	.nz_or_p => .pseudo,
1028	1036	},
1029		.data = .{ .x_cc = .{
1030		.scratch = if (cc == .z_and_np or cc == .nz_or_p)
1031		(try self.register_manager.allocReg(null, gp)).to8()
1032		else
1033		.none,
1034		.cc = cc,
	1037	.ops = switch (cc) {
	1038	else => switch (m) {
	1039	.sib => .rm_sib,
	1040	.rip => .rm_rip,
	1041	else => unreachable,
	1042	},
	1043	.z_and_np => unreachable,
	1044	.nz_or_p => switch (m) {
	1045	.sib => .pseudo_cmov_nz_or_p_rm_sib,
	1046	.rip => .pseudo_cmov_nz_or_p_rm_rip,
	1047	else => unreachable,
	1048	},
	1049	},
	1050	.data = .{ .rx = .{
	1051	.fixes = switch (cc) {
	1052	else => Mir.Inst.Fixes.fromCondition(cc),
	1053	.z_and_np => unreachable,
	1054	.nz_or_p => ._,
	1055	},
	1056	.r1 = reg,
1035	1057	.payload = switch (m) {
1036	1058	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1037	1059	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...	...	@@ -1041,60 +1063,106 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
1041	1063	});
1042	1064	}
1043	1065
1044		/// A `cc` of `.z_and_np` clobbers `reg2`!
1045		fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
	1066	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1046	1067	_ = try self.addInst(.{
1047		.tag = .cmovcc,
1048		.ops = .rr_cc,
1049		.data = .{ .rr_cc = .{
1050		.r1 = reg1,
1051		.r2 = reg2,
1052		.cc = cc,
1053		} },
	1068	.tag = switch (cc) {
	1069	else => .set,
	1070	.z_and_np, .nz_or_p => .pseudo,
	1071	},
	1072	.ops = switch (cc) {
	1073	else => .r,
	1074	.z_and_np => .pseudo_set_z_and_np_r,
	1075	.nz_or_p => .pseudo_set_nz_or_p_r,
	1076	},
	1077	.data = switch (cc) {
	1078	else => .{ .r = .{
	1079	.fixes = Mir.Inst.Fixes.fromCondition(cc),
	1080	.r1 = reg,
	1081	} },
	1082	.z_and_np, .nz_or_p => .{ .r_scratch = .{
	1083	.r1 = reg,
	1084	.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
	1085	} },
	1086	},
1054	1087	});
1055	1088	}
1056	1089
1057		fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
1058		assert(cc != .z_and_np); // not supported
	1090	fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
	1091	const payload = switch (m) {
	1092	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1093	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1094	else => unreachable,
	1095	};
1059	1096	_ = try self.addInst(.{
1060		.tag = .cmovcc,
1061		.ops = switch (m) {
1062		.sib => .rm_sib_cc,
1063		.rip => .rm_rip_cc,
1064		else => unreachable,
	1097	.tag = switch (cc) {
	1098	else => .set,
	1099	.z_and_np, .nz_or_p => .pseudo,
1065	1100	},
1066		.data = .{ .rx_cc = .{
1067		.r = reg,
1068		.cc = cc,
1069		.payload = switch (m) {
1070		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1071		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1101	.ops = switch (cc) {
	1102	else => switch (m) {
	1103	.sib => .m_sib,
	1104	.rip => .m_rip,
1072	1105	else => unreachable,
1073	1106	},
1074		} },
	1107	.z_and_np => switch (m) {
	1108	.sib => .pseudo_set_z_and_np_m_sib,
	1109	.rip => .pseudo_set_z_and_np_m_rip,
	1110	else => unreachable,
	1111	},
	1112	.nz_or_p => switch (m) {
	1113	.sib => .pseudo_set_nz_or_p_m_sib,
	1114	.rip => .pseudo_set_nz_or_p_m_rip,
	1115	else => unreachable,
	1116	},
	1117	},
	1118	.data = switch (cc) {
	1119	else => .{ .x = .{
	1120	.fixes = Mir.Inst.Fixes.fromCondition(cc),
	1121	.payload = payload,
	1122	} },
	1123	.z_and_np, .nz_or_p => .{ .x_scratch = .{
	1124	.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
	1125	.payload = payload,
	1126	} },
	1127	},
1075	1128	});
1076	1129	}
1077	1130
1078	1131	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1079	1132	return self.addInst(.{
1080		.tag = .jmp_reloc,
1081		.ops = undefined,
1082		.data = .{ .inst = target },
	1133	.tag = .jmp,
	1134	.ops = .inst,
	1135	.data = .{ .inst = .{
	1136	.inst = target,
	1137	} },
1083	1138	});
1084	1139	}
1085	1140
1086	1141	fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
1087	1142	return self.addInst(.{
1088		.tag = .jcc,
1089		.ops = .inst_cc,
1090		.data = .{ .inst_cc = .{ .inst = target, .cc = cc } },
	1143	.tag = switch (cc) {
	1144	else => .j,
	1145	.z_and_np, .nz_or_p => .pseudo,
	1146	},
	1147	.ops = switch (cc) {
	1148	else => .inst,
	1149	.z_and_np => .pseudo_j_z_and_np_inst,
	1150	.nz_or_p => .pseudo_j_nz_or_p_inst,
	1151	},
	1152	.data = .{ .inst = .{
	1153	.fixes = switch (cc) {
	1154	else => Mir.Inst.Fixes.fromCondition(cc),
	1155	.z_and_np, .nz_or_p => ._,
	1156	},
	1157	.inst = target,
	1158	} },
1091	1159	});
1092	1160	}
1093	1161
1094	1162	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1095	1163	return self.addInst(.{
1096		.tag = .dead,
1097		.ops = undefined,
	1164	.tag = .pseudo,
	1165	.ops = .pseudo_dead_none,
1098	1166	.data = undefined,
1099	1167	});
1100	1168	}
...	...	@@ -1107,11 +1175,19 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
1107	1175	});
1108	1176	}
1109	1177
	1178	fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
	1179	_ = try self.addInst(.{
	1180	.tag = .pseudo,
	1181	.ops = ops,
	1182	.data = undefined,
	1183	});
	1184	}
	1185
1110	1186	fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
1111	1187	_ = try self.addInst(.{
1112	1188	.tag = tag,
1113	1189	.ops = .r,
1114		.data = .{ .r = reg },
	1190	.data = .{ .r = .{ .r1 = reg } },
1115	1191	});
1116	1192	}
1117	1193
...	...	@@ -1122,9 +1198,11 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
1122	1198	.signed => .i_s,
1123	1199	.unsigned => .i_u,
1124	1200	},
1125		.data = .{ .i = switch (imm) {
1126		.signed => \|s\| @bitCast(u32, s),
1127		.unsigned => \|u\| @intCast(u32, u),
	1201	.data = .{ .i = .{
	1202	.i = switch (imm) {
	1203	.signed => \|s\| @bitCast(u32, s),
	1204	.unsigned => \|u\| @intCast(u32, u),
	1205	},
1128	1206	} },
1129	1207	});
1130	1208	}
...	...	@@ -1147,14 +1225,14 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1147	1225	.ops = ops,
1148	1226	.data = switch (ops) {
1149	1227	.ri_s, .ri_u => .{ .ri = .{
1150		.r = reg,
	1228	.r1 = reg,
1151	1229	.i = switch (imm) {
1152	1230	.signed => \|s\| @bitCast(u32, s),
1153	1231	.unsigned => \|u\| @intCast(u32, u),
1154	1232	},
1155	1233	} },
1156	1234	.ri64 => .{ .rx = .{
1157		.r = reg,
	1235	.r1 = reg,
1158	1236	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
1159	1237	} },
1160	1238	else => unreachable,
...	...	@@ -1249,10 +1327,12 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
1249	1327	.rip => .m_rip,
1250	1328	else => unreachable,
1251	1329	},
1252		.data = .{ .payload = switch (m) {
1253		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1254		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1255		else => unreachable,
	1330	.data = .{ .x = .{
	1331	.payload = switch (m) {
	1332	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1333	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1334	else => unreachable,
	1335	},
1256	1336	} },
1257	1337	});
1258	1338	}
...	...	@@ -1266,7 +1346,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
1266	1346	else => unreachable,
1267	1347	},
1268	1348	.data = .{ .rx = .{
1269		.r = reg,
	1349	.r1 = reg,
1270	1350	.payload = switch (m) {
1271	1351	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1272	1352	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...	...	@@ -1291,7 +1371,7 @@ fn asmRegisterMemoryImmediate(
1291	1371	else => unreachable,
1292	1372	},
1293	1373	.data = .{ .rix = .{
1294		.r = reg,
	1374	.r1 = reg,
1295	1375	.i = @intCast(u8, imm.unsigned),
1296	1376	.payload = switch (m) {
1297	1377	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
...	...	@@ -1339,7 +1419,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
1339	1419	else => unreachable,
1340	1420	},
1341	1421	.data = .{ .rx = .{
1342		.r = reg,
	1422	.r1 = reg,
1343	1423	.payload = switch (m) {
1344	1424	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1345	1425	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...	...	@@ -1413,11 +1493,15 @@ fn asmMemoryRegisterImmediate(
1413	1493	.rip => .mri_rip,
1414	1494	else => unreachable,
1415	1495	},
1416		.data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
1417		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1418		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1419		else => unreachable,
1420		} } },
	1496	.data = .{ .rix = .{
	1497	.r1 = reg,
	1498	.i = @intCast(u8, imm.unsigned),
	1499	.payload = switch (m) {
	1500	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1501	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1502	else => unreachable,
	1503	},
	1504	} },
1421	1505	});
1422	1506	}
1423	1507
...	...	@@ -1450,7 +1534,7 @@ fn gen(self: *Self) InnerError!void {
1450	1534	else => unreachable,
1451	1535	}
1452	1536
1453		try self.asmOpOnly(.dbg_prologue_end);
	1537	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1454	1538
1455	1539	try self.genBody(self.air.getMainBody());
1456	1540
...	...	@@ -1462,11 +1546,11 @@ fn gen(self: *Self) InnerError!void {
1462	1546	// }
1463	1547	// Eliding the reloc will cause a miscompilation in this case.
1464	1548	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
1465		self.mir_instructions.items(.data)[jmp_reloc].inst =
	1549	self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
1466	1550	@intCast(u32, self.mir_instructions.len);
1467	1551	}
1468	1552
1469		try self.asmOpOnly(.dbg_epilogue_begin);
	1553	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1470	1554	const backpatch_stack_dealloc = try self.asmPlaceholder();
1471	1555	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
1472	1556	try self.asmRegister(.pop, .rbp);
...	...	@@ -1480,46 +1564,54 @@ fn gen(self: *Self) InnerError!void {
1480	1564	self.mir_instructions.set(backpatch_frame_align, .{
1481	1565	.tag = .@"and",
1482	1566	.ops = .ri_s,
1483		.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
	1567	.data = .{ .ri = .{
	1568	.r1 = .rsp,
	1569	.i = frame_layout.stack_mask,
	1570	} },
1484	1571	});
1485	1572	}
1486	1573	if (need_stack_adjust) {
1487	1574	self.mir_instructions.set(backpatch_stack_alloc, .{
1488	1575	.tag = .sub,
1489	1576	.ops = .ri_s,
1490		.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
	1577	.data = .{ .ri = .{
	1578	.r1 = .rsp,
	1579	.i = frame_layout.stack_adjust,
	1580	} },
1491	1581	});
1492	1582	}
1493	1583	if (need_frame_align or need_stack_adjust) {
1494	1584	self.mir_instructions.set(backpatch_stack_dealloc, .{
1495	1585	.tag = .mov,
1496	1586	.ops = .rr,
1497		.data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
	1587	.data = .{ .rr = .{
	1588	.r1 = .rsp,
	1589	.r2 = .rbp,
	1590	} },
1498	1591	});
1499	1592	}
1500	1593	if (need_save_reg) {
1501		const save_reg_list = frame_layout.save_reg_list.asInt();
1502	1594	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
1503		.tag = .push_regs,
1504		.ops = undefined,
1505		.data = .{ .payload = save_reg_list },
	1595	.tag = .pseudo,
	1596	.ops = .pseudo_push_reg_list,
	1597	.data = .{ .reg_list = frame_layout.save_reg_list },
1506	1598	});
1507	1599	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
1508		.tag = .pop_regs,
1509		.ops = undefined,
1510		.data = .{ .payload = save_reg_list },
	1600	.tag = .pseudo,
	1601	.ops = .pseudo_pop_reg_list,
	1602	.data = .{ .reg_list = frame_layout.save_reg_list },
1511	1603	});
1512	1604	}
1513	1605	} else {
1514		try self.asmOpOnly(.dbg_prologue_end);
	1606	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1515	1607	try self.genBody(self.air.getMainBody());
1516		try self.asmOpOnly(.dbg_epilogue_begin);
	1608	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1517	1609	}
1518	1610
1519	1611	// Drop them off at the rbrace.
1520	1612	_ = try self.addInst(.{
1521		.tag = .dbg_line,
1522		.ops = undefined,
	1613	.tag = .pseudo,
	1614	.ops = .pseudo_dbg_line_line_column,
1523	1615	.data = .{ .line_column = .{
1524	1616	.line = self.end_di_line,
1525	1617	.column = self.end_di_column,
...	...	@@ -2446,11 +2538,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2446	2538	.register => \|dst_reg\| {
2447	2539	const min_abi_size = @min(dst_abi_size, src_abi_size);
2448	2540	const tag: Mir.Inst.Tag = switch (signedness) {
2449		.signed => .movsx,
2450		.unsigned => if (min_abi_size > 2) .mov else .movzx,
	2541	.signed => if (min_abi_size >= 4) .movsxd else .movsx,
	2542	.unsigned => if (min_abi_size >= 4) .mov else .movzx,
2451	2543	};
2452	2544	const dst_alias = switch (tag) {
2453		.movsx => dst_reg.to64(),
	2545	.movsx, .movsxd => dst_reg.to64(),
2454	2546	.mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
2455	2547	else => unreachable,
2456	2548	};
...	...	@@ -5247,7 +5339,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5247	5339	const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
5248	5340	if (signedness == .signed and field_byte_size < 8) {
5249	5341	try self.asmRegisterRegister(
5250		.movsx,
	5342	if (field_byte_size >= 4) .movsxd else .movsx,
5251	5343	dst_mcv.register,
5252	5344	registerAlias(dst_mcv.register, field_byte_size),
5253	5345	);
...	...	@@ -7194,10 +7286,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7194	7286	const atom_index = try self.owner.getSymbolIndex(self);
7195	7287	const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
7196	7288	_ = try self.addInst(.{
7197		.tag = .mov_linker,
	7289	.tag = .mov,
7198	7290	.ops = .import_reloc,
7199	7291	.data = .{ .rx = .{
7200		.r = .rax,
	7292	.r1 = .rax,
7201	7293	.payload = try self.addExtra(Mir.Reloc{
7202	7294	.atom_index = atom_index,
7203	7295	.sym_index = sym_index,
...	...	@@ -7209,9 +7301,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7209	7301	const atom_index = try self.owner.getSymbolIndex(self);
7210	7302	const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
7211	7303	_ = try self.addInst(.{
7212		.tag = .call_extern,
7213		.ops = undefined,
7214		.data = .{ .relocation = .{
	7304	.tag = .call,
	7305	.ops = .extern_fn_reloc,
	7306	.data = .{ .reloc = .{
7215	7307	.atom_index = atom_index,
7216	7308	.sym_index = sym_index,
7217	7309	} },
...	...	@@ -7489,8 +7581,8 @@ fn genTry(
7489	7581	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
7490	7582	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
7491	7583	_ = try self.addInst(.{
7492		.tag = .dbg_line,
7493		.ops = undefined,
	7584	.tag = .pseudo,
	7585	.ops = .pseudo_dbg_line_line_column,
7494	7586	.data = .{ .line_column = .{
7495	7587	.line = dbg_stmt.line,
7496	7588	.column = dbg_stmt.column,
...	...	@@ -8021,14 +8113,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
8021	8113	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
8022	8114	const next_inst = @intCast(u32, self.mir_instructions.len);
8023	8115	switch (self.mir_instructions.items(.tag)[reloc]) {
8024		.jcc => {
8025		self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
8026		},
8027		.jmp_reloc => {
8028		self.mir_instructions.items(.data)[reloc].inst = next_inst;
	8116	.j, .jmp => {},
	8117	.pseudo => switch (self.mir_instructions.items(.ops)[reloc]) {
	8118	.pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {},
	8119	else => unreachable,
8029	8120	},
8030	8121	else => unreachable,
8031	8122	}
	8123	self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
8032	8124	}
8033	8125
8034	8126	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -8577,10 +8669,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8577	8669	.load_direct => \|sym_index\| if (!ty.isRuntimeFloat()) {
8578	8670	const atom_index = try self.owner.getSymbolIndex(self);
8579	8671	_ = try self.addInst(.{
8580		.tag = .mov_linker,
	8672	.tag = .mov,
8581	8673	.ops = .direct_reloc,
8582	8674	.data = .{ .rx = .{
8583		.r = dst_reg.to64(),
	8675	.r1 = dst_reg.to64(),
8584	8676	.payload = try self.addExtra(Mir.Reloc{
8585	8677	.atom_index = atom_index,
8586	8678	.sym_index = sym_index,
...	...	@@ -8618,8 +8710,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8618	8710	const atom_index = try self.owner.getSymbolIndex(self);
8619	8711	_ = try self.addInst(.{
8620	8712	.tag = switch (src_mcv) {
8621		.lea_direct => .lea_linker,
8622		.lea_got => .mov_linker,
	8713	.lea_direct => .lea,
	8714	.lea_got => .mov,
8623	8715	else => unreachable,
8624	8716	},
8625	8717	.ops = switch (src_mcv) {
...	...	@@ -8628,7 +8720,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8628	8720	else => unreachable,
8629	8721	},
8630	8722	.data = .{ .rx = .{
8631		.r = dst_reg.to64(),
	8723	.r1 = dst_reg.to64(),
8632	8724	.payload = try self.addExtra(Mir.Reloc{
8633	8725	.atom_index = atom_index,
8634	8726	.sym_index = sym_index,
...	...	@@ -8640,10 +8732,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8640	8732	const atom_index = try self.owner.getSymbolIndex(self);
8641	8733	if (self.bin_file.cast(link.File.MachO)) \|_\| {
8642	8734	_ = try self.addInst(.{
8643		.tag = .lea_linker,
	8735	.tag = .lea,
8644	8736	.ops = .tlv_reloc,
8645	8737	.data = .{ .rx = .{
8646		.r = .rdi,
	8738	.r1 = .rdi,
8647	8739	.payload = try self.addExtra(Mir.Reloc{
8648	8740	.atom_index = atom_index,
8649	8741	.sym_index = sym_index,
...	...	@@ -8847,9 +8939,9 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
8847	8939	try self.genSetReg(.rsi, Type.usize, src_ptr);
8848	8940	try self.genSetReg(.rcx, Type.usize, len);
8849	8941	_ = try self.addInst(.{
8850		.tag = .movs,
8851		.ops = .string,
8852		.data = .{ .string = .{ .repeat = .rep, .width = .b } },
	8942	.tag = .mov,
	8943	.ops = .none,
	8944	.data = .{ .none = .{ .fixes = .@"rep _sb" } },
8853	8945	});
8854	8946	}
8855	8947
...	...	@@ -8859,9 +8951,9 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
8859	8951	try self.genSetReg(.al, Type.u8, value);
8860	8952	try self.genSetReg(.rcx, Type.usize, len);
8861	8953	_ = try self.addInst(.{
8862		.tag = .stos,
8863		.ops = .string,
8864		.data = .{ .string = .{ .repeat = .rep, .width = .b } },
	8954	.tag = .sto,
	8955	.ops = .none,
	8956	.data = .{ .none = .{ .fixes = .@"rep _sb" } },
8865	8957	});
8866	8958	}
8867	8959
...	...	@@ -9135,22 +9227,22 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
9135	9227	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
9136	9228
9137	9229	try self.spillEflagsIfOccupied();
9138		if (val_abi_size <= 8) {
9139		_ = try self.addInst(.{
9140		.tag = .cmpxchg,
9141		.ops = .lock_mr_sib,
9142		.data = .{ .rx = .{
9143		.r = registerAlias(new_reg.?, val_abi_size),
9144		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9145		} },
9146		});
9147		} else {
9148		_ = try self.addInst(.{
9149		.tag = .cmpxchgb,
9150		.ops = .lock_m_sib,
9151		.data = .{ .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)) },
9152		});
9153		}
	9230	_ = try self.addInst(if (val_abi_size <= 8) .{
	9231	.tag = .cmpxchg,
	9232	.ops = .mr_sib,
	9233	.data = .{ .rx = .{
	9234	.fixes = .@"lock _",
	9235	.r1 = registerAlias(new_reg.?, val_abi_size),
	9236	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	9237	} },
	9238	} else .{
	9239	.tag = .cmpxchg,
	9240	.ops = .m_sib,
	9241	.data = .{ .x = .{
	9242	.fixes = .@"lock _16b",
	9243	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	9244	} },
	9245	});
9154	9246
9155	9247	const result: MCValue = result: {
9156	9248	if (self.liveness.isUnused(inst)) break :result .unreach;
...	...	@@ -9252,13 +9344,14 @@ fn atomicOp(
9252	9344	}
9253	9345	_ = try self.addInst(.{
9254	9346	.tag = tag,
9255		.ops = switch (tag) {
9256		.mov, .xchg => .mr_sib,
9257		.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
9258		else => unreachable,
9259		},
	9347	.ops = .mr_sib,
9260	9348	.data = .{ .rx = .{
9261		.r = registerAlias(dst_reg, val_abi_size),
	9349	.fixes = switch (tag) {
	9350	.mov, .xchg => ._,
	9351	.xadd, .add, .sub, .@"and", .@"or", .xor => .@"lock _",
	9352	else => unreachable,
	9353	},
	9354	.r1 = registerAlias(dst_reg, val_abi_size),
9262	9355	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9263	9356	} },
9264	9357	});
...	...	@@ -9330,9 +9423,10 @@ fn atomicOp(
9330	9423	};
9331	9424	_ = try self.addInst(.{
9332	9425	.tag = .cmpxchg,
9333		.ops = .lock_mr_sib,
	9426	.ops = .mr_sib,
9334	9427	.data = .{ .rx = .{
9335		.r = registerAlias(tmp_reg, val_abi_size),
	9428	.fixes = .@"lock _",
	9429	.r1 = registerAlias(tmp_reg, val_abi_size),
9336	9430	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9337	9431	} },
9338	9432	});
...	...	@@ -9397,9 +9491,14 @@ fn atomicOp(
9397	9491	val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
9398	9492	}),
9399	9493	};
9400		_ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
9401		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9402		} });
	9494	_ = try self.addInst(.{
	9495	.tag = .cmpxchg,
	9496	.ops = .m_sib,
	9497	.data = .{ .x = .{
	9498	.fixes = .@"lock _16b",
	9499	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	9500	} },
	9501	});
9403	9502	_ = try self.asmJccReloc(loop, .ne);
9404	9503
9405	9504	if (unused) return .unreach;

src/arch/x86_64/Emit.zig+34-31

...	...	@@ -41,7 +41,7 @@ pub fn emitMir(emit: *Emit) Error!void {
41	41	.offset = end_offset - 4,
42	42	.length = @intCast(u5, end_offset - start_offset),
43	43	}),
44		.@"extern" => \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	44	.linker_extern_fn => \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
45	45	// Add relocation to the decl.
46	46	const atom_index = macho_file.getAtomIndexForSymbol(
47	47	.{ .sym_index = symbol.atom_index, .file = null },
...	...	@@ -129,36 +129,39 @@ pub fn emitMir(emit: *Emit) Error!void {
129	129	const mir_inst = emit.lower.mir.instructions.get(mir_index);
130	130	switch (mir_inst.tag) {
131	131	else => unreachable,
132		.dead => {},
133		.dbg_line => try emit.dbgAdvancePCAndLine(
134		mir_inst.data.line_column.line,
135		mir_inst.data.line_column.column,
136		),
137		.dbg_prologue_end => {
138		switch (emit.debug_output) {
139		.dwarf => \|dw\| {
140		try dw.setPrologueEnd();
141		log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
142		emit.prev_di_line, emit.prev_di_column,
143		});
144		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
145		},
146		.plan9 => {},
147		.none => {},
148		}
149		},
150		.dbg_epilogue_begin => {
151		switch (emit.debug_output) {
152		.dwarf => \|dw\| {
153		try dw.setEpilogueBegin();
154		log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
155		emit.prev_di_line, emit.prev_di_column,
156		});
157		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
158		},
159		.plan9 => {},
160		.none => {},
161		}
	132	.pseudo => switch (mir_inst.ops) {
	133	else => unreachable,
	134	.pseudo_dbg_prologue_end_none => {
	135	switch (emit.debug_output) {
	136	.dwarf => \|dw\| {
	137	try dw.setPrologueEnd();
	138	log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
	139	emit.prev_di_line, emit.prev_di_column,
	140	});
	141	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	142	},
	143	.plan9 => {},
	144	.none => {},
	145	}
	146	},
	147	.pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(
	148	mir_inst.data.line_column.line,
	149	mir_inst.data.line_column.column,
	150	),
	151	.pseudo_dbg_epilogue_begin_none => {
	152	switch (emit.debug_output) {
	153	.dwarf => \|dw\| {
	154	try dw.setEpilogueBegin();
	155	log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
	156	emit.prev_di_line, emit.prev_di_column,
	157	});
	158	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	159	},
	160	.plan9 => {},
	161	.none => {},
	162	}
	163	},
	164	.pseudo_dead_none => {},
162	165	},
163	166	}
164	167	}

src/arch/x86_64/Encoding.zig+1-1

...	...	@@ -705,7 +705,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
705	705	}
706	706
707	707	const mnemonic_to_encodings_map = init: {
708		@setEvalBranchQuota(100_000);
	708	@setEvalBranchQuota(20_000);
709	709	const encodings = @import("encodings.zig");
710	710	var entries = encodings.table;
711	711	std.sort.sort(encodings.Entry, &entries, {}, struct {

src/arch/x86_64/Lower.zig+205-508

...	...	@@ -35,7 +35,7 @@ pub const Reloc = struct {
35	35
36	36	const Target = union(enum) {
37	37	inst: Mir.Inst.Index,
38		@"extern": Mir.Reloc,
	38	linker_extern_fn: Mir.Reloc,
39	39	linker_got: Mir.Reloc,
40	40	linker_direct: Mir.Reloc,
41	41	linker_import: Mir.Reloc,
...	...	@@ -59,280 +59,119 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
59	59
60	60	const inst = lower.mir.instructions.get(index);
61	61	switch (inst.tag) {
62		.adc,
63		.add,
64		.@"and",
65		.bsf,
66		.bsr,
67		.bswap,
68		.bt,
69		.btc,
70		.btr,
71		.bts,
72		.call,
73		.cbw,
74		.cwde,
75		.cdqe,
76		.cwd,
77		.cdq,
78		.cqo,
79		.cmp,
80		.cmpxchg,
81		.div,
82		.fisttp,
83		.fld,
84		.idiv,
85		.imul,
86		.int3,
87		.jmp,
88		.lea,
89		.lfence,
90		.lzcnt,
91		.mfence,
92		.mov,
93		.movbe,
94		.movd,
95		.movq,
96		.movzx,
97		.mul,
98		.neg,
99		.nop,
100		.not,
101		.@"or",
102		.pop,
103		.popcnt,
104		.push,
105		.rcl,
106		.rcr,
107		.ret,
108		.rol,
109		.ror,
110		.sal,
111		.sar,
112		.sbb,
113		.sfence,
114		.shl,
115		.shld,
116		.shr,
117		.shrd,
118		.sub,
119		.syscall,
120		.@"test",
121		.tzcnt,
122		.ud2,
123		.xadd,
124		.xchg,
125		.xor,
126
127		.addps,
128		.addss,
129		.andnps,
130		.andps,
131		.cmpss,
132		.cvtsi2ss,
133		.divps,
134		.divss,
135		.maxps,
136		.maxss,
137		.minps,
138		.minss,
139		.movaps,
140		.movhlps,
141		.movss,
142		.movups,
143		.mulps,
144		.mulss,
145		.orps,
146		.pextrw,
147		.pinsrw,
148		.sqrtps,
149		.sqrtss,
150		.subps,
151		.subss,
152		.ucomiss,
153		.unpckhps,
154		.unpcklps,
155		.xorps,
156
157		.addpd,
158		.addsd,
159		.andnpd,
160		.andpd,
161		.cmpsd,
162		.cvtsd2ss,
163		.cvtsi2sd,
164		.cvtss2sd,
165		.divpd,
166		.divsd,
167		.maxpd,
168		.maxsd,
169		.minpd,
170		.minsd,
171		.movsd,
172		.mulpd,
173		.mulsd,
174		.orpd,
175		.pshufhw,
176		.pshuflw,
177		.psrld,
178		.psrlq,
179		.psrlw,
180		.punpckhbw,
181		.punpckhdq,
182		.punpckhqdq,
183		.punpckhwd,
184		.punpcklbw,
185		.punpckldq,
186		.punpcklqdq,
187		.punpcklwd,
188		.sqrtpd,
189		.sqrtsd,
190		.subpd,
191		.subsd,
192		.ucomisd,
193		.unpckhpd,
194		.unpcklpd,
195		.xorpd,
196
197		.movddup,
198		.movshdup,
199		.movsldup,
200
201		.pextrb,
202		.pextrd,
203		.pextrq,
204		.pinsrb,
205		.pinsrd,
206		.pinsrq,
207		.roundpd,
208		.roundps,
209		.roundsd,
210		.roundss,
211
212		.vaddpd,
213		.vaddps,
214		.vaddsd,
215		.vaddss,
216		.vcvtsd2ss,
217		.vcvtsi2sd,
218		.vcvtsi2ss,
219		.vcvtss2sd,
220		.vdivpd,
221		.vdivps,
222		.vdivsd,
223		.vdivss,
224		.vmaxpd,
225		.vmaxps,
226		.vmaxsd,
227		.vmaxss,
228		.vminpd,
229		.vminps,
230		.vminsd,
231		.vminss,
232		.vmovapd,
233		.vmovaps,
234		.vmovddup,
235		.vmovhlps,
236		.vmovsd,
237		.vmovshdup,
238		.vmovsldup,
239		.vmovss,
240		.vmovupd,
241		.vmovups,
242		.vmulpd,
243		.vmulps,
244		.vmulsd,
245		.vmulss,
246		.vpextrb,
247		.vpextrd,
248		.vpextrq,
249		.vpextrw,
250		.vpinsrb,
251		.vpinsrd,
252		.vpinsrq,
253		.vpinsrw,
254		.vpshufhw,
255		.vpshuflw,
256		.vpsrld,
257		.vpsrlq,
258		.vpsrlw,
259		.vpunpckhbw,
260		.vpunpckhdq,
261		.vpunpckhqdq,
262		.vpunpckhwd,
263		.vpunpcklbw,
264		.vpunpckldq,
265		.vpunpcklqdq,
266		.vpunpcklwd,
267		.vroundpd,
268		.vroundps,
269		.vroundsd,
270		.vroundss,
271		.vsqrtpd,
272		.vsqrtps,
273		.vsqrtsd,
274		.vsqrtss,
275		.vsubpd,
276		.vsubps,
277		.vsubsd,
278		.vsubss,
279		.vunpckhpd,
280		.vunpckhps,
281		.vunpcklpd,
282		.vunpcklps,
283
284		.vcvtph2ps,
285		.vcvtps2ph,
286
287		.vfmadd132pd,
288		.vfmadd213pd,
289		.vfmadd231pd,
290		.vfmadd132ps,
291		.vfmadd213ps,
292		.vfmadd231ps,
293		.vfmadd132sd,
294		.vfmadd213sd,
295		.vfmadd231sd,
296		.vfmadd132ss,
297		.vfmadd213ss,
298		.vfmadd231ss,
299		=> try lower.mirGeneric(inst),
300
301		.cmps,
302		.lods,
303		.movs,
304		.scas,
305		.stos,
306		=> try lower.mirString(inst),
307
308		.cmpxchgb => try lower.mirCmpxchgBytes(inst),
309
310		.jmp_reloc => try lower.emitInstWithReloc(.none, .jmp, &.{
311		.{ .imm = Immediate.s(0) },
312		}, .{ .inst = inst.data.inst }),
313
314		.call_extern => try lower.emitInstWithReloc(.none, .call, &.{
315		.{ .imm = Immediate.s(0) },
316		}, .{ .@"extern" = inst.data.relocation }),
317
318		.lea_linker => try lower.mirLinker(.lea, inst),
319		.mov_linker => try lower.mirLinker(.mov, inst),
320
321		.mov_moffs => try lower.mirMovMoffs(inst),
322
323		.movsx => try lower.mirMovsx(inst),
324		.cmovcc => try lower.mirCmovcc(inst),
325		.setcc => try lower.mirSetcc(inst),
326		.jcc => try lower.mirJcc(index, inst),
	62	else => try lower.generic(inst),
	63	.pseudo => switch (inst.ops) {
	64	.pseudo_cmov_z_and_np_rr => {
	65	try lower.emit(.none, .cmovnz, &.{
	66	.{ .reg = inst.data.rr.r2 },
	67	.{ .reg = inst.data.rr.r1 },
	68	});
	69	try lower.emit(.none, .cmovnp, &.{
	70	.{ .reg = inst.data.rr.r1 },
	71	.{ .reg = inst.data.rr.r2 },
	72	});
	73	},
	74	.pseudo_cmov_nz_or_p_rr => {
	75	try lower.emit(.none, .cmovnz, &.{
	76	.{ .reg = inst.data.rr.r1 },
	77	.{ .reg = inst.data.rr.r2 },
	78	});
	79	try lower.emit(.none, .cmovp, &.{
	80	.{ .reg = inst.data.rr.r1 },
	81	.{ .reg = inst.data.rr.r2 },
	82	});
	83	},
	84	.pseudo_cmov_nz_or_p_rm_sib,
	85	.pseudo_cmov_nz_or_p_rm_rip,
	86	=> {
	87	try lower.emit(.none, .cmovnz, &.{
	88	.{ .reg = inst.data.rx.r1 },
	89	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	90	});
	91	try lower.emit(.none, .cmovp, &.{
	92	.{ .reg = inst.data.rx.r1 },
	93	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	94	});
	95	},
	96	.pseudo_set_z_and_np_r => {
	97	try lower.emit(.none, .setz, &.{
	98	.{ .reg = inst.data.r_scratch.r1 },
	99	});
	100	try lower.emit(.none, .setnp, &.{
	101	.{ .reg = inst.data.r_scratch.scratch_reg },
	102	});
	103	try lower.emit(.none, .@"and", &.{
	104	.{ .reg = inst.data.r_scratch.r1 },
	105	.{ .reg = inst.data.r_scratch.scratch_reg },
	106	});
	107	},
	108	.pseudo_set_z_and_np_m_sib,
	109	.pseudo_set_z_and_np_m_rip,
	110	=> {
	111	try lower.emit(.none, .setz, &.{
	112	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	113	});
	114	try lower.emit(.none, .setnp, &.{
	115	.{ .reg = inst.data.x_scratch.scratch_reg },
	116	});
	117	try lower.emit(.none, .@"and", &.{
	118	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	119	.{ .reg = inst.data.x_scratch.scratch_reg },
	120	});
	121	},
	122	.pseudo_set_nz_or_p_r => {
	123	try lower.emit(.none, .setnz, &.{
	124	.{ .reg = inst.data.r_scratch.r1 },
	125	});
	126	try lower.emit(.none, .setp, &.{
	127	.{ .reg = inst.data.r_scratch.scratch_reg },
	128	});
	129	try lower.emit(.none, .@"or", &.{
	130	.{ .reg = inst.data.r_scratch.r1 },
	131	.{ .reg = inst.data.r_scratch.scratch_reg },
	132	});
	133	},
	134	.pseudo_set_nz_or_p_m_sib,
	135	.pseudo_set_nz_or_p_m_rip,
	136	=> {
	137	try lower.emit(.none, .setnz, &.{
	138	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	139	});
	140	try lower.emit(.none, .setp, &.{
	141	.{ .reg = inst.data.x_scratch.scratch_reg },
	142	});
	143	try lower.emit(.none, .@"or", &.{
	144	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	145	.{ .reg = inst.data.x_scratch.scratch_reg },
	146	});
	147	},
	148	.pseudo_j_z_and_np_inst => {
	149	try lower.emit(.none, .jnz, &.{
	150	.{ .imm = lower.reloc(.{ .inst = index + 1 }) },
	151	});
	152	try lower.emit(.none, .jnp, &.{
	153	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	154	});
	155	},
	156	.pseudo_j_nz_or_p_inst => {
	157	try lower.emit(.none, .jnz, &.{
	158	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	159	});
	160	try lower.emit(.none, .jp, &.{
	161	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	162	});
	163	},
327	164
328		.push_regs => try lower.mirRegisterList(.push, inst),
329		.pop_regs => try lower.mirRegisterList(.pop, inst),
	165	.pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
	166	.pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
330	167
331		.dbg_line,
332		.dbg_prologue_end,
333		.dbg_epilogue_begin,
334		.dead,
335		=> {},
	168	.pseudo_dbg_prologue_end_none,
	169	.pseudo_dbg_line_line_column,
	170	.pseudo_dbg_epilogue_begin_none,
	171	.pseudo_dead_none,
	172	=> {},
	173	else => unreachable,
	174	},
336	175	}
337	176
338	177	return .{
...	...	@@ -348,15 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
348	187	return error.LowerFail;
349	188	}
350	189
351		fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
352		return switch (cc) {
353		inline else => \|c\| if (@hasField(Mnemonic, @tagName(base) ++ @tagName(c)))
354		@field(Mnemonic, @tagName(base) ++ @tagName(c))
355		else
356		unreachable,
357		};
358		}
359
360	190	fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
361	191	return switch (ops) {
362	192	.rri_s,
...	...	@@ -364,8 +194,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
364	194	.i_s,
365	195	.mi_sib_s,
366	196	.mi_rip_s,
367		.lock_mi_sib_s,
368		.lock_mi_rip_s,
369	197	=> Immediate.s(@bitCast(i32, i)),
370	198
371	199	.rrri,
...	...	@@ -374,8 +202,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
374	202	.i_u,
375	203	.mi_sib_u,
376	204	.mi_rip_u,
377		.lock_mi_sib_u,
378		.lock_mi_rip_u,
379	205	.rmi_sib,
380	206	.rmi_rip,
381	207	.mri_sib,
...	...	@@ -395,10 +221,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
395	221	fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
396	222	return lower.mir.resolveFrameLoc(switch (ops) {
397	223	.rm_sib,
398		.rm_sib_cc,
399	224	.rmi_sib,
400	225	.m_sib,
401		.m_sib_cc,
402	226	.mi_sib_u,
403	227	.mi_sib_s,
404	228	.mr_sib,
...	...	@@ -406,17 +230,15 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
406	230	.mri_sib,
407	231	.rrm_sib,
408	232	.rrmi_sib,
409		.lock_m_sib,
410		.lock_mi_sib_u,
411		.lock_mi_sib_s,
412		.lock_mr_sib,
	233
	234	.pseudo_cmov_nz_or_p_rm_sib,
	235	.pseudo_set_z_and_np_m_sib,
	236	.pseudo_set_nz_or_p_m_sib,
413	237	=> lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
414	238
415	239	.rm_rip,
416		.rm_rip_cc,
417	240	.rmi_rip,
418	241	.m_rip,
419		.m_rip_cc,
420	242	.mi_rip_u,
421	243	.mi_rip_s,
422	244	.mr_rip,
...	...	@@ -424,66 +246,83 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
424	246	.mri_rip,
425	247	.rrm_rip,
426	248	.rrmi_rip,
427		.lock_m_rip,
428		.lock_mi_rip_u,
429		.lock_mi_rip_s,
430		.lock_mr_rip,
	249
	250	.pseudo_cmov_nz_or_p_rm_rip,
	251	.pseudo_set_z_and_np_m_rip,
	252	.pseudo_set_nz_or_p_m_rip,
431	253	=> lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
432	254
433	255	.rax_moffs,
434	256	.moffs_rax,
435		.lock_moffs_rax,
436	257	=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
437	258
438	259	else => unreachable,
439	260	});
440	261	}
441	262
442		fn emitInst(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
443		lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
444		lower.result_insts_len += 1;
445		}
446
447		fn emitInstWithReloc(
448		lower: *Lower,
449		prefix: Prefix,
450		mnemonic: Mnemonic,
451		ops: []const Operand,
452		target: Reloc.Target,
453		) Error!void {
	263	fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
454	264	lower.result_relocs[lower.result_relocs_len] = .{
455	265	.lowered_inst_index = lower.result_insts_len,
456	266	.target = target,
457	267	};
458	268	lower.result_relocs_len += 1;
459		try lower.emitInst(prefix, mnemonic, ops);
	269	return Immediate.s(0);
	270	}
	271
	272	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
	273	lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
	274	lower.result_insts_len += 1;
460	275	}
461	276
462		fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
463		try lower.emitInst(switch (inst.ops) {
464		else => .none,
465		.lock_m_sib,
466		.lock_m_rip,
467		.lock_mi_sib_u,
468		.lock_mi_rip_u,
469		.lock_mi_sib_s,
470		.lock_mi_rip_s,
471		.lock_mr_sib,
472		.lock_mr_rip,
473		.lock_moffs_rax,
474		=> .lock,
475		}, switch (inst.tag) {
476		inline else => \|tag\| if (@hasField(Mnemonic, @tagName(tag)))
477		@field(Mnemonic, @tagName(tag))
	277	fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
	278	const fixes = switch (inst.ops) {
	279	.none => inst.data.none.fixes,
	280	.inst => inst.data.inst.fixes,
	281	.i_s, .i_u => inst.data.i.fixes,
	282	.r => inst.data.r.fixes,
	283	.rr => inst.data.rr.fixes,
	284	.rrr => inst.data.rrr.fixes,
	285	.rrri => inst.data.rrri.fixes,
	286	.rri_s, .rri_u => inst.data.rri.fixes,
	287	.ri_s, .ri_u => inst.data.ri.fixes,
	288	.ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
	289	.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => ._,
	290	.mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
	291	.rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
	292	.rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
	293	.m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
	294	.extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
	295	else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
	296	};
	297	try lower.emit(switch (fixes) {
	298	inline else => \|tag\| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) \|space\|
	299	@field(Prefix, @tagName(tag)[0..space])
478	300	else
479		unreachable,
	301	.none,
	302	}, mnemonic: {
	303	comptime var max_len = 0;
	304	inline for (@typeInfo(Mnemonic).Enum.fields) \|field\| max_len = @max(field.name.len, max_len);
	305	var buf: [max_len]u8 = undefined;
	306
	307	const fixes_name = @tagName(fixes);
	308	const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) \|i\| i + 1 else 0..];
	309	const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
	310	const parts = .{ pattern[0..wildcard_i], @tagName(inst.tag), pattern[wildcard_i + 1 ..] };
	311	const err_msg = "unsupported mnemonic: ";
	312	const mnemonic = std.fmt.bufPrint(&buf, "{s}{s}{s}", parts) catch
	313	return lower.fail(err_msg ++ "'{s}{s}{s}'", parts);
	314	break :mnemonic std.meta.stringToEnum(Mnemonic, mnemonic) orelse
	315	return lower.fail(err_msg ++ "'{s}'", .{mnemonic});
480	316	}, switch (inst.ops) {
481	317	.none => &.{},
	318	.inst => &.{
	319	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	320	},
482	321	.i_s, .i_u => &.{
483		.{ .imm = lower.imm(inst.ops, inst.data.i) },
	322	.{ .imm = lower.imm(inst.ops, inst.data.i.i) },
484	323	},
485	324	.r => &.{
486		.{ .reg = inst.data.r },
	325	.{ .reg = inst.data.r.r1 },
487	326	},
488	327	.rr => &.{
489	328	.{ .reg = inst.data.rr.r1 },
...	...	@@ -501,11 +340,11 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
501	340	.{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
502	341	},
503	342	.ri_s, .ri_u => &.{
504		.{ .reg = inst.data.ri.r },
	343	.{ .reg = inst.data.ri.r1 },
505	344	.{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
506	345	},
507	346	.ri64 => &.{
508		.{ .reg = inst.data.rx.r },
	347	.{ .reg = inst.data.rx.r1 },
509	348	.{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },
510	349	},
511	350	.rri_s, .rri_u => &.{
...	...	@@ -513,33 +352,25 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
513	352	.{ .reg = inst.data.rri.r2 },
514	353	.{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
515	354	},
516		.m_sib, .lock_m_sib, .m_rip, .lock_m_rip => &.{
517		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	355	.m_sib, .m_rip => &.{
	356	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
518	357	},
519		.mi_sib_s,
520		.lock_mi_sib_s,
521		.mi_sib_u,
522		.lock_mi_sib_u,
523		.mi_rip_u,
524		.lock_mi_rip_u,
525		.mi_rip_s,
526		.lock_mi_rip_s,
527		=> &.{
	358	.mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
528	359	.{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
529	360	.{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
530	361	},
531	362	.rm_sib, .rm_rip => &.{
532		.{ .reg = inst.data.rx.r },
	363	.{ .reg = inst.data.rx.r1 },
533	364	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
534	365	},
535	366	.rmi_sib, .rmi_rip => &.{
536		.{ .reg = inst.data.rix.r },
	367	.{ .reg = inst.data.rix.r1 },
537	368	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
538	369	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
539	370	},
540		.mr_sib, .lock_mr_sib, .mr_rip, .lock_mr_rip => &.{
	371	.mr_sib, .mr_rip => &.{
541	372	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
542		.{ .reg = inst.data.rx.r },
	373	.{ .reg = inst.data.rx.r1 },
543	374	},
544	375	.mrr_sib, .mrr_rip => &.{
545	376	.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
...	...	@@ -548,7 +379,7 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
548	379	},
549	380	.mri_sib, .mri_rip => &.{
550	381	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
551		.{ .reg = inst.data.rix.r },
	382	.{ .reg = inst.data.rix.r1 },
552	383	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
553	384	},
554	385	.rrm_sib, .rrm_rip => &.{
...	...	@@ -562,180 +393,46 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
562	393	.{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
563	394	.{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
564	395	},
565		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
566		});
567		}
568
569		fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {
570		switch (inst.ops) {
571		.string => try lower.emitInst(switch (inst.data.string.repeat) {
572		inline else => \|repeat\| @field(Prefix, @tagName(repeat)),
573		}, switch (inst.tag) {
574		inline .cmps, .lods, .movs, .scas, .stos => \|tag\| switch (inst.data.string.width) {
575		inline else => \|width\| @field(Mnemonic, @tagName(tag) ++ @tagName(width)),
576		},
577		else => unreachable,
578		}, &.{}),
579		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
580		}
581		}
582
583		fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
584		const ops: [1]Operand = switch (inst.ops) {
585		.m_sib, .lock_m_sib, .m_rip, .lock_m_rip => .{
586		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
587		},
588		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
589		};
590		try lower.emitInst(switch (inst.ops) {
591		.m_sib, .m_rip => .none,
592		.lock_m_sib, .lock_m_rip => .lock,
593		else => unreachable,
594		}, switch (@divExact(ops[0].bitSize(), 8)) {
595		8 => .cmpxchg8b,
596		16 => .cmpxchg16b,
597		else => return lower.fail("invalid operand for {s}", .{@tagName(inst.tag)}),
598		}, &ops);
599		}
600
601		fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
602		try lower.emitInst(switch (inst.ops) {
603		.rax_moffs, .moffs_rax => .none,
604		.lock_moffs_rax => .lock,
605		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
606		}, .mov, switch (inst.ops) {
607	396	.rax_moffs => &.{
608	397	.{ .reg = .rax },
609		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	398	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
610	399	},
611		.moffs_rax, .lock_moffs_rax => &.{
612		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	400	.moffs_rax => &.{
	401	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
613	402	.{ .reg = .rax },
614	403	},
615		else => unreachable,
616		});
617		}
618
619		fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
620		const ops: [2]Operand = switch (inst.ops) {
621		.rr => .{
622		.{ .reg = inst.data.rr.r1 },
623		.{ .reg = inst.data.rr.r2 },
624		},
625		.rm_sib, .rm_rip => .{
626		.{ .reg = inst.data.rx.r },
627		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
628		},
629		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
630		};
631		try lower.emitInst(.none, switch (ops[0].bitSize()) {
632		32, 64 => switch (ops[1].bitSize()) {
633		32 => .movsxd,
634		else => .movsx,
635		},
636		else => .movsx,
637		}, &ops);
638		}
639
640		fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
641		const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
642		.rr_cc => .{ .cc = inst.data.rr_cc.cc, .ops = .{
643		.{ .reg = inst.data.rr_cc.r1 },
644		.{ .reg = inst.data.rr_cc.r2 },
645		} },
646		.rm_sib_cc, .rm_rip_cc => .{ .cc = inst.data.rx_cc.cc, .ops = .{
647		.{ .reg = inst.data.rx_cc.r },
648		.{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
649		} },
650		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
651		};
652		switch (data.cc) {
653		else => \|cc\| try lower.emitInst(.none, mnem_cc(.cmov, cc), &data.ops),
654		.z_and_np => {
655		try lower.emitInst(.none, mnem_cc(.cmov, .nz), &.{ data.ops[1], data.ops[0] });
656		try lower.emitInst(.none, mnem_cc(.cmov, .np), &data.ops);
	404	.extern_fn_reloc => &.{
	405	.{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
657	406	},
658		.nz_or_p => {
659		try lower.emitInst(.none, mnem_cc(.cmov, .nz), &data.ops);
660		try lower.emitInst(.none, mnem_cc(.cmov, .p), &data.ops);
	407	.got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
	408	const reg = inst.data.rx.r1;
	409	const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
	410	_ = lower.reloc(switch (inst.ops) {
	411	.got_reloc => .{ .linker_got = extra },
	412	.direct_reloc => .{ .linker_direct = extra },
	413	.import_reloc => .{ .linker_import = extra },
	414	.tlv_reloc => .{ .linker_tlv = extra },
	415	else => unreachable,
	416	});
	417	break :ops &.{
	418	.{ .reg = reg },
	419	.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
	420	};
661	421	},
662		}
663		}
664
665		fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
666		const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
667		.r_cc => .{ .cc = inst.data.r_cc.cc, .ops = .{
668		.{ .reg = inst.data.r_cc.r },
669		.{ .reg = inst.data.r_cc.scratch },
670		} },
671		.m_sib_cc, .m_rip_cc => .{ .cc = inst.data.x_cc.cc, .ops = .{
672		.{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
673		.{ .reg = inst.data.x_cc.scratch },
674		} },
675	422	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
676		};
677		switch (data.cc) {
678		else => \|cc\| try lower.emitInst(.none, mnem_cc(.set, cc), data.ops[0..1]),
679		.z_and_np => {
680		try lower.emitInst(.none, mnem_cc(.set, .z), data.ops[0..1]);
681		try lower.emitInst(.none, mnem_cc(.set, .np), data.ops[1..2]);
682		try lower.emitInst(.none, .@"and", data.ops[0..2]);
683		},
684		.nz_or_p => {
685		try lower.emitInst(.none, mnem_cc(.set, .nz), data.ops[0..1]);
686		try lower.emitInst(.none, mnem_cc(.set, .p), data.ops[1..2]);
687		try lower.emitInst(.none, .@"or", data.ops[0..2]);
688		},
689		}
690		}
691
692		fn mirJcc(lower: *Lower, index: Mir.Inst.Index, inst: Mir.Inst) Error!void {
693		switch (inst.data.inst_cc.cc) {
694		else => \|cc\| try lower.emitInstWithReloc(.none, mnem_cc(.j, cc), &.{
695		.{ .imm = Immediate.s(0) },
696		}, .{ .inst = inst.data.inst_cc.inst }),
697		.z_and_np => {
698		try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
699		.{ .imm = Immediate.s(0) },
700		}, .{ .inst = index + 1 });
701		try lower.emitInstWithReloc(.none, mnem_cc(.j, .np), &.{
702		.{ .imm = Immediate.s(0) },
703		}, .{ .inst = inst.data.inst_cc.inst });
704		},
705		.nz_or_p => {
706		try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
707		.{ .imm = Immediate.s(0) },
708		}, .{ .inst = inst.data.inst_cc.inst });
709		try lower.emitInstWithReloc(.none, mnem_cc(.j, .p), &.{
710		.{ .imm = Immediate.s(0) },
711		}, .{ .inst = inst.data.inst_cc.inst });
712		},
713		}
	423	});
714	424	}
715	425
716		fn mirRegisterList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
717		const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
	426	fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
718	427	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
719		var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
	428	var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) {
720	429	.push => .reverse,
721	430	.pop => .forward,
722	431	else => unreachable,
723	432	} });
724		while (it.next()) \|i\| try lower.emitInst(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
725		}
726
727		fn mirLinker(lower: *Lower, mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
728		const reloc = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
729		try lower.emitInstWithReloc(.none, mnemonic, &.{
730		.{ .reg = inst.data.rx.r },
731		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(inst.data.rx.r.bitSize()), 0) },
732		}, switch (inst.ops) {
733		.got_reloc => .{ .linker_got = reloc },
734		.direct_reloc => .{ .linker_direct = reloc },
735		.import_reloc => .{ .linker_import = reloc },
736		.tlv_reloc => .{ .linker_tlv = reloc },
737		else => unreachable,
738		});
	433	while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{
	434	.reg = callee_preserved_regs[i],
	435	}});
739	436	}
740	437
741	438	const abi = @import("abi.zig");

src/arch/x86_64/Mir.zig+348-169

...	...	@@ -32,6 +32,210 @@ pub const Inst = struct {
32	32
33	33	pub const Index = u32;
34	34
	35	pub const Fixes = enum(u8) {
	36	/// ___
	37	@"_",
	38
	39	/// ___ Above
	40	_a,
	41	/// ___ Above Or Equal
	42	_ae,
	43	/// ___ Below
	44	_b,
	45	/// ___ Below Or Equal
	46	_be,
	47	/// ___ Carry
	48	_c,
	49	/// ___ Equal
	50	_e,
	51	/// ___ Greater
	52	_g,
	53	/// ___ Greater Or Equal
	54	_ge,
	55	/// ___ Less
	56	_l,
	57	/// ___ Less Or Equal
	58	_le,
	59	/// ___ Not Above
	60	_na,
	61	/// ___ Not Above Or Equal
	62	_nae,
	63	/// ___ Not Below
	64	_nb,
	65	/// ___ Not Below Or Equal
	66	_nbe,
	67	/// ___ Not Carry
	68	_nc,
	69	/// ___ Not Equal
	70	_ne,
	71	/// ___ Not Greater
	72	_ng,
	73	/// ___ Not Greater Or Equal
	74	_nge,
	75	/// ___ Not Less
	76	_nl,
	77	/// ___ Not Less Or Equal
	78	_nle,
	79	/// ___ Not Overflow
	80	_no,
	81	/// ___ Not Parity
	82	_np,
	83	/// ___ Not Sign
	84	_ns,
	85	/// ___ Not Zero
	86	_nz,
	87	/// ___ Overflow
	88	_o,
	89	/// ___ Parity
	90	_p,
	91	/// ___ Parity Even
	92	_pe,
	93	/// ___ Parity Odd
	94	_po,
	95	/// ___ Sign
	96	_s,
	97	/// ___ Zero
	98	_z,
	99
	100	/// ___ String
	101	//_s,
	102	/// ___ String Byte
	103	_sb,
	104	/// ___ String Word
	105	_sw,
	106	/// ___ String Doubleword
	107	_sd,
	108	/// ___ String Quadword
	109	_sq,
	110
	111	/// Repeat ___ String
	112	@"rep _s",
	113	/// Repeat ___ String Byte
	114	@"rep _sb",
	115	/// Repeat ___ String Word
	116	@"rep _sw",
	117	/// Repeat ___ String Doubleword
	118	@"rep _sd",
	119	/// Repeat ___ String Quadword
	120	@"rep _sq",
	121
	122	/// Repeat Equal ___ String
	123	@"repe _s",
	124	/// Repeat Equal ___ String Byte
	125	@"repe _sb",
	126	/// Repeat Equal ___ String Word
	127	@"repe _sw",
	128	/// Repeat Equal ___ String Doubleword
	129	@"repe _sd",
	130	/// Repeat Equal ___ String Quadword
	131	@"repe _sq",
	132
	133	/// Repeat Not Equal ___ String
	134	@"repne _s",
	135	/// Repeat Not Equal ___ String Byte
	136	@"repne _sb",
	137	/// Repeat Not Equal ___ String Word
	138	@"repne _sw",
	139	/// Repeat Not Equal ___ String Doubleword
	140	@"repne _sd",
	141	/// Repeat Not Equal ___ String Quadword
	142	@"repne _sq",
	143
	144	/// Repeat Not Zero ___ String
	145	@"repnz _s",
	146	/// Repeat Not Zero ___ String Byte
	147	@"repnz _sb",
	148	/// Repeat Not Zero ___ String Word
	149	@"repnz _sw",
	150	/// Repeat Not Zero ___ String Doubleword
	151	@"repnz _sd",
	152	/// Repeat Not Zero ___ String Quadword
	153	@"repnz _sq",
	154
	155	/// Repeat Zero ___ String
	156	@"repz _s",
	157	/// Repeat Zero ___ String Byte
	158	@"repz _sb",
	159	/// Repeat Zero ___ String Word
	160	@"repz _sw",
	161	/// Repeat Zero ___ String Doubleword
	162	@"repz _sd",
	163	/// Repeat Zero ___ String Quadword
	164	@"repz _sq",
	165
	166	/// Locked ___
	167	@"lock _",
	168	/// ___ 8 Bytes
	169	_8b,
	170	/// Locked ___ 8 Bytes
	171	@"lock _8b",
	172	/// ___ 16 Bytes
	173	_16b,
	174	/// Locked ___ 16 Bytes
	175	@"lock _16b",
	176
	177	/// Packed ___
	178	p_,
	179	/// Packed ___ Byte
	180	p_b,
	181	/// Packed ___ Word
	182	p_w,
	183	/// Packed ___ Doubleword
	184	p_d,
	185	/// Packed ___ Quadword
	186	p_q,
	187	/// Packed ___ Double Quadword
	188	p_dq,
	189
	190	/// ___ Scalar Single-Precision Values
	191	_ss,
	192	/// ___ Packed Single-Precision Values
	193	_ps,
	194	/// ___ Scalar Double-Precision Values
	195	//_sd,
	196	/// ___ Packed Double-Precision Values
	197	_pd,
	198
	199	/// VEX-Encoded ___
	200	v_,
	201	/// VEX-Encoded Packed ___
	202	vp_,
	203	/// VEX-Encoded Packed ___ Byte
	204	vp_b,
	205	/// VEX-Encoded Packed ___ Word
	206	vp_w,
	207	/// VEX-Encoded Packed ___ Doubleword
	208	vp_d,
	209	/// VEX-Encoded Packed ___ Quadword
	210	vp_q,
	211	/// VEX-Encoded Packed ___ Double Quadword
	212	vp_dq,
	213	/// VEX-Encoded ___ Scalar Single-Precision Values
	214	v_ss,
	215	/// VEX-Encoded ___ Packed Single-Precision Values
	216	v_ps,
	217	/// VEX-Encoded ___ Scalar Double-Precision Values
	218	v_sd,
	219	/// VEX-Encoded ___ Packed Double-Precision Values
	220	v_pd,
	221
	222	/// Mask ___ Byte
	223	k_b,
	224	/// Mask ___ Word
	225	k_w,
	226	/// Mask ___ Doubleword
	227	k_d,
	228	/// Mask ___ Quadword
	229	k_q,
	230
	231	pub fn fromCondition(cc: bits.Condition) Fixes {
	232	return switch (cc) {
	233	inline else => \|cc_tag\| @field(Fixes, "_" ++ @tagName(cc_tag)),
	234	.z_and_np, .nz_or_p => unreachable,
	235	};
	236	}
	237	};
	238
35	239	pub const Tag = enum(u8) {
36	240	/// Add with carry
37	241	adc,
...	...	@@ -57,22 +261,24 @@ pub const Inst = struct {
57	261	call,
58	262	/// Convert byte to word
59	263	cbw,
60		/// Convert word to doubleword
61		cwde,
62		/// Convert doubleword to quadword
63		cdqe,
64		/// Convert word to doubleword
65		cwd,
66	264	/// Convert doubleword to quadword
67	265	cdq,
68	266	/// Convert doubleword to quadword
69		cqo,
	267	cdqe,
	268	/// Conditional move
	269	cmov,
70	270	/// Logical compare
	271	/// Compare string
71	272	cmp,
72	273	/// Compare and exchange
73		cmpxchg,
74	274	/// Compare and exchange bytes
75		cmpxchgb,
	275	cmpxchg,
	276	/// Convert doubleword to quadword
	277	cqo,
	278	/// Convert word to doubleword
	279	cwd,
	280	/// Convert word to doubleword
	281	cwde,
76	282	/// Unsigned division
77	283	div,
78	284	/// Store integer with truncation
...	...	@@ -85,10 +291,14 @@ pub const Inst = struct {
85	291	imul,
86	292	///
87	293	int3,
	294	/// Conditional jump
	295	j,
88	296	/// Jump
89	297	jmp,
90	298	/// Load effective address
91	299	lea,
	300	/// Load string
	301	lod,
92	302	/// Load fence
93	303	lfence,
94	304	/// Count the number of leading zero bits
...	...	@@ -96,6 +306,7 @@ pub const Inst = struct {
96	306	/// Memory fence
97	307	mfence,
98	308	/// Move
	309	/// Move data from string to string
99	310	mov,
100	311	/// Move data after swapping bytes
101	312	movbe,
...	...	@@ -105,6 +316,8 @@ pub const Inst = struct {
105	316	movq,
106	317	/// Move with sign extension
107	318	movsx,
	319	/// Move with sign extension
	320	movsxd,
108	321	/// Move with zero extension
109	322	movzx,
110	323	/// Multiply
...	...	@@ -139,6 +352,10 @@ pub const Inst = struct {
139	352	sar,
140	353	/// Integer subtraction with borrow
141	354	sbb,
	355	/// Scan string
	356	sca,
	357	/// Set byte on condition
	358	set,
142	359	/// Store fence
143	360	sfence,
144	361	/// Logical shift left
...	...	@@ -151,6 +368,8 @@ pub const Inst = struct {
151	368	shrd,
152	369	/// Subtract
153	370	sub,
	371	/// Store string
	372	sto,
154	373	/// Syscall
155	374	syscall,
156	375	/// Test condition
...	...	@@ -505,57 +724,10 @@ pub const Inst = struct {
505	724	/// Fused multiply-add of scalar single-precision floating-point values
506	725	vfmadd231ss,
507	726
508		/// Compare string operands
509		cmps,
510		/// Load string
511		lods,
512		/// Move data from string to string
513		movs,
514		/// Scan string
515		scas,
516		/// Store string
517		stos,
518
519		/// Conditional move
520		cmovcc,
521		/// Conditional jump
522		jcc,
523		/// Set byte on condition
524		setcc,
525
526		/// Mov absolute to/from memory wrt segment register to/from rax
527		mov_moffs,
528
529		/// Jump with relocation to another local MIR instruction
530		/// Uses `inst` payload.
531		jmp_reloc,
532
533		/// Call to an extern symbol via linker relocation.
534		/// Uses `relocation` payload.
535		call_extern,
536
537		/// Load effective address of a symbol not yet allocated in VM.
538		lea_linker,
539		/// Move address of a symbol not yet allocated in VM.
540		mov_linker,
541
542		/// End of prologue
543		dbg_prologue_end,
544		/// Start of epilogue
545		dbg_epilogue_begin,
546		/// Update debug line
547		/// Uses `line_column` payload containing the line and column.
548		dbg_line,
549		/// Push registers
550		/// Uses `payload` payload containing `RegisterList.asInt` directly.
551		push_regs,
552		/// Pop registers
553		/// Uses `payload` payload containing `RegisterList.asInt` directly.
554		pop_regs,
555
556		/// Tombstone
557		/// Emitter should skip this instruction.
558		dead,
	727	/// A pseudo instruction that requires special lowering.
	728	/// This should be the only tag in this enum that doesn't
	729	/// directly correspond to one or more instruction mnemonics.
	730	pseudo,
559	731	};
560	732
561	733	pub const Ops = enum(u8) {
...	...	@@ -579,12 +751,6 @@ pub const Inst = struct {
579	751	/// Register, register, immediate (unsigned) operands.
580	752	/// Uses `rri` payload.
581	753	rri_u,
582		/// Register with condition code (CC).
583		/// Uses `r_cc` payload.
584		r_cc,
585		/// Register, register with condition code (CC).
586		/// Uses `rr_cc` payload.
587		rr_cc,
588	754	/// Register, immediate (sign-extended) operands.
589	755	/// Uses `ri` payload.
590	756	ri_s,
...	...	@@ -609,12 +775,6 @@ pub const Inst = struct {
609	775	/// Register, memory (RIP) operands.
610	776	/// Uses `rx` payload.
611	777	rm_rip,
612		/// Register, memory (SIB) operands with condition code (CC).
613		/// Uses `rx_cc` payload.
614		rm_sib_cc,
615		/// Register, memory (RIP) operands with condition code (CC).
616		/// Uses `rx_cc` payload.
617		rm_rip_cc,
618	778	/// Register, memory (SIB), immediate (byte) operands.
619	779	/// Uses `rix` payload with extra data of type `MemorySib`.
620	780	rmi_sib,
...	...	@@ -634,17 +794,11 @@ pub const Inst = struct {
634	794	/// Uses `rix` payload with extra data of type `MemoryRip`.
635	795	rmi_rip,
636	796	/// Single memory (SIB) operand.
637		/// Uses `payload` with extra data of type `MemorySib`.
	797	/// Uses `x` with extra data of type `MemorySib`.
638	798	m_sib,
639	799	/// Single memory (RIP) operand.
640		/// Uses `payload` with extra data of type `MemoryRip`.
	800	/// Uses `x` with extra data of type `MemoryRip`.
641	801	m_rip,
642		/// Single memory (SIB) operand with condition code (CC).
643		/// Uses `x_cc` with extra data of type `MemorySib`.
644		m_sib_cc,
645		/// Single memory (RIP) operand with condition code (CC).
646		/// Uses `x_cc` with extra data of type `MemoryRip`.
647		m_rip_cc,
648	802	/// Memory (SIB), immediate (unsigned) operands.
649	803	/// Uses `ix` payload with extra data of type `MemorySib`.
650	804	mi_sib_u,
...	...	@@ -676,49 +830,17 @@ pub const Inst = struct {
676	830	/// Uses `rix` payload with extra data of type `MemoryRip`.
677	831	mri_rip,
678	832	/// Rax, Memory moffs.
679		/// Uses `payload` with extra data of type `MemoryMoffs`.
	833	/// Uses `x` with extra data of type `MemoryMoffs`.
680	834	rax_moffs,
681	835	/// Memory moffs, rax.
682		/// Uses `payload` with extra data of type `MemoryMoffs`.
	836	/// Uses `x` with extra data of type `MemoryMoffs`.
683	837	moffs_rax,
684		/// Single memory (SIB) operand with lock prefix.
685		/// Uses `payload` with extra data of type `MemorySib`.
686		lock_m_sib,
687		/// Single memory (RIP) operand with lock prefix.
688		/// Uses `payload` with extra data of type `MemoryRip`.
689		lock_m_rip,
690		/// Memory (SIB), immediate (unsigned) operands with lock prefix.
691		/// Uses `xi` payload with extra data of type `MemorySib`.
692		lock_mi_sib_u,
693		/// Memory (RIP), immediate (unsigned) operands with lock prefix.
694		/// Uses `xi` payload with extra data of type `MemoryRip`.
695		lock_mi_rip_u,
696		/// Memory (SIB), immediate (sign-extend) operands with lock prefix.
697		/// Uses `xi` payload with extra data of type `MemorySib`.
698		lock_mi_sib_s,
699		/// Memory (RIP), immediate (sign-extend) operands with lock prefix.
700		/// Uses `xi` payload with extra data of type `MemoryRip`.
701		lock_mi_rip_s,
702		/// Memory (SIB), register operands with lock prefix.
703		/// Uses `rx` payload with extra data of type `MemorySib`.
704		lock_mr_sib,
705		/// Memory (RIP), register operands with lock prefix.
706		/// Uses `rx` payload with extra data of type `MemoryRip`.
707		lock_mr_rip,
708		/// Memory moffs, rax with lock prefix.
709		/// Uses `payload` with extra data of type `MemoryMoffs`.
710		lock_moffs_rax,
711	838	/// References another Mir instruction directly.
712	839	/// Uses `inst` payload.
713	840	inst,
714		/// References another Mir instruction directly with condition code (CC).
715		/// Uses `inst_cc` payload.
716		inst_cc,
717		/// String repeat and width
718		/// Uses `string` payload.
719		string,
	841	/// Linker relocation - external function.
720	842	/// Uses `reloc` payload.
721		reloc,
	843	extern_fn_reloc,
722	844	/// Linker relocation - GOT indirection.
723	845	/// Uses `rx` payload with extra data of type `Reloc`.
724	846	got_reloc,
...	...	@@ -731,74 +853,125 @@ pub const Inst = struct {
731	853	/// Linker relocation - threadlocal variable via GOT indirection.
732	854	/// Uses `rx` payload with extra data of type `Reloc`.
733	855	tlv_reloc,
	856
	857	// Pseudo instructions:
	858
	859	/// Conditional move if zero flag set and parity flag not set
	860	/// Clobbers the source operand!
	861	/// Uses `rr` payload.
	862	pseudo_cmov_z_and_np_rr,
	863	/// Conditional move if zero flag not set or parity flag set
	864	/// Uses `rr` payload.
	865	pseudo_cmov_nz_or_p_rr,
	866	/// Conditional move if zero flag not set or parity flag set
	867	/// Uses `rx` payload.
	868	pseudo_cmov_nz_or_p_rm_sib,
	869	/// Conditional move if zero flag not set or parity flag set
	870	/// Uses `rx` payload.
	871	pseudo_cmov_nz_or_p_rm_rip,
	872	/// Set byte if zero flag set and parity flag not set
	873	/// Requires a scratch register!
	874	/// Uses `r_scratch` payload.
	875	pseudo_set_z_and_np_r,
	876	/// Set byte if zero flag set and parity flag not set
	877	/// Requires a scratch register!
	878	/// Uses `x_scratch` payload.
	879	pseudo_set_z_and_np_m_sib,
	880	/// Set byte if zero flag set and parity flag not set
	881	/// Requires a scratch register!
	882	/// Uses `x_scratch` payload.
	883	pseudo_set_z_and_np_m_rip,
	884	/// Set byte if zero flag not set or parity flag set
	885	/// Requires a scratch register!
	886	/// Uses `r_scratch` payload.
	887	pseudo_set_nz_or_p_r,
	888	/// Set byte if zero flag not set or parity flag set
	889	/// Requires a scratch register!
	890	/// Uses `x_scratch` payload.
	891	pseudo_set_nz_or_p_m_sib,
	892	/// Set byte if zero flag not set or parity flag set
	893	/// Requires a scratch register!
	894	/// Uses `x_scratch` payload.
	895	pseudo_set_nz_or_p_m_rip,
	896	/// Jump if zero flag set and parity flag not set
	897	/// Uses `inst` payload.
	898	pseudo_j_z_and_np_inst,
	899	/// Jump if zero flag not set or parity flag set
	900	/// Uses `inst` payload.
	901	pseudo_j_nz_or_p_inst,
	902
	903	/// Push registers
	904	/// Uses `reg_list` payload.
	905	pseudo_push_reg_list,
	906	/// Pop registers
	907	/// Uses `reg_list` payload.
	908	pseudo_pop_reg_list,
	909
	910	/// End of prologue
	911	pseudo_dbg_prologue_end_none,
	912	/// Update debug line
	913	/// Uses `line_column` payload.
	914	pseudo_dbg_line_line_column,
	915	/// Start of epilogue
	916	pseudo_dbg_epilogue_begin_none,
	917
	918	/// Tombstone
	919	/// Emitter should skip this instruction.
	920	pseudo_dead_none,
734	921	};
735	922
736	923	pub const Data = union {
	924	none: struct {
	925	fixes: Fixes = ._,
	926	},
737	927	/// References another Mir instruction.
738		inst: Index,
739		/// Another instruction with condition code (CC).
740		/// Used by `jcc`.
741		inst_cc: struct {
742		/// Another instruction.
	928	inst: struct {
	929	fixes: Fixes = ._,
743	930	inst: Index,
744		/// A condition code for use with EFLAGS register.
745		cc: bits.Condition,
746	931	},
747	932	/// A 32-bit immediate value.
748		i: u32,
749		r: Register,
	933	i: struct {
	934	fixes: Fixes = ._,
	935	i: u32,
	936	},
	937	r: struct {
	938	fixes: Fixes = ._,
	939	r1: Register,
	940	},
750	941	rr: struct {
	942	fixes: Fixes = ._,
751	943	r1: Register,
752	944	r2: Register,
753	945	},
754	946	rrr: struct {
	947	fixes: Fixes = ._,
755	948	r1: Register,
756	949	r2: Register,
757	950	r3: Register,
758	951	},
759	952	rrri: struct {
	953	fixes: Fixes = ._,
760	954	r1: Register,
761	955	r2: Register,
762	956	r3: Register,
763	957	i: u8,
764	958	},
765	959	rri: struct {
	960	fixes: Fixes = ._,
766	961	r1: Register,
767	962	r2: Register,
768	963	i: u32,
769	964	},
770		/// Condition code (CC), followed by custom payload found in extra.
771		x_cc: struct {
772		scratch: Register,
773		cc: bits.Condition,
774		payload: u32,
775		},
776		/// Register with condition code (CC).
777		r_cc: struct {
778		r: Register,
779		scratch: Register,
780		cc: bits.Condition,
781		},
782		/// Register, register with condition code (CC).
783		rr_cc: struct {
784		r1: Register,
785		r2: Register,
786		cc: bits.Condition,
787		},
788	965	/// Register, immediate.
789	966	ri: struct {
790		r: Register,
	967	fixes: Fixes = ._,
	968	r1: Register,
791	969	i: u32,
792	970	},
793	971	/// Register, followed by custom payload found in extra.
794	972	rx: struct {
795		r: Register,
796		payload: u32,
797		},
798		/// Register with condition code (CC), followed by custom payload found in extra.
799		rx_cc: struct {
800		r: Register,
801		cc: bits.Condition,
	973	fixes: Fixes = ._,
	974	r1: Register,
802	975	payload: u32,
803	976	},
804	977	/// Immediate, followed by Custom payload found in extra.
...	...	@@ -808,39 +981,54 @@ pub const Inst = struct {
808	981	},
809	982	/// Register, register, followed by Custom payload found in extra.
810	983	rrx: struct {
	984	fixes: Fixes = ._,
811	985	r1: Register,
812	986	r2: Register,
813	987	payload: u32,
814	988	},
815	989	/// Register, byte immediate, followed by Custom payload found in extra.
816	990	rix: struct {
817		r: Register,
	991	fixes: Fixes = ._,
	992	r1: Register,
818	993	i: u8,
819	994	payload: u32,
820	995	},
821	996	/// Register, register, byte immediate, followed by Custom payload found in extra.
822	997	rrix: struct {
	998	fixes: Fixes = ._,
823	999	r1: Register,
824	1000	r2: Register,
825	1001	i: u8,
826	1002	payload: u32,
827	1003	},
828		/// String instruction prefix and width.
829		string: struct {
830		repeat: bits.StringRepeat,
831		width: bits.StringWidth,
	1004	/// Register, scratch register
	1005	r_scratch: struct {
	1006	fixes: Fixes = ._,
	1007	r1: Register,
	1008	scratch_reg: Register,
	1009	},
	1010	/// Scratch register, followed by Custom payload found in extra.
	1011	x_scratch: struct {
	1012	fixes: Fixes = ._,
	1013	scratch_reg: Register,
	1014	payload: u32,
	1015	},
	1016	/// Custom payload found in extra.
	1017	x: struct {
	1018	fixes: Fixes = ._,
	1019	payload: u32,
832	1020	},
833	1021	/// Relocation for the linker where:
834	1022	/// * `atom_index` is the index of the source
835	1023	/// * `sym_index` is the index of the target
836		relocation: Reloc,
	1024	reloc: Reloc,
837	1025	/// Debug line and column position
838	1026	line_column: struct {
839	1027	line: u32,
840	1028	column: u32,
841	1029	},
842		/// Index into `extra`. Meaning of what can be found there is context-dependent.
843		payload: u32,
	1030	/// Register list
	1031	reg_list: RegisterList,
844	1032	};
845	1033
846	1034	// Make sure we don't accidentally make instructions bigger than expected.
...	...	@@ -852,6 +1040,7 @@ pub const Inst = struct {
852	1040	}
853	1041	};
854	1042
	1043	/// A linker symbol not yet allocated in VM.
855	1044	pub const Reloc = struct {
856	1045	/// Index of the containing atom.
857	1046	atom_index: u32,
...	...	@@ -887,16 +1076,6 @@ pub const RegisterList = struct {
887	1076	return self.bitset.iterator(options);
888	1077	}
889	1078
890		pub fn asInt(self: Self) u32 {
891		return self.bitset.mask;
892		}
893
894		pub fn fromInt(mask: u32) Self {
895		return .{
896		.bitset = BitSet{ .mask = @intCast(BitSet.MaskInt, mask) },
897		};
898		}
899
900	1079	pub fn count(self: Self) u32 {
901	1080	return @intCast(u32, self.bitset.count());
902	1081	}

src/arch/x86_64/bits.zig-3

...	...	@@ -6,9 +6,6 @@ const Allocator = std.mem.Allocator;
6	6	const ArrayList = std.ArrayList;
7	7	const DW = std.dwarf;
8	8
9		pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
10		pub const StringWidth = enum(u2) { b, w, d, q };
11
12	9	/// EFLAGS condition codes
13	10	pub const Condition = enum(u5) {
14	11	/// above