Commit f40539e5d83365412bf8c6973ce867125ea36faf | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-05-08 19:04:05+02:00
committer	GitHub <noreply@github.com> 2023-05-08 19:04:05+02:00
log	f40539e5d83365412bf8c6973ce867125ea36faf
tree	c8b6c458a2a20c1846d57721de33d2befed750e7
parent	16314e0e199051289075c9e3dab425ce3fb52719
parent	1f5aa7747f5710e281cd2190508ce562a4bfd35f
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15597 from jacobly0/x86_64-behavior

x86_64: more behavior than ever before

18 files changed, 4760 insertions(+), 2753 deletions(-)

lib
- std
  - target
    x86.zig
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig
    encodings.zig
- link
  - Dwarf.zig
test
- behavior

lib/std/target/x86.zig+3-3

...	...	@@ -326,7 +326,7 @@ pub const all_features = blk: {
326	326	};
327	327	result[@enumToInt(Feature.avx512ifma)] = .{
328	328	.llvm_name = "avx512ifma",
329		.description = "Enable AVX-512 Integer Fused Multiple-Add",
	329	.description = "Enable AVX-512 Integer Fused Multiply-Add",
330	330	.dependencies = featureSet(&[_]Feature{
331	331	.avx512f,
332	332	}),
...	...	@@ -599,14 +599,14 @@ pub const all_features = blk: {
599	599	};
600	600	result[@enumToInt(Feature.fma)] = .{
601	601	.llvm_name = "fma",
602		.description = "Enable three-operand fused multiple-add",
	602	.description = "Enable three-operand fused multiply-add",
603	603	.dependencies = featureSet(&[_]Feature{
604	604	.avx,
605	605	}),
606	606	};
607	607	result[@enumToInt(Feature.fma4)] = .{
608	608	.llvm_name = "fma4",
609		.description = "Enable four-operand fused multiple-add",
	609	.description = "Enable four-operand fused multiply-add",
610	610	.dependencies = featureSet(&[_]Feature{
611	611	.avx,
612	612	.sse4a,

src/arch/x86_64/CodeGen.zig+2140-944

...	...	@@ -205,16 +205,7 @@ pub const MCValue = union(enum) {
205	205
206	206	fn isMemory(mcv: MCValue) bool {
207	207	return switch (mcv) {
208		.memory,
209		.load_direct,
210		.lea_direct,
211		.load_got,
212		.lea_got,
213		.load_tlv,
214		.lea_tlv,
215		.load_frame,
216		.lea_frame,
217		=> true,
	208	.memory, .indirect, .load_frame => true,
218	209	else => false,
219	210	};
220	211	}
...	...	@@ -937,7 +928,7 @@ fn formatWipMir(
937	928	.target = data.self.target,
938	929	.src_loc = data.self.src_loc,
939	930	};
940		for (lower.lowerMir(data.self.mir_instructions.get(data.inst)) catch \|err\| switch (err) {
	931	for ((lower.lowerMir(data.inst) catch \|err\| switch (err) {
941	932	error.LowerFail => {
942	933	defer {
943	934	lower.err_msg.?.deinit(data.self.gpa);
...	...	@@ -955,7 +946,7 @@ fn formatWipMir(
955	946	return;
956	947	},
957	948	else => \|e\| return e,
958		}) \|lower_inst\| try writer.print(" \| {}", .{lower_inst});
	949	}).insts) \|lowered_inst\| try writer.print(" \| {}", .{lowered_inst});
959	950	}
960	951	fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
961	952	return .{ .data = .{ .self = self, .inst = inst } };
...	...	@@ -982,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
982	973	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
983	974	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
984	975	self.mir_instructions.appendAssumeCapacity(inst);
985		switch (inst.tag) {
986		else => wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)}),
987		.dbg_line,
988		.dbg_prologue_end,
989		.dbg_epilogue_begin,
990		.dead,
991		=> {},
992		}
	976	if (inst.tag != .pseudo or switch (inst.ops) {
	977	else => true,
	978	.pseudo_dbg_prologue_end_none,
	979	.pseudo_dbg_line_line_column,
	980	.pseudo_dbg_epilogue_begin_none,
	981	.pseudo_dead_none,
	982	=> false,
	983	}) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
993	984	return result_index;
994	985	}
995	986
...	...	@@ -1012,131 +1003,248 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1012	1003	return result;
1013	1004	}
1014	1005
1015		fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
	1006	/// A `cc` of `.z_and_np` clobbers `reg2`!
	1007	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1016	1008	_ = try self.addInst(.{
1017		.tag = .setcc,
1018		.ops = .r_cc,
1019		.data = .{ .r_cc = .{ .r = reg, .cc = cc } },
	1009	.tag = switch (cc) {
	1010	else => .cmov,
	1011	.z_and_np, .nz_or_p => .pseudo,
	1012	},
	1013	.ops = switch (cc) {
	1014	else => .rr,
	1015	.z_and_np => .pseudo_cmov_z_and_np_rr,
	1016	.nz_or_p => .pseudo_cmov_nz_or_p_rr,
	1017	},
	1018	.data = .{ .rr = .{
	1019	.fixes = switch (cc) {
	1020	else => Mir.Inst.Fixes.fromCondition(cc),
	1021	.z_and_np, .nz_or_p => ._,
	1022	},
	1023	.r1 = reg1,
	1024	.r2 = reg2,
	1025	} },
1020	1026	});
1021	1027	}
1022	1028
1023		fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
	1029	/// A `cc` of `.z_and_np` is not supported by this encoding!
	1030	fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
1024	1031	_ = try self.addInst(.{
1025		.tag = .setcc,
1026		.ops = switch (m) {
1027		.sib => .m_sib_cc,
1028		.rip => .m_rip_cc,
1029		else => unreachable,
	1032	.tag = switch (cc) {
	1033	else => .cmov,
	1034	.z_and_np => unreachable,
	1035	.nz_or_p => .pseudo,
1030	1036	},
1031		.data = .{ .x_cc = .{ .cc = cc, .payload = switch (m) {
1032		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1033		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1034		else => unreachable,
1035		} } },
	1037	.ops = switch (cc) {
	1038	else => switch (m) {
	1039	.sib => .rm_sib,
	1040	.rip => .rm_rip,
	1041	else => unreachable,
	1042	},
	1043	.z_and_np => unreachable,
	1044	.nz_or_p => switch (m) {
	1045	.sib => .pseudo_cmov_nz_or_p_rm_sib,
	1046	.rip => .pseudo_cmov_nz_or_p_rm_rip,
	1047	else => unreachable,
	1048	},
	1049	},
	1050	.data = .{ .rx = .{
	1051	.fixes = switch (cc) {
	1052	else => Mir.Inst.Fixes.fromCondition(cc),
	1053	.z_and_np => unreachable,
	1054	.nz_or_p => ._,
	1055	},
	1056	.r1 = reg,
	1057	.payload = switch (m) {
	1058	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1059	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1060	else => unreachable,
	1061	},
	1062	} },
1036	1063	});
1037	1064	}
1038	1065
1039		fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
	1066	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1040	1067	_ = try self.addInst(.{
1041		.tag = .cmovcc,
1042		.ops = .rr_cc,
1043		.data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc } },
	1068	.tag = switch (cc) {
	1069	else => .set,
	1070	.z_and_np, .nz_or_p => .pseudo,
	1071	},
	1072	.ops = switch (cc) {
	1073	else => .r,
	1074	.z_and_np => .pseudo_set_z_and_np_r,
	1075	.nz_or_p => .pseudo_set_nz_or_p_r,
	1076	},
	1077	.data = switch (cc) {
	1078	else => .{ .r = .{
	1079	.fixes = Mir.Inst.Fixes.fromCondition(cc),
	1080	.r1 = reg,
	1081	} },
	1082	.z_and_np, .nz_or_p => .{ .r_scratch = .{
	1083	.r1 = reg,
	1084	.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
	1085	} },
	1086	},
1044	1087	});
1045	1088	}
1046	1089
1047		fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
	1090	fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
	1091	const payload = switch (m) {
	1092	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1093	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1094	else => unreachable,
	1095	};
1048	1096	_ = try self.addInst(.{
1049		.tag = .cmovcc,
1050		.ops = switch (m) {
1051		.sib => .rm_sib_cc,
1052		.rip => .rm_rip_cc,
1053		else => unreachable,
	1097	.tag = switch (cc) {
	1098	else => .set,
	1099	.z_and_np, .nz_or_p => .pseudo,
	1100	},
	1101	.ops = switch (cc) {
	1102	else => switch (m) {
	1103	.sib => .m_sib,
	1104	.rip => .m_rip,
	1105	else => unreachable,
	1106	},
	1107	.z_and_np => switch (m) {
	1108	.sib => .pseudo_set_z_and_np_m_sib,
	1109	.rip => .pseudo_set_z_and_np_m_rip,
	1110	else => unreachable,
	1111	},
	1112	.nz_or_p => switch (m) {
	1113	.sib => .pseudo_set_nz_or_p_m_sib,
	1114	.rip => .pseudo_set_nz_or_p_m_rip,
	1115	else => unreachable,
	1116	},
	1117	},
	1118	.data = switch (cc) {
	1119	else => .{ .x = .{
	1120	.fixes = Mir.Inst.Fixes.fromCondition(cc),
	1121	.payload = payload,
	1122	} },
	1123	.z_and_np, .nz_or_p => .{ .x_scratch = .{
	1124	.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
	1125	.payload = payload,
	1126	} },
1054	1127	},
1055		.data = .{ .rx_cc = .{ .r = reg, .cc = cc, .payload = switch (m) {
1056		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1057		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1058		else => unreachable,
1059		} } },
1060	1128	});
1061	1129	}
1062	1130
1063	1131	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1064	1132	return self.addInst(.{
1065		.tag = .jmp_reloc,
1066		.ops = undefined,
1067		.data = .{ .inst = target },
	1133	.tag = .jmp,
	1134	.ops = .inst,
	1135	.data = .{ .inst = .{
	1136	.inst = target,
	1137	} },
1068	1138	});
1069	1139	}
1070	1140
1071	1141	fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
1072	1142	return self.addInst(.{
1073		.tag = .jcc,
1074		.ops = .inst_cc,
1075		.data = .{ .inst_cc = .{ .inst = target, .cc = cc } },
	1143	.tag = switch (cc) {
	1144	else => .j,
	1145	.z_and_np, .nz_or_p => .pseudo,
	1146	},
	1147	.ops = switch (cc) {
	1148	else => .inst,
	1149	.z_and_np => .pseudo_j_z_and_np_inst,
	1150	.nz_or_p => .pseudo_j_nz_or_p_inst,
	1151	},
	1152	.data = .{ .inst = .{
	1153	.fixes = switch (cc) {
	1154	else => Mir.Inst.Fixes.fromCondition(cc),
	1155	.z_and_np, .nz_or_p => ._,
	1156	},
	1157	.inst = target,
	1158	} },
1076	1159	});
1077	1160	}
1078	1161
1079	1162	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1080	1163	return self.addInst(.{
1081		.tag = .dead,
1082		.ops = undefined,
	1164	.tag = .pseudo,
	1165	.ops = .pseudo_dead_none,
1083	1166	.data = undefined,
1084	1167	});
1085	1168	}
1086	1169
1087		fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
	1170	fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1088	1171	_ = try self.addInst(.{
1089		.tag = tag,
	1172	.tag = tag[1],
1090	1173	.ops = .none,
	1174	.data = .{ .none = .{
	1175	.fixes = tag[0],
	1176	} },
	1177	});
	1178	}
	1179
	1180	fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
	1181	_ = try self.addInst(.{
	1182	.tag = .pseudo,
	1183	.ops = ops,
1091	1184	.data = undefined,
1092	1185	});
1093	1186	}
1094	1187
1095		fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
	1188	fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1096	1189	_ = try self.addInst(.{
1097		.tag = tag,
	1190	.tag = tag[1],
1098	1191	.ops = .r,
1099		.data = .{ .r = reg },
	1192	.data = .{ .r = .{
	1193	.fixes = tag[0],
	1194	.r1 = reg,
	1195	} },
1100	1196	});
1101	1197	}
1102	1198
1103		fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
	1199	fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1104	1200	_ = try self.addInst(.{
1105		.tag = tag,
	1201	.tag = tag[1],
1106	1202	.ops = switch (imm) {
1107	1203	.signed => .i_s,
1108	1204	.unsigned => .i_u,
1109	1205	},
1110		.data = .{ .i = switch (imm) {
1111		.signed => \|s\| @bitCast(u32, s),
1112		.unsigned => \|u\| @intCast(u32, u),
	1206	.data = .{ .i = .{
	1207	.fixes = tag[0],
	1208	.i = switch (imm) {
	1209	.signed => \|s\| @bitCast(u32, s),
	1210	.unsigned => \|u\| @intCast(u32, u),
	1211	},
1113	1212	} },
1114	1213	});
1115	1214	}
1116	1215
1117		fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {
	1216	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1118	1217	_ = try self.addInst(.{
1119		.tag = tag,
	1218	.tag = tag[1],
1120	1219	.ops = .rr,
1121		.data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },
	1220	.data = .{ .rr = .{
	1221	.fixes = tag[0],
	1222	.r1 = reg1,
	1223	.r2 = reg2,
	1224	} },
1122	1225	});
1123	1226	}
1124	1227
1125		fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
	1228	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1126	1229	const ops: Mir.Inst.Ops = switch (imm) {
1127	1230	.signed => .ri_s,
1128	1231	.unsigned => \|u\| if (math.cast(u32, u)) \|_\| .ri_u else .ri64,
1129	1232	};
1130	1233	_ = try self.addInst(.{
1131		.tag = tag,
	1234	.tag = tag[1],
1132	1235	.ops = ops,
1133	1236	.data = switch (ops) {
1134		.ri_s, .ri_u => .{ .ri = .{ .r = reg, .i = switch (imm) {
1135		.signed => \|s\| @bitCast(u32, s),
1136		.unsigned => \|u\| @intCast(u32, u),
1137		} } },
	1237	.ri_s, .ri_u => .{ .ri = .{
	1238	.fixes = tag[0],
	1239	.r1 = reg,
	1240	.i = switch (imm) {
	1241	.signed => \|s\| @bitCast(u32, s),
	1242	.unsigned => \|u\| @intCast(u32, u),
	1243	},
	1244	} },
1138	1245	.ri64 => .{ .rx = .{
1139		.r = reg,
	1246	.fixes = tag[0],
	1247	.r1 = reg,
1140	1248	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
1141	1249	} },
1142	1250	else => unreachable,
...	...	@@ -1146,111 +1254,214 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1146	1254
1147	1255	fn asmRegisterRegisterRegister(
1148	1256	self: *Self,
1149		tag: Mir.Inst.Tag,
	1257	tag: Mir.Inst.FixedTag,
1150	1258	reg1: Register,
1151	1259	reg2: Register,
1152	1260	reg3: Register,
1153	1261	) !void {
1154	1262	_ = try self.addInst(.{
1155		.tag = tag,
	1263	.tag = tag[1],
1156	1264	.ops = .rrr,
1157		.data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },
	1265	.data = .{ .rrr = .{
	1266	.fixes = tag[0],
	1267	.r1 = reg1,
	1268	.r2 = reg2,
	1269	.r3 = reg3,
	1270	} },
	1271	});
	1272	}
	1273
	1274	fn asmRegisterRegisterRegisterImmediate(
	1275	self: *Self,
	1276	tag: Mir.Inst.FixedTag,
	1277	reg1: Register,
	1278	reg2: Register,
	1279	reg3: Register,
	1280	imm: Immediate,
	1281	) !void {
	1282	_ = try self.addInst(.{
	1283	.tag = tag[1],
	1284	.ops = .rrri,
	1285	.data = .{ .rrri = .{
	1286	.fixes = tag[0],
	1287	.r1 = reg1,
	1288	.r2 = reg2,
	1289	.r3 = reg3,
	1290	.i = @intCast(u8, imm.unsigned),
	1291	} },
1158	1292	});
1159	1293	}
1160	1294
1161	1295	fn asmRegisterRegisterImmediate(
1162	1296	self: *Self,
1163		tag: Mir.Inst.Tag,
	1297	tag: Mir.Inst.FixedTag,
1164	1298	reg1: Register,
1165	1299	reg2: Register,
1166	1300	imm: Immediate,
1167	1301	) !void {
1168	1302	_ = try self.addInst(.{
1169		.tag = tag,
	1303	.tag = tag[1],
1170	1304	.ops = switch (imm) {
1171	1305	.signed => .rri_s,
1172	1306	.unsigned => .rri_u,
1173	1307	},
1174		.data = .{ .rri = .{ .r1 = reg1, .r2 = reg2, .i = switch (imm) {
1175		.signed => \|s\| @bitCast(u32, s),
1176		.unsigned => \|u\| @intCast(u32, u),
1177		} } },
	1308	.data = .{ .rri = .{
	1309	.fixes = tag[0],
	1310	.r1 = reg1,
	1311	.r2 = reg2,
	1312	.i = switch (imm) {
	1313	.signed => \|s\| @bitCast(u32, s),
	1314	.unsigned => \|u\| @intCast(u32, u),
	1315	},
	1316	} },
	1317	});
	1318	}
	1319
	1320	fn asmRegisterRegisterMemory(
	1321	self: *Self,
	1322	tag: Mir.Inst.FixedTag,
	1323	reg1: Register,
	1324	reg2: Register,
	1325	m: Memory,
	1326	) !void {
	1327	_ = try self.addInst(.{
	1328	.tag = tag[1],
	1329	.ops = switch (m) {
	1330	.sib => .rrm_sib,
	1331	.rip => .rrm_rip,
	1332	else => unreachable,
	1333	},
	1334	.data = .{ .rrx = .{
	1335	.fixes = tag[0],
	1336	.r1 = reg1,
	1337	.r2 = reg2,
	1338	.payload = switch (m) {
	1339	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1340	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1341	else => unreachable,
	1342	},
	1343	} },
1178	1344	});
1179	1345	}
1180	1346
1181		fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
	1347	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1182	1348	_ = try self.addInst(.{
1183		.tag = tag,
	1349	.tag = tag[1],
1184	1350	.ops = switch (m) {
1185	1351	.sib => .m_sib,
1186	1352	.rip => .m_rip,
1187	1353	else => unreachable,
1188	1354	},
1189		.data = .{ .payload = switch (m) {
1190		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1191		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1192		else => unreachable,
	1355	.data = .{ .x = .{
	1356	.fixes = tag[0],
	1357	.payload = switch (m) {
	1358	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1359	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1360	else => unreachable,
	1361	},
1193	1362	} },
1194	1363	});
1195	1364	}
1196	1365
1197		fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
	1366	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1198	1367	_ = try self.addInst(.{
1199		.tag = tag,
	1368	.tag = tag[1],
1200	1369	.ops = switch (m) {
1201	1370	.sib => .rm_sib,
1202	1371	.rip => .rm_rip,
1203	1372	else => unreachable,
1204	1373	},
1205		.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
1206		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1207		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1208		else => unreachable,
1209		} } },
	1374	.data = .{ .rx = .{
	1375	.fixes = tag[0],
	1376	.r1 = reg,
	1377	.payload = switch (m) {
	1378	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1379	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1380	else => unreachable,
	1381	},
	1382	} },
1210	1383	});
1211	1384	}
1212	1385
1213	1386	fn asmRegisterMemoryImmediate(
1214	1387	self: *Self,
1215		tag: Mir.Inst.Tag,
	1388	tag: Mir.Inst.FixedTag,
1216	1389	reg: Register,
1217	1390	m: Memory,
1218	1391	imm: Immediate,
1219	1392	) !void {
1220	1393	_ = try self.addInst(.{
1221		.tag = tag,
	1394	.tag = tag[1],
1222	1395	.ops = switch (m) {
1223	1396	.sib => .rmi_sib,
1224	1397	.rip => .rmi_rip,
1225	1398	else => unreachable,
1226	1399	},
1227		.data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
1228		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1229		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1400	.data = .{ .rix = .{
	1401	.fixes = tag[0],
	1402	.r1 = reg,
	1403	.i = @intCast(u8, imm.unsigned),
	1404	.payload = switch (m) {
	1405	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1406	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1407	else => unreachable,
	1408	},
	1409	} },
	1410	});
	1411	}
	1412
	1413	fn asmRegisterRegisterMemoryImmediate(
	1414	self: *Self,
	1415	tag: Mir.Inst.FixedTag,
	1416	reg1: Register,
	1417	reg2: Register,
	1418	m: Memory,
	1419	imm: Immediate,
	1420	) !void {
	1421	_ = try self.addInst(.{
	1422	.tag = tag[1],
	1423	.ops = switch (m) {
	1424	.sib => .rrmi_sib,
	1425	.rip => .rrmi_rip,
1230	1426	else => unreachable,
1231		} } },
	1427	},
	1428	.data = .{ .rrix = .{
	1429	.fixes = tag[0],
	1430	.r1 = reg1,
	1431	.r2 = reg2,
	1432	.i = @intCast(u8, imm.unsigned),
	1433	.payload = switch (m) {
	1434	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1435	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1436	else => unreachable,
	1437	},
	1438	} },
1232	1439	});
1233	1440	}
1234	1441
1235		fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
	1442	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1236	1443	_ = try self.addInst(.{
1237		.tag = tag,
	1444	.tag = tag[1],
1238	1445	.ops = switch (m) {
1239	1446	.sib => .mr_sib,
1240	1447	.rip => .mr_rip,
1241	1448	else => unreachable,
1242	1449	},
1243		.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
1244		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1245		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1246		else => unreachable,
1247		} } },
	1450	.data = .{ .rx = .{
	1451	.fixes = tag[0],
	1452	.r1 = reg,
	1453	.payload = switch (m) {
	1454	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1455	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1456	else => unreachable,
	1457	},
	1458	} },
1248	1459	});
1249	1460	}
1250	1461
1251		fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
	1462	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1252	1463	_ = try self.addInst(.{
1253		.tag = tag,
	1464	.tag = tag[1],
1254	1465	.ops = switch (m) {
1255	1466	.sib => switch (imm) {
1256	1467	.signed => .mi_sib_s,
...	...	@@ -1262,67 +1473,81 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
1262	1473	},
1263	1474	else => unreachable,
1264	1475	},
1265		.data = .{ .ix = .{ .i = switch (imm) {
1266		.signed => \|s\| @bitCast(u32, s),
1267		.unsigned => \|u\| @intCast(u32, u),
1268		}, .payload = switch (m) {
1269		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1270		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1271		else => unreachable,
1272		} } },
	1476	.data = .{ .x = .{
	1477	.fixes = tag[0],
	1478	.payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
	1479	.signed => \|s\| @bitCast(u32, s),
	1480	.unsigned => \|u\| @intCast(u32, u),
	1481	} }),
	1482	} },
1273	1483	});
	1484	_ = switch (m) {
	1485	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1486	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1487	else => unreachable,
	1488	};
1274	1489	}
1275	1490
1276	1491	fn asmMemoryRegisterRegister(
1277	1492	self: *Self,
1278		tag: Mir.Inst.Tag,
	1493	tag: Mir.Inst.FixedTag,
1279	1494	m: Memory,
1280	1495	reg1: Register,
1281	1496	reg2: Register,
1282	1497	) !void {
1283	1498	_ = try self.addInst(.{
1284		.tag = tag,
	1499	.tag = tag[1],
1285	1500	.ops = switch (m) {
1286	1501	.sib => .mrr_sib,
1287	1502	.rip => .mrr_rip,
1288	1503	else => unreachable,
1289	1504	},
1290		.data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {
1291		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1292		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1293		else => unreachable,
1294		} } },
	1505	.data = .{ .rrx = .{
	1506	.fixes = tag[0],
	1507	.r1 = reg1,
	1508	.r2 = reg2,
	1509	.payload = switch (m) {
	1510	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1511	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1512	else => unreachable,
	1513	},
	1514	} },
1295	1515	});
1296	1516	}
1297	1517
1298	1518	fn asmMemoryRegisterImmediate(
1299	1519	self: *Self,
1300		tag: Mir.Inst.Tag,
	1520	tag: Mir.Inst.FixedTag,
1301	1521	m: Memory,
1302	1522	reg: Register,
1303	1523	imm: Immediate,
1304	1524	) !void {
1305	1525	_ = try self.addInst(.{
1306		.tag = tag,
	1526	.tag = tag[1],
1307	1527	.ops = switch (m) {
1308	1528	.sib => .mri_sib,
1309	1529	.rip => .mri_rip,
1310	1530	else => unreachable,
1311	1531	},
1312		.data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
1313		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1314		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1315		else => unreachable,
1316		} } },
	1532	.data = .{ .rix = .{
	1533	.fixes = tag[0],
	1534	.r1 = reg,
	1535	.i = @intCast(u8, imm.unsigned),
	1536	.payload = switch (m) {
	1537	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1538	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1539	else => unreachable,
	1540	},
	1541	} },
1317	1542	});
1318	1543	}
1319	1544
1320	1545	fn gen(self: *Self) InnerError!void {
1321	1546	const cc = self.fn_type.fnCallingConvention();
1322	1547	if (cc != .Naked) {
1323		try self.asmRegister(.push, .rbp);
	1548	try self.asmRegister(.{ ._, .push }, .rbp);
1324	1549	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
1325		try self.asmRegisterRegister(.mov, .rbp, .rsp);
	1550	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1326	1551	const backpatch_frame_align = try self.asmPlaceholder();
1327	1552	const backpatch_stack_alloc = try self.asmPlaceholder();
1328	1553
...	...	@@ -1346,7 +1571,7 @@ fn gen(self: *Self) InnerError!void {
1346	1571	else => unreachable,
1347	1572	}
1348	1573
1349		try self.asmOpOnly(.dbg_prologue_end);
	1574	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1350	1575
1351	1576	try self.genBody(self.air.getMainBody());
1352	1577
...	...	@@ -1358,15 +1583,15 @@ fn gen(self: *Self) InnerError!void {
1358	1583	// }
1359	1584	// Eliding the reloc will cause a miscompilation in this case.
1360	1585	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
1361		self.mir_instructions.items(.data)[jmp_reloc].inst =
	1586	self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
1362	1587	@intCast(u32, self.mir_instructions.len);
1363	1588	}
1364	1589
1365		try self.asmOpOnly(.dbg_epilogue_begin);
	1590	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1366	1591	const backpatch_stack_dealloc = try self.asmPlaceholder();
1367	1592	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
1368		try self.asmRegister(.pop, .rbp);
1369		try self.asmOpOnly(.ret);
	1593	try self.asmRegister(.{ ._, .pop }, .rbp);
	1594	try self.asmOpOnly(.{ ._, .ret });
1370	1595
1371	1596	const frame_layout = try self.computeFrameLayout();
1372	1597	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
...	...	@@ -1376,46 +1601,54 @@ fn gen(self: *Self) InnerError!void {
1376	1601	self.mir_instructions.set(backpatch_frame_align, .{
1377	1602	.tag = .@"and",
1378	1603	.ops = .ri_s,
1379		.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
	1604	.data = .{ .ri = .{
	1605	.r1 = .rsp,
	1606	.i = frame_layout.stack_mask,
	1607	} },
1380	1608	});
1381	1609	}
1382	1610	if (need_stack_adjust) {
1383	1611	self.mir_instructions.set(backpatch_stack_alloc, .{
1384	1612	.tag = .sub,
1385	1613	.ops = .ri_s,
1386		.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
	1614	.data = .{ .ri = .{
	1615	.r1 = .rsp,
	1616	.i = frame_layout.stack_adjust,
	1617	} },
1387	1618	});
1388	1619	}
1389	1620	if (need_frame_align or need_stack_adjust) {
1390	1621	self.mir_instructions.set(backpatch_stack_dealloc, .{
1391	1622	.tag = .mov,
1392	1623	.ops = .rr,
1393		.data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
	1624	.data = .{ .rr = .{
	1625	.r1 = .rsp,
	1626	.r2 = .rbp,
	1627	} },
1394	1628	});
1395	1629	}
1396	1630	if (need_save_reg) {
1397		const save_reg_list = frame_layout.save_reg_list.asInt();
1398	1631	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
1399		.tag = .push_regs,
1400		.ops = undefined,
1401		.data = .{ .payload = save_reg_list },
	1632	.tag = .pseudo,
	1633	.ops = .pseudo_push_reg_list,
	1634	.data = .{ .reg_list = frame_layout.save_reg_list },
1402	1635	});
1403	1636	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
1404		.tag = .pop_regs,
1405		.ops = undefined,
1406		.data = .{ .payload = save_reg_list },
	1637	.tag = .pseudo,
	1638	.ops = .pseudo_pop_reg_list,
	1639	.data = .{ .reg_list = frame_layout.save_reg_list },
1407	1640	});
1408	1641	}
1409	1642	} else {
1410		try self.asmOpOnly(.dbg_prologue_end);
	1643	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1411	1644	try self.genBody(self.air.getMainBody());
1412		try self.asmOpOnly(.dbg_epilogue_begin);
	1645	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1413	1646	}
1414	1647
1415	1648	// Drop them off at the rbrace.
1416	1649	_ = try self.addInst(.{
1417		.tag = .dbg_line,
1418		.ops = undefined,
	1650	.tag = .pseudo,
	1651	.ops = .pseudo_dbg_line_line_column,
1419	1652	.data = .{ .line_column = .{
1420	1653	.line = self.end_di_line,
1421	1654	.column = self.end_di_column,
...	...	@@ -1480,12 +1713,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1480	1713	.log,
1481	1714	.log2,
1482	1715	.log10,
1483		.floor,
1484		.ceil,
1485	1716	.round,
1486		.trunc_float,
1487	1717	=> try self.airUnaryMath(inst),
1488	1718
	1719	.floor => try self.airRound(inst, 0b1_0_01),
	1720	.ceil => try self.airRound(inst, 0b1_0_10),
	1721	.trunc_float => try self.airRound(inst, 0b1_0_11),
1489	1722	.sqrt => try self.airSqrt(inst),
1490	1723	.neg, .fabs => try self.airFloatSign(inst),
1491	1724
...	...	@@ -1731,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1731	1964	};
1732	1965	const tag_val = Value.initPayload(&tag_pl.base);
1733	1966	const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
1734		try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);
	1967	try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
1735	1968	const skip_reloc = try self.asmJccReloc(undefined, .ne);
1736	1969
1737	1970	try self.genSetMem(
...	...	@@ -1751,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1751	1984	try self.airTrap();
1752	1985
1753	1986	for (exitlude_jump_relocs) \|reloc\| try self.performReloc(reloc);
1754		try self.asmOpOnly(.ret);
	1987	try self.asmOpOnly(.{ ._, .ret });
1755	1988	},
1756	1989	else => return self.fail(
1757	1990	"TODO implement {s} for {}",
...	...	@@ -1919,6 +2152,11 @@ fn computeFrameLayout(self: *Self) !FrameLayout {
1919	2152	};
1920	2153	}
1921	2154
	2155	fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 {
	2156	const alloc_align = @as(u32, 1) << self.frame_allocs.get(@enumToInt(frame_addr.index)).abi_align;
	2157	return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1));
	2158	}
	2159
1922	2160	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
1923	2161	const frame_allocs_slice = self.frame_allocs.slice();
1924	2162	const frame_size = frame_allocs_slice.items(.abi_size);
...	...	@@ -1962,24 +2200,36 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
1962	2200	return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
1963	2201	}
1964	2202
1965		fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
1966		const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {
	2203	fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
	2204	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) orelse {
1967	2205	const mod = self.bin_file.options.module.?;
1968		return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});
	2206	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(mod)});
1969	2207	};
1970	2208
1971		if (reg_ok) {
1972		// Make sure the type can fit in a register before we try to allocate one.
1973		const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1974		const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1975		if (abi_size <= ptr_bytes) {
1976		if (self.register_manager.tryAllocReg(inst, regClassForType(elem_ty))) \|reg\| {
	2209	if (reg_ok) need_mem: {
	2210	if (abi_size <= @as(u32, switch (ty.zigTypeTag()) {
	2211	.Float => switch (ty.floatBits(self.target.*)) {
	2212	16, 32, 64, 128 => 16,
	2213	80 => break :need_mem,
	2214	else => unreachable,
	2215	},
	2216	.Vector => switch (ty.childType().zigTypeTag()) {
	2217	.Float => switch (ty.childType().floatBits(self.target.*)) {
	2218	16, 32, 64 => if (self.hasFeature(.avx)) 32 else 16,
	2219	80, 128 => break :need_mem,
	2220	else => unreachable,
	2221	},
	2222	else => break :need_mem,
	2223	},
	2224	else => 8,
	2225	})) {
	2226	if (self.register_manager.tryAllocReg(inst, regClassForType(ty))) \|reg\| {
1977	2227	return MCValue{ .register = registerAlias(reg, abi_size) };
1978	2228	}
1979	2229	}
1980	2230	}
1981	2231
1982		const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));
	2232	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, self.target.*));
1983	2233	return .{ .load_frame = .{ .index = frame_index } };
1984	2234	}
1985	2235
...	...	@@ -2172,44 +2422,127 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
2172	2422	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2173	2423	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2174	2424	const dst_ty = self.air.typeOfIndex(inst);
	2425	const dst_bits = dst_ty.floatBits(self.target.*);
2175	2426	const src_ty = self.air.typeOf(ty_op.operand);
2176		if (dst_ty.floatBits(self.target.) != 32 or src_ty.floatBits(self.target.) != 64 or
2177		!Target.x86.featureSetHas(self.target.cpu.features, .sse2))
2178		return self.fail("TODO implement airFptrunc from {} to {}", .{
2179		src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2180		});
	2427	const src_bits = src_ty.floatBits(self.target.*);
2181	2428
2182	2429	const src_mcv = try self.resolveInst(ty_op.operand);
2183	2430	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2184	2431	src_mcv
2185	2432	else
2186	2433	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2187		const dst_lock = self.register_manager.lockReg(dst_mcv.register);
	2434	const dst_reg = dst_mcv.getReg().?.to128();
	2435	const dst_lock = self.register_manager.lockReg(dst_reg);
2188	2436	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2189	2437
2190		try self.genBinOpMir(.cvtsd2ss, src_ty, dst_mcv, src_mcv);
	2438	if (dst_bits == 16 and self.hasFeature(.f16c)) {
	2439	switch (src_bits) {
	2440	32 => {
	2441	const mat_src_reg = if (src_mcv.isRegister())
	2442	src_mcv.getReg().?
	2443	else
	2444	try self.copyToTmpRegister(src_ty, src_mcv);
	2445	try self.asmRegisterRegisterImmediate(
	2446	.{ .v_, .cvtps2ph },
	2447	dst_reg,
	2448	mat_src_reg.to128(),
	2449	Immediate.u(0b1_00),
	2450	);
	2451	},
	2452	else => return self.fail("TODO implement airFptrunc from {} to {}", .{
	2453	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	2454	}),
	2455	}
	2456	} else if (src_bits == 64 and dst_bits == 32) {
	2457	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	2458	.{ .v_, .cvtsd2ss },
	2459	dst_reg,
	2460	dst_reg,
	2461	src_mcv.mem(.qword),
	2462	) else try self.asmRegisterRegisterRegister(
	2463	.{ .v_, .cvtsd2ss },
	2464	dst_reg,
	2465	dst_reg,
	2466	(if (src_mcv.isRegister())
	2467	src_mcv.getReg().?
	2468	else
	2469	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
	2470	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
	2471	.{ ._, .cvtsd2ss },
	2472	dst_reg,
	2473	src_mcv.mem(.qword),
	2474	) else try self.asmRegisterRegister(
	2475	.{ ._, .cvtsd2ss },
	2476	dst_reg,
	2477	(if (src_mcv.isRegister())
	2478	src_mcv.getReg().?
	2479	else
	2480	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
	2481	);
	2482	} else return self.fail("TODO implement airFptrunc from {} to {}", .{
	2483	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	2484	});
2191	2485	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
2192	2486	}
2193	2487
2194	2488	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2195	2489	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2196	2490	const dst_ty = self.air.typeOfIndex(inst);
	2491	const dst_bits = dst_ty.floatBits(self.target.*);
2197	2492	const src_ty = self.air.typeOf(ty_op.operand);
2198		if (dst_ty.floatBits(self.target.) != 64 or src_ty.floatBits(self.target.) != 32 or
2199		!Target.x86.featureSetHas(self.target.cpu.features, .sse2))
2200		return self.fail("TODO implement airFpext from {} to {}", .{
2201		src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2202		});
	2493	const src_bits = src_ty.floatBits(self.target.*);
2203	2494
2204	2495	const src_mcv = try self.resolveInst(ty_op.operand);
2205	2496	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2206	2497	src_mcv
2207	2498	else
2208	2499	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2209		const dst_lock = self.register_manager.lockReg(dst_mcv.register);
	2500	const dst_reg = dst_mcv.getReg().?.to128();
	2501	const dst_lock = self.register_manager.lockReg(dst_reg);
2210	2502	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2211	2503
2212		try self.genBinOpMir(.cvtss2sd, src_ty, dst_mcv, src_mcv);
	2504	if (src_bits == 16 and self.hasFeature(.f16c)) {
	2505	const mat_src_reg = if (src_mcv.isRegister())
	2506	src_mcv.getReg().?
	2507	else
	2508	try self.copyToTmpRegister(src_ty, src_mcv);
	2509	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
	2510	switch (dst_bits) {
	2511	32 => {},
	2512	64 => try self.asmRegisterRegisterRegister(.{ .v_, .cvtss2sd }, dst_reg, dst_reg, dst_reg),
	2513	else => return self.fail("TODO implement airFpext from {} to {}", .{
	2514	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	2515	}),
	2516	}
	2517	} else if (src_bits == 32 and dst_bits == 64) {
	2518	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	2519	.{ .v_, .cvtss2sd },
	2520	dst_reg,
	2521	dst_reg,
	2522	src_mcv.mem(.dword),
	2523	) else try self.asmRegisterRegisterRegister(
	2524	.{ .v_, .cvtss2sd },
	2525	dst_reg,
	2526	dst_reg,
	2527	(if (src_mcv.isRegister())
	2528	src_mcv.getReg().?
	2529	else
	2530	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
	2531	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
	2532	.{ ._, .cvtss2sd },
	2533	dst_reg,
	2534	src_mcv.mem(.dword),
	2535	) else try self.asmRegisterRegister(
	2536	.{ ._, .cvtss2sd },
	2537	dst_reg,
	2538	(if (src_mcv.isRegister())
	2539	src_mcv.getReg().?
	2540	else
	2541	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
	2542	);
	2543	} else return self.fail("TODO implement airFpext from {} to {}", .{
	2544	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
	2545	});
2213	2546	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
2214	2547	}
2215	2548
...	...	@@ -2241,11 +2574,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2241	2574	switch (dst_mcv) {
2242	2575	.register => \|dst_reg\| {
2243	2576	const min_abi_size = @min(dst_abi_size, src_abi_size);
2244		const tag: Mir.Inst.Tag = switch (signedness) {
2245		.signed => .movsx,
2246		.unsigned => if (min_abi_size > 2) .mov else .movzx,
	2577	const tag: Mir.Inst.FixedTag = switch (signedness) {
	2578	.signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
	2579	.unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx },
2247	2580	};
2248		const dst_alias = switch (tag) {
	2581	const dst_alias = switch (tag[1]) {
2249	2582	.movsx => dst_reg.to64(),
2250	2583	.mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
2251	2584	else => unreachable,
...	...	@@ -2274,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2274	2607	try self.genCopy(min_ty, dst_mcv, src_mcv);
2275	2608	const extra = dst_abi_size * 8 - dst_int_info.bits;
2276	2609	if (extra > 0) {
2277		try self.genShiftBinOpMir(switch (signedness) {
2278		.signed => .sal,
2279		.unsigned => .shl,
2280		}, dst_ty, dst_mcv, .{ .immediate = extra });
2281		try self.genShiftBinOpMir(switch (signedness) {
2282		.signed => .sar,
2283		.unsigned => .shr,
2284		}, dst_ty, dst_mcv, .{ .immediate = extra });
	2610	try self.genShiftBinOpMir(
	2611	switch (signedness) {
	2612	.signed => .{ ._l, .sa },
	2613	.unsigned => .{ ._l, .sh },
	2614	},
	2615	dst_ty,
	2616	dst_mcv,
	2617	.{ .immediate = extra },
	2618	);
	2619	try self.genShiftBinOpMir(
	2620	switch (signedness) {
	2621	.signed => .{ ._r, .sa },
	2622	.unsigned => .{ ._r, .sh },
	2623	},
	2624	dst_ty,
	2625	dst_mcv,
	2626	.{ .immediate = extra },
	2627	);
2285	2628	}
2286	2629	},
2287	2630	}
...	...	@@ -2466,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2466	2809	const reg_bits = self.regBitSize(ty);
2467	2810	const cc: Condition = if (ty.isSignedInt()) cc: {
2468	2811	try self.genSetReg(limit_reg, ty, dst_mcv);
2469		try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2470		try self.genBinOpMir(.xor, ty, limit_mcv, .{
	2812	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	2813	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2471	2814	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2472	2815	});
2473	2816	break :cc .o;
...	...	@@ -2477,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2477	2820	});
2478	2821	break :cc .c;
2479	2822	};
2480		try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
	2823	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
2481	2824
2482	2825	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2483	2826	try self.asmCmovccRegisterRegister(
...	...	@@ -2517,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2517	2860	const reg_bits = self.regBitSize(ty);
2518	2861	const cc: Condition = if (ty.isSignedInt()) cc: {
2519	2862	try self.genSetReg(limit_reg, ty, dst_mcv);
2520		try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2521		try self.genBinOpMir(.xor, ty, limit_mcv, .{
	2863	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	2864	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2522	2865	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2523	2866	});
2524	2867	break :cc .o;
...	...	@@ -2526,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2526	2869	try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
2527	2870	break :cc .c;
2528	2871	};
2529		try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
	2872	try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
2530	2873
2531	2874	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2532	2875	try self.asmCmovccRegisterRegister(
...	...	@@ -2568,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
2568	2911	const reg_bits = self.regBitSize(ty);
2569	2912	const cc: Condition = if (ty.isSignedInt()) cc: {
2570	2913	try self.genSetReg(limit_reg, ty, lhs_mcv);
2571		try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
2572		try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2573		try self.genBinOpMir(.xor, ty, limit_mcv, .{
	2914	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
	2915	try self.genShiftBinOpMir(.{ ._, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
	2916	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2574	2917	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2575	2918	});
2576	2919	break :cc .o;
...	...	@@ -2683,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2683	3026	};
2684	3027	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
2685	3028
2686		try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
	3029	try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs);
2687	3030	const cc = Condition.ne;
2688	3031
2689	3032	const tuple_ty = self.air.typeOfIndex(inst);
...	...	@@ -2770,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(
2770	3113	src_mcv;
2771	3114	try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
2772	3115	try self.truncateRegister(hi_limb_ty, scratch_reg);
2773		try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
	3116	try self.genBinOpMir(.{ ._, .cmp }, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
2774	3117
2775	3118	const eq_reg = temp_regs[2];
2776	3119	if (overflow_cc) \|_\| {
2777	3120	try self.asmSetccRegister(eq_reg.to8(), .ne);
2778		try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
	3121	try self.genBinOpMir(
	3122	.{ ._, .@"or" },
	3123	Type.u8,
	3124	.{ .register = overflow_reg },
	3125	.{ .register = eq_reg },
	3126	);
2779	3127	}
2780	3128
2781	3129	const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
...	...	@@ -2904,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2904	3252	/// Generates signed or unsigned integer multiplication/division.
2905	3253	/// Clobbers .rax and .rdx registers.
2906	3254	/// Quotient is saved in .rax and remainder in .rdx.
2907		fn genIntMulDivOpMir(
2908		self: *Self,
2909		tag: Mir.Inst.Tag,
2910		ty: Type,
2911		lhs: MCValue,
2912		rhs: MCValue,
2913		) !void {
	3255	fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
2914	3256	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
2915	3257	if (abi_size > 8) {
2916	3258	return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
2917	3259	}
2918	3260
2919	3261	try self.genSetReg(.rax, ty, lhs);
2920		switch (tag) {
	3262	switch (tag[1]) {
2921	3263	else => unreachable,
2922		.mul, .imul => {},
2923		.div => try self.asmRegisterRegister(.xor, .edx, .edx),
2924		.idiv => switch (self.regBitSize(ty)) {
2925		8 => try self.asmOpOnly(.cbw),
2926		16 => try self.asmOpOnly(.cwd),
2927		32 => try self.asmOpOnly(.cdq),
2928		64 => try self.asmOpOnly(.cqo),
	3264	.mul => {},
	3265	.div => switch (tag[0]) {
	3266	._ => try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx),
	3267	.i_ => switch (self.regBitSize(ty)) {
	3268	8 => try self.asmOpOnly(.{ ._, .cbw }),
	3269	16 => try self.asmOpOnly(.{ ._, .cwd }),
	3270	32 => try self.asmOpOnly(.{ ._, .cdq }),
	3271	64 => try self.asmOpOnly(.{ ._, .cqo }),
	3272	else => unreachable,
	3273	},
2929	3274	else => unreachable,
2930	3275	},
2931	3276	}
...	...	@@ -2963,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2963	3308	const divisor_lock = self.register_manager.lockReg(divisor);
2964	3309	defer if (divisor_lock) \|lock\| self.register_manager.unlockReg(lock);
2965	3310
2966		try self.genIntMulDivOpMir(switch (int_info.signedness) {
2967		.signed => .idiv,
2968		.unsigned => .div,
2969		}, ty, .{ .register = dividend }, .{ .register = divisor });
	3311	try self.genIntMulDivOpMir(
	3312	switch (int_info.signedness) {
	3313	.signed => .{ .i_, .div },
	3314	.unsigned => .{ ._, .div },
	3315	},
	3316	ty,
	3317	.{ .register = dividend },
	3318	.{ .register = divisor },
	3319	);
2970	3320
2971	3321	try self.asmRegisterRegister(
2972		.xor,
	3322	.{ ._, .xor },
2973	3323	registerAlias(divisor, abi_size),
2974	3324	registerAlias(dividend, abi_size),
2975	3325	);
2976	3326	try self.asmRegisterImmediate(
2977		.sar,
	3327	.{ ._r, .sa },
2978	3328	registerAlias(divisor, abi_size),
2979	3329	Immediate.u(int_info.bits - 1),
2980	3330	);
2981	3331	try self.asmRegisterRegister(
2982		.@"test",
	3332	.{ ._, .@"test" },
2983	3333	registerAlias(.rdx, abi_size),
2984	3334	registerAlias(.rdx, abi_size),
2985	3335	);
...	...	@@ -2988,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2988	3338	registerAlias(.rdx, abi_size),
2989	3339	.z,
2990	3340	);
2991		try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });
	3341	try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
2992	3342	return MCValue{ .register = divisor };
2993	3343	}
2994	3344
...	...	@@ -3110,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
3110	3460	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
3111	3461	if (err_off > 0) {
3112	3462	const shift = @intCast(u6, err_off * 8);
3113		try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });
	3463	try self.genShiftBinOpMir(
	3464	.{ ._r, .sh },
	3465	err_union_ty,
	3466	result,
	3467	.{ .immediate = shift },
	3468	);
3114	3469	} else {
3115	3470	try self.truncateRegister(Type.anyerror, result.register);
3116	3471	}
...	...	@@ -3162,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(
3162	3517	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
3163	3518	if (payload_off > 0) {
3164	3519	const shift = @intCast(u6, payload_off * 8);
3165		try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });
	3520	try self.genShiftBinOpMir(
	3521	.{ ._r, .sh },
	3522	err_union_ty,
	3523	result_mcv,
	3524	.{ .immediate = shift },
	3525	);
3166	3526	} else {
3167	3527	try self.truncateRegister(payload_ty, result_mcv.register);
3168	3528	}
...	...	@@ -3199,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3199	3559	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3200	3560	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3201	3561	try self.asmRegisterMemory(
3202		.mov,
	3562	.{ ._, .mov },
3203	3563	registerAlias(dst_reg, err_abi_size),
3204	3564	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3205	3565	.base = .{ .reg = src_reg },
...	...	@@ -3237,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
3237	3597	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3238	3598	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3239	3599	try self.asmRegisterMemory(
3240		.lea,
	3600	.{ ._, .lea },
3241	3601	registerAlias(dst_reg, dst_abi_size),
3242	3602	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3243	3603	);
...	...	@@ -3263,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3263	3623	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3264	3624	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3265	3625	try self.asmMemoryImmediate(
3266		.mov,
	3626	.{ ._, .mov },
3267	3627	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3268	3628	.base = .{ .reg = src_reg },
3269	3629	.disp = err_off,
...	...	@@ -3284,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3284	3644	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3285	3645	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3286	3646	try self.asmRegisterMemory(
3287		.lea,
	3647	.{ ._, .lea },
3288	3648	registerAlias(dst_reg, dst_abi_size),
3289	3649	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3290	3650	);
...	...	@@ -3335,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
3335	3695	else => unreachable,
3336	3696
3337	3697	.register => \|opt_reg\| try self.asmRegisterImmediate(
3338		.bts,
	3698	.{ ._s, .bt },
3339	3699	opt_reg,
3340	3700	Immediate.u(@intCast(u6, pl_abi_size * 8)),
3341	3701	),
3342	3702
3343	3703	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
3344		.mov,
	3704	.{ ._, .mov },
3345	3705	Memory.sib(.byte, .{
3346	3706	.base = .{ .frame = frame_addr.index },
3347	3707	.disp = frame_addr.off + pl_abi_size,
...	...	@@ -3453,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
3453	3813
3454	3814	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3455	3815	try self.asmRegisterMemory(
3456		.lea,
	3816	.{ ._, .lea },
3457	3817	registerAlias(dst_reg, dst_abi_size),
3458	3818	Memory.sib(.qword, .{
3459	3819	.base = .{ .reg = src_reg },
...	...	@@ -3527,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
3527	3887	try self.genSetReg(addr_reg, Type.usize, slice_mcv);
3528	3888	// TODO we could allocate register here, but need to expect addr register and potentially
3529	3889	// offset register.
3530		try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{
	3890	try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
3531	3891	.register = offset_reg,
3532	3892	});
3533	3893	return MCValue{ .register = addr_reg.to64() };
...	...	@@ -3585,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3585	3945	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
3586	3946	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
3587	3947	try self.asmRegisterMemory(
3588		.lea,
	3948	.{ ._, .lea },
3589	3949	addr_reg,
3590	3950	Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
3591	3951	);
3592	3952	},
3593	3953	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
3594		.lea,
	3954	.{ ._, .lea },
3595	3955	addr_reg,
3596	3956	Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
3597	3957	),
...	...	@@ -3607,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3607	3967	// TODO we could allocate register here, but need to expect addr register and potentially
3608	3968	// offset register.
3609	3969	const dst_mcv = try self.allocRegOrMem(inst, false);
3610		try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
	3970	try self.genBinOpMir(
	3971	.{ ._, .add },
	3972	Type.usize,
	3973	.{ .register = addr_reg },
	3974	.{ .register = offset_reg },
	3975	);
3611	3976	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3612	3977
3613	3978	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -3641,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
3641	4006	try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3642	4007	const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);
3643	4008	defer self.register_manager.unlockReg(elem_ptr_lock);
3644		try self.asmRegisterRegister(.add, elem_ptr_reg, offset_reg);
	4009	try self.asmRegisterRegister(
	4010	.{ ._, .add },
	4011	elem_ptr_reg,
	4012	offset_reg,
	4013	);
3645	4014
3646	4015	const dst_mcv = try self.allocRegOrMem(inst, true);
3647	4016	const dst_lock = switch (dst_mcv) {
...	...	@@ -3681,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
3681	4050	defer self.register_manager.unlockReg(offset_reg_lock);
3682	4051
3683	4052	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
3684		try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
	4053	try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg });
3685	4054
3686	4055	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
3687	4056	}
...	...	@@ -3714,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3714	4083	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
3715	4084	// TODO reusing the operand
3716	4085	const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
3717		try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
	4086	try self.genBinOpMir(
	4087	.{ ._, .add },
	4088	ptr_union_ty,
	4089	.{ .register = reg },
	4090	.{ .immediate = layout.payload_size },
	4091	);
3718	4092	break :blk MCValue{ .register = reg };
3719	4093	} else ptr;
3720	4094
...	...	@@ -3767,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3767	4141	else
3768	4142	0;
3769	4143	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
3770		try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });
	4144	try self.genShiftBinOpMir(.{ ._r, .sh }, Type.usize, result, .{ .immediate = shift });
3771	4145	break :blk MCValue{
3772	4146	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
3773	4147	};
...	...	@@ -3798,24 +4172,53 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3798	4172
3799	4173	const dst_reg = try self.register_manager.allocReg(inst, gp);
3800	4174	const dst_mcv = MCValue{ .register = dst_reg };
3801		const dst_lock = self.register_manager.lockReg(dst_reg);
3802		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	4175	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	4176	defer self.register_manager.unlockReg(dst_lock);
3803	4177
3804		if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
3805		try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
3806		const extra_bits = self.regExtraBits(src_ty);
3807		if (extra_bits > 0) {
3808		try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
3809		}
	4178	const src_bits = src_ty.bitSize(self.target.*);
	4179	if (self.hasFeature(.lzcnt)) {
	4180	if (src_bits <= 64) {
	4181	try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
	4182
	4183	const extra_bits = self.regExtraBits(src_ty);
	4184	if (extra_bits > 0) {
	4185	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
	4186	}
	4187	} else if (src_bits <= 128) {
	4188	const tmp_reg = try self.register_manager.allocReg(null, gp);
	4189	const tmp_mcv = MCValue{ .register = tmp_reg };
	4190	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	4191	defer self.register_manager.unlockReg(tmp_lock);
	4192
	4193	try self.genBinOpMir(.{ ._, .lzcnt }, Type.u64, dst_mcv, mat_src_mcv);
	4194	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
	4195	try self.genBinOpMir(
	4196	.{ ._, .lzcnt },
	4197	Type.u64,
	4198	tmp_mcv,
	4199	mat_src_mcv.address().offset(8).deref(),
	4200	);
	4201	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
	4202
	4203	if (src_bits < 128) {
	4204	try self.genBinOpMir(
	4205	.{ ._, .sub },
	4206	dst_ty,
	4207	dst_mcv,
	4208	.{ .immediate = 128 - src_bits },
	4209	);
	4210	}
	4211	} else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3810	4212	break :result dst_mcv;
3811	4213	}
3812	4214
3813		const src_bits = src_ty.bitSize(self.target.*);
	4215	if (src_bits > 64)
	4216	return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3814	4217	if (math.isPowerOfTwo(src_bits)) {
3815	4218	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3816	4219	.immediate = src_bits ^ (src_bits - 1),
3817	4220	});
3818		try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
	4221	try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
3819	4222
3820	4223	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3821	4224	try self.asmCmovccRegisterRegister(
...	...	@@ -3824,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3824	4227	.z,
3825	4228	);
3826	4229
3827		try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
	4230	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
3828	4231	} else {
3829	4232	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3830	4233	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
3831	4234	});
3832		try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
	4235	try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
3833	4236
3834	4237	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3835	4238	try self.asmCmovccRegisterRegister(
...	...	@@ -3839,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3839	4242	);
3840	4243
3841	4244	try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
3842		try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
	4245	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
3843	4246	}
3844	4247	break :result dst_mcv;
3845	4248	};
...	...	@@ -3869,27 +4272,55 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3869	4272	const dst_lock = self.register_manager.lockReg(dst_reg);
3870	4273	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
3871	4274
3872		if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
3873		const extra_bits = self.regExtraBits(src_ty);
3874		const masked_mcv = if (extra_bits > 0) masked: {
3875		const mask_mcv = MCValue{
3876		.immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) <<
3877		@intCast(u6, src_bits),
3878		};
3879		const tmp_mcv = tmp: {
3880		if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
3881		try self.genSetReg(dst_reg, src_ty, src_mcv);
3882		break :tmp dst_mcv;
3883		};
3884		try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
3885		break :masked tmp_mcv;
3886		} else mat_src_mcv;
3887		try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
	4275	if (self.hasFeature(.bmi)) {
	4276	if (src_bits <= 64) {
	4277	const extra_bits = self.regExtraBits(src_ty);
	4278	const masked_mcv = if (extra_bits > 0) masked: {
	4279	const tmp_mcv = tmp: {
	4280	if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
	4281	break :tmp src_mcv;
	4282	try self.genSetReg(dst_reg, src_ty, src_mcv);
	4283	break :tmp dst_mcv;
	4284	};
	4285	try self.genBinOpMir(
	4286	.{ ._, .@"or" },
	4287	src_ty,
	4288	tmp_mcv,
	4289	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
	4290	@intCast(u6, src_bits) },
	4291	);
	4292	break :masked tmp_mcv;
	4293	} else mat_src_mcv;
	4294	try self.genBinOpMir(.{ ._, .tzcnt }, src_ty, dst_mcv, masked_mcv);
	4295	} else if (src_bits <= 128) {
	4296	const tmp_reg = try self.register_manager.allocReg(null, gp);
	4297	const tmp_mcv = MCValue{ .register = tmp_reg };
	4298	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	4299	defer self.register_manager.unlockReg(tmp_lock);
	4300
	4301	const masked_mcv = if (src_bits < 128) masked: {
	4302	try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
	4303	try self.genBinOpMir(
	4304	.{ ._, .@"or" },
	4305	Type.u64,
	4306	dst_mcv,
	4307	.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
	4308	);
	4309	break :masked dst_mcv;
	4310	} else mat_src_mcv.address().offset(8).deref();
	4311	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
	4312	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
	4313	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
	4314	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
	4315	} else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3888	4316	break :result dst_mcv;
3889	4317	}
3890	4318
	4319	if (src_bits > 64)
	4320	return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
	4321
3891	4322	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
3892		try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
	4323	try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
3893	4324
3894	4325	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3895	4326	try self.asmCmovccRegisterRegister(
...	...	@@ -3909,7 +4340,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3909	4340	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
3910	4341	const src_mcv = try self.resolveInst(ty_op.operand);
3911	4342
3912		if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
	4343	if (self.hasFeature(.popcnt)) {
3913	4344	const mat_src_mcv = switch (src_mcv) {
3914	4345	.immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
3915	4346	else => src_mcv,
...	...	@@ -3927,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3927	4358	.{ .register = try self.register_manager.allocReg(inst, gp) };
3928	4359
3929	4360	const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
3930		try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);
	4361	try self.genBinOpMir(.{ ._, .popcnt }, popcnt_ty, dst_mcv, mat_src_mcv);
3931	4362	break :result dst_mcv;
3932	4363	}
3933	4364
...	...	@@ -3958,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3958	4389	undefined;
3959	4390
3960	4391	// dst = operand
3961		try self.asmRegisterRegister(.mov, tmp, dst);
	4392	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
3962	4393	// tmp = operand
3963		try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
	4394	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
3964	4395	// tmp = operand >> 1
3965	4396	if (src_abi_size > 4) {
3966		try self.asmRegisterImmediate(.mov, imm, imm_0_1);
3967		try self.asmRegisterRegister(.@"and", tmp, imm);
3968		} else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
	4397	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
	4398	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4399	} else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
3969	4400	// tmp = (operand >> 1) & 0x55...55
3970		try self.asmRegisterRegister(.sub, dst, tmp);
	4401	try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp);
3971	4402	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
3972		try self.asmRegisterRegister(.mov, tmp, dst);
	4403	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
3973	4404	// tmp = temp1
3974		try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
	4405	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
3975	4406	// dst = temp1 >> 2
3976	4407	if (src_abi_size > 4) {
3977		try self.asmRegisterImmediate(.mov, imm, imm_00_11);
3978		try self.asmRegisterRegister(.@"and", tmp, imm);
3979		try self.asmRegisterRegister(.@"and", dst, imm);
	4408	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
	4409	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4410	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
3980	4411	} else {
3981		try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
3982		try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
	4412	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
	4413	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
3983	4414	}
3984	4415	// tmp = temp1 & 0x33...33
3985	4416	// dst = (temp1 >> 2) & 0x33...33
3986		try self.asmRegisterRegister(.add, tmp, dst);
	4417	try self.asmRegisterRegister(.{ ._, .add }, tmp, dst);
3987	4418	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
3988		try self.asmRegisterRegister(.mov, dst, tmp);
	4419	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
3989	4420	// dst = temp2
3990		try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));
	4421	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
3991	4422	// tmp = temp2 >> 4
3992		try self.asmRegisterRegister(.add, dst, tmp);
	4423	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
3993	4424	// dst = temp2 + (temp2 >> 4)
3994	4425	if (src_abi_size > 4) {
3995		try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
3996		try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);
3997		try self.asmRegisterRegister(.@"and", dst, imm);
3998		try self.asmRegisterRegister(.imul, dst, tmp);
	4426	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
	4427	try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001);
	4428	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
	4429	try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp);
3999	4430	} else {
4000		try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
	4431	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4001	4432	if (src_abi_size > 1) {
4002		try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);
	4433	try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001);
4003	4434	}
4004	4435	}
4005	4436	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
4006	4437	// dst = temp3 * 0x01...01
4007	4438	if (src_abi_size > 1) {
4008		try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));
	4439	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
4009	4440	}
4010	4441	// dst = (temp3 * 0x01...01) >> (bits - 8)
4011	4442	}
...	...	@@ -4034,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4034	4465	16 => if ((mem_ok or src_mcv.isRegister()) and
4035	4466	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
4036	4467	{
4037		try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });
	4468	try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 });
4038	4469	return src_mcv;
4039	4470	},
4040	4471	32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
4041		try self.genUnOpMir(.bswap, src_ty, src_mcv);
	4472	try self.genUnOpMir(.{ ._, .bswap }, src_ty, src_mcv);
4042	4473	return src_mcv;
4043	4474	},
4044	4475	}
...	...	@@ -4055,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4055	4486	try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
4056	4487	switch (src_bits) {
4057	4488	else => unreachable,
4058		16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
4059		32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),
	4489	16 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
	4490	32, 64 => try self.genUnOpMir(.{ ._, .bswap }, src_ty, dst_mcv),
4060	4491	}
4061		} else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
	4492	} else try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4062	4493	return dst_mcv;
4063	4494	}
4064	4495
...	...	@@ -4067,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4067	4498	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4068	4499	defer self.register_manager.unlockReg(dst_lock);
4069	4500
4070		try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
	4501	try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4071	4502	return dst_mcv;
4072	4503	}
4073	4504
...	...	@@ -4081,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
4081	4512	switch (self.regExtraBits(src_ty)) {
4082	4513	0 => {},
4083	4514	else => \|extra\| try self.genBinOpMir(
4084		if (src_ty.isSignedInt()) .sar else .shr,
	4515	if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4085	4516	src_ty,
4086	4517	dst_mcv,
4087	4518	.{ .immediate = extra },
...	...	@@ -4121,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4121	4552	const imm_0_1 = Immediate.u(mask / 0b1_1);
4122	4553
4123	4554	// dst = temp1 = bswap(operand)
4124		try self.asmRegisterRegister(.mov, tmp, dst);
	4555	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4125	4556	// tmp = temp1
4126		try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));
	4557	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
4127	4558	// dst = temp1 >> 4
4128	4559	if (src_abi_size > 4) {
4129		try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
4130		try self.asmRegisterRegister(.@"and", tmp, imm);
4131		try self.asmRegisterRegister(.@"and", dst, imm);
	4560	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
	4561	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4562	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4132	4563	} else {
4133		try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);
4134		try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
	4564	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
	4565	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4135	4566	}
4136	4567	// tmp = temp1 & 0x0F...0F
4137	4568	// dst = (temp1 >> 4) & 0x0F...0F
4138		try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));
	4569	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
4139	4570	// tmp = (temp1 & 0x0F...0F) << 4
4140		try self.asmRegisterRegister(.@"or", dst, tmp);
	4571	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
4141	4572	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
4142		try self.asmRegisterRegister(.mov, tmp, dst);
	4573	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4143	4574	// tmp = temp2
4144		try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
	4575	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
4145	4576	// dst = temp2 >> 2
4146	4577	if (src_abi_size > 4) {
4147		try self.asmRegisterImmediate(.mov, imm, imm_00_11);
4148		try self.asmRegisterRegister(.@"and", tmp, imm);
4149		try self.asmRegisterRegister(.@"and", dst, imm);
	4578	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
	4579	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
	4580	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4150	4581	} else {
4151		try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
4152		try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
	4582	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
	4583	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
4153	4584	}
4154	4585	// tmp = temp2 & 0x33...33
4155	4586	// dst = (temp2 >> 2) & 0x33...33
4156	4587	try self.asmRegisterMemory(
4157		.lea,
	4588	.{ ._, .lea },
4158	4589	if (src_abi_size > 4) tmp.to64() else tmp.to32(),
4159	4590	Memory.sib(.qword, .{
4160	4591	.base = .{ .reg = dst.to64() },
...	...	@@ -4162,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4162	4593	}),
4163	4594	);
4164	4595	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
4165		try self.asmRegisterRegister(.mov, dst, tmp);
	4596	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
4166	4597	// dst = temp3
4167		try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
	4598	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
4168	4599	// tmp = temp3 >> 1
4169	4600	if (src_abi_size > 4) {
4170		try self.asmRegisterImmediate(.mov, imm, imm_0_1);
4171		try self.asmRegisterRegister(.@"and", dst, imm);
4172		try self.asmRegisterRegister(.@"and", tmp, imm);
	4601	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
	4602	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
	4603	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4173	4604	} else {
4174		try self.asmRegisterImmediate(.@"and", dst, imm_0_1);
4175		try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
	4605	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1);
	4606	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
4176	4607	}
4177	4608	// dst = temp3 & 0x55...55
4178	4609	// tmp = (temp3 >> 1) & 0x55...55
4179	4610	try self.asmRegisterMemory(
4180		.lea,
	4611	.{ ._, .lea },
4181	4612	if (src_abi_size > 4) dst.to64() else dst.to32(),
4182	4613	Memory.sib(.qword, .{
4183	4614	.base = .{ .reg = tmp.to64() },
...	...	@@ -4190,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4190	4621	switch (self.regExtraBits(src_ty)) {
4191	4622	0 => {},
4192	4623	else => \|extra\| try self.genBinOpMir(
4193		if (src_ty.isSignedInt()) .sar else .shr,
	4624	if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4194	4625	src_ty,
4195	4626	dst_mcv,
4196	4627	.{ .immediate = extra },
...	...	@@ -4246,19 +4677,20 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
4246	4677	const tag = self.air.instructions.items(.tag)[inst];
4247	4678	try self.genBinOpMir(switch (ty_bits) {
4248	4679	// No point using an extra prefix byte for *pd which performs the same operation.
4249		32, 64 => switch (tag) {
4250		.neg => .xorps,
4251		.fabs => .andnps,
	4680	16, 32, 64, 128 => switch (tag) {
	4681	.neg => .{ ._ps, .xor },
	4682	.fabs => .{ ._ps, .andn },
4252	4683	else => unreachable,
4253	4684	},
4254		else => return self.fail("TODO implement airFloatSign for {}", .{
	4685	80 => return self.fail("TODO implement airFloatSign for {}", .{
4255	4686	ty.fmt(self.bin_file.options.module.?),
4256	4687	}),
	4688	else => unreachable,
4257	4689	}, vec_ty, dst_mcv, sign_mcv);
4258	4690	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4259	4691	}
4260	4692
4261		fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
	4693	fn airRound(self: *Self, inst: Air.Inst.Index, mode: u4) !void {
4262	4694	const un_op = self.air.instructions.items(.data)[inst].un_op;
4263	4695	const ty = self.air.typeOf(un_op);
4264	4696
...	...	@@ -4267,40 +4699,246 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4267	4699	src_mcv
4268	4700	else
4269	4701	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
4270
4271		try self.genBinOpMir(switch (ty.zigTypeTag()) {
4272		.Float => switch (ty.floatBits(self.target.*)) {
4273		32 => .sqrtss,
4274		64 => .sqrtsd,
4275		else => return self.fail("TODO implement airSqrt for {}", .{
4276		ty.fmt(self.bin_file.options.module.?),
4277		}),
4278		},
4279		else => return self.fail("TODO implement airSqrt for {}", .{
4280		ty.fmt(self.bin_file.options.module.?),
4281		}),
4282		}, ty, dst_mcv, src_mcv);
	4702	const dst_reg = dst_mcv.getReg().?;
	4703	const dst_lock = self.register_manager.lockReg(dst_reg);
	4704	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	4705	try self.genRound(ty, dst_reg, src_mcv, mode);
4283	4706	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4284	4707	}
4285	4708
4286		fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
4287		const un_op = self.air.instructions.items(.data)[inst].un_op;
4288		_ = un_op;
4289		return self.fail("TODO implement airUnaryMath for {}", .{
4290		self.air.instructions.items(.tag)[inst],
4291		});
4292		//return self.finishAir(inst, result, .{ un_op, .none, .none });
4293		}
	4709	fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4) !void {
	4710	if (!self.hasFeature(.sse4_1))
	4711	return self.fail("TODO implement genRound without sse4_1 feature", .{});
4294	4712
4295		fn reuseOperand(
4296		self: *Self,
4297		inst: Air.Inst.Index,
4298		operand: Air.Inst.Ref,
4299		op_index: Liveness.OperandInt,
4300		mcv: MCValue,
4301		) bool {
4302		return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
4303		}
	4713	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
	4714	.Float => switch (ty.floatBits(self.target.*)) {
	4715	32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
	4716	64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
	4717	16, 80, 128 => null,
	4718	else => unreachable,
	4719	},
	4720	.Vector => switch (ty.childType().zigTypeTag()) {
	4721	.Float => switch (ty.childType().floatBits(self.target.*)) {
	4722	32 => switch (ty.vectorLen()) {
	4723	1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
	4724	2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round },
	4725	5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null,
	4726	else => null,
	4727	},
	4728	64 => switch (ty.vectorLen()) {
	4729	1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
	4730	2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round },
	4731	3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null,
	4732	else => null,
	4733	},
	4734	16, 80, 128 => null,
	4735	else => unreachable,
	4736	},
	4737	else => null,
	4738	},
	4739	else => unreachable,
	4740	})) \|tag\| tag else return self.fail("TODO implement genRound for {}", .{
	4741	ty.fmt(self.bin_file.options.module.?),
	4742	});
	4743
	4744	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
	4745	const dst_alias = registerAlias(dst_reg, abi_size);
	4746	switch (mir_tag[0]) {
	4747	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	4748	mir_tag,
	4749	dst_alias,
	4750	dst_alias,
	4751	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	4752	Immediate.u(mode),
	4753	) else try self.asmRegisterRegisterRegisterImmediate(
	4754	mir_tag,
	4755	dst_alias,
	4756	dst_alias,
	4757	registerAlias(if (src_mcv.isRegister())
	4758	src_mcv.getReg().?
	4759	else
	4760	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	4761	Immediate.u(mode),
	4762	),
	4763	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	4764	mir_tag,
	4765	dst_alias,
	4766	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	4767	Immediate.u(mode),
	4768	) else try self.asmRegisterRegisterImmediate(
	4769	mir_tag,
	4770	dst_alias,
	4771	registerAlias(if (src_mcv.isRegister())
	4772	src_mcv.getReg().?
	4773	else
	4774	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	4775	Immediate.u(mode),
	4776	),
	4777	}
	4778	}
	4779
	4780	fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
	4781	const un_op = self.air.instructions.items(.data)[inst].un_op;
	4782	const ty = self.air.typeOf(un_op);
	4783	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
	4784
	4785	const src_mcv = try self.resolveInst(un_op);
	4786	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
	4787	src_mcv
	4788	else
	4789	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
	4790	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
	4791	const dst_lock = self.register_manager.lockReg(dst_reg);
	4792	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	4793
	4794	const result: MCValue = result: {
	4795	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
	4796	.Float => switch (ty.floatBits(self.target.*)) {
	4797	16 => if (self.hasFeature(.f16c)) {
	4798	const mat_src_reg = if (src_mcv.isRegister())
	4799	src_mcv.getReg().?
	4800	else
	4801	try self.copyToTmpRegister(ty, src_mcv);
	4802	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
	4803	try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg);
	4804	try self.asmRegisterRegisterImmediate(
	4805	.{ .v_, .cvtps2ph },
	4806	dst_reg,
	4807	dst_reg,
	4808	Immediate.u(0b1_00),
	4809	);
	4810	break :result dst_mcv;
	4811	} else null,
	4812	32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
	4813	64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
	4814	80, 128 => null,
	4815	else => unreachable,
	4816	},
	4817	.Vector => switch (ty.childType().zigTypeTag()) {
	4818	.Float => switch (ty.childType().floatBits(self.target.*)) {
	4819	16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
	4820	1 => {
	4821	try self.asmRegisterRegister(
	4822	.{ .v_, .cvtph2ps },
	4823	dst_reg,
	4824	(if (src_mcv.isRegister())
	4825	src_mcv.getReg().?
	4826	else
	4827	try self.copyToTmpRegister(ty, src_mcv)).to128(),
	4828	);
	4829	try self.asmRegisterRegisterRegister(
	4830	.{ .v_ss, .sqrt },
	4831	dst_reg,
	4832	dst_reg,
	4833	dst_reg,
	4834	);
	4835	try self.asmRegisterRegisterImmediate(
	4836	.{ .v_, .cvtps2ph },
	4837	dst_reg,
	4838	dst_reg,
	4839	Immediate.u(0b1_00),
	4840	);
	4841	break :result dst_mcv;
	4842	},
	4843	2...8 => {
	4844	const wide_reg = registerAlias(dst_reg, abi_size * 2);
	4845	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	4846	.{ .v_, .cvtph2ps },
	4847	wide_reg,
	4848	src_mcv.mem(Memory.PtrSize.fromSize(
	4849	@intCast(u32, @divExact(wide_reg.bitSize(), 16)),
	4850	)),
	4851	) else try self.asmRegisterRegister(
	4852	.{ .v_, .cvtph2ps },
	4853	wide_reg,
	4854	(if (src_mcv.isRegister())
	4855	src_mcv.getReg().?
	4856	else
	4857	try self.copyToTmpRegister(ty, src_mcv)).to128(),
	4858	);
	4859	try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg);
	4860	try self.asmRegisterRegisterImmediate(
	4861	.{ .v_, .cvtps2ph },
	4862	dst_reg,
	4863	wide_reg,
	4864	Immediate.u(0b1_00),
	4865	);
	4866	break :result dst_mcv;
	4867	},
	4868	else => null,
	4869	} else null,
	4870	32 => switch (ty.vectorLen()) {
	4871	1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
	4872	2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt },
	4873	5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null,
	4874	else => null,
	4875	},
	4876	64 => switch (ty.vectorLen()) {
	4877	1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
	4878	2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt },
	4879	3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null,
	4880	else => null,
	4881	},
	4882	80, 128 => null,
	4883	else => unreachable,
	4884	},
	4885	else => unreachable,
	4886	},
	4887	else => unreachable,
	4888	})) \|tag\| tag else return self.fail("TODO implement airSqrt for {}", .{
	4889	ty.fmt(self.bin_file.options.module.?),
	4890	});
	4891	switch (mir_tag[0]) {
	4892	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	4893	mir_tag,
	4894	dst_reg,
	4895	dst_reg,
	4896	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	4897	) else try self.asmRegisterRegisterRegister(
	4898	mir_tag,
	4899	dst_reg,
	4900	dst_reg,
	4901	registerAlias(if (src_mcv.isRegister())
	4902	src_mcv.getReg().?
	4903	else
	4904	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	4905	),
	4906	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
	4907	mir_tag,
	4908	dst_reg,
	4909	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	4910	) else try self.asmRegisterRegister(
	4911	mir_tag,
	4912	dst_reg,
	4913	registerAlias(if (src_mcv.isRegister())
	4914	src_mcv.getReg().?
	4915	else
	4916	try self.copyToTmpRegister(ty, src_mcv), abi_size),
	4917	),
	4918	}
	4919	break :result dst_mcv;
	4920	};
	4921	return self.finishAir(inst, result, .{ un_op, .none, .none });
	4922	}
	4923
	4924	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
	4925	const un_op = self.air.instructions.items(.data)[inst].un_op;
	4926	_ = un_op;
	4927	return self.fail("TODO implement airUnaryMath for {}", .{
	4928	self.air.instructions.items(.tag)[inst],
	4929	});
	4930	//return self.finishAir(inst, result, .{ un_op, .none, .none });
	4931	}
	4932
	4933	fn reuseOperand(
	4934	self: *Self,
	4935	inst: Air.Inst.Index,
	4936	operand: Air.Inst.Ref,
	4937	op_index: Liveness.OperandInt,
	4938	mcv: MCValue,
	4939	) bool {
	4940	return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
	4941	}
4304	4942
4305	4943	fn reuseOperandAdvanced(
4306	4944	self: *Self,
...	...	@@ -4366,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4366	5004	if (load_abi_size <= 8) {
4367	5005	const load_reg = registerAlias(dst_reg, load_abi_size);
4368	5006	try self.asmRegisterMemory(
4369		.mov,
	5007	.{ ._, .mov },
4370	5008	load_reg,
4371	5009	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4372	5010	.base = .{ .reg = ptr_reg },
4373	5011	.disp = val_byte_off,
4374	5012	}),
4375	5013	);
4376		try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
	5014	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
4377	5015	} else {
4378	5016	const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
4379	5017	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	...	@@ -4381,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4381	5019
4382	5020	const dst_alias = registerAlias(dst_reg, val_abi_size);
4383	5021	try self.asmRegisterMemory(
4384		.mov,
	5022	.{ ._, .mov },
4385	5023	dst_alias,
4386	5024	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4387	5025	.base = .{ .reg = ptr_reg },
...	...	@@ -4389,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4389	5027	}),
4390	5028	);
4391	5029	try self.asmRegisterMemory(
4392		.mov,
	5030	.{ ._, .mov },
4393	5031	tmp_reg,
4394	5032	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4395	5033	.base = .{ .reg = ptr_reg },
4396	5034	.disp = val_byte_off + 1,
4397	5035	}),
4398	5036	);
4399		try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
	5037	try self.asmRegisterRegisterImmediate(
	5038	.{ ._rd, .sh },
	5039	dst_alias,
	5040	tmp_reg,
	5041	Immediate.u(val_bit_off),
	5042	);
4400	5043	}
4401	5044
4402	5045	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
...	...	@@ -4502,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
4502	5145	const part_mask_not = part_mask ^
4503	5146	(@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));
4504	5147	if (limb_abi_size <= 4) {
4505		try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not));
	5148	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
4506	5149	} else if (math.cast(i32, @bitCast(i64, part_mask_not))) \|small\| {
4507		try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small));
	5150	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
4508	5151	} else {
4509	5152	const part_mask_reg = try self.register_manager.allocReg(null, gp);
4510		try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not));
4511		try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
	5153	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
	5154	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
4512	5155	}
4513	5156
4514	5157	if (src_bit_size <= 64) {
...	...	@@ -4519,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
4519	5162
4520	5163	try self.genSetReg(tmp_reg, src_ty, src_mcv);
4521	5164	switch (limb_i) {
4522		0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
4523		1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
4524		.immediate = limb_abi_bits - src_bit_off,
4525		}),
	5165	0 => try self.genShiftBinOpMir(
	5166	.{ ._l, .sh },
	5167	src_ty,
	5168	tmp_mcv,
	5169	.{ .immediate = src_bit_off },
	5170	),
	5171	1 => try self.genShiftBinOpMir(
	5172	.{ ._r, .sh },
	5173	src_ty,
	5174	tmp_mcv,
	5175	.{ .immediate = limb_abi_bits - src_bit_off },
	5176	),
4526	5177	else => unreachable,
4527	5178	}
4528		try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
4529		try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
	5179	try self.genBinOpMir(.{ ._, .@"and" }, src_ty, tmp_mcv, .{ .immediate = part_mask });
	5180	try self.asmMemoryRegister(
	5181	.{ ._, .@"or" },
	5182	limb_mem,
	5183	registerAlias(tmp_reg, limb_abi_size),
	5184	);
4530	5185	} else return self.fail("TODO: implement packed store of {}", .{
4531	5186	src_ty.fmt(self.bin_file.options.module.?),
4532	5187	});
...	...	@@ -4626,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
4626	5281	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4627	5282	else => mcv,
4628	5283	});
4629		try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
	5284	try self.genBinOpMir(.{ ._, .add }, Type.usize, dst_mcv, .{ .register = offset_reg });
4630	5285	break :result dst_mcv;
4631	5286	},
4632	5287	.indirect => \|reg_off\| break :result .{ .indirect = .{
...	...	@@ -4710,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4710	5365	if (load_abi_size <= 8) {
4711	5366	const load_reg = registerAlias(dst_reg, load_abi_size);
4712	5367	try self.asmRegisterMemory(
4713		.mov,
	5368	.{ ._, .mov },
4714	5369	load_reg,
4715	5370	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4716	5371	.base = .{ .frame = frame_addr.index },
4717	5372	.disp = frame_addr.off + field_byte_off,
4718	5373	}),
4719	5374	);
4720		try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
	5375	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
4721	5376	} else {
4722	5377	const tmp_reg = registerAlias(
4723	5378	try self.register_manager.allocReg(null, gp),
...	...	@@ -4728,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4728	5383
4729	5384	const dst_alias = registerAlias(dst_reg, field_abi_size);
4730	5385	try self.asmRegisterMemory(
4731		.mov,
	5386	.{ ._, .mov },
4732	5387	dst_alias,
4733	5388	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4734	5389	.base = .{ .frame = frame_addr.index },
...	...	@@ -4736,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4736	5391	}),
4737	5392	);
4738	5393	try self.asmRegisterMemory(
4739		.mov,
	5394	.{ ._, .mov },
4740	5395	tmp_reg,
4741	5396	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4742	5397	.base = .{ .frame = frame_addr.index },
...	...	@@ -4744,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4744	5399	}),
4745	5400	);
4746	5401	try self.asmRegisterRegisterImmediate(
4747		.shrd,
	5402	.{ ._rd, .sh },
4748	5403	dst_alias,
4749	5404	tmp_reg,
4750	5405	Immediate.u(field_bit_off),
...	...	@@ -4752,7 +5407,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4752	5407	}
4753	5408
4754	5409	if (field_extra_bits > 0) try self.truncateRegister(field_ty, dst_reg);
4755		break :result .{ .register = dst_reg };
	5410
	5411	const dst_mcv = MCValue{ .register = dst_reg };
	5412	const dst_rc = regClassForType(field_ty);
	5413	if (dst_rc.eql(gp)) break :result dst_mcv;
	5414
	5415	const result_reg = try self.register_manager.allocReg(inst, dst_rc);
	5416	try self.genSetReg(result_reg, field_ty, dst_mcv);
	5417	break :result .{ .register = result_reg };
4756	5418	},
4757	5419	.register => \|reg\| {
4758	5420	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
...	...	@@ -4773,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4773	5435	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
4774	5436
4775	5437	// Shift by struct_field_offset.
4776		try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_off });
	5438	try self.genShiftBinOpMir(
	5439	.{ ._r, .sh },
	5440	Type.usize,
	5441	dst_mcv,
	5442	.{ .immediate = field_off },
	5443	);
4777	5444
4778	5445	// Mask to field_bit_size bits
4779	5446	const field_bit_size = field_ty.bitSize(self.target.*);
4780	5447	const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);
4781	5448
4782	5449	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
4783		try self.genBinOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg });
	5450	try self.genBinOpMir(.{ ._, .@"and" }, Type.usize, dst_mcv, .{ .register = tmp_reg });
4784	5451
4785	5452	const signedness =
4786	5453	if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;
4787	5454	const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
4788	5455	if (signedness == .signed and field_byte_size < 8) {
4789	5456	try self.asmRegisterRegister(
4790		.movsx,
	5457	if (field_byte_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
4791	5458	dst_mcv.register,
4792	5459	registerAlias(dst_mcv.register, field_byte_size),
4793	5460	);
...	...	@@ -4899,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
4899	5566
4900	5567	if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {
4901	5568	const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);
4902		try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask });
4903		} else try self.genUnOpMir(.not, limb_ty, limb_mcv);
	5569	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
	5570	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
4904	5571	}
4905	5572	},
4906		.neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),
	5573	.neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
4907	5574	else => unreachable,
4908	5575	}
4909	5576	return dst_mcv;
4910	5577	}
4911	5578
4912		fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
	5579	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
4913	5580	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
4914	5581	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
4915	5582	mir_tag,
...	...	@@ -4952,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
4952	5619	/// Clobbers .rcx for non-immediate shift value.
4953	5620	fn genShiftBinOpMir(
4954	5621	self: *Self,
4955		tag: Mir.Inst.Tag,
	5622	tag: Mir.Inst.FixedTag,
4956	5623	ty: Type,
4957	5624	lhs_mcv: MCValue,
4958	5625	shift_mcv: MCValue,
...	...	@@ -5037,16 +5704,16 @@ fn genShiftBinOpMir(
5037	5704	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5038	5705	defer self.register_manager.unlockReg(tmp_lock);
5039	5706
5040		const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {
5041		.shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },
5042		.shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },
	5707	const info: struct { offsets: [2]i32, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) {
	5708	._l => .{ .offsets = .{ 0, 8 }, .double_tag = .{ ._ld, .sh } },
	5709	._r => .{ .offsets = .{ 8, 0 }, .double_tag = .{ ._rd, .sh } },
5043	5710	else => unreachable,
5044	5711	};
5045	5712	switch (lhs_mcv) {
5046	5713	.load_frame => \|dst_frame_addr\| switch (rhs_mcv) {
5047	5714	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
5048	5715	try self.asmRegisterMemory(
5049		.mov,
	5716	.{ ._, .mov },
5050	5717	tmp_reg,
5051	5718	Memory.sib(.qword, .{
5052	5719	.base = .{ .frame = dst_frame_addr.index },
...	...	@@ -5073,7 +5740,7 @@ fn genShiftBinOpMir(
5073	5740	} else {
5074	5741	assert(rhs_imm < 128);
5075	5742	try self.asmRegisterMemory(
5076		.mov,
	5743	.{ ._, .mov },
5077	5744	tmp_reg,
5078	5745	Memory.sib(.qword, .{
5079	5746	.base = .{ .frame = dst_frame_addr.index },
...	...	@@ -5084,34 +5751,30 @@ fn genShiftBinOpMir(
5084	5751	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
5085	5752	}
5086	5753	try self.asmMemoryRegister(
5087		.mov,
	5754	.{ ._, .mov },
5088	5755	Memory.sib(.qword, .{
5089	5756	.base = .{ .frame = dst_frame_addr.index },
5090	5757	.disp = dst_frame_addr.off + info.offsets[1],
5091	5758	}),
5092	5759	tmp_reg,
5093	5760	);
5094		switch (tag) {
5095		.shl, .sal, .shr => {
5096		try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
5097		try self.asmMemoryRegister(
5098		.mov,
5099		Memory.sib(.qword, .{
5100		.base = .{ .frame = dst_frame_addr.index },
5101		.disp = dst_frame_addr.off + info.offsets[0],
5102		}),
5103		tmp_reg,
5104		);
5105		},
5106		.sar => try self.asmMemoryImmediate(
5107		tag,
	5761	if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
	5762	tag,
	5763	Memory.sib(.qword, .{
	5764	.base = .{ .frame = dst_frame_addr.index },
	5765	.disp = dst_frame_addr.off + info.offsets[0],
	5766	}),
	5767	Immediate.u(63),
	5768	) else {
	5769	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
	5770	try self.asmMemoryRegister(
	5771	.{ ._, .mov },
5108	5772	Memory.sib(.qword, .{
5109	5773	.base = .{ .frame = dst_frame_addr.index },
5110	5774	.disp = dst_frame_addr.off + info.offsets[0],
5111	5775	}),
5112		Immediate.u(63),
5113		),
5114		else => unreachable,
	5776	tmp_reg,
	5777	);
5115	5778	}
5116	5779	},
5117	5780	else => {
...	...	@@ -5125,7 +5788,7 @@ fn genShiftBinOpMir(
5125	5788
5126	5789	try self.genSetReg(.cl, Type.u8, rhs_mcv);
5127	5790	try self.asmRegisterMemory(
5128		.mov,
	5791	.{ ._, .mov },
5129	5792	first_reg,
5130	5793	Memory.sib(.qword, .{
5131	5794	.base = .{ .frame = dst_frame_addr.index },
...	...	@@ -5133,32 +5796,28 @@ fn genShiftBinOpMir(
5133	5796	}),
5134	5797	);
5135	5798	try self.asmRegisterMemory(
5136		.mov,
	5799	.{ ._, .mov },
5137	5800	second_reg,
5138	5801	Memory.sib(.qword, .{
5139	5802	.base = .{ .frame = dst_frame_addr.index },
5140	5803	.disp = dst_frame_addr.off + info.offsets[1],
5141	5804	}),
5142	5805	);
5143		switch (tag) {
5144		.shl, .sal, .shr => try self.asmRegisterRegister(
5145		.xor,
5146		tmp_reg.to32(),
5147		tmp_reg.to32(),
5148		),
5149		.sar => {
5150		try self.asmRegisterRegister(.mov, tmp_reg, first_reg);
5151		try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
5152		},
5153		else => unreachable,
5154		}
	5806	if (tag[0] == ._r and tag[1] == .sa) {
	5807	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
	5808	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
	5809	} else try self.asmRegisterRegister(
	5810	.{ ._, .xor },
	5811	tmp_reg.to32(),
	5812	tmp_reg.to32(),
	5813	);
5155	5814	try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
5156	5815	try self.asmRegisterRegister(tag, first_reg, .cl);
5157		try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));
	5816	try self.asmRegisterImmediate(.{ ._, .cmp }, .cl, Immediate.u(64));
5158	5817	try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
5159	5818	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
5160	5819	try self.asmMemoryRegister(
5161		.mov,
	5820	.{ ._, .mov },
5162	5821	Memory.sib(.qword, .{
5163	5822	.base = .{ .frame = dst_frame_addr.index },
5164	5823	.disp = dst_frame_addr.off + info.offsets[1],
...	...	@@ -5166,7 +5825,7 @@ fn genShiftBinOpMir(
5166	5825	second_reg,
5167	5826	);
5168	5827	try self.asmMemoryRegister(
5169		.mov,
	5828	.{ ._, .mov },
5170	5829	Memory.sib(.qword, .{
5171	5830	.base = .{ .frame = dst_frame_addr.index },
5172	5831	.disp = dst_frame_addr.off + info.offsets[0],
...	...	@@ -5191,7 +5850,7 @@ fn genShiftBinOpMir(
5191	5850	/// Asserts .rcx is free.
5192	5851	fn genShiftBinOp(
5193	5852	self: *Self,
5194		tag: Air.Inst.Tag,
	5853	air_tag: Air.Inst.Tag,
5195	5854	maybe_inst: ?Air.Inst.Index,
5196	5855	lhs_mcv: MCValue,
5197	5856	rhs_mcv: MCValue,
...	...	@@ -5236,14 +5895,14 @@ fn genShiftBinOp(
5236	5895	};
5237	5896
5238	5897	const signedness = lhs_ty.intInfo(self.target.*).signedness;
5239		try self.genShiftBinOpMir(switch (tag) {
	5898	try self.genShiftBinOpMir(switch (air_tag) {
5240	5899	.shl, .shl_exact => switch (signedness) {
5241		.signed => .sal,
5242		.unsigned => .shl,
	5900	.signed => .{ ._l, .sa },
	5901	.unsigned => .{ ._l, .sh },
5243	5902	},
5244	5903	.shr, .shr_exact => switch (signedness) {
5245		.signed => .sar,
5246		.unsigned => .shr,
	5904	.signed => .{ ._r, .sa },
	5905	.unsigned => .{ ._r, .sh },
5247	5906	},
5248	5907	else => unreachable,
5249	5908	}, lhs_ty, dst_mcv, rhs_mcv);
...	...	@@ -5303,20 +5962,18 @@ fn genMulDivBinOp(
5303	5962	try self.register_manager.getReg(.rax, track_inst_rax);
5304	5963	try self.register_manager.getReg(.rdx, track_inst_rdx);
5305	5964
5306		const mir_tag: Mir.Inst.Tag = switch (signedness) {
	5965	try self.genIntMulDivOpMir(switch (signedness) {
5307	5966	.signed => switch (tag) {
5308		.mul, .mulwrap => .imul,
5309		.div_trunc, .div_exact, .rem => .idiv,
	5967	.mul, .mulwrap => .{ .i_, .mul },
	5968	.div_trunc, .div_exact, .rem => .{ .i_, .div },
5310	5969	else => unreachable,
5311	5970	},
5312	5971	.unsigned => switch (tag) {
5313		.mul, .mulwrap => .mul,
5314		.div_trunc, .div_exact, .rem => .div,
	5972	.mul, .mulwrap => .{ ._, .mul },
	5973	.div_trunc, .div_exact, .rem => .{ ._, .div },
5315	5974	else => unreachable,
5316	5975	},
5317		};
5318
5319		try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs);
	5976	}, ty, lhs, rhs);
5320	5977
5321	5978	if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
5322	5979	.mul, .mulwrap, .div_trunc, .div_exact => .rax,
...	...	@@ -5326,7 +5983,7 @@ fn genMulDivBinOp(
5326	5983
5327	5984	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
5328	5985	try self.asmMemoryRegister(
5329		.mov,
	5986	.{ ._, .mov },
5330	5987	Memory.sib(.qword, .{
5331	5988	.base = .{ .frame = dst_mcv.load_frame.index },
5332	5989	.disp = dst_mcv.load_frame.off,
...	...	@@ -5334,7 +5991,7 @@ fn genMulDivBinOp(
5334	5991	.rax,
5335	5992	);
5336	5993	try self.asmMemoryRegister(
5337		.mov,
	5994	.{ ._, .mov },
5338	5995	Memory.sib(.qword, .{
5339	5996	.base = .{ .frame = dst_mcv.load_frame.index },
5340	5997	.disp = dst_mcv.load_frame.off + 8,
...	...	@@ -5375,12 +6032,12 @@ fn genMulDivBinOp(
5375	6032	try self.copyToRegisterWithInstTracking(inst, ty, lhs)
5376	6033	else
5377	6034	.{ .register = try self.copyToTmpRegister(ty, lhs) };
5378		try self.genBinOpMir(.sub, ty, result, div_floor);
	6035	try self.genBinOpMir(.{ ._, .sub }, ty, result, div_floor);
5379	6036
5380	6037	return result;
5381	6038	},
5382	6039	.unsigned => {
5383		try self.genIntMulDivOpMir(.div, ty, lhs, rhs);
	6040	try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, rhs);
5384	6041	return .{ .register = registerAlias(.rdx, abi_size) };
5385	6042	},
5386	6043	}
...	...	@@ -5422,7 +6079,7 @@ fn genMulDivBinOp(
5422	6079	switch (signedness) {
5423	6080	.signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),
5424	6081	.unsigned => {
5425		try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs);
	6082	try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, actual_rhs);
5426	6083	return .{ .register = registerAlias(.rax, abi_size) };
5427	6084	},
5428	6085	}
...	...	@@ -5432,25 +6089,22 @@ fn genMulDivBinOp(
5432	6089	}
5433	6090	}
5434	6091
5435		/// Result is always a register.
5436	6092	fn genBinOp(
5437	6093	self: *Self,
5438	6094	maybe_inst: ?Air.Inst.Index,
5439		tag: Air.Inst.Tag,
	6095	air_tag: Air.Inst.Tag,
5440	6096	lhs_air: Air.Inst.Ref,
5441	6097	rhs_air: Air.Inst.Ref,
5442	6098	) !MCValue {
5443		const lhs = try self.resolveInst(lhs_air);
5444		const rhs = try self.resolveInst(rhs_air);
	6099	const lhs_mcv = try self.resolveInst(lhs_air);
	6100	const rhs_mcv = try self.resolveInst(rhs_air);
5445	6101	const lhs_ty = self.air.typeOf(lhs_air);
5446	6102	const rhs_ty = self.air.typeOf(rhs_air);
5447		if (lhs_ty.zigTypeTag() == .Vector) {
5448		return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
5449		}
	6103	const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
5450	6104
5451		switch (lhs) {
	6105	switch (lhs_mcv) {
5452	6106	.immediate => \|imm\| switch (imm) {
5453		0 => switch (tag) {
	6107	0 => switch (air_tag) {
5454	6108	.sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air),
5455	6109	else => {},
5456	6110	},
...	...	@@ -5459,9 +6113,10 @@ fn genBinOp(
5459	6113	else => {},
5460	6114	}
5461	6115
5462		const is_commutative = switch (tag) {
	6116	const is_commutative = switch (air_tag) {
5463	6117	.add,
5464	6118	.addwrap,
	6119	.mul,
5465	6120	.bool_or,
5466	6121	.bit_or,
5467	6122	.bool_and,
...	...	@@ -5473,48 +6128,42 @@ fn genBinOp(
5473	6128
5474	6129	else => false,
5475	6130	};
5476		const dst_mem_ok = switch (tag) {
5477		.add,
5478		.addwrap,
5479		.sub,
5480		.subwrap,
5481		.mul,
5482		.div_float,
5483		.div_exact,
5484		.div_trunc,
5485		.div_floor,
5486		=> !lhs_ty.isRuntimeFloat(),
5487
5488		else => true,
	6131	const vec_op = switch (lhs_ty.zigTypeTag()) {
	6132	else => false,
	6133	.Float, .Vector => true,
5489	6134	};
5490	6135
5491		const lhs_lock: ?RegisterLock = switch (lhs) {
	6136	const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
5492	6137	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
5493	6138	else => null,
5494	6139	};
5495	6140	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
5496	6141
5497		const rhs_lock: ?RegisterLock = switch (rhs) {
	6142	const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
5498	6143	.register => \|reg\| self.register_manager.lockReg(reg),
5499	6144	else => null,
5500	6145	};
5501	6146	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
5502	6147
5503		var flipped: bool = false;
	6148	var flipped = false;
	6149	var copied_to_dst = true;
5504	6150	const dst_mcv: MCValue = dst: {
5505	6151	if (maybe_inst) \|inst\| {
5506		if ((dst_mem_ok or lhs.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs)) {
5507		break :dst lhs;
	6152	if ((!vec_op or lhs_mcv.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs_mcv)) {
	6153	break :dst lhs_mcv;
5508	6154	}
5509		if (is_commutative and (dst_mem_ok or rhs.isRegister()) and
5510		self.reuseOperand(inst, rhs_air, 1, rhs))
	6155	if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and
	6156	self.reuseOperand(inst, rhs_air, 1, rhs_mcv))
5511	6157	{
5512	6158	flipped = true;
5513		break :dst rhs;
	6159	break :dst rhs_mcv;
5514	6160	}
5515	6161	}
5516	6162	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
5517		try self.genCopy(lhs_ty, dst_mcv, lhs);
	6163	if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
	6164	copied_to_dst = false
	6165	else
	6166	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
5518	6167	break :dst dst_mcv;
5519	6168	};
5520	6169	const dst_lock: ?RegisterLock = switch (dst_mcv) {
...	...	@@ -5523,160 +6172,52 @@ fn genBinOp(
5523	6172	};
5524	6173	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5525	6174
5526		const src_mcv = if (flipped) lhs else rhs;
5527		switch (tag) {
5528		.add,
5529		.addwrap,
5530		=> try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
5531		else => .add,
5532		.Float => switch (lhs_ty.floatBits(self.target.*)) {
5533		32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5534		.addss
5535		else
5536		return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5537		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5538		}),
5539		64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5540		.addsd
5541		else
5542		return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5543		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5544		}),
5545		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5546		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5547		}),
5548		},
5549		}, lhs_ty, dst_mcv, src_mcv),
5550
5551		.sub,
5552		.subwrap,
5553		=> try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
5554		else => .sub,
5555		.Float => switch (lhs_ty.floatBits(self.target.*)) {
5556		32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5557		.subss
5558		else
5559		return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5560		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5561		}),
5562		64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5563		.subsd
5564		else
5565		return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5566		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5567		}),
5568		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5569		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5570		}),
5571		},
5572		}, lhs_ty, dst_mcv, src_mcv),
5573
5574		.mul => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
5575		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5576		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5577		}),
5578		.Float => switch (lhs_ty.floatBits(self.target.*)) {
5579		32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5580		.mulss
5581		else
5582		return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5583		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5584		}),
5585		64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5586		.mulsd
5587		else
5588		return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5589		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5590		}),
5591		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5592		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5593		}),
5594		},
5595		}, lhs_ty, dst_mcv, src_mcv),
	6175	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
	6176	if (!vec_op) {
	6177	switch (air_tag) {
	6178	.add,
	6179	.addwrap,
	6180	=> try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv),
5596	6181
5597		.div_float,
5598		.div_exact,
5599		.div_trunc,
5600		.div_floor,
5601		=> {
5602		try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
5603		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5604		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5605		}),
5606		.Float => switch (lhs_ty.floatBits(self.target.*)) {
5607		32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5608		.divss
5609		else
5610		return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5611		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5612		}),
5613		64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5614		.divsd
5615		else
5616		return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5617		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5618		}),
5619		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5620		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5621		}),
5622		},
5623		}, lhs_ty, dst_mcv, src_mcv);
5624		switch (tag) {
5625		.div_float,
5626		.div_exact,
5627		=> {},
5628		.div_trunc,
5629		.div_floor,
5630		=> if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
5631		const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
5632		const dst_alias = registerAlias(dst_mcv.register, abi_size);
5633		try self.asmRegisterRegisterImmediate(switch (lhs_ty.floatBits(self.target.*)) {
5634		32 => .roundss,
5635		64 => .roundsd,
5636		else => unreachable,
5637		}, dst_alias, dst_alias, Immediate.u(switch (tag) {
5638		.div_trunc => 0b1_0_11,
5639		.div_floor => 0b1_0_01,
	6182	.sub,
	6183	.subwrap,
	6184	=> try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv),
	6185
	6186	.ptr_add,
	6187	.ptr_sub,
	6188	=> {
	6189	const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
	6190	const tmp_mcv = MCValue{ .register = tmp_reg };
	6191	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6192	defer self.register_manager.unlockReg(tmp_lock);
	6193
	6194	const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
	6195	try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
	6196	try self.genBinOpMir(
	6197	switch (air_tag) {
	6198	.ptr_add => .{ ._, .add },
	6199	.ptr_sub => .{ ._, .sub },
5640	6200	else => unreachable,
5641		}));
5642		} else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{
5643		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5644		}),
5645		else => unreachable,
5646		}
5647		},
5648
5649		.ptr_add,
5650		.ptr_sub,
5651		=> {
5652		const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
5653		const tmp_mcv = MCValue{ .register = tmp_reg };
5654		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5655		defer self.register_manager.unlockReg(tmp_lock);
5656
5657		const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
5658		try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
5659		try self.genBinOpMir(switch (tag) {
5660		.ptr_add => .add,
5661		.ptr_sub => .sub,
5662		else => unreachable,
5663		}, lhs_ty, dst_mcv, tmp_mcv);
5664		},
	6201	},
	6202	lhs_ty,
	6203	dst_mcv,
	6204	tmp_mcv,
	6205	);
	6206	},
5665	6207
5666		.bool_or,
5667		.bit_or,
5668		=> try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),
	6208	.bool_or,
	6209	.bit_or,
	6210	=> try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv),
5669	6211
5670		.bool_and,
5671		.bit_and,
5672		=> try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),
	6212	.bool_and,
	6213	.bit_and,
	6214	=> try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv),
5673	6215
5674		.xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
	6216	.xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv),
5675	6217
5676		.min,
5677		.max,
5678		=> switch (lhs_ty.zigTypeTag()) {
5679		.Int => {
	6218	.min,
	6219	.max,
	6220	=> {
5680	6221	const mat_src_mcv: MCValue = if (switch (src_mcv) {
5681	6222	.immediate,
5682	6223	.eflags,
...	...	@@ -5698,16 +6239,16 @@ fn genBinOp(
5698	6239	};
5699	6240	defer if (mat_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
5700	6241
5701		try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);
	6242	try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
5702	6243
5703	6244	const int_info = lhs_ty.intInfo(self.target.*);
5704	6245	const cc: Condition = switch (int_info.signedness) {
5705		.unsigned => switch (tag) {
	6246	.unsigned => switch (air_tag) {
5706	6247	.min => .a,
5707	6248	.max => .b,
5708	6249	else => unreachable,
5709	6250	},
5710		.signed => switch (tag) {
	6251	.signed => switch (air_tag) {
5711	6252	.min => .g,
5712	6253	.max => .l,
5713	6254	else => unreachable,
...	...	@@ -5766,32 +6307,404 @@ fn genBinOp(
5766	6307	}
5767	6308	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
5768	6309	},
5769		.Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {
5770		32 => switch (tag) {
5771		.min => .minss,
5772		.max => .maxss,
5773		else => unreachable,
5774		},
5775		64 => switch (tag) {
5776		.min => .minsd,
5777		.max => .maxsd,
5778		else => unreachable,
5779		},
5780		else => return self.fail("TODO implement genBinOp for {s} {}", .{
5781		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5782		}),
5783		}, lhs_ty, dst_mcv, src_mcv),
	6310
5784	6311	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5785		@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
	6312	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
5786	6313	}),
5787		},
	6314	}
	6315	return dst_mcv;
	6316	}
5788	6317
	6318	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
	6319	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
	6320	else => unreachable,
	6321	.Float => switch (lhs_ty.floatBits(self.target.*)) {
	6322	16 => if (self.hasFeature(.f16c)) {
	6323	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	6324	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6325	defer self.register_manager.unlockReg(tmp_lock);
	6326
	6327	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	6328	.{ .vp_w, .insr },
	6329	dst_reg,
	6330	dst_reg,
	6331	src_mcv.mem(.word),
	6332	Immediate.u(1),
	6333	) else try self.asmRegisterRegisterRegister(
	6334	.{ .vp_, .unpcklwd },
	6335	dst_reg,
	6336	dst_reg,
	6337	(if (src_mcv.isRegister())
	6338	src_mcv.getReg().?
	6339	else
	6340	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
	6341	);
	6342	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6343	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
	6344	try self.asmRegisterRegisterRegister(
	6345	switch (air_tag) {
	6346	.add => .{ .v_ss, .add },
	6347	.sub => .{ .v_ss, .sub },
	6348	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
	6349	.max => .{ .v_ss, .max },
	6350	.min => .{ .v_ss, .max },
	6351	else => unreachable,
	6352	},
	6353	dst_reg,
	6354	dst_reg,
	6355	tmp_reg,
	6356	);
	6357	try self.asmRegisterRegisterImmediate(
	6358	.{ .v_, .cvtps2ph },
	6359	dst_reg,
	6360	dst_reg,
	6361	Immediate.u(0b1_00),
	6362	);
	6363	return dst_mcv;
	6364	} else null,
	6365	32 => switch (air_tag) {
	6366	.add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
	6367	.sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
	6368	.mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
	6369	.div_float,
	6370	.div_trunc,
	6371	.div_floor,
	6372	.div_exact,
	6373	=> if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
	6374	.max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
	6375	.min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
	6376	else => unreachable,
	6377	},
	6378	64 => switch (air_tag) {
	6379	.add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
	6380	.sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
	6381	.mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
	6382	.div_float,
	6383	.div_trunc,
	6384	.div_floor,
	6385	.div_exact,
	6386	=> if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
	6387	.max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
	6388	.min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
	6389	else => unreachable,
	6390	},
	6391	80, 128 => null,
	6392	else => unreachable,
	6393	},
	6394	.Vector => switch (lhs_ty.childType().zigTypeTag()) {
	6395	else => null,
	6396	.Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
	6397	16 => if (self.hasFeature(.f16c)) switch (lhs_ty.vectorLen()) {
	6398	1 => {
	6399	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	6400	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6401	defer self.register_manager.unlockReg(tmp_lock);
	6402
	6403	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	6404	.{ .vp_w, .insr },
	6405	dst_reg,
	6406	dst_reg,
	6407	src_mcv.mem(.word),
	6408	Immediate.u(1),
	6409	) else try self.asmRegisterRegisterRegister(
	6410	.{ .vp_, .unpcklwd },
	6411	dst_reg,
	6412	dst_reg,
	6413	(if (src_mcv.isRegister())
	6414	src_mcv.getReg().?
	6415	else
	6416	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
	6417	);
	6418	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6419	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
	6420	try self.asmRegisterRegisterRegister(
	6421	switch (air_tag) {
	6422	.add => .{ .v_ss, .add },
	6423	.sub => .{ .v_ss, .sub },
	6424	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
	6425	.max => .{ .v_ss, .max },
	6426	.min => .{ .v_ss, .max },
	6427	else => unreachable,
	6428	},
	6429	dst_reg,
	6430	dst_reg,
	6431	tmp_reg,
	6432	);
	6433	try self.asmRegisterRegisterImmediate(
	6434	.{ .v_, .cvtps2ph },
	6435	dst_reg,
	6436	dst_reg,
	6437	Immediate.u(0b1_00),
	6438	);
	6439	return dst_mcv;
	6440	},
	6441	2 => {
	6442	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	6443	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6444	defer self.register_manager.unlockReg(tmp_lock);
	6445
	6446	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	6447	.{ .vp_d, .insr },
	6448	dst_reg,
	6449	src_mcv.mem(.dword),
	6450	Immediate.u(1),
	6451	) else try self.asmRegisterRegisterRegister(
	6452	.{ .v_ps, .unpckl },
	6453	dst_reg,
	6454	dst_reg,
	6455	(if (src_mcv.isRegister())
	6456	src_mcv.getReg().?
	6457	else
	6458	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
	6459	);
	6460	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6461	try self.asmRegisterRegisterRegister(
	6462	.{ .v_ps, .movhl },
	6463	tmp_reg,
	6464	dst_reg,
	6465	dst_reg,
	6466	);
	6467	try self.asmRegisterRegisterRegister(
	6468	switch (air_tag) {
	6469	.add => .{ .v_ps, .add },
	6470	.sub => .{ .v_ps, .sub },
	6471	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
	6472	.max => .{ .v_ps, .max },
	6473	.min => .{ .v_ps, .max },
	6474	else => unreachable,
	6475	},
	6476	dst_reg,
	6477	dst_reg,
	6478	tmp_reg,
	6479	);
	6480	try self.asmRegisterRegisterImmediate(
	6481	.{ .v_, .cvtps2ph },
	6482	dst_reg,
	6483	dst_reg,
	6484	Immediate.u(0b1_00),
	6485	);
	6486	return dst_mcv;
	6487	},
	6488	3...4 => {
	6489	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
	6490	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6491	defer self.register_manager.unlockReg(tmp_lock);
	6492
	6493	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
	6494	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	6495	.{ .v_, .cvtph2ps },
	6496	tmp_reg,
	6497	src_mcv.mem(.qword),
	6498	) else try self.asmRegisterRegister(
	6499	.{ .v_, .cvtph2ps },
	6500	tmp_reg,
	6501	(if (src_mcv.isRegister())
	6502	src_mcv.getReg().?
	6503	else
	6504	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
	6505	);
	6506	try self.asmRegisterRegisterRegister(
	6507	switch (air_tag) {
	6508	.add => .{ .v_ps, .add },
	6509	.sub => .{ .v_ps, .sub },
	6510	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
	6511	.max => .{ .v_ps, .max },
	6512	.min => .{ .v_ps, .max },
	6513	else => unreachable,
	6514	},
	6515	dst_reg,
	6516	dst_reg,
	6517	tmp_reg,
	6518	);
	6519	try self.asmRegisterRegisterImmediate(
	6520	.{ .v_, .cvtps2ph },
	6521	dst_reg,
	6522	dst_reg,
	6523	Immediate.u(0b1_00),
	6524	);
	6525	return dst_mcv;
	6526	},
	6527	5...8 => {
	6528	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256();
	6529	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	6530	defer self.register_manager.unlockReg(tmp_lock);
	6531
	6532	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg.to256(), dst_reg);
	6533	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	6534	.{ .v_, .cvtph2ps },
	6535	tmp_reg,
	6536	src_mcv.mem(.xword),
	6537	) else try self.asmRegisterRegister(
	6538	.{ .v_, .cvtph2ps },
	6539	tmp_reg,
	6540	(if (src_mcv.isRegister())
	6541	src_mcv.getReg().?
	6542	else
	6543	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
	6544	);
	6545	try self.asmRegisterRegisterRegister(
	6546	switch (air_tag) {
	6547	.add => .{ .v_ps, .add },
	6548	.sub => .{ .v_ps, .sub },
	6549	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
	6550	.max => .{ .v_ps, .max },
	6551	.min => .{ .v_ps, .max },
	6552	else => unreachable,
	6553	},
	6554	dst_reg.to256(),
	6555	dst_reg.to256(),
	6556	tmp_reg,
	6557	);
	6558	try self.asmRegisterRegisterImmediate(
	6559	.{ .v_, .cvtps2ph },
	6560	dst_reg,
	6561	dst_reg.to256(),
	6562	Immediate.u(0b1_00),
	6563	);
	6564	return dst_mcv;
	6565	},
	6566	else => null,
	6567	} else null,
	6568	32 => switch (lhs_ty.vectorLen()) {
	6569	1 => switch (air_tag) {
	6570	.add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
	6571	.sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
	6572	.mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
	6573	.div_float,
	6574	.div_trunc,
	6575	.div_floor,
	6576	.div_exact,
	6577	=> if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
	6578	.max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
	6579	.min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
	6580	else => unreachable,
	6581	},
	6582	2...4 => switch (air_tag) {
	6583	.add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add },
	6584	.sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub },
	6585	.mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul },
	6586	.div_float,
	6587	.div_trunc,
	6588	.div_floor,
	6589	.div_exact,
	6590	=> if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
	6591	.max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
	6592	.min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
	6593	else => unreachable,
	6594	},
	6595	5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
	6596	.add => .{ .v_ps, .add },
	6597	.sub => .{ .v_ps, .sub },
	6598	.mul => .{ .v_ps, .mul },
	6599	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
	6600	.max => .{ .v_ps, .max },
	6601	.min => .{ .v_ps, .min },
	6602	else => unreachable,
	6603	} else null,
	6604	else => null,
	6605	},
	6606	64 => switch (lhs_ty.vectorLen()) {
	6607	1 => switch (air_tag) {
	6608	.add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
	6609	.sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
	6610	.mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
	6611	.div_float,
	6612	.div_trunc,
	6613	.div_floor,
	6614	.div_exact,
	6615	=> if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
	6616	.max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
	6617	.min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
	6618	else => unreachable,
	6619	},
	6620	2 => switch (air_tag) {
	6621	.add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add },
	6622	.sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub },
	6623	.mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul },
	6624	.div_float,
	6625	.div_trunc,
	6626	.div_floor,
	6627	.div_exact,
	6628	=> if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
	6629	.max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
	6630	.min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
	6631	else => unreachable,
	6632	},
	6633	3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
	6634	.add => .{ .v_pd, .add },
	6635	.sub => .{ .v_pd, .sub },
	6636	.mul => .{ .v_pd, .mul },
	6637	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
	6638	.max => .{ .v_pd, .max },
	6639	.min => .{ .v_pd, .min },
	6640	else => unreachable,
	6641	} else null,
	6642	else => null,
	6643	},
	6644	80, 128 => null,
	6645	else => unreachable,
	6646	},
	6647	},
	6648	})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
	6649	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
	6650	});
	6651	if (self.hasFeature(.avx)) {
	6652	const src1_alias =
	6653	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
	6654	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	6655	mir_tag,
	6656	dst_reg,
	6657	src1_alias,
	6658	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6659	) else try self.asmRegisterRegisterRegister(
	6660	mir_tag,
	6661	dst_reg,
	6662	src1_alias,
	6663	registerAlias(if (src_mcv.isRegister())
	6664	src_mcv.getReg().?
	6665	else
	6666	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	6667	);
	6668	} else {
	6669	assert(copied_to_dst);
	6670	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	6671	mir_tag,
	6672	dst_reg,
	6673	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6674	) else try self.asmRegisterRegister(
	6675	mir_tag,
	6676	dst_reg,
	6677	registerAlias(if (src_mcv.isRegister())
	6678	src_mcv.getReg().?
	6679	else
	6680	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
	6681	);
	6682	}
	6683	switch (air_tag) {
	6684	.add, .sub, .mul, .div_float, .div_exact => {},
	6685	.div_trunc, .div_floor => try self.genRound(
	6686	lhs_ty,
	6687	dst_reg,
	6688	.{ .register = dst_reg },
	6689	switch (air_tag) {
	6690	.div_trunc => 0b1_0_11,
	6691	.div_floor => 0b1_0_01,
	6692	else => unreachable,
	6693	},
	6694	),
	6695	.max, .min => {}, // TODO: unordered select
5789	6696	else => unreachable,
5790	6697	}
5791	6698	return dst_mcv;
5792	6699	}
5793	6700
5794		fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
	6701	fn genBinOpMir(
	6702	self: *Self,
	6703	mir_tag: Mir.Inst.FixedTag,
	6704	ty: Type,
	6705	dst_mcv: MCValue,
	6706	src_mcv: MCValue,
	6707	) !void {
5795	6708	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5796	6709	switch (dst_mcv) {
5797	6710	.none,
...	...	@@ -5818,20 +6731,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5818	6731	.register_overflow,
5819	6732	.reserved_frame,
5820	6733	=> unreachable,
5821		.register => \|src_reg\| switch (ty.zigTypeTag()) {
5822		.Float => {
5823		if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
5824		return self.fail("TODO genBinOpMir for {s} {} without sse", .{
5825		@tagName(mir_tag), ty.fmt(self.bin_file.options.module.?),
5826		});
5827		return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5828		},
5829		else => try self.asmRegisterRegister(
5830		mir_tag,
5831		dst_alias,
5832		registerAlias(src_reg, abi_size),
5833		),
5834		},
	6734	.register => \|src_reg\| try self.asmRegisterRegister(
	6735	mir_tag,
	6736	dst_alias,
	6737	registerAlias(src_reg, abi_size),
	6738	),
5835	6739	.immediate => \|imm\| switch (self.regBitSize(ty)) {
5836	6740	8 => try self.asmRegisterImmediate(
5837	6741	mir_tag,
...	...	@@ -6005,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
6005	6909	};
6006	6910	var off: i32 = 0;
6007	6911	while (off < abi_size) : (off += 8) {
6008		const mir_limb_tag = switch (off) {
	6912	const mir_limb_tag: Mir.Inst.FixedTag = switch (off) {
6009	6913	0 => mir_tag,
6010		else => switch (mir_tag) {
6011		.add => .adc,
6012		.sub, .cmp => .sbb,
	6914	else => switch (mir_tag[1]) {
	6915	.add => .{ ._, .adc },
	6916	.sub, .cmp => .{ ._, .sbb },
6013	6917	.@"or", .@"and", .xor => mir_tag,
6014	6918	else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{
6015		@tagName(mir_tag),
	6919	@tagName(mir_tag[1]),
6016	6920	}),
6017	6921	},
6018	6922	};
...	...	@@ -6184,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6184	7088	.reserved_frame,
6185	7089	=> unreachable,
6186	7090	.register => \|src_reg\| try self.asmRegisterRegister(
6187		.imul,
	7091	.{ .i_, .mul },
6188	7092	dst_alias,
6189	7093	registerAlias(src_reg, abi_size),
6190	7094	),
6191	7095	.immediate => \|imm\| {
6192	7096	if (math.cast(i32, imm)) \|small\| {
6193	7097	try self.asmRegisterRegisterImmediate(
6194		.imul,
	7098	.{ .i_, .mul },
6195	7099	dst_alias,
6196	7100	dst_alias,
6197	7101	Immediate.s(small),
...	...	@@ -6211,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6211	7115	.lea_tlv,
6212	7116	.lea_frame,
6213	7117	=> try self.asmRegisterRegister(
6214		.imul,
	7118	.{ .i_, .mul },
6215	7119	dst_alias,
6216	7120	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
6217	7121	),
6218	7122	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
6219		.imul,
	7123	.{ .i_, .mul },
6220	7124	dst_alias,
6221	7125	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
6222	7126	.memory => \|addr\| .{
6223	7127	.base = .{ .reg = .ds },
6224	7128	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
6225	7129	return self.asmRegisterRegister(
6226		.imul,
	7130	.{ .i_, .mul },
6227	7131	dst_alias,
6228	7132	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
6229	7133	),
...	...	@@ -6348,12 +7252,12 @@ fn genVarDbgInfo(
6348	7252	}
6349	7253
6350	7254	fn airTrap(self: *Self) !void {
6351		try self.asmOpOnly(.ud2);
	7255	try self.asmOpOnly(.{ ._, .ud2 });
6352	7256	return self.finishAirBookkeeping();
6353	7257	}
6354	7258
6355	7259	fn airBreakpoint(self: *Self) !void {
6356		try self.asmOpOnly(.int3);
	7260	try self.asmOpOnly(.{ ._, .int3 });
6357	7261	return self.finishAirBookkeeping();
6358	7262	}
6359	7263
...	...	@@ -6374,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {
6374	7278	switch (order) {
6375	7279	.Unordered, .Monotonic => unreachable,
6376	7280	.Acquire, .Release, .AcqRel => {},
6377		.SeqCst => try self.asmOpOnly(.mfence),
	7281	.SeqCst => try self.asmOpOnly(.{ ._, .mfence }),
6378	7282	}
6379	7283	return self.finishAirBookkeeping();
6380	7284	}
...	...	@@ -6468,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6468	7372	const atom = elf_file.getAtom(atom_index);
6469	7373	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
6470	7374	const got_addr = atom.getOffsetTableAddress(elf_file);
6471		try self.asmMemory(.call, Memory.sib(.qword, .{
	7375	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
6472	7376	.base = .{ .reg = .ds },
6473	7377	.disp = @intCast(i32, got_addr),
6474	7378	}));
...	...	@@ -6476,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6476	7380	const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
6477	7381	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
6478	7382	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
6479		try self.asmRegister(.call, .rax);
	7383	try self.asmRegister(.{ ._, .call }, .rax);
6480	7384	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
6481	7385	const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
6482	7386	const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
6483	7387	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
6484		try self.asmRegister(.call, .rax);
	7388	try self.asmRegister(.{ ._, .call }, .rax);
6485	7389	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
6486	7390	const decl_block_index = try p9.seeDecl(func.owner_decl);
6487	7391	const decl_block = p9.getDeclBlock(decl_block_index);
...	...	@@ -6490,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6490	7394	const got_addr = p9.bases.data;
6491	7395	const got_index = decl_block.got_index.?;
6492	7396	const fn_got_addr = got_addr + got_index * ptr_bytes;
6493		try self.asmMemory(.call, Memory.sib(.qword, .{
	7397	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
6494	7398	.base = .{ .reg = .ds },
6495	7399	.disp = @intCast(i32, fn_got_addr),
6496	7400	}));
...	...	@@ -6503,22 +7407,24 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6503	7407	const atom_index = try self.owner.getSymbolIndex(self);
6504	7408	const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
6505	7409	_ = try self.addInst(.{
6506		.tag = .mov_linker,
	7410	.tag = .mov,
6507	7411	.ops = .import_reloc,
6508		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
6509		.reg = @enumToInt(Register.rax),
6510		.atom_index = atom_index,
6511		.sym_index = sym_index,
6512		}) },
	7412	.data = .{ .rx = .{
	7413	.r1 = .rax,
	7414	.payload = try self.addExtra(Mir.Reloc{
	7415	.atom_index = atom_index,
	7416	.sym_index = sym_index,
	7417	}),
	7418	} },
6513	7419	});
6514		try self.asmRegister(.call, .rax);
	7420	try self.asmRegister(.{ ._, .call }, .rax);
6515	7421	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
6516	7422	const atom_index = try self.owner.getSymbolIndex(self);
6517	7423	const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
6518	7424	_ = try self.addInst(.{
6519		.tag = .call_extern,
6520		.ops = undefined,
6521		.data = .{ .relocation = .{
	7425	.tag = .call,
	7426	.ops = .extern_fn_reloc,
	7427	.data = .{ .reloc = .{
6522	7428	.atom_index = atom_index,
6523	7429	.sym_index = sym_index,
6524	7430	} },
...	...	@@ -6533,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6533	7439	assert(ty.zigTypeTag() == .Pointer);
6534	7440	const mcv = try self.resolveInst(callee);
6535	7441	try self.genSetReg(.rax, Type.usize, mcv);
6536		try self.asmRegister(.call, .rax);
	7442	try self.asmRegister(.{ ._, .call }, .rax);
6537	7443	}
6538	7444
6539	7445	var bt = self.liveness.iterateBigTomb(inst);
...	...	@@ -6588,8 +7494,6 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
6588	7494	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
6589	7495	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
6590	7496	const ty = self.air.typeOf(bin_op.lhs);
6591		const ty_abi_size = ty.abiSize(self.target.*);
6592		const can_reuse = ty_abi_size <= 8;
6593	7497
6594	7498	try self.spillEflagsIfOccupied();
6595	7499	self.eflags_inst = inst;
...	...	@@ -6608,52 +7512,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
6608	7512	};
6609	7513	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
6610	7514
6611		const dst_mem_ok = !ty.isRuntimeFloat();
6612		var flipped = false;
6613		const dst_mcv: MCValue = if (can_reuse and !lhs_mcv.isImmediate() and
6614		(dst_mem_ok or lhs_mcv.isRegister()) and self.liveness.operandDies(inst, 0))
6615		lhs_mcv
6616		else if (can_reuse and !rhs_mcv.isImmediate() and
6617		(dst_mem_ok or rhs_mcv.isRegister()) and self.liveness.operandDies(inst, 1))
6618		dst: {
6619		flipped = true;
6620		break :dst rhs_mcv;
6621		} else if (dst_mem_ok) dst: {
6622		const dst_mcv = try self.allocTempRegOrMem(ty, true);
6623		try self.genCopy(ty, dst_mcv, lhs_mcv);
6624		break :dst dst_mcv;
6625		} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
6626		const dst_lock = switch (dst_mcv) {
6627		.register => \|reg\| self.register_manager.lockReg(reg),
6628		else => null,
6629		};
6630		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	7515	const result = MCValue{
	7516	.eflags = switch (ty.zigTypeTag()) {
	7517	else => result: {
	7518	var flipped = false;
	7519	const dst_mcv: MCValue = if (lhs_mcv.isRegister() or lhs_mcv.isMemory())
	7520	lhs_mcv
	7521	else if (rhs_mcv.isRegister() or rhs_mcv.isMemory()) dst: {
	7522	flipped = true;
	7523	break :dst rhs_mcv;
	7524	} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
	7525	const dst_lock = switch (dst_mcv) {
	7526	.register => \|reg\| self.register_manager.lockReg(reg),
	7527	else => null,
	7528	};
	7529	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	7530	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
6631	7531
6632		const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
6633		try self.genBinOpMir(switch (ty.zigTypeTag()) {
6634		else => .cmp,
6635		.Float => switch (ty.floatBits(self.target.*)) {
6636		32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
6637		.ucomiss
6638		else
6639		return self.fail("TODO implement airCmp for {} without sse", .{
6640		ty.fmt(self.bin_file.options.module.?),
6641		}),
6642		64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
6643		.ucomisd
6644		else
6645		return self.fail("TODO implement airCmp for {} without sse2", .{
6646		ty.fmt(self.bin_file.options.module.?),
6647		}),
6648		else => return self.fail("TODO implement airCmp for {}", .{
6649		ty.fmt(self.bin_file.options.module.?),
6650		}),
6651		},
6652		}, ty, dst_mcv, src_mcv);
	7532	try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
	7533	break :result Condition.fromCompareOperator(
	7534	if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
	7535	if (flipped) op.reverse() else op,
	7536	);
	7537	},
	7538	.Float => result: {
	7539	const flipped = switch (op) {
	7540	.lt, .lte => true,
	7541	.eq, .gte, .gt, .neq => false,
	7542	};
6653	7543
6654		const signedness = if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
6655		const result = MCValue{
6656		.eflags = Condition.fromCompareOperator(signedness, if (flipped) op.reverse() else op),
	7544	const dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
	7545	const dst_reg = if (dst_mcv.isRegister())
	7546	dst_mcv.getReg().?
	7547	else
	7548	try self.copyToTmpRegister(ty, dst_mcv);
	7549	const dst_lock = self.register_manager.lockReg(dst_reg);
	7550	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	7551	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
	7552
	7553	switch (ty.floatBits(self.target.*)) {
	7554	16 => if (self.hasFeature(.f16c)) {
	7555	const tmp1_reg = (try self.register_manager.allocReg(null, sse)).to128();
	7556	const tmp1_mcv = MCValue{ .register = tmp1_reg };
	7557	const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg);
	7558	defer self.register_manager.unlockReg(tmp1_lock);
	7559
	7560	const tmp2_reg = (try self.register_manager.allocReg(null, sse)).to128();
	7561	const tmp2_mcv = MCValue{ .register = tmp2_reg };
	7562	const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
	7563	defer self.register_manager.unlockReg(tmp2_lock);
	7564
	7565	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	7566	.{ .vp_w, .insr },
	7567	tmp1_reg,
	7568	dst_reg.to128(),
	7569	src_mcv.mem(.word),
	7570	Immediate.u(1),
	7571	) else try self.asmRegisterRegisterRegister(
	7572	.{ .vp_, .unpcklwd },
	7573	tmp1_reg,
	7574	dst_reg.to128(),
	7575	(if (src_mcv.isRegister())
	7576	src_mcv.getReg().?
	7577	else
	7578	try self.copyToTmpRegister(ty, src_mcv)).to128(),
	7579	);
	7580	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, tmp1_reg, tmp1_reg);
	7581	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg);
	7582	try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv);
	7583	} else return self.fail("TODO implement airCmp for {}", .{
	7584	ty.fmt(self.bin_file.options.module.?),
	7585	}),
	7586	32 => try self.genBinOpMir(
	7587	.{ ._ss, .ucomi },
	7588	ty,
	7589	.{ .register = dst_reg },
	7590	src_mcv,
	7591	),
	7592	64 => try self.genBinOpMir(
	7593	.{ ._sd, .ucomi },
	7594	ty,
	7595	.{ .register = dst_reg },
	7596	src_mcv,
	7597	),
	7598	else => return self.fail("TODO implement airCmp for {}", .{
	7599	ty.fmt(self.bin_file.options.module.?),
	7600	}),
	7601	}
	7602
	7603	break :result switch (if (flipped) op.reverse() else op) {
	7604	.lt, .lte => unreachable, // required to have been canonicalized to gt(e)
	7605	.gt => .a,
	7606	.gte => .ae,
	7607	.eq => .z_and_np,
	7608	.neq => .nz_or_p,
	7609	};
	7610	},
	7611	},
6657	7612	};
6658	7613	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6659	7614	}
...	...	@@ -6683,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
6683	7638	else => try self.copyToTmpRegister(op_ty, op_mcv),
6684	7639	};
6685	7640	try self.asmRegisterMemory(
6686		.cmp,
	7641	.{ ._, .cmp },
6687	7642	registerAlias(dst_reg, op_abi_size),
6688	7643	Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
6689	7644	);
...	...	@@ -6757,8 +7712,8 @@ fn genTry(
6757	7712	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
6758	7713	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
6759	7714	_ = try self.addInst(.{
6760		.tag = .dbg_line,
6761		.ops = undefined,
	7715	.tag = .pseudo,
	7716	.ops = .pseudo_dbg_line_line_column,
6762	7717	.data = .{ .line_column = .{
6763	7718	.line = dbg_stmt.line,
6764	7719	.column = dbg_stmt.column,
...	...	@@ -6803,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
6803	7758	},
6804	7759	.register => \|reg\| {
6805	7760	try self.spillEflagsIfOccupied();
6806		try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));
	7761	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
6807	7762	return self.asmJccReloc(undefined, .e);
6808	7763	},
6809	7764	.immediate,
...	...	@@ -6906,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6906	7861	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6907	7862	const alias_reg = registerAlias(opt_reg, some_abi_size);
6908	7863	assert(some_abi_size * 8 == alias_reg.bitSize());
6909		try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);
	7864	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
6910	7865	return .{ .eflags = .z };
6911	7866	}
6912	7867	assert(some_info.ty.tag() == .bool);
6913	7868	const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
6914	7869	try self.asmRegisterImmediate(
6915		.bt,
	7870	.{ ._, .bt },
6916	7871	registerAlias(opt_reg, opt_abi_size),
6917	7872	Immediate.u(@intCast(u6, some_info.off * 8)),
6918	7873	);
...	...	@@ -6931,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6931	7886	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
6932	7887	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6933	7888	try self.asmMemoryImmediate(
6934		.cmp,
	7889	.{ ._, .cmp },
6935	7890	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6936	7891	.base = .{ .reg = addr_reg },
6937	7892	.disp = some_info.off,
...	...	@@ -6944,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6944	7899	.indirect, .load_frame => {
6945	7900	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6946	7901	try self.asmMemoryImmediate(
6947		.cmp,
	7902	.{ ._, .cmp },
6948	7903	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
6949	7904	.indirect => \|reg_off\| .{
6950	7905	.base = .{ .reg = reg_off.reg },
...	...	@@ -6986,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
6986	7941
6987	7942	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6988	7943	try self.asmMemoryImmediate(
6989		.cmp,
	7944	.{ ._, .cmp },
6990	7945	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6991	7946	.base = .{ .reg = ptr_reg },
6992	7947	.disp = some_info.off,
...	...	@@ -7017,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
7017	7972	const tmp_reg = try self.copyToTmpRegister(ty, operand);
7018	7973	if (err_off > 0) {
7019	7974	const shift = @intCast(u6, err_off * 8);
7020		try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });
	7975	try self.genShiftBinOpMir(
	7976	.{ ._r, .sh },
	7977	ty,
	7978	.{ .register = tmp_reg },
	7979	.{ .immediate = shift },
	7980	);
7021	7981	} else {
7022	7982	try self.truncateRegister(Type.anyerror, tmp_reg);
7023	7983	}
7024		try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
	7984	try self.genBinOpMir(
	7985	.{ ._, .cmp },
	7986	Type.anyerror,
	7987	.{ .register = tmp_reg },
	7988	.{ .immediate = 0 },
	7989	);
7025	7990	},
7026	7991	.load_frame => \|frame_addr\| try self.genBinOpMir(
7027		.cmp,
	7992	.{ ._, .cmp },
7028	7993	Type.anyerror,
7029	7994	.{ .load_frame = .{
7030	7995	.index = frame_addr.index,
...	...	@@ -7249,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
7249	8214	try self.spillEflagsIfOccupied();
7250	8215	for (items, relocs, 0..) \|item, *reloc, i\| {
7251	8216	const item_mcv = try self.resolveInst(item);
7252		try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
	8217	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
7253	8218	reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
7254	8219	}
7255	8220
...	...	@@ -7289,14 +8254,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
7289	8254	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
7290	8255	const next_inst = @intCast(u32, self.mir_instructions.len);
7291	8256	switch (self.mir_instructions.items(.tag)[reloc]) {
7292		.jcc => {
7293		self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
7294		},
7295		.jmp_reloc => {
7296		self.mir_instructions.items(.data)[reloc].inst = next_inst;
	8257	.j, .jmp => {},
	8258	.pseudo => switch (self.mir_instructions.items(.ops)[reloc]) {
	8259	.pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {},
	8260	else => unreachable,
7297	8261	},
7298	8262	else => unreachable,
7299	8263	}
	8264	self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
7300	8265	}
7301	8266
7302	8267	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -7460,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7460	8425	.qword
7461	8426	else
7462	8427	null;
7463		const mnem = mnem: {
	8428	const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
7464	8429	if (mnem_size) \|_\| {
7465	8430	if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) \|mnem\| {
7466	8431	break :mnem mnem;
...	...	@@ -7468,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7468	8433	}
7469	8434	break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
7470	8435	return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
7471		};
	8436	} };
7472	8437
7473	8438	var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
7474	8439	var ops = [1]encoder.Instruction.Operand{.none} ** 4;
...	...	@@ -7519,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7519	8484	} else if (op_it.next()) \|op_str\| return self.fail("Extra operand: '{s}'", .{op_str});
7520	8485
7521	8486	(switch (ops[0]) {
7522		.none => self.asmOpOnly(mnem),
	8487	.none => self.asmOpOnly(mnem_tag),
7523	8488	.reg => \|reg0\| switch (ops[1]) {
7524		.none => self.asmRegister(mnem, reg0),
	8489	.none => self.asmRegister(mnem_tag, reg0),
7525	8490	.reg => \|reg1\| switch (ops[2]) {
7526		.none => self.asmRegisterRegister(mnem, reg1, reg0),
	8491	.none => self.asmRegisterRegister(mnem_tag, reg1, reg0),
7527	8492	.reg => \|reg2\| switch (ops[3]) {
7528		.none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),
	8493	.none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),
7529	8494	else => error.InvalidInstruction,
7530	8495	},
7531	8496	.mem => \|mem2\| switch (ops[3]) {
7532		.none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),
	8497	.none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),
7533	8498	else => error.InvalidInstruction,
7534	8499	},
7535	8500	else => error.InvalidInstruction,
7536	8501	},
7537	8502	.mem => \|mem1\| switch (ops[2]) {
7538		.none => self.asmMemoryRegister(mnem, mem1, reg0),
	8503	.none => self.asmMemoryRegister(mnem_tag, mem1, reg0),
7539	8504	else => error.InvalidInstruction,
7540	8505	},
7541	8506	else => error.InvalidInstruction,
7542	8507	},
7543	8508	.mem => \|mem0\| switch (ops[1]) {
7544		.none => self.asmMemory(mnem, mem0),
	8509	.none => self.asmMemory(mnem_tag, mem0),
7545	8510	.reg => \|reg1\| switch (ops[2]) {
7546		.none => self.asmRegisterMemory(mnem, reg1, mem0),
	8511	.none => self.asmRegisterMemory(mnem_tag, reg1, mem0),
7547	8512	else => error.InvalidInstruction,
7548	8513	},
7549	8514	else => error.InvalidInstruction,
7550	8515	},
7551	8516	.imm => \|imm0\| switch (ops[1]) {
7552		.none => self.asmImmediate(mnem, imm0),
	8517	.none => self.asmImmediate(mnem_tag, imm0),
7553	8518	.reg => \|reg1\| switch (ops[2]) {
7554		.none => self.asmRegisterImmediate(mnem, reg1, imm0),
	8519	.none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),
7555	8520	.reg => \|reg2\| switch (ops[3]) {
7556		.none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),
	8521	.none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),
7557	8522	else => error.InvalidInstruction,
7558	8523	},
7559	8524	.mem => \|mem2\| switch (ops[3]) {
7560		.none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),
	8525	.none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),
7561	8526	else => error.InvalidInstruction,
7562	8527	},
7563	8528	else => error.InvalidInstruction,
7564	8529	},
7565	8530	.mem => \|mem1\| switch (ops[2]) {
7566		.none => self.asmMemoryImmediate(mnem, mem1, imm0),
	8531	.none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),
7567	8532	else => error.InvalidInstruction,
7568	8533	},
7569	8534	else => error.InvalidInstruction,
...	...	@@ -7572,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7572	8537	error.InvalidInstruction => return self.fail(
7573	8538	"Invalid instruction: '{s} {s} {s} {s} {s}'",
7574	8539	.{
7575		@tagName(mnem),
	8540	@tagName(mnem_tag[1]),
7576	8541	@tagName(ops[0]),
7577	8542	@tagName(ops[1]),
7578	8543	@tagName(ops[2]),
...	...	@@ -7603,19 +8568,55 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7603	8568	return self.finishAirResult(inst, result);
7604	8569	}
7605	8570
7606		fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
7607		return switch (ty.zigTypeTag()) {
7608		else => .mov,
	8571	fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
	8572	switch (ty.zigTypeTag()) {
	8573	else => return .{ ._, .mov },
7609	8574	.Float => switch (ty.floatBits(self.target.*)) {
7610	8575	16 => unreachable, // needs special handling
7611		32 => .movss,
7612		64 => .movsd,
7613		128 => .movaps,
7614		else => return self.fail("TODO movMirTag from {}", .{
7615		ty.fmt(self.bin_file.options.module.?),
7616		}),
	8576	32 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
	8577	64 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
	8578	128 => return if (self.hasFeature(.avx))
	8579	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
	8580	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	8581	else => {},
7617	8582	},
7618		};
	8583	.Vector => switch (ty.childType().zigTypeTag()) {
	8584	.Float => switch (ty.childType().floatBits(self.target.*)) {
	8585	16 => switch (ty.vectorLen()) {
	8586	1 => unreachable, // needs special handling
	8587	2 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
	8588	3...4 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
	8589	5...8 => return if (self.hasFeature(.avx))
	8590	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
	8591	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	8592	9...16 => if (self.hasFeature(.avx))
	8593	return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
	8594	else => {},
	8595	},
	8596	32 => switch (ty.vectorLen()) {
	8597	1 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
	8598	2...4 => return if (self.hasFeature(.avx))
	8599	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
	8600	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	8601	5...8 => if (self.hasFeature(.avx))
	8602	return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
	8603	else => {},
	8604	},
	8605	64 => switch (ty.vectorLen()) {
	8606	1 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
	8607	2 => return if (self.hasFeature(.avx))
	8608	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
	8609	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
	8610	3...4 => if (self.hasFeature(.avx))
	8611	return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
	8612	else => {},
	8613	},
	8614	else => {},
	8615	},
	8616	else => {},
	8617	},
	8618	}
	8619	return self.fail("TODO movMirTag for {}", .{ty.fmt(self.bin_file.options.module.?)});
7619	8620	}
7620	8621
7621	8622	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
...	...	@@ -7685,7 +8686,8 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
7685	8686
7686	8687	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
7687	8688	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
7688		if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
	8689	if (abi_size * 8 > dst_reg.bitSize())
	8690	return self.fail("genSetReg called with a value larger than dst_reg", .{});
7689	8691	switch (src_mcv) {
7690	8692	.none,
7691	8693	.unreach,
...	...	@@ -7700,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7700	8702	if (imm == 0) {
7701	8703	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7702	8704	// register is the fastest way to zero a register.
7703		try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
	8705	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
7704	8706	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
7705	8707	// 32-bit moves zero-extend to 64-bit.
7706		try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
	8708	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
7707	8709	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7708	8710	try self.asmRegisterImmediate(
7709		.mov,
	8711	.{ ._, .mov },
7710	8712	registerAlias(dst_reg, abi_size),
7711	8713	Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7712	8714	);
7713	8715	} else {
7714	8716	try self.asmRegisterImmediate(
7715		.mov,
	8717	.{ ._, .mov },
7716	8718	registerAlias(dst_reg, abi_size),
7717	8719	Immediate.u(imm),
7718	8720	);
...	...	@@ -7721,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7721	8723	.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
7722	8724	if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
7723	8725	switch (ty.zigTypeTag()) {
7724		else => .mov,
7725		.Float, .Vector => .movaps,
	8726	else => .{ ._, .mov },
	8727	.Float, .Vector => .{ ._ps, .mova },
7726	8728	}
7727	8729	else switch (abi_size) {
7728	8730	2 => return try self.asmRegisterRegisterImmediate(
7729		if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,
7730		registerAlias(dst_reg, abi_size),
7731		registerAlias(src_reg, abi_size),
	8731	if (dst_reg.class() == .floating_point) .{ .p_w, .insr } else .{ .p_w, .extr },
	8732	registerAlias(dst_reg, 4),
	8733	registerAlias(src_reg, 4),
7732	8734	Immediate.u(0),
7733	8735	),
7734		4 => .movd,
7735		8 => .movq,
	8736	4 => .{ ._d, .mov },
	8737	8 => .{ ._q, .mov },
7736	8738	else => return self.fail(
7737	8739	"unsupported register copy from {s} to {s}",
7738	8740	.{ @tagName(src_reg), @tagName(dst_reg) },
...	...	@@ -7759,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7759	8761	});
7760	8762	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7761	8763	try self.asmRegisterMemoryImmediate(
7762		.pinsrw,
	8764	.{ .p_w, .insr },
7763	8765	registerAlias(dst_reg, abi_size),
7764	8766	src_mem,
7765	8767	Immediate.u(0),
...	...	@@ -7769,10 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7769	8771	switch (src_mcv) {
7770	8772	.register_offset => \|reg_off\| switch (reg_off.off) {
7771	8773	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
7772		else => .lea,
	8774	else => .{ ._, .lea },
7773	8775	},
7774		.indirect, .load_frame => try self.movMirTag(ty),
7775		.lea_frame => .lea,
	8776	.indirect => try self.movMirTag(ty, false),
	8777	.load_frame => \|frame_addr\| try self.movMirTag(
	8778	ty,
	8779	self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
	8780	),
	8781	.lea_frame => .{ ._, .lea },
7776	8782	else => unreachable,
7777	8783	},
7778	8784	registerAlias(dst_reg, abi_size),
...	...	@@ -7788,14 +8794,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7788	8794	});
7789	8795	return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7790	8796	self.asmRegisterMemoryImmediate(
7791		.pinsrw,
	8797	.{ .p_w, .insr },
7792	8798	registerAlias(dst_reg, abi_size),
7793	8799	src_mem,
7794	8800	Immediate.u(0),
7795	8801	)
7796	8802	else
7797	8803	self.asmRegisterMemory(
7798		try self.movMirTag(ty),
	8804	try self.movMirTag(ty, mem.isAlignedGeneric(
	8805	u32,
	8806	@bitCast(u32, small_addr),
	8807	ty.abiAlignment(self.target.*),
	8808	)),
7799	8809	registerAlias(dst_reg, abi_size),
7800	8810	src_mem,
7801	8811	);
...	...	@@ -7803,13 +8813,15 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7803	8813	.load_direct => \|sym_index\| if (!ty.isRuntimeFloat()) {
7804	8814	const atom_index = try self.owner.getSymbolIndex(self);
7805	8815	_ = try self.addInst(.{
7806		.tag = .mov_linker,
	8816	.tag = .mov,
7807	8817	.ops = .direct_reloc,
7808		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7809		.reg = @enumToInt(dst_reg.to64()),
7810		.atom_index = atom_index,
7811		.sym_index = sym_index,
7812		}) },
	8818	.data = .{ .rx = .{
	8819	.r1 = dst_reg.to64(),
	8820	.payload = try self.addExtra(Mir.Reloc{
	8821	.atom_index = atom_index,
	8822	.sym_index = sym_index,
	8823	}),
	8824	} },
7813	8825	});
7814	8826	return;
7815	8827	},
...	...	@@ -7826,14 +8838,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7826	8838	});
7827	8839	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7828	8840	try self.asmRegisterMemoryImmediate(
7829		.pinsrw,
	8841	.{ .p_w, .insr },
7830	8842	registerAlias(dst_reg, abi_size),
7831	8843	src_mem,
7832	8844	Immediate.u(0),
7833	8845	)
7834	8846	else
7835	8847	try self.asmRegisterMemory(
7836		try self.movMirTag(ty),
	8848	try self.movMirTag(ty, false),
7837	8849	registerAlias(dst_reg, abi_size),
7838	8850	src_mem,
7839	8851	);
...	...	@@ -7842,8 +8854,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7842	8854	const atom_index = try self.owner.getSymbolIndex(self);
7843	8855	_ = try self.addInst(.{
7844	8856	.tag = switch (src_mcv) {
7845		.lea_direct => .lea_linker,
7846		.lea_got => .mov_linker,
	8857	.lea_direct => .lea,
	8858	.lea_got => .mov,
7847	8859	else => unreachable,
7848	8860	},
7849	8861	.ops = switch (src_mcv) {
...	...	@@ -7851,27 +8863,31 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7851	8863	.lea_got => .got_reloc,
7852	8864	else => unreachable,
7853	8865	},
7854		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7855		.reg = @enumToInt(dst_reg.to64()),
7856		.atom_index = atom_index,
7857		.sym_index = sym_index,
7858		}) },
	8866	.data = .{ .rx = .{
	8867	.r1 = dst_reg.to64(),
	8868	.payload = try self.addExtra(Mir.Reloc{
	8869	.atom_index = atom_index,
	8870	.sym_index = sym_index,
	8871	}),
	8872	} },
7859	8873	});
7860	8874	},
7861	8875	.lea_tlv => \|sym_index\| {
7862	8876	const atom_index = try self.owner.getSymbolIndex(self);
7863	8877	if (self.bin_file.cast(link.File.MachO)) \|_\| {
7864	8878	_ = try self.addInst(.{
7865		.tag = .lea_linker,
	8879	.tag = .lea,
7866	8880	.ops = .tlv_reloc,
7867		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7868		.reg = @enumToInt(Register.rdi),
7869		.atom_index = atom_index,
7870		.sym_index = sym_index,
7871		}) },
	8881	.data = .{ .rx = .{
	8882	.r1 = .rdi,
	8883	.payload = try self.addExtra(Mir.Reloc{
	8884	.atom_index = atom_index,
	8885	.sym_index = sym_index,
	8886	}),
	8887	} },
7872	8888	});
7873	8889	// TODO: spill registers before calling
7874		try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
	8890	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
7875	8891	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
7876	8892	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
7877	8893	@tagName(self.bin_file.tag),
...	...	@@ -7898,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
7898	8914	else
7899	8915	Immediate.u(@intCast(u32, imm));
7900	8916	try self.asmMemoryImmediate(
7901		.mov,
	8917	.{ ._, .mov },
7902	8918	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
7903	8919	immediate,
7904	8920	);
...	...	@@ -7906,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
7906	8922	3, 5...7 => unreachable,
7907	8923	else => if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
7908	8924	try self.asmMemoryImmediate(
7909		.mov,
	8925	.{ ._, .mov },
7910	8926	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
7911	8927	Immediate.s(small),
7912	8928	);
7913	8929	} else {
7914	8930	var offset: i32 = 0;
7915	8931	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
7916		.mov,
	8932	.{ ._, .mov },
7917	8933	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
7918	8934	if (ty.isSignedInt())
7919	8935	Immediate.s(@truncate(
...	...	@@ -7936,14 +8952,31 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
7936	8952	);
7937	8953	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7938	8954	try self.asmMemoryRegisterImmediate(
7939		.pextrw,
	8955	.{ .p_w, .extr },
7940	8956	dst_mem,
7941		registerAlias(src_reg, abi_size),
	8957	src_reg.to128(),
7942	8958	Immediate.u(0),
7943	8959	)
7944	8960	else
7945	8961	try self.asmMemoryRegister(
7946		try self.movMirTag(ty),
	8962	try self.movMirTag(ty, switch (base) {
	8963	.none => mem.isAlignedGeneric(
	8964	u32,
	8965	@bitCast(u32, disp),
	8966	ty.abiAlignment(self.target.*),
	8967	),
	8968	.reg => \|reg\| switch (reg) {
	8969	.es, .cs, .ss, .ds => mem.isAlignedGeneric(
	8970	u32,
	8971	@bitCast(u32, disp),
	8972	ty.abiAlignment(self.target.*),
	8973	),
	8974	else => false,
	8975	},
	8976	.frame => \|frame_index\| self.getFrameAddrAlignment(
	8977	.{ .index = frame_index, .off = disp },
	8978	) >= ty.abiAlignment(self.target.*),
	8979	}),
7947	8980	dst_mem,
7948	8981	registerAlias(src_reg, abi_size),
7949	8982	);
...	...	@@ -8015,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(
8015	9048	while (remainder > 0) {
8016	9049	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
8017	9050	try self.asmMemoryRegister(
8018		.mov,
	9051	.{ ._, .mov },
8019	9052	Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
8020	9053	.base = dst_reg,
8021	9054	.disp = -next_offset,
...	...	@@ -8024,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(
8024	9057	);
8025	9058
8026	9059	if (nearest_power_of_two > 1) {
8027		try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
	9060	try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{
8028	9061	.immediate = nearest_power_of_two * 8,
8029	9062	});
8030	9063	}
...	...	@@ -8035,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(
8035	9068	} else {
8036	9069	try self.asmMemoryRegister(
8037	9070	switch (src_reg.class()) {
8038		.general_purpose, .segment => .mov,
8039		.floating_point => .movss,
	9071	.general_purpose, .segment => .{ ._, .mov },
	9072	.floating_point => .{ ._ss, .mov },
8040	9073	},
8041	9074	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
8042	9075	registerAlias(src_reg, abi_size),
...	...	@@ -8049,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
8049	9082	try self.genSetReg(.rdi, Type.usize, dst_ptr);
8050	9083	try self.genSetReg(.rsi, Type.usize, src_ptr);
8051	9084	try self.genSetReg(.rcx, Type.usize, len);
8052		_ = try self.addInst(.{
8053		.tag = .movs,
8054		.ops = .string,
8055		.data = .{ .string = .{ .repeat = .rep, .width = .b } },
8056		});
	9085	try self.asmOpOnly(.{ .@"rep _sb", .mov });
8057	9086	}
8058	9087
8059	9088	fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
...	...	@@ -8061,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
8061	9090	try self.genSetReg(.rdi, Type.usize, dst_ptr);
8062	9091	try self.genSetReg(.al, Type.u8, value);
8063	9092	try self.genSetReg(.rcx, Type.usize, len);
8064		_ = try self.addInst(.{
8065		.tag = .stos,
8066		.ops = .string,
8067		.data = .{ .string = .{ .repeat = .rep, .width = .b } },
8068		});
	9093	try self.asmOpOnly(.{ .@"rep _sb", .sto });
8069	9094	}
8070	9095
8071	9096	fn genLazySymbolRef(
...	...	@@ -8083,14 +9108,14 @@ fn genLazySymbolRef(
8083	9108	const got_mem =
8084	9109	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });
8085	9110	switch (tag) {
8086		.lea, .mov => try self.asmRegisterMemory(.mov, reg.to64(), got_mem),
8087		.call => try self.asmMemory(.call, got_mem),
	9111	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
	9112	.call => try self.asmMemory(.{ ._, .call }, got_mem),
8088	9113	else => unreachable,
8089	9114	}
8090	9115	switch (tag) {
8091	9116	.lea, .call => {},
8092	9117	.mov => try self.asmRegisterMemory(
8093		tag,
	9118	.{ ._, tag },
8094	9119	reg.to64(),
8095	9120	Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),
8096	9121	),
...	...	@@ -8107,7 +9132,7 @@ fn genLazySymbolRef(
8107	9132	}
8108	9133	switch (tag) {
8109	9134	.lea, .mov => {},
8110		.call => try self.asmRegister(.call, reg),
	9135	.call => try self.asmRegister(.{ ._, .call }, reg),
8111	9136	else => unreachable,
8112	9137	}
8113	9138	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
...	...	@@ -8121,7 +9146,7 @@ fn genLazySymbolRef(
8121	9146	}
8122	9147	switch (tag) {
8123	9148	.lea, .mov => {},
8124		.call => try self.asmRegister(.call, reg),
	9149	.call => try self.asmRegister(.{ ._, .call }, reg),
8125	9150	else => unreachable,
8126	9151	}
8127	9152	} else {
...	...	@@ -8164,7 +9189,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
8164	9189	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
8165	9190
8166	9191	const dest = try self.allocRegOrMem(inst, true);
8167		try self.genCopy(self.air.typeOfIndex(inst), dest, operand);
	9192	try self.genCopy(if (!dest.isMemory() or operand.isMemory()) dst_ty else src_ty, dest, operand);
8168	9193	break :result dest;
8169	9194	};
8170	9195	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -8226,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
8226	9251
8227	9252	try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
8228	9253	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
8229		.cvtsi2ss
	9254	.{ ._, .cvtsi2ss }
8230	9255	else
8231	9256	return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
8232	9257	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
8233	9258	}),
8234	9259	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
8235		.cvtsi2sd
	9260	.{ ._, .cvtsi2sd }
8236	9261	else
8237	9262	return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
8238	9263	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
...	...	@@ -8272,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
8272	9297	},
8273	9298	};
8274	9299	try self.asmMemory(
8275		.fld,
	9300	.{ .f_, .ld },
8276	9301	Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
8277	9302	.base = .{ .frame = frame_addr.index },
8278	9303	.disp = frame_addr.off,
...	...	@@ -8282,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
8282	9307	// convert
8283	9308	const stack_dst = try self.allocRegOrMem(inst, false);
8284	9309	try self.asmMemory(
8285		.fisttp,
	9310	.{ .f_p, .istt },
8286	9311	Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
8287	9312	.base = .{ .frame = stack_dst.load_frame.index },
8288	9313	.disp = stack_dst.load_frame.off,
...	...	@@ -8338,16 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
8338	9363	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
8339	9364
8340	9365	try self.spillEflagsIfOccupied();
8341		if (val_abi_size <= 8) {
8342		_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
8343		.r = registerAlias(new_reg.?, val_abi_size),
8344		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8345		} } });
8346		} else {
8347		_ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
8348		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8349		} });
8350		}
	9366	if (val_abi_size <= 8) try self.asmMemoryRegister(
	9367	.{ .@"lock _", .cmpxchg },
	9368	ptr_mem,
	9369	registerAlias(new_reg.?, val_abi_size),
	9370	) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
8351	9371
8352	9372	const result: MCValue = result: {
8353	9373	if (self.liveness.isUnused(inst)) break :result .unreach;
...	...	@@ -8445,16 +9465,17 @@ fn atomicOp(
8445	9465
8446	9466	try self.genSetReg(dst_reg, val_ty, val_mcv);
8447	9467	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
8448		try self.genUnOpMir(.neg, val_ty, dst_mcv);
	9468	try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv);
8449	9469	}
8450		_ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
8451		.mov, .xchg => .mr_sib,
8452		.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
8453		else => unreachable,
8454		}, .data = .{ .rx = .{
8455		.r = registerAlias(dst_reg, val_abi_size),
8456		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8457		} } });
	9470	try self.asmMemoryRegister(
	9471	switch (tag) {
	9472	.mov, .xchg => .{ ._, tag },
	9473	.xadd, .add, .sub, .@"and", .@"or", .xor => .{ .@"lock _", tag },
	9474	else => unreachable,
	9475	},
	9476	ptr_mem,
	9477	registerAlias(dst_reg, val_abi_size),
	9478	);
8458	9479
8459	9480	return if (unused) .unreach else dst_mcv;
8460	9481	},
...	...	@@ -8464,22 +9485,22 @@ fn atomicOp(
8464	9485	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
8465	9486	defer self.register_manager.unlockReg(tmp_lock);
8466	9487
8467		try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
	9488	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
8468	9489	const loop = @intCast(u32, self.mir_instructions.len);
8469	9490	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8470	9491	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
8471	9492	}
8472	9493	if (rmw_op) \|op\| switch (op) {
8473	9494	.Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
8474		.Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
8475		.Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
8476		.And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
	9495	.Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
	9496	.Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
	9497	.And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv),
8477	9498	.Nand => {
8478		try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);
8479		try self.genUnOpMir(.not, val_ty, tmp_mcv);
	9499	try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv);
	9500	try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv);
8480	9501	},
8481		.Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),
8482		.Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),
	9502	.Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv),
	9503	.Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv),
8483	9504	.Min, .Max => {
8484	9505	const cc: Condition = switch (if (val_ty.isAbiInt())
8485	9506	val_ty.intInfo(self.target.*).signedness
...	...	@@ -8497,7 +9518,7 @@ fn atomicOp(
8497	9518	},
8498	9519	};
8499	9520
8500		try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);
	9521	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
8501	9522	const cmov_abi_size = @max(val_abi_size, 2);
8502	9523	switch (val_mcv) {
8503	9524	.register => \|val_reg\| try self.asmCmovccRegisterRegister(
...	...	@@ -8521,19 +9542,20 @@ fn atomicOp(
8521	9542	}
8522	9543	},
8523	9544	};
8524		_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
8525		.r = registerAlias(tmp_reg, val_abi_size),
8526		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8527		} } });
	9545	try self.asmMemoryRegister(
	9546	.{ .@"lock _", .cmpxchg },
	9547	ptr_mem,
	9548	registerAlias(tmp_reg, val_abi_size),
	9549	);
8528	9550	_ = try self.asmJccReloc(loop, .ne);
8529	9551	return if (unused) .unreach else .{ .register = .rax };
8530	9552	} else {
8531		try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
	9553	try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
8532	9554	.base = ptr_mem.sib.base,
8533	9555	.scale_index = ptr_mem.scaleIndex(),
8534	9556	.disp = ptr_mem.sib.disp + 0,
8535	9557	}));
8536		try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{
	9558	try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
8537	9559	.base = ptr_mem.sib.base,
8538	9560	.scale_index = ptr_mem.scaleIndex(),
8539	9561	.disp = ptr_mem.sib.disp + 8,
...	...	@@ -8548,53 +9570,51 @@ fn atomicOp(
8548	9570	const val_lo_mem = val_mem_mcv.mem(.qword);
8549	9571	const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
8550	9572	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8551		try self.asmRegisterRegister(.mov, .rbx, .rax);
8552		try self.asmRegisterRegister(.mov, .rcx, .rdx);
	9573	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
	9574	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
8553	9575	}
8554	9576	if (rmw_op) \|op\| switch (op) {
8555	9577	.Xchg => {
8556		try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);
8557		try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);
	9578	try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem);
	9579	try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem);
8558	9580	},
8559	9581	.Add => {
8560		try self.asmRegisterMemory(.add, .rbx, val_lo_mem);
8561		try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);
	9582	try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem);
	9583	try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem);
8562	9584	},
8563	9585	.Sub => {
8564		try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);
8565		try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);
	9586	try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem);
	9587	try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem);
8566	9588	},
8567	9589	.And => {
8568		try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
8569		try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
	9590	try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
	9591	try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
8570	9592	},
8571	9593	.Nand => {
8572		try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
8573		try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
8574		try self.asmRegister(.not, .rbx);
8575		try self.asmRegister(.not, .rcx);
	9594	try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
	9595	try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
	9596	try self.asmRegister(.{ ._, .not }, .rbx);
	9597	try self.asmRegister(.{ ._, .not }, .rcx);
8576	9598	},
8577	9599	.Or => {
8578		try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);
8579		try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);
	9600	try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem);
	9601	try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem);
8580	9602	},
8581	9603	.Xor => {
8582		try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);
8583		try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);
	9604	try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
	9605	try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
8584	9606	},
8585	9607	else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
8586	9608	val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
8587	9609	}),
8588	9610	};
8589		_ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
8590		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8591		} });
	9611	try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
8592	9612	_ = try self.asmJccReloc(loop, .ne);
8593	9613
8594	9614	if (unused) return .unreach;
8595	9615	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8596	9616	try self.asmMemoryRegister(
8597		.mov,
	9617	.{ ._, .mov },
8598	9618	Memory.sib(.qword, .{
8599	9619	.base = .{ .frame = dst_mcv.load_frame.index },
8600	9620	.disp = dst_mcv.load_frame.off + 0,
...	...	@@ -8602,7 +9622,7 @@ fn atomicOp(
8602	9622	.rax,
8603	9623	);
8604	9624	try self.asmMemoryRegister(
8605		.mov,
	9625	.{ ._, .mov },
8606	9626	Memory.sib(.qword, .{
8607	9627	.base = .{ .frame = dst_mcv.load_frame.index },
8608	9628	.disp = dst_mcv.load_frame.off + 8,
...	...	@@ -8754,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8754	9774	.off = elem_abi_size,
8755	9775	} });
8756	9776
8757		try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
8758		try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
	9777	try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
	9778	try self.asmRegisterRegisterImmediate(
	9779	.{ .i_, .mul },
	9780	len_reg,
	9781	len_reg,
	9782	Immediate.u(elem_abi_size),
	9783	);
8759	9784	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
8760	9785
8761	9786	try self.performReloc(skip_reloc);
...	...	@@ -8893,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8893	9918	try self.truncateRegister(err_ty, err_reg.to32());
8894	9919
8895	9920	try self.asmRegisterMemory(
8896		.mov,
	9921	.{ ._, .mov },
8897	9922	start_reg.to32(),
8898	9923	Memory.sib(.dword, .{
8899	9924	.base = .{ .reg = addr_reg.to64() },
...	...	@@ -8902,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8902	9927	}),
8903	9928	);
8904	9929	try self.asmRegisterMemory(
8905		.mov,
	9930	.{ ._, .mov },
8906	9931	end_reg.to32(),
8907	9932	Memory.sib(.dword, .{
8908	9933	.base = .{ .reg = addr_reg.to64() },
...	...	@@ -8910,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8910	9935	.disp = 8,
8911	9936	}),
8912	9937	);
8913		try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
	9938	try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
8914	9939	try self.asmRegisterMemory(
8915		.lea,
	9940	.{ ._, .lea },
8916	9941	start_reg.to64(),
8917	9942	Memory.sib(.byte, .{
8918	9943	.base = .{ .reg = addr_reg.to64() },
...	...	@@ -8921,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8921	9946	}),
8922	9947	);
8923	9948	try self.asmRegisterMemory(
8924		.lea,
	9949	.{ ._, .lea },
8925	9950	end_reg.to32(),
8926	9951	Memory.sib(.byte, .{
8927	9952	.base = .{ .reg = end_reg.to64() },
...	...	@@ -8931,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8931	9956
8932	9957	const dst_mcv = try self.allocRegOrMem(inst, false);
8933	9958	try self.asmMemoryRegister(
8934		.mov,
	9959	.{ ._, .mov },
8935	9960	Memory.sib(.qword, .{
8936	9961	.base = .{ .frame = dst_mcv.load_frame.index },
8937	9962	.disp = dst_mcv.load_frame.off,
...	...	@@ -8939,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8939	9964	start_reg.to64(),
8940	9965	);
8941	9966	try self.asmMemoryRegister(
8942		.mov,
	9967	.{ ._, .mov },
8943	9968	Memory.sib(.qword, .{
8944	9969	.base = .{ .frame = dst_mcv.load_frame.index },
8945	9970	.disp = dst_mcv.load_frame.off + 8,
...	...	@@ -9035,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9035	10060	try self.truncateRegister(elem_ty, elem_reg);
9036	10061	}
9037	10062	if (elem_bit_off > 0) try self.genShiftBinOpMir(
9038		.shl,
	10063	.{ ._l, .sh },
9039	10064	elem_ty,
9040	10065	.{ .register = elem_reg },
9041	10066	.{ .immediate = elem_bit_off },
9042	10067	);
9043	10068	try self.genBinOpMir(
9044		.@"or",
	10069	.{ ._, .@"or" },
9045	10070	elem_ty,
9046	10071	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
9047	10072	.{ .register = elem_reg },
...	...	@@ -9052,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9052	10077	try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));
9053	10078	}
9054	10079	try self.genShiftBinOpMir(
9055		.shr,
	10080	.{ ._r, .sh },
9056	10081	elem_ty,
9057	10082	.{ .register = reg },
9058	10083	.{ .immediate = elem_abi_bits - elem_bit_off },
9059	10084	);
9060	10085	try self.genBinOpMir(
9061		.@"or",
	10086	.{ ._, .@"or" },
9062	10087	elem_ty,
9063	10088	.{ .load_frame = .{
9064	10089	.index = frame_index,
...	...	@@ -9130,9 +10155,150 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
9130	10155	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
9131	10156	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
9132	10157	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
9133		_ = extra;
9134		return self.fail("TODO implement airMulAdd for x86_64", .{});
9135		//return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
	10158	const ty = self.air.typeOfIndex(inst);
	10159
	10160	if (!self.hasFeature(.fma)) return self.fail("TODO implement airMulAdd for {}", .{
	10161	ty.fmt(self.bin_file.options.module.?),
	10162	});
	10163
	10164	const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
	10165	var mcvs: [3]MCValue = undefined;
	10166	var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
	10167	defer for (locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
	10168	var order = [1]u2{0} ** 3;
	10169	var unused = std.StaticBitSet(3).initFull();
	10170	for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {
	10171	const op_index = @intCast(u2, op_i);
	10172	mcv.* = try self.resolveInst(op);
	10173	if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
	10174	order[op_index] = 1;
	10175	unused.unset(0);
	10176	} else if (unused.isSet(2) and mcv.isMemory()) {
	10177	order[op_index] = 3;
	10178	unused.unset(2);
	10179	}
	10180	switch (mcv.*) {
	10181	.register => \|reg\| lock.* = self.register_manager.lockReg(reg),
	10182	else => {},
	10183	}
	10184	}
	10185	for (&order, &mcvs, &locks) \|mop_index, mcv, *lock\| {
	10186	if (mop_index.* != 0) continue;
	10187	mop_index.* = 1 + @intCast(u2, unused.toggleFirstSet().?);
	10188	if (mop_index.* > 1 and mcv.isRegister()) continue;
	10189	const reg = try self.copyToTmpRegister(ty, mcv.*);
	10190	mcv.* = .{ .register = reg };
	10191	if (lock.*) \|old_lock\| self.register_manager.unlockReg(old_lock);
	10192	lock.* = self.register_manager.lockRegAssumeUnused(reg);
	10193	}
	10194
	10195	const mir_tag = if (@as(
	10196	?Mir.Inst.FixedTag,
	10197	if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
	10198	switch (ty.zigTypeTag()) {
	10199	.Float => switch (ty.floatBits(self.target.*)) {
	10200	32 => .{ .v_ss, .fmadd132 },
	10201	64 => .{ .v_sd, .fmadd132 },
	10202	16, 80, 128 => null,
	10203	else => unreachable,
	10204	},
	10205	.Vector => switch (ty.childType().zigTypeTag()) {
	10206	.Float => switch (ty.childType().floatBits(self.target.*)) {
	10207	32 => switch (ty.vectorLen()) {
	10208	1 => .{ .v_ss, .fmadd132 },
	10209	2...8 => .{ .v_ps, .fmadd132 },
	10210	else => null,
	10211	},
	10212	64 => switch (ty.vectorLen()) {
	10213	1 => .{ .v_sd, .fmadd132 },
	10214	2...4 => .{ .v_pd, .fmadd132 },
	10215	else => null,
	10216	},
	10217	16, 80, 128 => null,
	10218	else => unreachable,
	10219	},
	10220	else => unreachable,
	10221	},
	10222	else => unreachable,
	10223	}
	10224	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
	10225	switch (ty.zigTypeTag()) {
	10226	.Float => switch (ty.floatBits(self.target.*)) {
	10227	32 => .{ .v_ss, .fmadd213 },
	10228	64 => .{ .v_sd, .fmadd213 },
	10229	16, 80, 128 => null,
	10230	else => unreachable,
	10231	},
	10232	.Vector => switch (ty.childType().zigTypeTag()) {
	10233	.Float => switch (ty.childType().floatBits(self.target.*)) {
	10234	32 => switch (ty.vectorLen()) {
	10235	1 => .{ .v_ss, .fmadd213 },
	10236	2...8 => .{ .v_ps, .fmadd213 },
	10237	else => null,
	10238	},
	10239	64 => switch (ty.vectorLen()) {
	10240	1 => .{ .v_sd, .fmadd213 },
	10241	2...4 => .{ .v_pd, .fmadd213 },
	10242	else => null,
	10243	},
	10244	16, 80, 128 => null,
	10245	else => unreachable,
	10246	},
	10247	else => unreachable,
	10248	},
	10249	else => unreachable,
	10250	}
	10251	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
	10252	switch (ty.zigTypeTag()) {
	10253	.Float => switch (ty.floatBits(self.target.*)) {
	10254	32 => .{ .v_ss, .fmadd231 },
	10255	64 => .{ .v_sd, .fmadd231 },
	10256	16, 80, 128 => null,
	10257	else => unreachable,
	10258	},
	10259	.Vector => switch (ty.childType().zigTypeTag()) {
	10260	.Float => switch (ty.childType().floatBits(self.target.*)) {
	10261	32 => switch (ty.vectorLen()) {
	10262	1 => .{ .v_ss, .fmadd231 },
	10263	2...8 => .{ .v_ps, .fmadd231 },
	10264	else => null,
	10265	},
	10266	64 => switch (ty.vectorLen()) {
	10267	1 => .{ .v_sd, .fmadd231 },
	10268	2...4 => .{ .v_pd, .fmadd231 },
	10269	else => null,
	10270	},
	10271	16, 80, 128 => null,
	10272	else => unreachable,
	10273	},
	10274	else => unreachable,
	10275	},
	10276	else => unreachable,
	10277	}
	10278	else
	10279	unreachable,
	10280	)) \|tag\| tag else return self.fail("TODO implement airMulAdd for {}", .{
	10281	ty.fmt(self.bin_file.options.module.?),
	10282	});
	10283
	10284	var mops: [3]MCValue = undefined;
	10285	for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;
	10286
	10287	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
	10288	const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
	10289	const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
	10290	if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
	10291	mir_tag,
	10292	mop1_reg,
	10293	mop2_reg,
	10294	registerAlias(mops[2].getReg().?, abi_size),
	10295	) else try self.asmRegisterRegisterMemory(
	10296	mir_tag,
	10297	mop1_reg,
	10298	mop2_reg,
	10299	mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
	10300	);
	10301	return self.finishAir(inst, mops[0], ops);
9136	10302	}
9137	10303
9138	10304	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
...	...	@@ -9471,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
9471	10637	switch (int_info.signedness) {
9472	10638	.signed => {
9473	10639	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
9474		try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });
9475		try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });
	10640	try self.genShiftBinOpMir(
	10641	.{ ._l, .sa },
	10642	Type.isize,
	10643	.{ .register = reg },
	10644	.{ .immediate = shift },
	10645	);
	10646	try self.genShiftBinOpMir(
	10647	.{ ._r, .sa },
	10648	Type.isize,
	10649	.{ .register = reg },
	10650	.{ .immediate = shift },
	10651	);
9476	10652	},
9477	10653	.unsigned => {
9478	10654	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
9479	10655	const mask = (~@as(u64, 0)) >> shift;
9480	10656	if (int_info.bits <= 32) {
9481		try self.genBinOpMir(.@"and", Type.u32, .{ .register = reg }, .{ .immediate = mask });
	10657	try self.genBinOpMir(
	10658	.{ ._, .@"and" },
	10659	Type.u32,
	10660	.{ .register = reg },
	10661	.{ .immediate = mask },
	10662	);
9482	10663	} else {
9483	10664	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
9484		try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg });
	10665	try self.genBinOpMir(
	10666	.{ ._, .@"and" },
	10667	Type.usize,
	10668	.{ .register = reg },
	10669	.{ .register = tmp_reg },
	10670	);
9485	10671	}
9486	10672	},
9487	10673	}
...	...	@@ -9508,3 +10694,13 @@ fn regBitSize(self: *Self, ty: Type) u64 {
9508	10694	fn regExtraBits(self: *Self, ty: Type) u64 {
9509	10695	return self.regBitSize(ty) - ty.bitSize(self.target.*);
9510	10696	}
	10697
	10698	fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {
	10699	return Target.x86.featureSetHas(self.target.cpu.features, feature);
	10700	}
	10701	fn hasAnyFeatures(self: *Self, features: anytype) bool {
	10702	return Target.x86.featureSetHasAny(self.target.cpu.features, features);
	10703	}
	10704	fn hasAllFeatures(self: *Self, features: anytype) bool {
	10705	return Target.x86.featureSetHasAll(self.target.cpu.features, features);
	10706	}

src/arch/x86_64/Emit.zig+141-131

...	...	@@ -18,142 +18,152 @@ pub const Error = Lower.Error \|\| error{
18	18	};
19	19
20	20	pub fn emitMir(emit: *Emit) Error!void {
21		for (0..emit.lower.mir.instructions.len) \|i\| {
22		const index = @intCast(Mir.Inst.Index, i);
23		const inst = emit.lower.mir.instructions.get(index);
24
25		const start_offset = @intCast(u32, emit.code.items.len);
26		try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
27		for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer(), .{});
28		const end_offset = @intCast(u32, emit.code.items.len);
29
30		switch (inst.tag) {
31		else => {},
32
33		.jmp_reloc => try emit.relocs.append(emit.lower.allocator, .{
34		.source = start_offset,
35		.target = inst.data.inst,
36		.offset = end_offset - 4,
37		.length = 5,
38		}),
39
40		.call_extern => if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
41		// Add relocation to the decl.
42		const atom_index = macho_file.getAtomIndexForSymbol(
43		.{ .sym_index = inst.data.relocation.atom_index, .file = null },
44		).?;
45		const target = macho_file.getGlobalByIndex(inst.data.relocation.sym_index);
46		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
47		.type = .branch,
	21	for (0..emit.lower.mir.instructions.len) \|mir_i\| {
	22	const mir_index = @intCast(Mir.Inst.Index, mir_i);
	23	try emit.code_offset_mapping.putNoClobber(
	24	emit.lower.allocator,
	25	mir_index,
	26	@intCast(u32, emit.code.items.len),
	27	);
	28	const lowered = try emit.lower.lowerMir(mir_index);
	29	var lowered_relocs = lowered.relocs;
	30	for (lowered.insts, 0..) \|lowered_inst, lowered_index\| {
	31	const start_offset = @intCast(u32, emit.code.items.len);
	32	try lowered_inst.encode(emit.code.writer(), .{});
	33	const end_offset = @intCast(u32, emit.code.items.len);
	34	while (lowered_relocs.len > 0 and
	35	lowered_relocs[0].lowered_inst_index == lowered_index) : ({
	36	lowered_relocs = lowered_relocs[1..];
	37	}) switch (lowered_relocs[0].target) {
	38	.inst => \|target\| try emit.relocs.append(emit.lower.allocator, .{
	39	.source = start_offset,
48	40	.target = target,
49	41	.offset = end_offset - 4,
50		.addend = 0,
51		.pcrel = true,
52		.length = 2,
53		});
54		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
55		// Add relocation to the decl.
56		const atom_index = coff_file.getAtomIndexForSymbol(
57		.{ .sym_index = inst.data.relocation.atom_index, .file = null },
58		).?;
59		const target = coff_file.getGlobalByIndex(inst.data.relocation.sym_index);
60		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
61		.type = .direct,
62		.target = target,
63		.offset = end_offset - 4,
64		.addend = 0,
65		.pcrel = true,
66		.length = 2,
67		});
68		} else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),
69
70		.mov_linker, .lea_linker => if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
71		const metadata =
72		emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
73		const atom_index = macho_file.getAtomIndexForSymbol(.{
74		.sym_index = metadata.atom_index,
75		.file = null,
76		}).?;
77		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
78		.type = switch (inst.ops) {
79		.got_reloc => .got,
80		.direct_reloc => .signed,
81		.tlv_reloc => .tlv,
82		else => unreachable,
83		},
84		.target = .{ .sym_index = metadata.sym_index, .file = null },
85		.offset = @intCast(u32, end_offset - 4),
86		.addend = 0,
87		.pcrel = true,
88		.length = 2,
89		});
90		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
91		const metadata =
92		emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
93		const atom_index = coff_file.getAtomIndexForSymbol(.{
94		.sym_index = metadata.atom_index,
95		.file = null,
96		}).?;
97		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
98		.type = switch (inst.ops) {
99		.got_reloc => .got,
100		.direct_reloc => .direct,
101		.import_reloc => .import,
102		else => unreachable,
103		},
104		.target = switch (inst.ops) {
105		.got_reloc,
106		.direct_reloc,
107		=> .{ .sym_index = metadata.sym_index, .file = null },
108		.import_reloc => coff_file.getGlobalByIndex(metadata.sym_index),
109		else => unreachable,
110		},
111		.offset = @intCast(u32, end_offset - 4),
112		.addend = 0,
113		.pcrel = true,
114		.length = 2,
115		});
116		} else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),
117
118		.jcc => try emit.relocs.append(emit.lower.allocator, .{
119		.source = start_offset,
120		.target = inst.data.inst_cc.inst,
121		.offset = end_offset - 4,
122		.length = 6,
123		}),
124
125		.dbg_line => try emit.dbgAdvancePCAndLine(
126		inst.data.line_column.line,
127		inst.data.line_column.column,
128		),
	42	.length = @intCast(u5, end_offset - start_offset),
	43	}),
	44	.linker_extern_fn => \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	45	// Add relocation to the decl.
	46	const atom_index = macho_file.getAtomIndexForSymbol(
	47	.{ .sym_index = symbol.atom_index, .file = null },
	48	).?;
	49	const target = macho_file.getGlobalByIndex(symbol.sym_index);
	50	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	51	.type = .branch,
	52	.target = target,
	53	.offset = end_offset - 4,
	54	.addend = 0,
	55	.pcrel = true,
	56	.length = 2,
	57	});
	58	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	59	// Add relocation to the decl.
	60	const atom_index = coff_file.getAtomIndexForSymbol(
	61	.{ .sym_index = symbol.atom_index, .file = null },
	62	).?;
	63	const target = coff_file.getGlobalByIndex(symbol.sym_index);
	64	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	65	.type = .direct,
	66	.target = target,
	67	.offset = end_offset - 4,
	68	.addend = 0,
	69	.pcrel = true,
	70	.length = 2,
	71	});
	72	} else return emit.fail("TODO implement extern reloc for {s}", .{
	73	@tagName(emit.bin_file.tag),
	74	}),
	75	.linker_got,
	76	.linker_direct,
	77	.linker_import,
	78	.linker_tlv,
	79	=> \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	80	const atom_index = macho_file.getAtomIndexForSymbol(.{
	81	.sym_index = symbol.atom_index,
	82	.file = null,
	83	}).?;
	84	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	85	.type = switch (lowered_relocs[0].target) {
	86	.linker_got => .got,
	87	.linker_direct => .signed,
	88	.linker_tlv => .tlv,
	89	else => unreachable,
	90	},
	91	.target = .{ .sym_index = symbol.sym_index, .file = null },
	92	.offset = @intCast(u32, end_offset - 4),
	93	.addend = 0,
	94	.pcrel = true,
	95	.length = 2,
	96	});
	97	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	98	const atom_index = coff_file.getAtomIndexForSymbol(.{
	99	.sym_index = symbol.atom_index,
	100	.file = null,
	101	}).?;
	102	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	103	.type = switch (lowered_relocs[0].target) {
	104	.linker_got => .got,
	105	.linker_direct => .direct,
	106	.linker_import => .import,
	107	else => unreachable,
	108	},
	109	.target = switch (lowered_relocs[0].target) {
	110	.linker_got,
	111	.linker_direct,
	112	=> .{ .sym_index = symbol.sym_index, .file = null },
	113	.linker_import => coff_file.getGlobalByIndex(symbol.sym_index),
	114	else => unreachable,
	115	},
	116	.offset = @intCast(u32, end_offset - 4),
	117	.addend = 0,
	118	.pcrel = true,
	119	.length = 2,
	120	});
	121	} else return emit.fail("TODO implement linker reloc for {s}", .{
	122	@tagName(emit.bin_file.tag),
	123	}),
	124	};
	125	}
	126	std.debug.assert(lowered_relocs.len == 0);
129	127
130		.dbg_prologue_end => {
131		switch (emit.debug_output) {
132		.dwarf => \|dw\| {
133		try dw.setPrologueEnd();
134		log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
135		emit.prev_di_line, emit.prev_di_column,
136		});
137		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	128	if (lowered.insts.len == 0) {
	129	const mir_inst = emit.lower.mir.instructions.get(mir_index);
	130	switch (mir_inst.tag) {
	131	else => unreachable,
	132	.pseudo => switch (mir_inst.ops) {
	133	else => unreachable,
	134	.pseudo_dbg_prologue_end_none => {
	135	switch (emit.debug_output) {
	136	.dwarf => \|dw\| {
	137	try dw.setPrologueEnd();
	138	log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
	139	emit.prev_di_line, emit.prev_di_column,
	140	});
	141	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	142	},
	143	.plan9 => {},
	144	.none => {},
	145	}
138	146	},
139		.plan9 => {},
140		.none => {},
141		}
142		},
143
144		.dbg_epilogue_begin => {
145		switch (emit.debug_output) {
146		.dwarf => \|dw\| {
147		try dw.setEpilogueBegin();
148		log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
149		emit.prev_di_line, emit.prev_di_column,
150		});
151		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	147	.pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(
	148	mir_inst.data.line_column.line,
	149	mir_inst.data.line_column.column,
	150	),
	151	.pseudo_dbg_epilogue_begin_none => {
	152	switch (emit.debug_output) {
	153	.dwarf => \|dw\| {
	154	try dw.setEpilogueBegin();
	155	log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
	156	emit.prev_di_line, emit.prev_di_column,
	157	});
	158	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	159	},
	160	.plan9 => {},
	161	.none => {},
	162	}
152	163	},
153		.plan9 => {},
154		.none => {},
155		}
156		},
	164	.pseudo_dead_none => {},
	165	},
	166	}
157	167	}
158	168	}
159	169	try emit.fixupRelocs();

src/arch/x86_64/Encoding.zig+240-107

...	...	@@ -23,6 +23,7 @@ const Data = struct {
23	23	opc: [7]u8,
24	24	modrm_ext: u3,
25	25	mode: Mode,
	26	feature: Feature,
26	27	};
27	28
28	29	pub fn findByMnemonic(
...	...	@@ -57,9 +58,9 @@ pub fn findByMnemonic(
57	58	var shortest_len: ?usize = null;
58	59	next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) \|data\| {
59	60	switch (data.mode) {
60		.rex => if (!rex_required) continue,
61		.long, .sse_long, .sse2_long => {},
62		else => if (rex_required) continue,
	61	.none, .short => if (rex_required) continue,
	62	.rex, .rex_short => if (!rex_required) continue,
	63	else => {},
63	64	}
64	65	for (input_ops, data.ops) \|input_op, data_op\|
65	66	if (!input_op.isSubset(data_op)) continue :next;
...	...	@@ -88,28 +89,13 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
88	89	if (modrm_ext) \|ext\| if (ext != data.modrm_ext) continue;
89	90	if (!std.mem.eql(u8, opc, enc.opcode())) continue;
90	91	if (prefixes.rex.w) {
91		switch (data.mode) {
92		.short, .fpu, .sse, .sse2, .sse4_1, .none => continue,
93		.long, .sse_long, .sse2_long, .rex => {},
94		}
	92	if (!data.mode.isLong()) continue;
95	93	} else if (prefixes.rex.present and !prefixes.rex.isSet()) {
96		switch (data.mode) {
97		.rex => {},
98		else => continue,
99		}
	94	if (!data.mode.isRex()) continue;
100	95	} else if (prefixes.legacy.prefix_66) {
101		switch (enc.operandBitSize()) {
102		16 => {},
103		else => continue,
104		}
	96	if (!data.mode.isShort()) continue;
105	97	} else {
106		switch (data.mode) {
107		.none => switch (enc.operandBitSize()) {
108		16 => continue,
109		else => {},
110		},
111		else => continue,
112		}
	98	if (data.mode.isShort()) continue;
113	99	}
114	100	return enc;
115	101	};
...	...	@@ -130,30 +116,11 @@ pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 {
130	116
131	117	pub fn modRmExt(encoding: Encoding) u3 {
132	118	return switch (encoding.data.op_en) {
133		.m, .mi, .m1, .mc => encoding.data.modrm_ext,
	119	.m, .mi, .m1, .mc, .vmi => encoding.data.modrm_ext,
134	120	else => unreachable,
135	121	};
136	122	}
137	123
138		pub fn operandBitSize(encoding: Encoding) u64 {
139		switch (encoding.data.mode) {
140		.short => return 16,
141		.long, .sse_long, .sse2_long => return 64,
142		else => {},
143		}
144		const bit_size: u64 = switch (encoding.data.op_en) {
145		.np => switch (encoding.data.ops[0]) {
146		.o16 => 16,
147		.o32 => 32,
148		.o64 => 64,
149		else => 32,
150		},
151		.td => encoding.data.ops[1].bitSize(),
152		else => encoding.data.ops[0].bitSize(),
153		};
154		return bit_size;
155		}
156
157	124	pub fn format(
158	125	encoding: Encoding,
159	126	comptime fmt: []const u8,
...	...	@@ -162,14 +129,41 @@ pub fn format(
162	129	) !void {
163	130	_ = options;
164	131	_ = fmt;
165		switch (encoding.data.mode) {
166		.long, .sse_long, .sse2_long => try writer.writeAll("REX.W + "),
167		else => {},
168		}
169	132
170		for (encoding.opcode()) \|byte\| {
171		try writer.print("{x:0>2} ", .{byte});
172		}
	133	var opc = encoding.opcode();
	134	if (encoding.data.mode.isVex()) {
	135	try writer.writeAll("VEX.");
	136
	137	try writer.writeAll(switch (encoding.data.mode) {
	138	.vex_128_w0, .vex_128_w1, .vex_128_wig => "128",
	139	.vex_256_w0, .vex_256_w1, .vex_256_wig => "256",
	140	.vex_lig_w0, .vex_lig_w1, .vex_lig_wig => "LIG",
	141	.vex_lz_w0, .vex_lz_w1, .vex_lz_wig => "LZ",
	142	else => unreachable,
	143	});
	144
	145	switch (opc[0]) {
	146	else => {},
	147	0x66, 0xf3, 0xf2 => {
	148	try writer.print(".{X:0>2}", .{opc[0]});
	149	opc = opc[1..];
	150	},
	151	}
	152
	153	try writer.print(".{}", .{std.fmt.fmtSliceHexUpper(opc[0 .. opc.len - 1])});
	154	opc = opc[opc.len - 1 ..];
	155
	156	try writer.writeAll(".W");
	157	try writer.writeAll(switch (encoding.data.mode) {
	158	.vex_128_w0, .vex_256_w0, .vex_lig_w0, .vex_lz_w0 => "0",
	159	.vex_128_w1, .vex_256_w1, .vex_lig_w1, .vex_lz_w1 => "1",
	160	.vex_128_wig, .vex_256_wig, .vex_lig_wig, .vex_lz_wig => "IG",
	161	else => unreachable,
	162	});
	163
	164	try writer.writeByte(' ');
	165	} else if (encoding.data.mode.isLong()) try writer.writeAll("REX.W + ");
	166	for (opc) \|byte\| try writer.print("{x:0>2} ", .{byte});
173	167
174	168	switch (encoding.data.op_en) {
175	169	.np, .fd, .td, .i, .zi, .d => {},
...	...	@@ -183,16 +177,17 @@ pub fn format(
183	177	};
184	178	try writer.print("+{s} ", .{tag});
185	179	},
186		.m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
187		.mr, .rm, .rmi, .mri, .mrc => try writer.writeAll("/r "),
	180	.m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
	181	.mr, .rm, .rmi, .mri, .mrc, .rvm, .rvmi, .mvr => try writer.writeAll("/r "),
188	182	}
189	183
190	184	switch (encoding.data.op_en) {
191		.i, .d, .zi, .oi, .mi, .rmi, .mri => {
	185	.i, .d, .zi, .oi, .mi, .rmi, .mri, .vmi, .rvmi => {
192	186	const op = switch (encoding.data.op_en) {
193	187	.i, .d => encoding.data.ops[0],
194	188	.zi, .oi, .mi => encoding.data.ops[1],
195		.rmi, .mri => encoding.data.ops[2],
	189	.rmi, .mri, .vmi => encoding.data.ops[2],
	190	.rvmi => encoding.data.ops[3],
196	191	else => unreachable,
197	192	};
198	193	const tag = switch (op) {
...	...	@@ -207,7 +202,7 @@ pub fn format(
207	202	};
208	203	try writer.print("{s} ", .{tag});
209	204	},
210		.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc => {},
	205	.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rvm, .mvr => {},
211	206	}
212	207
213	208	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
...	...	@@ -267,44 +262,79 @@ pub const Mnemonic = enum {
267	262	// MMX
268	263	movd,
269	264	// SSE
270		addss,
	265	addps, addss,
271	266	andps,
272	267	andnps,
273	268	cmpss,
274	269	cvtsi2ss,
275		divss,
276		maxss, minss,
277		movaps, movss, movups,
278		mulss,
	270	divps, divss,
	271	maxps, maxss,
	272	minps, minss,
	273	movaps, movhlps, movss, movups,
	274	mulps, mulss,
279	275	orps,
280		pextrw,
281		pinsrw,
282		sqrtps,
283		sqrtss,
284		subss,
	276	pextrw, pinsrw,
	277	sqrtps, sqrtss,
	278	subps, subss,
285	279	ucomiss,
286	280	xorps,
287	281	// SSE2
288		addsd,
	282	addpd, addsd,
289	283	andpd,
290	284	andnpd,
291	285	//cmpsd,
292	286	cvtsd2ss, cvtsi2sd, cvtss2sd,
293		divsd,
294		maxsd, minsd,
	287	divpd, divsd,
	288	maxpd, maxsd,
	289	minpd, minsd,
295	290	movapd,
296	291	movq, //movd, movsd,
297	292	movupd,
298		mulsd,
	293	mulpd, mulsd,
299	294	orpd,
300		sqrtpd,
301		sqrtsd,
302		subsd,
	295	pshufhw, pshuflw,
	296	psrld, psrlq, psrlw,
	297	punpckhbw, punpckhdq, punpckhqdq, punpckhwd,
	298	punpcklbw, punpckldq, punpcklqdq, punpcklwd,
	299	sqrtpd, sqrtsd,
	300	subpd, subsd,
303	301	ucomisd,
304	302	xorpd,
	303	// SSE3
	304	movddup, movshdup, movsldup,
305	305	// SSE4.1
306		roundss,
307		roundsd,
	306	pextrb, pextrd, pextrq,
	307	pinsrb, pinsrd, pinsrq,
	308	roundpd, roundps, roundsd, roundss,
	309	// AVX
	310	vaddpd, vaddps, vaddsd, vaddss,
	311	vcvtsd2ss, vcvtsi2sd, vcvtsi2ss, vcvtss2sd,
	312	vdivpd, vdivps, vdivsd, vdivss,
	313	vmaxpd, vmaxps, vmaxsd, vmaxss,
	314	vminpd, vminps, vminsd, vminss,
	315	vmovapd, vmovaps,
	316	vmovddup, vmovhlps,
	317	vmovsd,
	318	vmovshdup, vmovsldup,
	319	vmovss,
	320	vmovupd, vmovups,
	321	vmulpd, vmulps, vmulsd, vmulss,
	322	vpextrb, vpextrd, vpextrq, vpextrw,
	323	vpinsrb, vpinsrd, vpinsrq, vpinsrw,
	324	vpshufhw, vpshuflw,
	325	vpsrld, vpsrlq, vpsrlw,
	326	vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
	327	vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
	328	vroundpd, vroundps, vroundsd, vroundss,
	329	vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
	330	vsubpd, vsubps, vsubsd, vsubss,
	331	// F16C
	332	vcvtph2ps, vcvtps2ph,
	333	// FMA
	334	vfmadd132pd, vfmadd213pd, vfmadd231pd,
	335	vfmadd132ps, vfmadd213ps, vfmadd231ps,
	336	vfmadd132sd, vfmadd213sd, vfmadd231sd,
	337	vfmadd132ss, vfmadd213ss, vfmadd231ss,
308	338	// zig fmt: on
309	339	};
310	340
...	...	@@ -317,6 +347,7 @@ pub const OpEn = enum {
317	347	fd, td,
318	348	m1, mc, mi, mr, rm,
319	349	rmi, mri, mrc,
	350	vmi, rvm, rvmi, mvr,
320	351	// zig fmt: on
321	352	};
322	353
...	...	@@ -331,12 +362,14 @@ pub const Op = enum {
331	362	cl,
332	363	r8, r16, r32, r64,
333	364	rm8, rm16, rm32, rm64,
334		m8, m16, m32, m64, m80, m128,
	365	r32_m8, r32_m16, r64_m16,
	366	m8, m16, m32, m64, m80, m128, m256,
335	367	rel8, rel16, rel32,
336	368	m,
337	369	moffs,
338	370	sreg,
339	371	xmm, xmm_m32, xmm_m64, xmm_m128,
	372	ymm, ymm_m256,
340	373	// zig fmt: on
341	374
342	375	pub fn fromOperand(operand: Instruction.Operand) Op {
...	...	@@ -348,6 +381,7 @@ pub const Op = enum {
348	381	.segment => return .sreg,
349	382	.floating_point => return switch (reg.bitSize()) {
350	383	128 => .xmm,
	384	256 => .ymm,
351	385	else => unreachable,
352	386	},
353	387	.general_purpose => {
...	...	@@ -381,6 +415,7 @@ pub const Op = enum {
381	415	64 => .m64,
382	416	80 => .m80,
383	417	128 => .m128,
	418	256 => .m256,
384	419	else => unreachable,
385	420	};
386	421	},
...	...	@@ -409,16 +444,52 @@ pub const Op = enum {
409	444	}
410	445	}
411	446
412		pub fn bitSize(op: Op) u64 {
	447	pub fn immBitSize(op: Op) u64 {
413	448	return switch (op) {
414	449	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
	450	.al, .cl, .r8, .rm8, .r32_m8 => unreachable,
	451	.ax, .r16, .rm16 => unreachable,
	452	.eax, .r32, .rm32, .r32_m16 => unreachable,
	453	.rax, .r64, .rm64, .r64_m16 => unreachable,
	454	.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
	455	.ymm, .ymm_m256 => unreachable,
	456	.m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
415	457	.unity => 1,
416		.imm8, .imm8s, .al, .cl, .r8, .m8, .rm8, .rel8 => 8,
417		.imm16, .imm16s, .ax, .r16, .m16, .rm16, .rel16 => 16,
418		.imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
419		.imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
	458	.imm8, .imm8s, .rel8 => 8,
	459	.imm16, .imm16s, .rel16 => 16,
	460	.imm32, .imm32s, .rel32 => 32,
	461	.imm64 => 64,
	462	};
	463	}
	464
	465	pub fn regBitSize(op: Op) u64 {
	466	return switch (op) {
	467	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
	468	.unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
	469	.rel8, .rel16, .rel32 => unreachable,
	470	.m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
	471	.al, .cl, .r8, .rm8 => 8,
	472	.ax, .r16, .rm16 => 16,
	473	.eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
	474	.rax, .r64, .rm64, .r64_m16 => 64,
	475	.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
	476	.ymm, .ymm_m256 => 256,
	477	};
	478	}
	479
	480	pub fn memBitSize(op: Op) u64 {
	481	return switch (op) {
	482	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
	483	.unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
	484	.rel8, .rel16, .rel32 => unreachable,
	485	.al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm, .ymm => unreachable,
	486	.m8, .rm8, .r32_m8 => 8,
	487	.m16, .rm16, .r32_m16, .r64_m16 => 16,
	488	.m32, .rm32, .xmm_m32 => 32,
	489	.m64, .rm64, .xmm_m64 => 64,
420	490	.m80 => 80,
421		.m128, .xmm, .xmm_m128 => 128,
	491	.m128, .xmm_m128 => 128,
	492	.m256, .ymm_m256 => 256,
422	493	};
423	494	}
424	495
...	...	@@ -441,7 +512,9 @@ pub const Op = enum {
441	512	.al, .ax, .eax, .rax,
442	513	.r8, .r16, .r32, .r64,
443	514	.rm8, .rm16, .rm32, .rm64,
	515	.r32_m8, .r32_m16, .r64_m16,
444	516	.xmm, .xmm_m32, .xmm_m64, .xmm_m128,
	517	.ymm, .ymm_m256,
445	518	=> true,
446	519	else => false,
447	520	};
...	...	@@ -465,9 +538,11 @@ pub const Op = enum {
465	538	// zig fmt: off
466	539	return switch (op) {
467	540	.rm8, .rm16, .rm32, .rm64,
468		.m8, .m16, .m32, .m64, .m80, .m128,
	541	.r32_m8, .r32_m16, .r64_m16,
	542	.m8, .m16, .m32, .m64, .m80, .m128, .m256,
469	543	.m,
470	544	.xmm_m32, .xmm_m64, .xmm_m128,
	545	.ymm_m256,
471	546	=> true,
472	547	else => false,
473	548	};
...	...	@@ -487,15 +562,10 @@ pub const Op = enum {
487	562	.al, .ax, .eax, .rax, .cl => .general_purpose,
488	563	.r8, .r16, .r32, .r64 => .general_purpose,
489	564	.rm8, .rm16, .rm32, .rm64 => .general_purpose,
	565	.r32_m8, .r32_m16, .r64_m16 => .general_purpose,
490	566	.sreg => .segment,
491	567	.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
492		};
493		}
494
495		pub fn isFloatingPointRegister(op: Op) bool {
496		return switch (op) {
497		.xmm, .xmm_m32, .xmm_m64, .xmm_m128 => true,
498		else => false,
	568	.ymm, .ymm_m256 => .floating_point,
499	569	};
500	570	}
501	571
...	...	@@ -512,30 +582,27 @@ pub const Op = enum {
512	582	if (op.isRegister() and target.isRegister()) {
513	583	return switch (target) {
514	584	.cl, .al, .ax, .eax, .rax => op == target,
515		else => op.class() == target.class() and switch (target.class()) {
516		.floating_point => true,
517		else => op.bitSize() == target.bitSize(),
518		},
	585	else => op.class() == target.class() and op.regBitSize() == target.regBitSize(),
519	586	};
520	587	}
521	588	if (op.isMemory() and target.isMemory()) {
522	589	switch (target) {
523	590	.m => return true,
524		else => return op.bitSize() == target.bitSize(),
	591	else => return op.memBitSize() == target.memBitSize(),
525	592	}
526	593	}
527	594	if (op.isImmediate() and target.isImmediate()) {
528	595	switch (target) {
529		.imm64 => if (op.bitSize() <= 64) return true,
530		.imm32s, .rel32 => if (op.bitSize() < 32 or (op.bitSize() == 32 and op.isSigned()))
	596	.imm64 => if (op.immBitSize() <= 64) return true,
	597	.imm32s, .rel32 => if (op.immBitSize() < 32 or (op.immBitSize() == 32 and op.isSigned()))
531	598	return true,
532		.imm32 => if (op.bitSize() <= 32) return true,
533		.imm16s, .rel16 => if (op.bitSize() < 16 or (op.bitSize() == 16 and op.isSigned()))
	599	.imm32 => if (op.immBitSize() <= 32) return true,
	600	.imm16s, .rel16 => if (op.immBitSize() < 16 or (op.immBitSize() == 16 and op.isSigned()))
534	601	return true,
535		.imm16 => if (op.bitSize() <= 16) return true,
536		.imm8s, .rel8 => if (op.bitSize() < 8 or (op.bitSize() == 8 and op.isSigned()))
	602	.imm16 => if (op.immBitSize() <= 16) return true,
	603	.imm8s, .rel8 => if (op.immBitSize() < 8 or (op.immBitSize() == 8 and op.isSigned()))
537	604	return true,
538		.imm8 => if (op.bitSize() <= 8) return true,
	605	.imm8 => if (op.immBitSize() <= 8) return true,
539	606	else => {},
540	607	}
541	608	return op == target;
...	...	@@ -547,16 +614,81 @@ pub const Op = enum {
547	614	};
548	615
549	616	pub const Mode = enum {
	617	// zig fmt: off
	618	none,
	619	short, long,
	620	rex, rex_short,
	621	vex_128_w0, vex_128_w1, vex_128_wig,
	622	vex_256_w0, vex_256_w1, vex_256_wig,
	623	vex_lig_w0, vex_lig_w1, vex_lig_wig,
	624	vex_lz_w0, vex_lz_w1, vex_lz_wig,
	625	// zig fmt: on
	626
	627	pub fn isShort(mode: Mode) bool {
	628	return switch (mode) {
	629	.short, .rex_short => true,
	630	else => false,
	631	};
	632	}
	633
	634	pub fn isLong(mode: Mode) bool {
	635	return switch (mode) {
	636	.long,
	637	.vex_128_w1,
	638	.vex_256_w1,
	639	.vex_lig_w1,
	640	.vex_lz_w1,
	641	=> true,
	642	else => false,
	643	};
	644	}
	645
	646	pub fn isRex(mode: Mode) bool {
	647	return switch (mode) {
	648	else => false,
	649	.rex, .rex_short => true,
	650	};
	651	}
	652
	653	pub fn isVex(mode: Mode) bool {
	654	return switch (mode) {
	655	// zig fmt: off
	656	else => false,
	657	.vex_128_w0, .vex_128_w1, .vex_128_wig,
	658	.vex_256_w0, .vex_256_w1, .vex_256_wig,
	659	.vex_lig_w0, .vex_lig_w1, .vex_lig_wig,
	660	.vex_lz_w0, .vex_lz_w1, .vex_lz_wig,
	661	=> true,
	662	// zig fmt: on
	663	};
	664	}
	665
	666	pub fn isVecLong(mode: Mode) bool {
	667	return switch (mode) {
	668	// zig fmt: off
	669	else => unreachable,
	670	.vex_128_w0, .vex_128_w1, .vex_128_wig,
	671	.vex_lig_w0, .vex_lig_w1, .vex_lig_wig,
	672	.vex_lz_w0, .vex_lz_w1, .vex_lz_wig,
	673	=> false,
	674	.vex_256_w0, .vex_256_w1, .vex_256_wig,
	675	=> true,
	676	// zig fmt: on
	677	};
	678	}
	679	};
	680
	681	pub const Feature = enum {
550	682	none,
551		short,
552		fpu,
553		rex,
554		long,
	683	avx,
	684	avx2,
	685	f16c,
	686	fma,
555	687	sse,
556		sse_long,
557	688	sse2,
558		sse2_long,
	689	sse3,
559	690	sse4_1,
	691	x87,
560	692	};
561	693
562	694	fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Operand) usize {
...	...	@@ -573,7 +705,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
573	705	}
574	706
575	707	const mnemonic_to_encodings_map = init: {
576		@setEvalBranchQuota(100_000);
	708	@setEvalBranchQuota(20_000);
577	709	const encodings = @import("encodings.zig");
578	710	var entries = encodings.table;
579	711	std.sort.sort(encodings.Entry, &entries, {}, struct {
...	...	@@ -593,6 +725,7 @@ const mnemonic_to_encodings_map = init: {
593	725	.opc = undefined,
594	726	.modrm_ext = entry[4],
595	727	.mode = entry[5],
	728	.feature = entry[6],
596	729	};
597	730	// TODO: use `@memcpy` for these. When I did that, I got a false positive
598	731	// compile error for this copy happening at compile time.

src/arch/x86_64/Lower.zig+283-323

...	...	@@ -5,13 +5,22 @@ mir: Mir,
5	5	target: *const std.Target,
6	6	err_msg: ?*ErrorMsg = null,
7	7	src_loc: Module.SrcLoc,
8		result: [
	8	result_insts_len: u8 = undefined,
	9	result_relocs_len: u8 = undefined,
	10	result_insts: [
9	11	std.mem.max(usize, &.{
10		abi.Win64.callee_preserved_regs.len,
11		abi.SysV.callee_preserved_regs.len,
	12	2, // cmovcc: cmovcc \ cmovcc
	13	3, // setcc: setcc \ setcc \ logicop
	14	2, // jcc: jcc \ jcc
	15	abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
	16	abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
12	17	})
13	18	]Instruction = undefined,
14		result_len: usize = undefined,
	19	result_relocs: [
	20	std.mem.max(usize, &.{
	21	2, // jcc: jcc \ jcc
	22	})
	23	]Reloc = undefined,
15	24
16	25	pub const Error = error{
17	26	OutOfMemory,
...	...	@@ -20,155 +29,155 @@ pub const Error = error{
20	29	CannotEncode,
21	30	};
22	31
23		/// The returned slice is overwritten by the next call to lowerMir.
24		pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
25		lower.result = undefined;
26		errdefer lower.result = undefined;
27		lower.result_len = 0;
28		defer lower.result_len = undefined;
	32	pub const Reloc = struct {
	33	lowered_inst_index: u8,
	34	target: Target,
	35
	36	const Target = union(enum) {
	37	inst: Mir.Inst.Index,
	38	linker_extern_fn: Mir.Reloc,
	39	linker_got: Mir.Reloc,
	40	linker_direct: Mir.Reloc,
	41	linker_import: Mir.Reloc,
	42	linker_tlv: Mir.Reloc,
	43	};
	44	};
29	45
	46	/// The returned slice is overwritten by the next call to lowerMir.
	47	pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
	48	insts: []const Instruction,
	49	relocs: []const Reloc,
	50	} {
	51	lower.result_insts = undefined;
	52	lower.result_relocs = undefined;
	53	errdefer lower.result_insts = undefined;
	54	errdefer lower.result_relocs = undefined;
	55	lower.result_insts_len = 0;
	56	lower.result_relocs_len = 0;
	57	defer lower.result_insts_len = undefined;
	58	defer lower.result_relocs_len = undefined;
	59
	60	const inst = lower.mir.instructions.get(index);
30	61	switch (inst.tag) {
31		.adc,
32		.add,
33		.@"and",
34		.bsf,
35		.bsr,
36		.bswap,
37		.bt,
38		.btc,
39		.btr,
40		.bts,
41		.call,
42		.cbw,
43		.cwde,
44		.cdqe,
45		.cwd,
46		.cdq,
47		.cqo,
48		.cmp,
49		.cmpxchg,
50		.div,
51		.fisttp,
52		.fld,
53		.idiv,
54		.imul,
55		.int3,
56		.jmp,
57		.lea,
58		.lfence,
59		.lzcnt,
60		.mfence,
61		.mov,
62		.movbe,
63		.movd,
64		.movq,
65		.movzx,
66		.mul,
67		.neg,
68		.nop,
69		.not,
70		.@"or",
71		.pop,
72		.popcnt,
73		.push,
74		.rcl,
75		.rcr,
76		.ret,
77		.rol,
78		.ror,
79		.sal,
80		.sar,
81		.sbb,
82		.sfence,
83		.shl,
84		.shld,
85		.shr,
86		.shrd,
87		.sub,
88		.syscall,
89		.@"test",
90		.tzcnt,
91		.ud2,
92		.xadd,
93		.xchg,
94		.xor,
95
96		.addss,
97		.andnps,
98		.andps,
99		.cmpss,
100		.cvtsi2ss,
101		.divss,
102		.maxss,
103		.minss,
104		.movaps,
105		.movss,
106		.movups,
107		.mulss,
108		.orps,
109		.pextrw,
110		.pinsrw,
111		.roundss,
112		.sqrtps,
113		.sqrtss,
114		.subss,
115		.ucomiss,
116		.xorps,
117		.addsd,
118		.andnpd,
119		.andpd,
120		.cmpsd,
121		.cvtsd2ss,
122		.cvtsi2sd,
123		.cvtss2sd,
124		.divsd,
125		.maxsd,
126		.minsd,
127		.movsd,
128		.mulsd,
129		.orpd,
130		.roundsd,
131		.sqrtpd,
132		.sqrtsd,
133		.subsd,
134		.ucomisd,
135		.xorpd,
136		=> try lower.mirGeneric(inst),
137
138		.cmps,
139		.lods,
140		.movs,
141		.scas,
142		.stos,
143		=> try lower.mirString(inst),
144
145		.cmpxchgb => try lower.mirCmpxchgBytes(inst),
146
147		.jmp_reloc => try lower.emit(.none, .jmp, &.{.{ .imm = Immediate.s(0) }}),
148
149		.call_extern => try lower.emit(.none, .call, &.{.{ .imm = Immediate.s(0) }}),
150
151		.lea_linker => try lower.mirLeaLinker(inst),
152		.mov_linker => try lower.mirMovLinker(inst),
153
154		.mov_moffs => try lower.mirMovMoffs(inst),
155
156		.movsx => try lower.mirMovsx(inst),
157		.cmovcc => try lower.mirCmovcc(inst),
158		.setcc => try lower.mirSetcc(inst),
159		.jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
	62	else => try lower.generic(inst),
	63	.pseudo => switch (inst.ops) {
	64	.pseudo_cmov_z_and_np_rr => {
	65	try lower.emit(.none, .cmovnz, &.{
	66	.{ .reg = inst.data.rr.r2 },
	67	.{ .reg = inst.data.rr.r1 },
	68	});
	69	try lower.emit(.none, .cmovnp, &.{
	70	.{ .reg = inst.data.rr.r1 },
	71	.{ .reg = inst.data.rr.r2 },
	72	});
	73	},
	74	.pseudo_cmov_nz_or_p_rr => {
	75	try lower.emit(.none, .cmovnz, &.{
	76	.{ .reg = inst.data.rr.r1 },
	77	.{ .reg = inst.data.rr.r2 },
	78	});
	79	try lower.emit(.none, .cmovp, &.{
	80	.{ .reg = inst.data.rr.r1 },
	81	.{ .reg = inst.data.rr.r2 },
	82	});
	83	},
	84	.pseudo_cmov_nz_or_p_rm_sib,
	85	.pseudo_cmov_nz_or_p_rm_rip,
	86	=> {
	87	try lower.emit(.none, .cmovnz, &.{
	88	.{ .reg = inst.data.rx.r1 },
	89	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	90	});
	91	try lower.emit(.none, .cmovp, &.{
	92	.{ .reg = inst.data.rx.r1 },
	93	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	94	});
	95	},
	96	.pseudo_set_z_and_np_r => {
	97	try lower.emit(.none, .setz, &.{
	98	.{ .reg = inst.data.r_scratch.r1 },
	99	});
	100	try lower.emit(.none, .setnp, &.{
	101	.{ .reg = inst.data.r_scratch.scratch_reg },
	102	});
	103	try lower.emit(.none, .@"and", &.{
	104	.{ .reg = inst.data.r_scratch.r1 },
	105	.{ .reg = inst.data.r_scratch.scratch_reg },
	106	});
	107	},
	108	.pseudo_set_z_and_np_m_sib,
	109	.pseudo_set_z_and_np_m_rip,
	110	=> {
	111	try lower.emit(.none, .setz, &.{
	112	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	113	});
	114	try lower.emit(.none, .setnp, &.{
	115	.{ .reg = inst.data.x_scratch.scratch_reg },
	116	});
	117	try lower.emit(.none, .@"and", &.{
	118	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	119	.{ .reg = inst.data.x_scratch.scratch_reg },
	120	});
	121	},
	122	.pseudo_set_nz_or_p_r => {
	123	try lower.emit(.none, .setnz, &.{
	124	.{ .reg = inst.data.r_scratch.r1 },
	125	});
	126	try lower.emit(.none, .setp, &.{
	127	.{ .reg = inst.data.r_scratch.scratch_reg },
	128	});
	129	try lower.emit(.none, .@"or", &.{
	130	.{ .reg = inst.data.r_scratch.r1 },
	131	.{ .reg = inst.data.r_scratch.scratch_reg },
	132	});
	133	},
	134	.pseudo_set_nz_or_p_m_sib,
	135	.pseudo_set_nz_or_p_m_rip,
	136	=> {
	137	try lower.emit(.none, .setnz, &.{
	138	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	139	});
	140	try lower.emit(.none, .setp, &.{
	141	.{ .reg = inst.data.x_scratch.scratch_reg },
	142	});
	143	try lower.emit(.none, .@"or", &.{
	144	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
	145	.{ .reg = inst.data.x_scratch.scratch_reg },
	146	});
	147	},
	148	.pseudo_j_z_and_np_inst => {
	149	try lower.emit(.none, .jnz, &.{
	150	.{ .imm = lower.reloc(.{ .inst = index + 1 }) },
	151	});
	152	try lower.emit(.none, .jnp, &.{
	153	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	154	});
	155	},
	156	.pseudo_j_nz_or_p_inst => {
	157	try lower.emit(.none, .jnz, &.{
	158	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	159	});
	160	try lower.emit(.none, .jp, &.{
	161	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	162	});
	163	},
160	164
161		.push_regs => try lower.mirPushPopRegisterList(inst, .push),
162		.pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
	165	.pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
	166	.pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
163	167
164		.dbg_line,
165		.dbg_prologue_end,
166		.dbg_epilogue_begin,
167		.dead,
168		=> {},
	168	.pseudo_dbg_prologue_end_none,
	169	.pseudo_dbg_line_line_column,
	170	.pseudo_dbg_epilogue_begin_none,
	171	.pseudo_dead_none,
	172	=> {},
	173	else => unreachable,
	174	},
169	175	}
170	176
171		return lower.result[0..lower.result_len];
	177	return .{
	178	.insts = lower.result_insts[0..lower.result_insts_len],
	179	.relocs = lower.result_relocs[0..lower.result_relocs_len],
	180	};
172	181	}
173	182
174	183	pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
...	...	@@ -178,12 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
178	187	return error.LowerFail;
179	188	}
180	189
181		fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
182		return switch (cc) {
183		inline else => \|c\| @field(Mnemonic, @tagName(base) ++ @tagName(c)),
184		};
185		}
186
187	190	fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
188	191	return switch (ops) {
189	192	.rri_s,
...	...	@@ -191,21 +194,22 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
191	194	.i_s,
192	195	.mi_sib_s,
193	196	.mi_rip_s,
194		.lock_mi_sib_s,
195		.lock_mi_rip_s,
196	197	=> Immediate.s(@bitCast(i32, i)),
197	198
	199	.rrri,
198	200	.rri_u,
199	201	.ri_u,
200	202	.i_u,
201	203	.mi_sib_u,
202	204	.mi_rip_u,
203		.lock_mi_sib_u,
204		.lock_mi_rip_u,
205	205	.rmi_sib,
206	206	.rmi_rip,
207	207	.mri_sib,
208	208	.mri_rip,
	209	.rrm_sib,
	210	.rrm_rip,
	211	.rrmi_sib,
	212	.rrmi_rip,
209	213	=> Immediate.u(i),
210	214
211	215	.ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
...	...	@@ -217,76 +221,108 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
217	221	fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
218	222	return lower.mir.resolveFrameLoc(switch (ops) {
219	223	.rm_sib,
220		.rm_sib_cc,
221	224	.rmi_sib,
222	225	.m_sib,
223		.m_sib_cc,
224	226	.mi_sib_u,
225	227	.mi_sib_s,
226	228	.mr_sib,
227	229	.mrr_sib,
228	230	.mri_sib,
229		.lock_m_sib,
230		.lock_mi_sib_u,
231		.lock_mi_sib_s,
232		.lock_mr_sib,
	231	.rrm_sib,
	232	.rrmi_sib,
	233
	234	.pseudo_cmov_nz_or_p_rm_sib,
	235	.pseudo_set_z_and_np_m_sib,
	236	.pseudo_set_nz_or_p_m_sib,
233	237	=> lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
234	238
235	239	.rm_rip,
236		.rm_rip_cc,
237	240	.rmi_rip,
238	241	.m_rip,
239		.m_rip_cc,
240	242	.mi_rip_u,
241	243	.mi_rip_s,
242	244	.mr_rip,
243	245	.mrr_rip,
244	246	.mri_rip,
245		.lock_m_rip,
246		.lock_mi_rip_u,
247		.lock_mi_rip_s,
248		.lock_mr_rip,
	247	.rrm_rip,
	248	.rrmi_rip,
	249
	250	.pseudo_cmov_nz_or_p_rm_rip,
	251	.pseudo_set_z_and_np_m_rip,
	252	.pseudo_set_nz_or_p_m_rip,
249	253	=> lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
250	254
251	255	.rax_moffs,
252	256	.moffs_rax,
253		.lock_moffs_rax,
254	257	=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
255	258
256	259	else => unreachable,
257	260	});
258	261	}
259	262
	263	fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
	264	lower.result_relocs[lower.result_relocs_len] = .{
	265	.lowered_inst_index = lower.result_insts_len,
	266	.target = target,
	267	};
	268	lower.result_relocs_len += 1;
	269	return Immediate.s(0);
	270	}
	271
260	272	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
261		lower.result[lower.result_len] = try Instruction.new(prefix, mnemonic, ops);
262		lower.result_len += 1;
	273	lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
	274	lower.result_insts_len += 1;
263	275	}
264	276
265		fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
266		try lower.emit(switch (inst.ops) {
267		else => .none,
268		.lock_m_sib,
269		.lock_m_rip,
270		.lock_mi_sib_u,
271		.lock_mi_rip_u,
272		.lock_mi_sib_s,
273		.lock_mi_rip_s,
274		.lock_mr_sib,
275		.lock_mr_rip,
276		.lock_moffs_rax,
277		=> .lock,
278		}, switch (inst.tag) {
279		inline else => \|tag\| if (@hasField(Mnemonic, @tagName(tag)))
280		@field(Mnemonic, @tagName(tag))
	277	fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
	278	const fixes = switch (inst.ops) {
	279	.none => inst.data.none.fixes,
	280	.inst => inst.data.inst.fixes,
	281	.i_s, .i_u => inst.data.i.fixes,
	282	.r => inst.data.r.fixes,
	283	.rr => inst.data.rr.fixes,
	284	.rrr => inst.data.rrr.fixes,
	285	.rrri => inst.data.rrri.fixes,
	286	.rri_s, .rri_u => inst.data.rri.fixes,
	287	.ri_s, .ri_u => inst.data.ri.fixes,
	288	.ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
	289	.mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
	290	.rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
	291	.rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
	292	.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
	293	.m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
	294	.extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
	295	else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
	296	};
	297	try lower.emit(switch (fixes) {
	298	inline else => \|tag\| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) \|space\|
	299	@field(Prefix, @tagName(tag)[0..space])
281	300	else
282		unreachable,
	301	.none,
	302	}, mnemonic: {
	303	comptime var max_len = 0;
	304	inline for (@typeInfo(Mnemonic).Enum.fields) \|field\| max_len = @max(field.name.len, max_len);
	305	var buf: [max_len]u8 = undefined;
	306
	307	const fixes_name = @tagName(fixes);
	308	const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) \|i\| i + 1 else 0..];
	309	const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
	310	const parts = .{ pattern[0..wildcard_i], @tagName(inst.tag), pattern[wildcard_i + 1 ..] };
	311	const err_msg = "unsupported mnemonic: ";
	312	const mnemonic = std.fmt.bufPrint(&buf, "{s}{s}{s}", parts) catch
	313	return lower.fail(err_msg ++ "'{s}{s}{s}'", parts);
	314	break :mnemonic std.meta.stringToEnum(Mnemonic, mnemonic) orelse
	315	return lower.fail(err_msg ++ "'{s}'", .{mnemonic});
283	316	}, switch (inst.ops) {
284	317	.none => &.{},
	318	.inst => &.{
	319	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
	320	},
285	321	.i_s, .i_u => &.{
286		.{ .imm = lower.imm(inst.ops, inst.data.i) },
	322	.{ .imm = lower.imm(inst.ops, inst.data.i.i) },
287	323	},
288	324	.r => &.{
289		.{ .reg = inst.data.r },
	325	.{ .reg = inst.data.r.r1 },
290	326	},
291	327	.rr => &.{
292	328	.{ .reg = inst.data.rr.r1 },
...	...	@@ -297,12 +333,18 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
297	333	.{ .reg = inst.data.rrr.r2 },
298	334	.{ .reg = inst.data.rrr.r3 },
299	335	},
	336	.rrri => &.{
	337	.{ .reg = inst.data.rrri.r1 },
	338	.{ .reg = inst.data.rrri.r2 },
	339	.{ .reg = inst.data.rrri.r3 },
	340	.{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
	341	},
300	342	.ri_s, .ri_u => &.{
301		.{ .reg = inst.data.ri.r },
	343	.{ .reg = inst.data.ri.r1 },
302	344	.{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
303	345	},
304	346	.ri64 => &.{
305		.{ .reg = inst.data.rx.r },
	347	.{ .reg = inst.data.rx.r1 },
306	348	.{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },
307	349	},
308	350	.rri_s, .rri_u => &.{
...	...	@@ -310,33 +352,28 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
310	352	.{ .reg = inst.data.rri.r2 },
311	353	.{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
312	354	},
313		.m_sib, .lock_m_sib, .m_rip, .lock_m_rip => &.{
314		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	355	.m_sib, .m_rip => &.{
	356	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
315	357	},
316		.mi_sib_s,
317		.lock_mi_sib_s,
318		.mi_sib_u,
319		.lock_mi_sib_u,
320		.mi_rip_u,
321		.lock_mi_rip_u,
322		.mi_rip_s,
323		.lock_mi_rip_s,
324		=> &.{
325		.{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
326		.{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
	358	.mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
	359	.{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
	360	.{ .imm = lower.imm(
	361	inst.ops,
	362	lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
	363	) },
327	364	},
328	365	.rm_sib, .rm_rip => &.{
329		.{ .reg = inst.data.rx.r },
	366	.{ .reg = inst.data.rx.r1 },
330	367	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
331	368	},
332	369	.rmi_sib, .rmi_rip => &.{
333		.{ .reg = inst.data.rix.r },
	370	.{ .reg = inst.data.rix.r1 },
334	371	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
335	372	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
336	373	},
337		.mr_sib, .lock_mr_sib, .mr_rip, .lock_mr_rip => &.{
	374	.mr_sib, .mr_rip => &.{
338	375	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
339		.{ .reg = inst.data.rx.r },
	376	.{ .reg = inst.data.rx.r1 },
340	377	},
341	378	.mrr_sib, .mrr_rip => &.{
342	379	.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
...	...	@@ -345,137 +382,60 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
345	382	},
346	383	.mri_sib, .mri_rip => &.{
347	384	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
348		.{ .reg = inst.data.rix.r },
	385	.{ .reg = inst.data.rix.r1 },
349	386	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
350	387	},
351		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
352		});
353		}
354
355		fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {
356		switch (inst.ops) {
357		.string => try lower.emit(switch (inst.data.string.repeat) {
358		inline else => \|repeat\| @field(Prefix, @tagName(repeat)),
359		}, switch (inst.tag) {
360		inline .cmps, .lods, .movs, .scas, .stos => \|tag\| switch (inst.data.string.width) {
361		inline else => \|width\| @field(Mnemonic, @tagName(tag) ++ @tagName(width)),
362		},
363		else => unreachable,
364		}, &.{}),
365		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
366		}
367		}
368
369		fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
370		const ops: [1]Operand = switch (inst.ops) {
371		.m_sib, .lock_m_sib, .m_rip, .lock_m_rip => .{
372		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	388	.rrm_sib, .rrm_rip => &.{
	389	.{ .reg = inst.data.rrx.r1 },
	390	.{ .reg = inst.data.rrx.r2 },
	391	.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
	392	},
	393	.rrmi_sib, .rrmi_rip => &.{
	394	.{ .reg = inst.data.rrix.r1 },
	395	.{ .reg = inst.data.rrix.r2 },
	396	.{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
	397	.{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
373	398	},
374		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
375		};
376		try lower.emit(switch (inst.ops) {
377		.m_sib, .m_rip => .none,
378		.lock_m_sib, .lock_m_rip => .lock,
379		else => unreachable,
380		}, switch (@divExact(ops[0].bitSize(), 8)) {
381		8 => .cmpxchg8b,
382		16 => .cmpxchg16b,
383		else => return lower.fail("invalid operand for {s}", .{@tagName(inst.tag)}),
384		}, &ops);
385		}
386
387		fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
388		try lower.emit(switch (inst.ops) {
389		.rax_moffs, .moffs_rax => .none,
390		.lock_moffs_rax => .lock,
391		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
392		}, .mov, switch (inst.ops) {
393	399	.rax_moffs => &.{
394	400	.{ .reg = .rax },
395		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	401	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
396	402	},
397		.moffs_rax, .lock_moffs_rax => &.{
398		.{ .mem = lower.mem(inst.ops, inst.data.payload) },
	403	.moffs_rax => &.{
	404	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
399	405	.{ .reg = .rax },
400	406	},
401		else => unreachable,
402		});
403		}
404
405		fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
406		const ops: [2]Operand = switch (inst.ops) {
407		.rr => .{
408		.{ .reg = inst.data.rr.r1 },
409		.{ .reg = inst.data.rr.r2 },
410		},
411		.rm_sib, .rm_rip => .{
412		.{ .reg = inst.data.rx.r },
413		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	407	.extern_fn_reloc => &.{
	408	.{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
414	409	},
415		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
416		};
417		try lower.emit(.none, switch (ops[0].bitSize()) {
418		32, 64 => switch (ops[1].bitSize()) {
419		32 => .movsxd,
420		else => .movsx,
	410	.got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
	411	const reg = inst.data.rx.r1;
	412	const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
	413	_ = lower.reloc(switch (inst.ops) {
	414	.got_reloc => .{ .linker_got = extra },
	415	.direct_reloc => .{ .linker_direct = extra },
	416	.import_reloc => .{ .linker_import = extra },
	417	.tlv_reloc => .{ .linker_tlv = extra },
	418	else => unreachable,
	419	});
	420	break :ops &.{
	421	.{ .reg = reg },
	422	.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
	423	};
421	424	},
422		else => .movsx,
423		}, &ops);
424		}
425
426		fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
427		switch (inst.ops) {
428		.rr_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rr_cc.cc), &.{
429		.{ .reg = inst.data.rr_cc.r1 },
430		.{ .reg = inst.data.rr_cc.r2 },
431		}),
432		.rm_sib_cc, .rm_rip_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rx_cc.cc), &.{
433		.{ .reg = inst.data.rx_cc.r },
434		.{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
435		}),
436	425	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
437		}
438		}
439
440		fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
441		switch (inst.ops) {
442		.r_cc => try lower.emit(.none, mnem_cc(.set, inst.data.r_cc.cc), &.{
443		.{ .reg = inst.data.r_cc.r },
444		}),
445		.m_sib_cc, .m_rip_cc => try lower.emit(.none, mnem_cc(.set, inst.data.x_cc.cc), &.{
446		.{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
447		}),
448		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
449		}
	426	});
450	427	}
451	428
452		fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
453		const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
	429	fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
454	430	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
455		var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
	431	var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) {
456	432	.push => .reverse,
457	433	.pop => .forward,
458	434	else => unreachable,
459	435	} });
460		while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
461		}
462
463		fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {
464		const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
465		const reg = @intToEnum(Register, metadata.reg);
466		try lower.emit(.none, .lea, &.{
467		.{ .reg = reg },
468		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
469		});
470		}
471
472		fn mirMovLinker(lower: *Lower, inst: Mir.Inst) Error!void {
473		const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
474		const reg = @intToEnum(Register, metadata.reg);
475		try lower.emit(.none, .mov, &.{
476		.{ .reg = reg },
477		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
478		});
	436	while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{
	437	.reg = callee_preserved_regs[i],
	438	}});
479	439	}
480	440
481	441	const abi = @import("abi.zig");

src/arch/x86_64/Mir.zig+560-276

...	...	@@ -32,12 +32,260 @@ pub const Inst = struct {
32	32
33	33	pub const Index = u32;
34	34
	35	pub const Fixes = enum(u8) {
	36	/// ___
	37	@"_",
	38
	39	/// Integer __
	40	i_,
	41
	42	/// ___ Left
	43	_l,
	44	/// ___ Left Double
	45	_ld,
	46	/// ___ Right
	47	_r,
	48	/// ___ Right Double
	49	_rd,
	50
	51	/// ___ Above
	52	_a,
	53	/// ___ Above Or Equal
	54	_ae,
	55	/// ___ Below
	56	_b,
	57	/// ___ Below Or Equal
	58	_be,
	59	/// ___ Carry
	60	_c,
	61	/// ___ Equal
	62	_e,
	63	/// ___ Greater
	64	_g,
	65	/// ___ Greater Or Equal
	66	_ge,
	67	/// ___ Less
	68	//_l,
	69	/// ___ Less Or Equal
	70	_le,
	71	/// ___ Not Above
	72	_na,
	73	/// ___ Not Above Or Equal
	74	_nae,
	75	/// ___ Not Below
	76	_nb,
	77	/// ___ Not Below Or Equal
	78	_nbe,
	79	/// ___ Not Carry
	80	_nc,
	81	/// ___ Not Equal
	82	_ne,
	83	/// ___ Not Greater
	84	_ng,
	85	/// ___ Not Greater Or Equal
	86	_nge,
	87	/// ___ Not Less
	88	_nl,
	89	/// ___ Not Less Or Equal
	90	_nle,
	91	/// ___ Not Overflow
	92	_no,
	93	/// ___ Not Parity
	94	_np,
	95	/// ___ Not Sign
	96	_ns,
	97	/// ___ Not Zero
	98	_nz,
	99	/// ___ Overflow
	100	_o,
	101	/// ___ Parity
	102	_p,
	103	/// ___ Parity Even
	104	_pe,
	105	/// ___ Parity Odd
	106	_po,
	107	/// ___ Sign
	108	_s,
	109	/// ___ Zero
	110	_z,
	111
	112	/// ___ Byte
	113	//_b,
	114	/// ___ Word
	115	_w,
	116	/// ___ Doubleword
	117	_d,
	118	/// ___ QuadWord
	119	_q,
	120
	121	/// ___ String
	122	//_s,
	123	/// ___ String Byte
	124	_sb,
	125	/// ___ String Word
	126	_sw,
	127	/// ___ String Doubleword
	128	_sd,
	129	/// ___ String Quadword
	130	_sq,
	131
	132	/// Repeat ___ String
	133	@"rep _s",
	134	/// Repeat ___ String Byte
	135	@"rep _sb",
	136	/// Repeat ___ String Word
	137	@"rep _sw",
	138	/// Repeat ___ String Doubleword
	139	@"rep _sd",
	140	/// Repeat ___ String Quadword
	141	@"rep _sq",
	142
	143	/// Repeat Equal ___ String
	144	@"repe _s",
	145	/// Repeat Equal ___ String Byte
	146	@"repe _sb",
	147	/// Repeat Equal ___ String Word
	148	@"repe _sw",
	149	/// Repeat Equal ___ String Doubleword
	150	@"repe _sd",
	151	/// Repeat Equal ___ String Quadword
	152	@"repe _sq",
	153
	154	/// Repeat Not Equal ___ String
	155	@"repne _s",
	156	/// Repeat Not Equal ___ String Byte
	157	@"repne _sb",
	158	/// Repeat Not Equal ___ String Word
	159	@"repne _sw",
	160	/// Repeat Not Equal ___ String Doubleword
	161	@"repne _sd",
	162	/// Repeat Not Equal ___ String Quadword
	163	@"repne _sq",
	164
	165	/// Repeat Not Zero ___ String
	166	@"repnz _s",
	167	/// Repeat Not Zero ___ String Byte
	168	@"repnz _sb",
	169	/// Repeat Not Zero ___ String Word
	170	@"repnz _sw",
	171	/// Repeat Not Zero ___ String Doubleword
	172	@"repnz _sd",
	173	/// Repeat Not Zero ___ String Quadword
	174	@"repnz _sq",
	175
	176	/// Repeat Zero ___ String
	177	@"repz _s",
	178	/// Repeat Zero ___ String Byte
	179	@"repz _sb",
	180	/// Repeat Zero ___ String Word
	181	@"repz _sw",
	182	/// Repeat Zero ___ String Doubleword
	183	@"repz _sd",
	184	/// Repeat Zero ___ String Quadword
	185	@"repz _sq",
	186
	187	/// Locked ___
	188	@"lock _",
	189	/// ___ And Complement
	190	//_c,
	191	/// Locked ___ And Complement
	192	@"lock _c",
	193	/// ___ And Reset
	194	//_r,
	195	/// Locked ___ And Reset
	196	@"lock _r",
	197	/// ___ And Set
	198	//_s,
	199	/// Locked ___ And Set
	200	@"lock _s",
	201	/// ___ 8 Bytes
	202	_8b,
	203	/// Locked ___ 8 Bytes
	204	@"lock _8b",
	205	/// ___ 16 Bytes
	206	_16b,
	207	/// Locked ___ 16 Bytes
	208	@"lock _16b",
	209
	210	/// Float ___
	211	f_,
	212	/// Float ___ Pop
	213	f_p,
	214
	215	/// Packed ___
	216	p_,
	217	/// Packed ___ Byte
	218	p_b,
	219	/// Packed ___ Word
	220	p_w,
	221	/// Packed ___ Doubleword
	222	p_d,
	223	/// Packed ___ Quadword
	224	p_q,
	225	/// Packed ___ Double Quadword
	226	p_dq,
	227
	228	/// ___ Scalar Single-Precision Values
	229	_ss,
	230	/// ___ Packed Single-Precision Values
	231	_ps,
	232	/// ___ Scalar Double-Precision Values
	233	//_sd,
	234	/// ___ Packed Double-Precision Values
	235	_pd,
	236
	237	/// VEX-Encoded ___
	238	v_,
	239	/// VEX-Encoded Packed ___
	240	vp_,
	241	/// VEX-Encoded Packed ___ Byte
	242	vp_b,
	243	/// VEX-Encoded Packed ___ Word
	244	vp_w,
	245	/// VEX-Encoded Packed ___ Doubleword
	246	vp_d,
	247	/// VEX-Encoded Packed ___ Quadword
	248	vp_q,
	249	/// VEX-Encoded Packed ___ Double Quadword
	250	vp_dq,
	251	/// VEX-Encoded ___ Scalar Single-Precision Values
	252	v_ss,
	253	/// VEX-Encoded ___ Packed Single-Precision Values
	254	v_ps,
	255	/// VEX-Encoded ___ Scalar Double-Precision Values
	256	v_sd,
	257	/// VEX-Encoded ___ Packed Double-Precision Values
	258	v_pd,
	259
	260	/// Mask ___ Byte
	261	k_b,
	262	/// Mask ___ Word
	263	k_w,
	264	/// Mask ___ Doubleword
	265	k_d,
	266	/// Mask ___ Quadword
	267	k_q,
	268
	269	pub fn fromCondition(cc: bits.Condition) Fixes {
	270	return switch (cc) {
	271	inline else => \|cc_tag\| @field(Fixes, "_" ++ @tagName(cc_tag)),
	272	.z_and_np, .nz_or_p => unreachable,
	273	};
	274	}
	275	};
	276
35	277	pub const Tag = enum(u8) {
36	278	/// Add with carry
37	279	adc,
38	280	/// Add
	281	/// Add packed single-precision floating-point values
	282	/// Add scalar single-precision floating-point values
	283	/// Add packed double-precision floating-point values
	284	/// Add scalar double-precision floating-point values
39	285	add,
40	286	/// Logical and
	287	/// Bitwise logical and of packed single-precision floating-point values
	288	/// Bitwise logical and of packed double-precision floating-point values
41	289	@"and",
42	290	/// Bit scan forward
43	291	bsf,
...	...	@@ -46,49 +294,55 @@ pub const Inst = struct {
46	294	/// Byte swap
47	295	bswap,
48	296	/// Bit test
49		bt,
50	297	/// Bit test and complement
51		btc,
52	298	/// Bit test and reset
53		btr,
54	299	/// Bit test and set
55		bts,
	300	bt,
56	301	/// Call
57	302	call,
58	303	/// Convert byte to word
59	304	cbw,
60		/// Convert word to doubleword
61		cwde,
62		/// Convert doubleword to quadword
63		cdqe,
64		/// Convert word to doubleword
65		cwd,
66	305	/// Convert doubleword to quadword
67	306	cdq,
68	307	/// Convert doubleword to quadword
69		cqo,
	308	cdqe,
	309	/// Conditional move
	310	cmov,
70	311	/// Logical compare
	312	/// Compare string
	313	/// Compare scalar single-precision floating-point values
	314	/// Compare scalar double-precision floating-point values
71	315	cmp,
72	316	/// Compare and exchange
73		cmpxchg,
74	317	/// Compare and exchange bytes
75		cmpxchgb,
	318	cmpxchg,
	319	/// Convert doubleword to quadword
	320	cqo,
	321	/// Convert word to doubleword
	322	cwd,
	323	/// Convert word to doubleword
	324	cwde,
76	325	/// Unsigned division
77		div,
78		/// Store integer with truncation
79		fisttp,
80		/// Load floating-point value
81		fld,
82	326	/// Signed division
83		idiv,
84		/// Signed multiplication
85		imul,
	327	/// Divide packed single-precision floating-point values
	328	/// Divide scalar single-precision floating-point values
	329	/// Divide packed double-precision floating-point values
	330	/// Divide scalar double-precision floating-point values
	331	div,
86	332	///
87	333	int3,
	334	/// Store integer with truncation
	335	istt,
	336	/// Conditional jump
	337	j,
88	338	/// Jump
89	339	jmp,
	340	/// Load floating-point value
	341	ld,
90	342	/// Load effective address
91	343	lea,
	344	/// Load string
	345	lod,
92	346	/// Load fence
93	347	lfence,
94	348	/// Count the number of leading zero bits
...	...	@@ -96,18 +350,24 @@ pub const Inst = struct {
96	350	/// Memory fence
97	351	mfence,
98	352	/// Move
	353	/// Move data from string to string
	354	/// Move scalar single-precision floating-point value
	355	/// Move scalar double-precision floating-point value
	356	/// Move doubleword
	357	/// Move quadword
99	358	mov,
100	359	/// Move data after swapping bytes
101	360	movbe,
102		/// Move doubleword
103		movd,
104		/// Move quadword
105		movq,
106	361	/// Move with sign extension
107	362	movsx,
108	363	/// Move with zero extension
109	364	movzx,
110	365	/// Multiply
	366	/// Signed multiplication
	367	/// Multiply packed single-precision floating-point values
	368	/// Multiply scalar single-precision floating-point values
	369	/// Multiply packed double-precision floating-point values
	370	/// Multiply scalar double-precision floating-point values
111	371	mul,
112	372	/// Two's complement negation
113	373	neg,
...	...	@@ -116,6 +376,8 @@ pub const Inst = struct {
116	376	/// One's complement negation
117	377	not,
118	378	/// Logical or
	379	/// Bitwise logical or of packed single-precision floating-point values
	380	/// Bitwise logical or of packed double-precision floating-point values
119	381	@"or",
120	382	/// Pop
121	383	pop,
...	...	@@ -124,33 +386,37 @@ pub const Inst = struct {
124	386	/// Push
125	387	push,
126	388	/// Rotate left through carry
127		rcl,
128	389	/// Rotate right through carry
129		rcr,
	390	rc,
130	391	/// Return
131	392	ret,
132	393	/// Rotate left
133		rol,
134	394	/// Rotate right
135		ror,
	395	ro,
136	396	/// Arithmetic shift left
137		sal,
138	397	/// Arithmetic shift right
139		sar,
	398	sa,
140	399	/// Integer subtraction with borrow
141	400	sbb,
	401	/// Scan string
	402	sca,
	403	/// Set byte on condition
	404	set,
142	405	/// Store fence
143	406	sfence,
144	407	/// Logical shift left
145		shl,
146	408	/// Double precision shift left
147		shld,
148	409	/// Logical shift right
149		shr,
150	410	/// Double precision shift right
151		shrd,
	411	sh,
152	412	/// Subtract
	413	/// Subtract packed single-precision floating-point values
	414	/// Subtract scalar single-precision floating-point values
	415	/// Subtract packed double-precision floating-point values
	416	/// Subtract scalar double-precision floating-point values
153	417	sub,
	418	/// Store string
	419	sto,
154	420	/// Syscall
155	421	syscall,
156	422	/// Test condition
...	...	@@ -164,142 +430,131 @@ pub const Inst = struct {
164	430	/// Exchange register/memory with register
165	431	xchg,
166	432	/// Logical exclusive-or
	433	/// Bitwise logical xor of packed single-precision floating-point values
	434	/// Bitwise logical xor of packed double-precision floating-point values
167	435	xor,
168	436
169		/// Add single precision floating point values
170		addss,
171		/// Bitwise logical and of packed single precision floating-point values
172		andps,
173		/// Bitwise logical and not of packed single precision floating-point values
174		andnps,
175		/// Compare scalar single-precision floating-point values
176		cmpss,
	437	/// Bitwise logical and not of packed single-precision floating-point values
	438	/// Bitwise logical and not of packed double-precision floating-point values
	439	andn,
177	440	/// Convert doubleword integer to scalar single-precision floating-point value
178	441	cvtsi2ss,
179		/// Divide scalar single-precision floating-point values
180		divss,
181		/// Return maximum single-precision floating-point value
182		maxss,
183		/// Return minimum single-precision floating-point value
184		minss,
	442	/// Maximum of packed single-precision floating-point values
	443	/// Maximum of scalar single-precision floating-point values
	444	/// Maximum of packed double-precision floating-point values
	445	/// Maximum of scalar double-precision floating-point values
	446	max,
	447	/// Minimum of packed single-precision floating-point values
	448	/// Minimum of scalar single-precision floating-point values
	449	/// Minimum of packed double-precision floating-point values
	450	/// Minimum of scalar double-precision floating-point values
	451	min,
185	452	/// Move aligned packed single-precision floating-point values
186		movaps,
187		/// Move scalar single-precision floating-point value
188		movss,
	453	/// Move aligned packed double-precision floating-point values
	454	mova,
	455	/// Move packed single-precision floating-point values high to low
	456	movhl,
189	457	/// Move unaligned packed single-precision floating-point values
190		movups,
191		/// Multiply scalar single-precision floating-point values
192		mulss,
193		/// Bitwise logical or of packed single precision floating-point values
194		orps,
	458	/// Move unaligned packed double-precision floating-point values
	459	movu,
	460	/// Extract byte
195	461	/// Extract word
196		pextrw,
	462	/// Extract doubleword
	463	/// Extract quadword
	464	extr,
	465	/// Insert byte
197	466	/// Insert word
198		pinsrw,
199		/// Round scalar single-precision floating-point values
200		roundss,
201		/// Square root of scalar single precision floating-point value
202		sqrtps,
203		/// Subtract scalar single-precision floating-point values
204		sqrtss,
205		/// Square root of single precision floating-point values
206		subss,
	467	/// Insert doubleword
	468	/// Insert quadword
	469	insr,
	470	/// Square root of packed single-precision floating-point values
	471	/// Square root of scalar single-precision floating-point value
	472	/// Square root of packed double-precision floating-point values
	473	/// Square root of scalar double-precision floating-point value
	474	sqrt,
207	475	/// Unordered compare scalar single-precision floating-point values
208		ucomiss,
209		/// Bitwise logical xor of packed single precision floating-point values
210		xorps,
211		/// Add double precision floating point values
212		addsd,
213		/// Bitwise logical and not of packed double precision floating-point values
214		andnpd,
215		/// Bitwise logical and of packed double precision floating-point values
216		andpd,
217		/// Compare scalar double-precision floating-point values
218		cmpsd,
	476	/// Unordered compare scalar double-precision floating-point values
	477	ucomi,
	478	/// Unpack and interleave high packed single-precision floating-point values
	479	/// Unpack and interleave high packed double-precision floating-point values
	480	unpckh,
	481	/// Unpack and interleave low packed single-precision floating-point values
	482	/// Unpack and interleave low packed double-precision floating-point values
	483	unpckl,
	484
219	485	/// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
220	486	cvtsd2ss,
221	487	/// Convert doubleword integer to scalar double-precision floating-point value
222	488	cvtsi2sd,
223	489	/// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
224	490	cvtss2sd,
225		/// Divide scalar double-precision floating-point values
226		divsd,
227		/// Return maximum double-precision floating-point value
228		maxsd,
229		/// Return minimum double-precision floating-point value
230		minsd,
231		/// Move scalar double-precision floating-point value
232		movsd,
233		/// Multiply scalar double-precision floating-point values
234		mulsd,
235		/// Bitwise logical or of packed double precision floating-point values
236		orpd,
237		/// Round scalar double-precision floating-point values
238		roundsd,
239		/// Square root of double precision floating-point values
240		sqrtpd,
241		/// Square root of scalar double precision floating-point value
242		sqrtsd,
243		/// Subtract scalar double-precision floating-point values
244		subsd,
245		/// Unordered compare scalar double-precision floating-point values
246		ucomisd,
247		/// Bitwise logical xor of packed double precision floating-point values
248		xorpd,
	491	/// Shuffle packed high words
	492	shufh,
	493	/// Shuffle packed low words
	494	shufl,
	495	/// Shift packed data right logical
	496	/// Shift packed data right logical
	497	/// Shift packed data right logical
	498	srl,
	499	/// Unpack high data
	500	unpckhbw,
	501	/// Unpack high data
	502	unpckhdq,
	503	/// Unpack high data
	504	unpckhqdq,
	505	/// Unpack high data
	506	unpckhwd,
	507	/// Unpack low data
	508	unpcklbw,
	509	/// Unpack low data
	510	unpckldq,
	511	/// Unpack low data
	512	unpcklqdq,
	513	/// Unpack low data
	514	unpcklwd,
249	515
250		/// Compare string operands
251		cmps,
252		/// Load string
253		lods,
254		/// Move data from string to string
255		movs,
256		/// Scan string
257		scas,
258		/// Store string
259		stos,
260
261		/// Conditional move
262		cmovcc,
263		/// Conditional jump
264		jcc,
265		/// Set byte on condition
266		setcc,
267
268		/// Mov absolute to/from memory wrt segment register to/from rax
269		mov_moffs,
270
271		/// Jump with relocation to another local MIR instruction
272		/// Uses `inst` payload.
273		jmp_reloc,
	516	/// Replicate double floating-point values
	517	movddup,
	518	/// Replicate single floating-point values
	519	movshdup,
	520	/// Replicate single floating-point values
	521	movsldup,
274	522
275		/// Call to an extern symbol via linker relocation.
276		/// Uses `relocation` payload.
277		call_extern,
	523	/// Round packed single-precision floating-point values
	524	/// Round scalar single-precision floating-point value
	525	/// Round packed double-precision floating-point values
	526	/// Round scalar double-precision floating-point value
	527	round,
278	528
279		/// Load effective address of a symbol not yet allocated in VM.
280		lea_linker,
281		/// Move address of a symbol not yet allocated in VM.
282		mov_linker,
	529	/// Convert 16-bit floating-point values to single-precision floating-point values
	530	cvtph2ps,
	531	/// Convert single-precision floating-point values to 16-bit floating-point values
	532	cvtps2ph,
283	533
284		/// End of prologue
285		dbg_prologue_end,
286		/// Start of epilogue
287		dbg_epilogue_begin,
288		/// Update debug line
289		/// Uses `line_column` payload containing the line and column.
290		dbg_line,
291		/// Push registers
292		/// Uses `payload` payload containing `RegisterList.asInt` directly.
293		push_regs,
294		/// Pop registers
295		/// Uses `payload` payload containing `RegisterList.asInt` directly.
296		pop_regs,
	534	/// Fused multiply-add of packed single-precision floating-point values
	535	/// Fused multiply-add of scalar single-precision floating-point values
	536	/// Fused multiply-add of packed double-precision floating-point values
	537	/// Fused multiply-add of scalar double-precision floating-point values
	538	fmadd132,
	539	/// Fused multiply-add of packed single-precision floating-point values
	540	/// Fused multiply-add of scalar single-precision floating-point values
	541	/// Fused multiply-add of packed double-precision floating-point values
	542	/// Fused multiply-add of scalar double-precision floating-point values
	543	fmadd213,
	544	/// Fused multiply-add of packed single-precision floating-point values
	545	/// Fused multiply-add of scalar single-precision floating-point values
	546	/// Fused multiply-add of packed double-precision floating-point values
	547	/// Fused multiply-add of scalar double-precision floating-point values
	548	fmadd231,
297	549
298		/// Tombstone
299		/// Emitter should skip this instruction.
300		dead,
	550	/// A pseudo instruction that requires special lowering.
	551	/// This should be the only tag in this enum that doesn't
	552	/// directly correspond to one or more instruction mnemonics.
	553	pseudo,
301	554	};
302	555
	556	pub const FixedTag = struct { Fixes, Tag };
	557
303	558	pub const Ops = enum(u8) {
304	559	/// No data associated with this instruction (only mnemonic is used).
305	560	none,
...	...	@@ -312,18 +567,15 @@ pub const Inst = struct {
312	567	/// Register, register, register operands.
313	568	/// Uses `rrr` payload.
314	569	rrr,
	570	/// Register, register, register, immediate (byte) operands.
	571	/// Uses `rrri` payload.
	572	rrri,
315	573	/// Register, register, immediate (sign-extended) operands.
316	574	/// Uses `rri` payload.
317	575	rri_s,
318	576	/// Register, register, immediate (unsigned) operands.
319	577	/// Uses `rri` payload.
320	578	rri_u,
321		/// Register with condition code (CC).
322		/// Uses `r_cc` payload.
323		r_cc,
324		/// Register, register with condition code (CC).
325		/// Uses `rr_cc` payload.
326		rr_cc,
327	579	/// Register, immediate (sign-extended) operands.
328	580	/// Uses `ri` payload.
329	581	ri_s,
...	...	@@ -348,41 +600,41 @@ pub const Inst = struct {
348	600	/// Register, memory (RIP) operands.
349	601	/// Uses `rx` payload.
350	602	rm_rip,
351		/// Register, memory (SIB) operands with condition code (CC).
352		/// Uses `rx_cc` payload.
353		rm_sib_cc,
354		/// Register, memory (RIP) operands with condition code (CC).
355		/// Uses `rx_cc` payload.
356		rm_rip_cc,
357	603	/// Register, memory (SIB), immediate (byte) operands.
358	604	/// Uses `rix` payload with extra data of type `MemorySib`.
359	605	rmi_sib,
	606	/// Register, register, memory (RIP).
	607	/// Uses `rrix` payload with extra data of type `MemoryRip`.
	608	rrm_rip,
	609	/// Register, register, memory (SIB).
	610	/// Uses `rrix` payload with extra data of type `MemorySib`.
	611	rrm_sib,
	612	/// Register, register, memory (RIP), immediate (byte) operands.
	613	/// Uses `rrix` payload with extra data of type `MemoryRip`.
	614	rrmi_rip,
	615	/// Register, register, memory (SIB), immediate (byte) operands.
	616	/// Uses `rrix` payload with extra data of type `MemorySib`.
	617	rrmi_sib,
360	618	/// Register, memory (RIP), immediate (byte) operands.
361	619	/// Uses `rix` payload with extra data of type `MemoryRip`.
362	620	rmi_rip,
363	621	/// Single memory (SIB) operand.
364		/// Uses `payload` with extra data of type `MemorySib`.
	622	/// Uses `x` with extra data of type `MemorySib`.
365	623	m_sib,
366	624	/// Single memory (RIP) operand.
367		/// Uses `payload` with extra data of type `MemoryRip`.
	625	/// Uses `x` with extra data of type `MemoryRip`.
368	626	m_rip,
369		/// Single memory (SIB) operand with condition code (CC).
370		/// Uses `x_cc` with extra data of type `MemorySib`.
371		m_sib_cc,
372		/// Single memory (RIP) operand with condition code (CC).
373		/// Uses `x_cc` with extra data of type `MemoryRip`.
374		m_rip_cc,
375	627	/// Memory (SIB), immediate (unsigned) operands.
376		/// Uses `ix` payload with extra data of type `MemorySib`.
	628	/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
377	629	mi_sib_u,
378	630	/// Memory (RIP), immediate (unsigned) operands.
379		/// Uses `ix` payload with extra data of type `MemoryRip`.
	631	/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
380	632	mi_rip_u,
381	633	/// Memory (SIB), immediate (sign-extend) operands.
382		/// Uses `ix` payload with extra data of type `MemorySib`.
	634	/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
383	635	mi_sib_s,
384	636	/// Memory (RIP), immediate (sign-extend) operands.
385		/// Uses `ix` payload with extra data of type `MemoryRip`.
	637	/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
386	638	mi_rip_s,
387	639	/// Memory (SIB), register operands.
388	640	/// Uses `rx` payload with extra data of type `MemorySib`.
...	...	@@ -403,161 +655,200 @@ pub const Inst = struct {
403	655	/// Uses `rix` payload with extra data of type `MemoryRip`.
404	656	mri_rip,
405	657	/// Rax, Memory moffs.
406		/// Uses `payload` with extra data of type `MemoryMoffs`.
	658	/// Uses `x` with extra data of type `MemoryMoffs`.
407	659	rax_moffs,
408	660	/// Memory moffs, rax.
409		/// Uses `payload` with extra data of type `MemoryMoffs`.
	661	/// Uses `x` with extra data of type `MemoryMoffs`.
410	662	moffs_rax,
411		/// Single memory (SIB) operand with lock prefix.
412		/// Uses `payload` with extra data of type `MemorySib`.
413		lock_m_sib,
414		/// Single memory (RIP) operand with lock prefix.
415		/// Uses `payload` with extra data of type `MemoryRip`.
416		lock_m_rip,
417		/// Memory (SIB), immediate (unsigned) operands with lock prefix.
418		/// Uses `xi` payload with extra data of type `MemorySib`.
419		lock_mi_sib_u,
420		/// Memory (RIP), immediate (unsigned) operands with lock prefix.
421		/// Uses `xi` payload with extra data of type `MemoryRip`.
422		lock_mi_rip_u,
423		/// Memory (SIB), immediate (sign-extend) operands with lock prefix.
424		/// Uses `xi` payload with extra data of type `MemorySib`.
425		lock_mi_sib_s,
426		/// Memory (RIP), immediate (sign-extend) operands with lock prefix.
427		/// Uses `xi` payload with extra data of type `MemoryRip`.
428		lock_mi_rip_s,
429		/// Memory (SIB), register operands with lock prefix.
430		/// Uses `rx` payload with extra data of type `MemorySib`.
431		lock_mr_sib,
432		/// Memory (RIP), register operands with lock prefix.
433		/// Uses `rx` payload with extra data of type `MemoryRip`.
434		lock_mr_rip,
435		/// Memory moffs, rax with lock prefix.
436		/// Uses `payload` with extra data of type `MemoryMoffs`.
437		lock_moffs_rax,
438	663	/// References another Mir instruction directly.
439	664	/// Uses `inst` payload.
440	665	inst,
441		/// References another Mir instruction directly with condition code (CC).
442		/// Uses `inst_cc` payload.
443		inst_cc,
444		/// String repeat and width
445		/// Uses `string` payload.
446		string,
	666	/// Linker relocation - external function.
447	667	/// Uses `reloc` payload.
448		reloc,
	668	extern_fn_reloc,
449	669	/// Linker relocation - GOT indirection.
450		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	670	/// Uses `rx` payload with extra data of type `Reloc`.
451	671	got_reloc,
452	672	/// Linker relocation - direct reference.
453		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	673	/// Uses `rx` payload with extra data of type `Reloc`.
454	674	direct_reloc,
455	675	/// Linker relocation - imports table indirection (binding).
456		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	676	/// Uses `rx` payload with extra data of type `Reloc`.
457	677	import_reloc,
458	678	/// Linker relocation - threadlocal variable via GOT indirection.
459		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	679	/// Uses `rx` payload with extra data of type `Reloc`.
460	680	tlv_reloc,
	681
	682	// Pseudo instructions:
	683
	684	/// Conditional move if zero flag set and parity flag not set
	685	/// Clobbers the source operand!
	686	/// Uses `rr` payload.
	687	pseudo_cmov_z_and_np_rr,
	688	/// Conditional move if zero flag not set or parity flag set
	689	/// Uses `rr` payload.
	690	pseudo_cmov_nz_or_p_rr,
	691	/// Conditional move if zero flag not set or parity flag set
	692	/// Uses `rx` payload.
	693	pseudo_cmov_nz_or_p_rm_sib,
	694	/// Conditional move if zero flag not set or parity flag set
	695	/// Uses `rx` payload.
	696	pseudo_cmov_nz_or_p_rm_rip,
	697	/// Set byte if zero flag set and parity flag not set
	698	/// Requires a scratch register!
	699	/// Uses `r_scratch` payload.
	700	pseudo_set_z_and_np_r,
	701	/// Set byte if zero flag set and parity flag not set
	702	/// Requires a scratch register!
	703	/// Uses `x_scratch` payload.
	704	pseudo_set_z_and_np_m_sib,
	705	/// Set byte if zero flag set and parity flag not set
	706	/// Requires a scratch register!
	707	/// Uses `x_scratch` payload.
	708	pseudo_set_z_and_np_m_rip,
	709	/// Set byte if zero flag not set or parity flag set
	710	/// Requires a scratch register!
	711	/// Uses `r_scratch` payload.
	712	pseudo_set_nz_or_p_r,
	713	/// Set byte if zero flag not set or parity flag set
	714	/// Requires a scratch register!
	715	/// Uses `x_scratch` payload.
	716	pseudo_set_nz_or_p_m_sib,
	717	/// Set byte if zero flag not set or parity flag set
	718	/// Requires a scratch register!
	719	/// Uses `x_scratch` payload.
	720	pseudo_set_nz_or_p_m_rip,
	721	/// Jump if zero flag set and parity flag not set
	722	/// Uses `inst` payload.
	723	pseudo_j_z_and_np_inst,
	724	/// Jump if zero flag not set or parity flag set
	725	/// Uses `inst` payload.
	726	pseudo_j_nz_or_p_inst,
	727
	728	/// Push registers
	729	/// Uses `reg_list` payload.
	730	pseudo_push_reg_list,
	731	/// Pop registers
	732	/// Uses `reg_list` payload.
	733	pseudo_pop_reg_list,
	734
	735	/// End of prologue
	736	pseudo_dbg_prologue_end_none,
	737	/// Update debug line
	738	/// Uses `line_column` payload.
	739	pseudo_dbg_line_line_column,
	740	/// Start of epilogue
	741	pseudo_dbg_epilogue_begin_none,
	742
	743	/// Tombstone
	744	/// Emitter should skip this instruction.
	745	pseudo_dead_none,
461	746	};
462	747
463	748	pub const Data = union {
	749	none: struct {
	750	fixes: Fixes = ._,
	751	},
464	752	/// References another Mir instruction.
465		inst: Index,
466		/// Another instruction with condition code (CC).
467		/// Used by `jcc`.
468		inst_cc: struct {
469		/// Another instruction.
	753	inst: struct {
	754	fixes: Fixes = ._,
470	755	inst: Index,
471		/// A condition code for use with EFLAGS register.
472		cc: bits.Condition,
473	756	},
474	757	/// A 32-bit immediate value.
475		i: u32,
476		r: Register,
	758	i: struct {
	759	fixes: Fixes = ._,
	760	i: u32,
	761	},
	762	r: struct {
	763	fixes: Fixes = ._,
	764	r1: Register,
	765	},
477	766	rr: struct {
	767	fixes: Fixes = ._,
478	768	r1: Register,
479	769	r2: Register,
480	770	},
481	771	rrr: struct {
	772	fixes: Fixes = ._,
482	773	r1: Register,
483	774	r2: Register,
484	775	r3: Register,
485	776	},
486		rri: struct {
	777	rrri: struct {
	778	fixes: Fixes = ._,
487	779	r1: Register,
488	780	r2: Register,
489		i: u32,
490		},
491		/// Condition code (CC), followed by custom payload found in extra.
492		x_cc: struct {
493		cc: bits.Condition,
494		payload: u32,
495		},
496		/// Register with condition code (CC).
497		r_cc: struct {
498		r: Register,
499		cc: bits.Condition,
	781	r3: Register,
	782	i: u8,
500	783	},
501		/// Register, register with condition code (CC).
502		rr_cc: struct {
	784	rri: struct {
	785	fixes: Fixes = ._,
503	786	r1: Register,
504	787	r2: Register,
505		cc: bits.Condition,
	788	i: u32,
506	789	},
507	790	/// Register, immediate.
508	791	ri: struct {
509		r: Register,
	792	fixes: Fixes = ._,
	793	r1: Register,
510	794	i: u32,
511	795	},
512	796	/// Register, followed by custom payload found in extra.
513	797	rx: struct {
514		r: Register,
515		payload: u32,
516		},
517		/// Register with condition code (CC), followed by custom payload found in extra.
518		rx_cc: struct {
519		r: Register,
520		cc: bits.Condition,
521		payload: u32,
522		},
523		/// Immediate, followed by Custom payload found in extra.
524		ix: struct {
525		i: u32,
	798	fixes: Fixes = ._,
	799	r1: Register,
526	800	payload: u32,
527	801	},
528	802	/// Register, register, followed by Custom payload found in extra.
529	803	rrx: struct {
	804	fixes: Fixes = ._,
530	805	r1: Register,
531	806	r2: Register,
532	807	payload: u32,
533	808	},
534	809	/// Register, byte immediate, followed by Custom payload found in extra.
535	810	rix: struct {
536		r: Register,
	811	fixes: Fixes = ._,
	812	r1: Register,
	813	i: u8,
	814	payload: u32,
	815	},
	816	/// Register, register, byte immediate, followed by Custom payload found in extra.
	817	rrix: struct {
	818	fixes: Fixes = ._,
	819	r1: Register,
	820	r2: Register,
537	821	i: u8,
538	822	payload: u32,
539	823	},
540		/// String instruction prefix and width.
541		string: struct {
542		repeat: bits.StringRepeat,
543		width: bits.StringWidth,
	824	/// Register, scratch register
	825	r_scratch: struct {
	826	fixes: Fixes = ._,
	827	r1: Register,
	828	scratch_reg: Register,
	829	},
	830	/// Scratch register, followed by Custom payload found in extra.
	831	x_scratch: struct {
	832	fixes: Fixes = ._,
	833	scratch_reg: Register,
	834	payload: u32,
	835	},
	836	/// Custom payload found in extra.
	837	x: struct {
	838	fixes: Fixes = ._,
	839	payload: u32,
544	840	},
545	841	/// Relocation for the linker where:
546	842	/// * `atom_index` is the index of the source
547	843	/// * `sym_index` is the index of the target
548		relocation: struct {
549		/// Index of the containing atom.
550		atom_index: u32,
551		/// Index into the linker's symbol table.
552		sym_index: u32,
553		},
	844	reloc: Reloc,
554	845	/// Debug line and column position
555	846	line_column: struct {
556	847	line: u32,
557	848	column: u32,
558	849	},
559		/// Index into `extra`. Meaning of what can be found there is context-dependent.
560		payload: u32,
	850	/// Register list
	851	reg_list: RegisterList,
561	852	};
562	853
563	854	// Make sure we don't accidentally make instructions bigger than expected.
...	...	@@ -569,9 +860,8 @@ pub const Inst = struct {
569	860	}
570	861	};
571	862
572		pub const LeaRegisterReloc = struct {
573		/// Destination register.
574		reg: u32,
	863	/// A linker symbol not yet allocated in VM.
	864	pub const Reloc = struct {
575	865	/// Index of the containing atom.
576	866	atom_index: u32,
577	867	/// Index into the linker's symbol table.
...	...	@@ -606,21 +896,15 @@ pub const RegisterList = struct {
606	896	return self.bitset.iterator(options);
607	897	}
608	898
609		pub fn asInt(self: Self) u32 {
610		return self.bitset.mask;
611		}
612
613		pub fn fromInt(mask: u32) Self {
614		return .{
615		.bitset = BitSet{ .mask = @intCast(BitSet.MaskInt, mask) },
616		};
617		}
618
619	899	pub fn count(self: Self) u32 {
620	900	return @intCast(u32, self.bitset.count());
621	901	}
622	902	};
623	903
	904	pub const Imm32 = struct {
	905	imm: u32,
	906	};
	907
624	908	pub const Imm64 = struct {
625	909	msb: u32,
626	910	lsb: u32,

src/arch/x86_64/bits.zig+21-7

...	...	@@ -6,9 +6,6 @@ const Allocator = std.mem.Allocator;
6	6	const ArrayList = std.ArrayList;
7	7	const DW = std.dwarf;
8	8
9		pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
10		pub const StringWidth = enum(u2) { b, w, d, q };
11
12	9	/// EFLAGS condition codes
13	10	pub const Condition = enum(u5) {
14	11	/// above
...	...	@@ -72,6 +69,12 @@ pub const Condition = enum(u5) {
72	69	/// zero
73	70	z,
74	71
	72	// Pseudo conditions
	73	/// zero and not parity
	74	z_and_np,
	75	/// not zero or parity
	76	nz_or_p,
	77
75	78	/// Converts a std.math.CompareOperator into a condition flag,
76	79	/// i.e. returns the condition that is true iff the result of the
77	80	/// comparison is true. Assumes signed comparison
...	...	@@ -143,6 +146,9 @@ pub const Condition = enum(u5) {
143	146	.po => .pe,
144	147	.s => .ns,
145	148	.z => .nz,
	149
	150	.z_and_np => .nz_or_p,
	151	.nz_or_p => .z_and_np,
146	152	};
147	153	}
148	154	};
...	...	@@ -476,7 +482,9 @@ pub const Memory = union(enum) {
476	482	dword,
477	483	qword,
478	484	tbyte,
479		dqword,
	485	xword,
	486	yword,
	487	zword,
480	488
481	489	pub fn fromSize(size: u32) PtrSize {
482	490	return switch (size) {
...	...	@@ -484,7 +492,9 @@ pub const Memory = union(enum) {
484	492	2...2 => .word,
485	493	3...4 => .dword,
486	494	5...8 => .qword,
487		9...16 => .dqword,
	495	9...16 => .xword,
	496	17...32 => .yword,
	497	33...64 => .zword,
488	498	else => unreachable,
489	499	};
490	500	}
...	...	@@ -496,7 +506,9 @@ pub const Memory = union(enum) {
496	506	32 => .dword,
497	507	64 => .qword,
498	508	80 => .tbyte,
499		128 => .dqword,
	509	128 => .xword,
	510	256 => .yword,
	511	512 => .zword,
500	512	else => unreachable,
501	513	};
502	514	}
...	...	@@ -508,7 +520,9 @@ pub const Memory = union(enum) {
508	520	.dword => 32,
509	521	.qword => 64,
510	522	.tbyte => 80,
511		.dqword => 128,
	523	.xword => 128,
	524	.yword => 256,
	525	.zword => 512,
512	526	};
513	527	}
514	528	};

src/arch/x86_64/encoder.zig+154-35

...	...	@@ -151,15 +151,12 @@ pub const Instruction = struct {
151	151	moffs.offset,
152	152	}),
153	153	},
154		.imm => \|imm\| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.bitSize())}),
	154	.imm => \|imm\| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.immBitSize())}),
155	155	}
156	156	}
157	157
158	158	pub fn fmtPrint(op: Operand, enc_op: Encoding.Op) std.fmt.Formatter(fmt) {
159		return .{ .data = .{
160		.op = op,
161		.enc_op = enc_op,
162		} };
	159	return .{ .data = .{ .op = op, .enc_op = enc_op } };
163	160	}
164	161	};
165	162
...	...	@@ -209,10 +206,16 @@ pub const Instruction = struct {
209	206	const enc = inst.encoding;
210	207	const data = enc.data;
211	208
212		try inst.encodeLegacyPrefixes(encoder);
213		try inst.encodeMandatoryPrefix(encoder);
214		try inst.encodeRexPrefix(encoder);
215		try inst.encodeOpcode(encoder);
	209	if (data.mode.isVex()) {
	210	try inst.encodeVexPrefix(encoder);
	211	const opc = inst.encoding.opcode();
	212	try encoder.opcode_1byte(opc[opc.len - 1]);
	213	} else {
	214	try inst.encodeLegacyPrefixes(encoder);
	215	try inst.encodeMandatoryPrefix(encoder);
	216	try inst.encodeRexPrefix(encoder);
	217	try inst.encodeOpcode(encoder);
	218	}
216	219
217	220	switch (data.op_en) {
218	221	.np, .o => {},
...	...	@@ -222,25 +225,28 @@ pub const Instruction = struct {
222	225	.td => try encoder.imm64(inst.ops[0].mem.moffs.offset),
223	226	else => {
224	227	const mem_op = switch (data.op_en) {
225		.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
226		.rm, .rmi => inst.ops[1],
	228	.m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
	229	.rm, .rmi, .vmi => inst.ops[1],
	230	.rvm, .rvmi => inst.ops[2],
227	231	else => unreachable,
228	232	};
229	233	switch (mem_op) {
230	234	.reg => \|reg\| {
231	235	const rm = switch (data.op_en) {
232		.m, .mi, .m1, .mc => enc.modRmExt(),
	236	.m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
233	237	.mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
234		.rm, .rmi => inst.ops[0].reg.lowEnc(),
	238	.rm, .rmi, .rvm, .rvmi => inst.ops[0].reg.lowEnc(),
	239	.mvr => inst.ops[2].reg.lowEnc(),
235	240	else => unreachable,
236	241	};
237	242	try encoder.modRm_direct(rm, reg.lowEnc());
238	243	},
239	244	.mem => \|mem\| {
240	245	const op = switch (data.op_en) {
241		.m, .mi, .m1, .mc => .none,
	246	.m, .mi, .m1, .mc, .vmi => .none,
242	247	.mr, .mri, .mrc => inst.ops[1],
243		.rm, .rmi => inst.ops[0],
	248	.rm, .rmi, .rvm, .rvmi => inst.ops[0],
	249	.mvr => inst.ops[2],
244	250	else => unreachable,
245	251	};
246	252	try encodeMemory(enc, mem, op, encoder);
...	...	@@ -250,7 +256,8 @@ pub const Instruction = struct {
250	256
251	257	switch (data.op_en) {
252	258	.mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
253		.rmi, .mri => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),
	259	.rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),
	260	.rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder),
254	261	else => {},
255	262	}
256	263	},
...	...	@@ -282,11 +289,9 @@ pub const Instruction = struct {
282	289	.rep, .repe, .repz => legacy.prefix_f3 = true,
283	290	}
284	291
285		if (data.mode == .none) {
286		const bit_size = enc.operandBitSize();
287		if (bit_size == 16) {
288		legacy.set16BitOverride();
289		}
	292	switch (data.mode) {
	293	.short, .rex_short => legacy.set16BitOverride(),
	294	else => {},
290	295	}
291	296
292	297	const segment_override: ?Register = switch (op_en) {
...	...	@@ -309,6 +314,7 @@ pub const Instruction = struct {
309	314	}
310	315	else
311	316	null,
	317	.vmi, .rvm, .rvmi, .mvr => unreachable,
312	318	};
313	319	if (segment_override) \|seg\| {
314	320	legacy.setSegmentOverride(seg);
...	...	@@ -322,10 +328,7 @@ pub const Instruction = struct {
322	328
323	329	var rex = Rex{};
324	330	rex.present = inst.encoding.data.mode == .rex;
325		switch (inst.encoding.data.mode) {
326		.long, .sse_long, .sse2_long => rex.w = true,
327		else => {},
328		}
	331	rex.w = inst.encoding.data.mode == .long;
329	332
330	333	switch (op_en) {
331	334	.np, .i, .zi, .fd, .td, .d => {},
...	...	@@ -346,11 +349,71 @@ pub const Instruction = struct {
346	349	rex.b = b_x_op.isBaseExtended();
347	350	rex.x = b_x_op.isIndexExtended();
348	351	},
	352	.vmi, .rvm, .rvmi, .mvr => unreachable,
349	353	}
350	354
351	355	try encoder.rex(rex);
352	356	}
353	357
	358	fn encodeVexPrefix(inst: Instruction, encoder: anytype) !void {
	359	const op_en = inst.encoding.data.op_en;
	360	const opc = inst.encoding.opcode();
	361	const mand_pre = inst.encoding.mandatoryPrefix();
	362
	363	var vex = Vex{};
	364
	365	vex.w = inst.encoding.data.mode.isLong();
	366
	367	switch (op_en) {
	368	.np, .i, .zi, .fd, .td, .d => {},
	369	.o, .oi => vex.b = inst.ops[0].reg.isExtended(),
	370	.m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .vmi, .rvm, .rvmi, .mvr => {
	371	const r_op = switch (op_en) {
	372	.rm, .rmi, .rvm, .rvmi => inst.ops[0],
	373	.mr, .mri, .mrc => inst.ops[1],
	374	.mvr => inst.ops[2],
	375	.m, .mi, .m1, .mc, .vmi => .none,
	376	else => unreachable,
	377	};
	378	vex.r = r_op.isBaseExtended();
	379
	380	const b_x_op = switch (op_en) {
	381	.rm, .rmi, .vmi => inst.ops[1],
	382	.m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
	383	.rvm, .rvmi => inst.ops[2],
	384	else => unreachable,
	385	};
	386	vex.b = b_x_op.isBaseExtended();
	387	vex.x = b_x_op.isIndexExtended();
	388	},
	389	}
	390
	391	vex.l = inst.encoding.data.mode.isVecLong();
	392
	393	vex.p = if (mand_pre) \|mand\| switch (mand) {
	394	0x66 => .@"66",
	395	0xf2 => .f2,
	396	0xf3 => .f3,
	397	else => unreachable,
	398	} else .none;
	399
	400	const leading: usize = if (mand_pre) \|_\| 1 else 0;
	401	assert(opc[leading] == 0x0f);
	402	vex.m = switch (opc[leading + 1]) {
	403	else => .@"0f",
	404	0x38 => .@"0f38",
	405	0x3a => .@"0f3a",
	406	};
	407
	408	switch (op_en) {
	409	else => {},
	410	.vmi => vex.v = inst.ops[0].reg,
	411	.rvm, .rvmi => vex.v = inst.ops[1].reg,
	412	}
	413
	414	try encoder.vex(vex);
	415	}
	416
354	417	fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {
355	418	const prefix = inst.encoding.mandatoryPrefix() orelse return;
356	419	try encoder.opcode_1byte(prefix);
...	...	@@ -443,8 +506,8 @@ pub const Instruction = struct {
443	506	}
444	507
445	508	fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {
446		const raw = imm.asUnsigned(kind.bitSize());
447		switch (kind.bitSize()) {
	509	const raw = imm.asUnsigned(kind.immBitSize());
	510	switch (kind.immBitSize()) {
448	511	8 => try encoder.imm8(@intCast(u8, raw)),
449	512	16 => try encoder.imm16(@intCast(u16, raw)),
450	513	32 => try encoder.imm32(@intCast(u32, raw)),
...	...	@@ -562,17 +625,48 @@ fn Encoder(comptime T: type, comptime opts: Options) type {
562	625	/// or one of reg, index, r/m, base, or opcode-reg might be extended.
563	626	///
564	627	/// See struct `Rex` for a description of each field.
565		pub fn rex(self: Self, byte: Rex) !void {
566		if (!byte.present and !byte.isSet()) return;
	628	pub fn rex(self: Self, fields: Rex) !void {
	629	if (!fields.present and !fields.isSet()) return;
567	630
568		var value: u8 = 0b0100_0000;
	631	var byte: u8 = 0b0100_0000;
569	632
570		if (byte.w) value \|= 0b1000;
571		if (byte.r) value \|= 0b0100;
572		if (byte.x) value \|= 0b0010;
573		if (byte.b) value \|= 0b0001;
	633	if (fields.w) byte \|= 0b1000;
	634	if (fields.r) byte \|= 0b0100;
	635	if (fields.x) byte \|= 0b0010;
	636	if (fields.b) byte \|= 0b0001;
574	637
575		try self.writer.writeByte(value);
	638	try self.writer.writeByte(byte);
	639	}
	640
	641	/// Encodes a VEX prefix given all the fields
	642	///
	643	/// See struct `Vex` for a description of each field.
	644	pub fn vex(self: Self, fields: Vex) !void {
	645	if (fields.is3Byte()) {
	646	try self.writer.writeByte(0b1100_0100);
	647
	648	try self.writer.writeByte(
	649	@as(u8, ~@boolToInt(fields.r)) << 7 \|
	650	@as(u8, ~@boolToInt(fields.x)) << 6 \|
	651	@as(u8, ~@boolToInt(fields.b)) << 5 \|
	652	@as(u8, @enumToInt(fields.m)) << 0,
	653	);
	654
	655	try self.writer.writeByte(
	656	@as(u8, @boolToInt(fields.w)) << 7 \|
	657	@as(u8, ~fields.v.enc()) << 3 \|
	658	@as(u8, @boolToInt(fields.l)) << 2 \|
	659	@as(u8, @enumToInt(fields.p)) << 0,
	660	);
	661	} else {
	662	try self.writer.writeByte(0b1100_0101);
	663	try self.writer.writeByte(
	664	@as(u8, ~@boolToInt(fields.r)) << 7 \|
	665	@as(u8, ~fields.v.enc()) << 3 \|
	666	@as(u8, @boolToInt(fields.l)) << 2 \|
	667	@as(u8, @enumToInt(fields.p)) << 0,
	668	);
	669	}
576	670	}
577	671
578	672	// ------
...	...	@@ -848,6 +942,31 @@ pub const Rex = struct {
848	942	}
849	943	};
850	944
	945	pub const Vex = struct {
	946	w: bool = false,
	947	r: bool = false,
	948	x: bool = false,
	949	b: bool = false,
	950	l: bool = false,
	951	p: enum(u2) {
	952	none = 0b00,
	953	@"66" = 0b01,
	954	f3 = 0b10,
	955	f2 = 0b11,
	956	} = .none,
	957	m: enum(u5) {
	958	@"0f" = 0b0_0001,
	959	@"0f38" = 0b0_0010,
	960	@"0f3a" = 0b0_0011,
	961	_,
	962	} = .@"0f",
	963	v: Register = .ymm0,
	964
	965	pub fn is3Byte(vex: Vex) bool {
	966	return vex.w or vex.x or vex.b or vex.m != .@"0f";
	967	}
	968	};
	969
851	970	// Tests
852	971	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
853	972	assert(expected.len > 0);

src/arch/x86_64/encodings.zig+1156-870

...	...	@@ -3,933 +3,1219 @@ const Mnemonic = Encoding.Mnemonic;
3	3	const OpEn = Encoding.OpEn;
4	4	const Op = Encoding.Op;
5	5	const Mode = Encoding.Mode;
	6	const Feature = Encoding.Feature;
6	7
7	8	const modrm_ext = u3;
8	9
9		pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mode };
	10	pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mode, Feature };
10	11
11	12	// TODO move this into a .zon file when Zig is capable of importing .zon files
12	13	// zig fmt: off
13	14	pub const table = [_]Entry{
14	15	// General-purpose
15		.{ .adc, .zi, &.{ .al, .imm8 }, &.{ 0x14 }, 0, .none },
16		.{ .adc, .zi, &.{ .ax, .imm16 }, &.{ 0x15 }, 0, .none },
17		.{ .adc, .zi, &.{ .eax, .imm32 }, &.{ 0x15 }, 0, .none },
18		.{ .adc, .zi, &.{ .rax, .imm32s }, &.{ 0x15 }, 0, .long },
19		.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .none },
20		.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .rex },
21		.{ .adc, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 2, .none },
22		.{ .adc, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 2, .none },
23		.{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long },
24		.{ .adc, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 2, .none },
25		.{ .adc, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 2, .none },
26		.{ .adc, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 2, .long },
27		.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .none },
28		.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .rex },
29		.{ .adc, .mr, &.{ .rm16, .r16 }, &.{ 0x11 }, 0, .none },
30		.{ .adc, .mr, &.{ .rm32, .r32 }, &.{ 0x11 }, 0, .none },
31		.{ .adc, .mr, &.{ .rm64, .r64 }, &.{ 0x11 }, 0, .long },
32		.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .none },
33		.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .rex },
34		.{ .adc, .rm, &.{ .r16, .rm16 }, &.{ 0x13 }, 0, .none },
35		.{ .adc, .rm, &.{ .r32, .rm32 }, &.{ 0x13 }, 0, .none },
36		.{ .adc, .rm, &.{ .r64, .rm64 }, &.{ 0x13 }, 0, .long },
37
38		.{ .add, .zi, &.{ .al, .imm8 }, &.{ 0x04 }, 0, .none },
39		.{ .add, .zi, &.{ .ax, .imm16 }, &.{ 0x05 }, 0, .none },
40		.{ .add, .zi, &.{ .eax, .imm32 }, &.{ 0x05 }, 0, .none },
41		.{ .add, .zi, &.{ .rax, .imm32s }, &.{ 0x05 }, 0, .long },
42		.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .none },
43		.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .rex },
44		.{ .add, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 0, .none },
45		.{ .add, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 0, .none },
46		.{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long },
47		.{ .add, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 0, .none },
48		.{ .add, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 0, .none },
49		.{ .add, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 0, .long },
50		.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .none },
51		.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .rex },
52		.{ .add, .mr, &.{ .rm16, .r16 }, &.{ 0x01 }, 0, .none },
53		.{ .add, .mr, &.{ .rm32, .r32 }, &.{ 0x01 }, 0, .none },
54		.{ .add, .mr, &.{ .rm64, .r64 }, &.{ 0x01 }, 0, .long },
55		.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .none },
56		.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .rex },
57		.{ .add, .rm, &.{ .r16, .rm16 }, &.{ 0x03 }, 0, .none },
58		.{ .add, .rm, &.{ .r32, .rm32 }, &.{ 0x03 }, 0, .none },
59		.{ .add, .rm, &.{ .r64, .rm64 }, &.{ 0x03 }, 0, .long },
60
61		.{ .@"and", .zi, &.{ .al, .imm8 }, &.{ 0x24 }, 0, .none },
62		.{ .@"and", .zi, &.{ .ax, .imm16 }, &.{ 0x25 }, 0, .none },
63		.{ .@"and", .zi, &.{ .eax, .imm32 }, &.{ 0x25 }, 0, .none },
64		.{ .@"and", .zi, &.{ .rax, .imm32s }, &.{ 0x25 }, 0, .long },
65		.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .none },
66		.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .rex },
67		.{ .@"and", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 4, .none },
68		.{ .@"and", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 4, .none },
69		.{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long },
70		.{ .@"and", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 4, .none },
71		.{ .@"and", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 4, .none },
72		.{ .@"and", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 4, .long },
73		.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .none },
74		.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .rex },
75		.{ .@"and", .mr, &.{ .rm16, .r16 }, &.{ 0x21 }, 0, .none },
76		.{ .@"and", .mr, &.{ .rm32, .r32 }, &.{ 0x21 }, 0, .none },
77		.{ .@"and", .mr, &.{ .rm64, .r64 }, &.{ 0x21 }, 0, .long },
78		.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .none },
79		.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .rex },
80		.{ .@"and", .rm, &.{ .r16, .rm16 }, &.{ 0x23 }, 0, .none },
81		.{ .@"and", .rm, &.{ .r32, .rm32 }, &.{ 0x23 }, 0, .none },
82		.{ .@"and", .rm, &.{ .r64, .rm64 }, &.{ 0x23 }, 0, .long },
83
84		.{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .none },
85		.{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none },
86		.{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long },
87
88		.{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .none },
89		.{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none },
90		.{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long },
91
92		.{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none },
93		.{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long },
94
95		.{ .bt, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xa3 }, 0, .none },
96		.{ .bt, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xa3 }, 0, .none },
97		.{ .bt, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xa3 }, 0, .long },
98		.{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .none },
99		.{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none },
100		.{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long },
101
102		.{ .btc, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xbb }, 0, .none },
103		.{ .btc, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xbb }, 0, .none },
104		.{ .btc, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xbb }, 0, .long },
105		.{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .none },
106		.{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none },
107		.{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long },
108
109		.{ .btr, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb3 }, 0, .none },
110		.{ .btr, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb3 }, 0, .none },
111		.{ .btr, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb3 }, 0, .long },
112		.{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .none },
113		.{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none },
114		.{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long },
115
116		.{ .bts, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xab }, 0, .none },
117		.{ .bts, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xab }, 0, .none },
118		.{ .bts, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xab }, 0, .long },
119		.{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .none },
120		.{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none },
121		.{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long },
	16	.{ .adc, .zi, &.{ .al, .imm8 }, &.{ 0x14 }, 0, .none, .none },
	17	.{ .adc, .zi, &.{ .ax, .imm16 }, &.{ 0x15 }, 0, .short, .none },
	18	.{ .adc, .zi, &.{ .eax, .imm32 }, &.{ 0x15 }, 0, .none, .none },
	19	.{ .adc, .zi, &.{ .rax, .imm32s }, &.{ 0x15 }, 0, .long, .none },
	20	.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .none, .none },
	21	.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .rex, .none },
	22	.{ .adc, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 2, .short, .none },
	23	.{ .adc, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 2, .none, .none },
	24	.{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long, .none },
	25	.{ .adc, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 2, .short, .none },
	26	.{ .adc, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 2, .none, .none },
	27	.{ .adc, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 2, .long, .none },
	28	.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .none, .none },
	29	.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .rex, .none },
	30	.{ .adc, .mr, &.{ .rm16, .r16 }, &.{ 0x11 }, 0, .short, .none },
	31	.{ .adc, .mr, &.{ .rm32, .r32 }, &.{ 0x11 }, 0, .none, .none },
	32	.{ .adc, .mr, &.{ .rm64, .r64 }, &.{ 0x11 }, 0, .long, .none },
	33	.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .none, .none },
	34	.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .rex, .none },
	35	.{ .adc, .rm, &.{ .r16, .rm16 }, &.{ 0x13 }, 0, .short, .none },
	36	.{ .adc, .rm, &.{ .r32, .rm32 }, &.{ 0x13 }, 0, .none, .none },
	37	.{ .adc, .rm, &.{ .r64, .rm64 }, &.{ 0x13 }, 0, .long, .none },
	38
	39	.{ .add, .zi, &.{ .al, .imm8 }, &.{ 0x04 }, 0, .none, .none },
	40	.{ .add, .zi, &.{ .ax, .imm16 }, &.{ 0x05 }, 0, .short, .none },
	41	.{ .add, .zi, &.{ .eax, .imm32 }, &.{ 0x05 }, 0, .none, .none },
	42	.{ .add, .zi, &.{ .rax, .imm32s }, &.{ 0x05 }, 0, .long, .none },
	43	.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .none, .none },
	44	.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .rex, .none },
	45	.{ .add, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 0, .short, .none },
	46	.{ .add, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 0, .none, .none },
	47	.{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long, .none },
	48	.{ .add, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 0, .short, .none },
	49	.{ .add, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 0, .none, .none },
	50	.{ .add, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 0, .long, .none },
	51	.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .none, .none },
	52	.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .rex, .none },
	53	.{ .add, .mr, &.{ .rm16, .r16 }, &.{ 0x01 }, 0, .short, .none },
	54	.{ .add, .mr, &.{ .rm32, .r32 }, &.{ 0x01 }, 0, .none, .none },
	55	.{ .add, .mr, &.{ .rm64, .r64 }, &.{ 0x01 }, 0, .long, .none },
	56	.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .none, .none },
	57	.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .rex, .none },
	58	.{ .add, .rm, &.{ .r16, .rm16 }, &.{ 0x03 }, 0, .short, .none },
	59	.{ .add, .rm, &.{ .r32, .rm32 }, &.{ 0x03 }, 0, .none, .none },
	60	.{ .add, .rm, &.{ .r64, .rm64 }, &.{ 0x03 }, 0, .long, .none },
	61
	62	.{ .@"and", .zi, &.{ .al, .imm8 }, &.{ 0x24 }, 0, .none, .none },
	63	.{ .@"and", .zi, &.{ .ax, .imm16 }, &.{ 0x25 }, 0, .short, .none },
	64	.{ .@"and", .zi, &.{ .eax, .imm32 }, &.{ 0x25 }, 0, .none, .none },
	65	.{ .@"and", .zi, &.{ .rax, .imm32s }, &.{ 0x25 }, 0, .long, .none },
	66	.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .none, .none },
	67	.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .rex, .none },
	68	.{ .@"and", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 4, .short, .none },
	69	.{ .@"and", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 4, .none, .none },
	70	.{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long, .none },
	71	.{ .@"and", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 4, .short, .none },
	72	.{ .@"and", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 4, .none, .none },
	73	.{ .@"and", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 4, .long, .none },
	74	.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .none, .none },
	75	.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .rex, .none },
	76	.{ .@"and", .mr, &.{ .rm16, .r16 }, &.{ 0x21 }, 0, .short, .none },
	77	.{ .@"and", .mr, &.{ .rm32, .r32 }, &.{ 0x21 }, 0, .none, .none },
	78	.{ .@"and", .mr, &.{ .rm64, .r64 }, &.{ 0x21 }, 0, .long, .none },
	79	.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .none, .none },
	80	.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .rex, .none },
	81	.{ .@"and", .rm, &.{ .r16, .rm16 }, &.{ 0x23 }, 0, .short, .none },
	82	.{ .@"and", .rm, &.{ .r32, .rm32 }, &.{ 0x23 }, 0, .none, .none },
	83	.{ .@"and", .rm, &.{ .r64, .rm64 }, &.{ 0x23 }, 0, .long, .none },
	84
	85	.{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .short, .none },
	86	.{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none, .none },
	87	.{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long, .none },
	88
	89	.{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .short, .none },
	90	.{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none, .none },
	91	.{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long, .none },
	92
	93	.{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none, .none },
	94	.{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long, .none },
	95
	96	.{ .bt, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xa3 }, 0, .short, .none },
	97	.{ .bt, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xa3 }, 0, .none, .none },
	98	.{ .bt, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xa3 }, 0, .long, .none },
	99	.{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .short, .none },
	100	.{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none, .none },
	101	.{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long, .none },
	102
	103	.{ .btc, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xbb }, 0, .short, .none },
	104	.{ .btc, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xbb }, 0, .none, .none },
	105	.{ .btc, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xbb }, 0, .long, .none },
	106	.{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .short, .none },
	107	.{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none, .none },
	108	.{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long, .none },
	109
	110	.{ .btr, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb3 }, 0, .short, .none },
	111	.{ .btr, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb3 }, 0, .none, .none },
	112	.{ .btr, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb3 }, 0, .long, .none },
	113	.{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .short, .none },
	114	.{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none, .none },
	115	.{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long, .none },
	116
	117	.{ .bts, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xab }, 0, .short, .none },
	118	.{ .bts, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xab }, 0, .none, .none },
	119	.{ .bts, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xab }, 0, .long, .none },
	120	.{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .short, .none },
	121	.{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none, .none },
	122	.{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long, .none },
122	123
123	124	// This is M encoding according to Intel, but D makes more sense here.
124		.{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none },
125		.{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none },
126
127		.{ .cbw, .np, &.{ .o16 }, &.{ 0x98 }, 0, .none },
128		.{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none },
129		.{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long },
130
131		.{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .none },
132		.{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none },
133		.{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long },
134
135		.{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none },
136		.{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none },
137		.{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long },
138		.{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },
139		.{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },
140		.{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },
141		.{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },
142		.{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },
143		.{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },
144		.{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none },
145		.{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none },
146		.{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long },
147		.{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },
148		.{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },
149		.{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },
150		.{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none },
151		.{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none },
152		.{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long },
153		.{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none },
154		.{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none },
155		.{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long },
156		.{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none },
157		.{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none },
158		.{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long },
159		.{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none },
160		.{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none },
161		.{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long },
162		.{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none },
163		.{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none },
164		.{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long },
165		.{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none },
166		.{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none },
167		.{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long },
168		.{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },
169		.{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },
170		.{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },
171		.{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },
172		.{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },
173		.{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },
174		.{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none },
175		.{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none },
176		.{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long },
177		.{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },
178		.{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },
179		.{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },
180		.{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none },
181		.{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none },
182		.{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long },
183		.{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none },
184		.{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none },
185		.{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long },
186		.{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none },
187		.{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none },
188		.{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long },
189		.{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none },
190		.{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none },
191		.{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long },
192		.{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none },
193		.{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none },
194		.{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long },
195		.{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .none },
196		.{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none },
197		.{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long },
198		.{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none },
199		.{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none },
200		.{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long },
201		.{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .none },
202		.{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none },
203		.{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long },
204		.{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none },
205		.{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none },
206		.{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long },
207		.{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .none },
208		.{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none },
209		.{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long },
210		.{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none },
211		.{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none },
212		.{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long },
213		.{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none },
214		.{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none },
215		.{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long },
216		.{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none },
217		.{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none },
218		.{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long },
219		.{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .none },
220		.{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none },
221		.{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long },
222		.{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none },
223		.{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none },
224		.{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long },
225
226		.{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none },
227		.{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .none },
228		.{ .cmp, .zi, &.{ .eax, .imm32 }, &.{ 0x3d }, 0, .none },
229		.{ .cmp, .zi, &.{ .rax, .imm32s }, &.{ 0x3d }, 0, .long },
230		.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .none },
231		.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .rex },
232		.{ .cmp, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 7, .none },
233		.{ .cmp, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 7, .none },
234		.{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long },
235		.{ .cmp, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 7, .none },
236		.{ .cmp, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 7, .none },
237		.{ .cmp, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 7, .long },
238		.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .none },
239		.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .rex },
240		.{ .cmp, .mr, &.{ .rm16, .r16 }, &.{ 0x39 }, 0, .none },
241		.{ .cmp, .mr, &.{ .rm32, .r32 }, &.{ 0x39 }, 0, .none },
242		.{ .cmp, .mr, &.{ .rm64, .r64 }, &.{ 0x39 }, 0, .long },
243		.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .none },
244		.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .rex },
245		.{ .cmp, .rm, &.{ .r16, .rm16 }, &.{ 0x3b }, 0, .none },
246		.{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none },
247		.{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long },
248
249		.{ .cmps, .np, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none },
250		.{ .cmps, .np, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .none },
251		.{ .cmps, .np, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none },
252		.{ .cmps, .np, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long },
253
254		.{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none },
255		.{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short },
256		.{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none },
257		.{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long },
258
259		.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none },
260		.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex },
261		.{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .none },
262		.{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none },
263		.{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long },
264
265		.{ .cmpxchg8b , .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none },
266		.{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long },
267
268		.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .none },
269		.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .rex },
270		.{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .none },
271		.{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none },
272		.{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long },
273
274		.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .fpu },
275		.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .fpu },
276		.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .fpu },
277
278		.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .fpu },
279		.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .fpu },
280		.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .fpu },
281
282		.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .none },
283		.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .rex },
284		.{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .none },
285		.{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none },
286		.{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long },
287
288		.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .none },
289		.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .rex },
290		.{ .imul, .m, &.{ .rm16, }, &.{ 0xf7 }, 5, .none },
291		.{ .imul, .m, &.{ .rm32, }, &.{ 0xf7 }, 5, .none },
292		.{ .imul, .m, &.{ .rm64, }, &.{ 0xf7 }, 5, .long },
293		.{ .imul, .rm, &.{ .r16, .rm16, }, &.{ 0x0f, 0xaf }, 0, .none },
294		.{ .imul, .rm, &.{ .r32, .rm32, }, &.{ 0x0f, 0xaf }, 0, .none },
295		.{ .imul, .rm, &.{ .r64, .rm64, }, &.{ 0x0f, 0xaf }, 0, .long },
296		.{ .imul, .rmi, &.{ .r16, .rm16, .imm8s }, &.{ 0x6b }, 0, .none },
297		.{ .imul, .rmi, &.{ .r32, .rm32, .imm8s }, &.{ 0x6b }, 0, .none },
298		.{ .imul, .rmi, &.{ .r64, .rm64, .imm8s }, &.{ 0x6b }, 0, .long },
299		.{ .imul, .rmi, &.{ .r16, .rm16, .imm16 }, &.{ 0x69 }, 0, .none },
300		.{ .imul, .rmi, &.{ .r32, .rm32, .imm32 }, &.{ 0x69 }, 0, .none },
301		.{ .imul, .rmi, &.{ .r64, .rm64, .imm32 }, &.{ 0x69 }, 0, .long },
302
303		.{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none },
304
305		.{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none },
306		.{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },
307		.{ .jb, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },
308		.{ .jbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none },
309		.{ .jc, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },
310		.{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .none },
311		.{ .je, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none },
312		.{ .jg, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none },
313		.{ .jge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none },
314		.{ .jl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none },
315		.{ .jle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none },
316		.{ .jna, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none },
317		.{ .jnae, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },
318		.{ .jnb, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },
319		.{ .jnbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none },
320		.{ .jnc, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },
321		.{ .jne, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none },
322		.{ .jng, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none },
323		.{ .jnge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none },
324		.{ .jnl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none },
325		.{ .jnle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none },
326		.{ .jno, .d, &.{ .rel32 }, &.{ 0x0f, 0x81 }, 0, .none },
327		.{ .jnp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none },
328		.{ .jns, .d, &.{ .rel32 }, &.{ 0x0f, 0x89 }, 0, .none },
329		.{ .jnz, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none },
330		.{ .jo, .d, &.{ .rel32 }, &.{ 0x0f, 0x80 }, 0, .none },
331		.{ .jp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none },
332		.{ .jpe, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none },
333		.{ .jpo, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none },
334		.{ .js, .d, &.{ .rel32 }, &.{ 0x0f, 0x88 }, 0, .none },
335		.{ .jz, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none },
336
337		.{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none },
338		.{ .jmp, .m, &.{ .rm64 }, &.{ 0xff }, 4, .none },
339
340		.{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .none },
341		.{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none },
342		.{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long },
343
344		.{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none },
345
346		.{ .lods, .np, &.{ .m8 }, &.{ 0xac }, 0, .none },
347		.{ .lods, .np, &.{ .m16 }, &.{ 0xad }, 0, .none },
348		.{ .lods, .np, &.{ .m32 }, &.{ 0xad }, 0, .none },
349		.{ .lods, .np, &.{ .m64 }, &.{ 0xad }, 0, .long },
350
351		.{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none },
352		.{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short },
353		.{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none },
354		.{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long },
355
356		.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
357		.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
358		.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
359
360		.{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none },
361
362		.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none },
363		.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex },
364		.{ .mov, .mr, &.{ .rm16, .r16 }, &.{ 0x89 }, 0, .none },
365		.{ .mov, .mr, &.{ .rm32, .r32 }, &.{ 0x89 }, 0, .none },
366		.{ .mov, .mr, &.{ .rm64, .r64 }, &.{ 0x89 }, 0, .long },
367		.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .none },
368		.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .rex },
369		.{ .mov, .rm, &.{ .r16, .rm16 }, &.{ 0x8b }, 0, .none },
370		.{ .mov, .rm, &.{ .r32, .rm32 }, &.{ 0x8b }, 0, .none },
371		.{ .mov, .rm, &.{ .r64, .rm64 }, &.{ 0x8b }, 0, .long },
372		.{ .mov, .mr, &.{ .rm16, .sreg }, &.{ 0x8c }, 0, .none },
373		.{ .mov, .mr, &.{ .rm64, .sreg }, &.{ 0x8c }, 0, .long },
374		.{ .mov, .rm, &.{ .sreg, .rm16 }, &.{ 0x8e }, 0, .none },
375		.{ .mov, .rm, &.{ .sreg, .rm64 }, &.{ 0x8e }, 0, .long },
376		.{ .mov, .fd, &.{ .al, .moffs }, &.{ 0xa0 }, 0, .none },
377		.{ .mov, .fd, &.{ .ax, .moffs }, &.{ 0xa1 }, 0, .none },
378		.{ .mov, .fd, &.{ .eax, .moffs }, &.{ 0xa1 }, 0, .none },
379		.{ .mov, .fd, &.{ .rax, .moffs }, &.{ 0xa1 }, 0, .long },
380		.{ .mov, .td, &.{ .moffs, .al }, &.{ 0xa2 }, 0, .none },
381		.{ .mov, .td, &.{ .moffs, .ax }, &.{ 0xa3 }, 0, .none },
382		.{ .mov, .td, &.{ .moffs, .eax }, &.{ 0xa3 }, 0, .none },
383		.{ .mov, .td, &.{ .moffs, .rax }, &.{ 0xa3 }, 0, .long },
384		.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .none },
385		.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .rex },
386		.{ .mov, .oi, &.{ .r16, .imm16 }, &.{ 0xb8 }, 0, .none },
387		.{ .mov, .oi, &.{ .r32, .imm32 }, &.{ 0xb8 }, 0, .none },
388		.{ .mov, .oi, &.{ .r64, .imm64 }, &.{ 0xb8 }, 0, .long },
389		.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .none },
390		.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .rex },
391		.{ .mov, .mi, &.{ .rm16, .imm16 }, &.{ 0xc7 }, 0, .none },
392		.{ .mov, .mi, &.{ .rm32, .imm32 }, &.{ 0xc7 }, 0, .none },
393		.{ .mov, .mi, &.{ .rm64, .imm32s }, &.{ 0xc7 }, 0, .long },
394
395		.{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
396		.{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
397		.{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long },
398		.{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
399		.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
400		.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long },
401
402		.{ .movs, .np, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none },
403		.{ .movs, .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .none },
404		.{ .movs, .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none },
405		.{ .movs, .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long },
406
407		.{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none },
408		.{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short },
409		.{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none },
410		.{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long },
411
412		.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .none },
413		.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex },
414		.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .none },
415		.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex },
416		.{ .movsx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xbe }, 0, .long },
417		.{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none },
418		.{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long },
	125	.{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
	126	.{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none, .none },
	127
	128	.{ .cbw, .np, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
	129	.{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
	130	.{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
	131
	132	.{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
	133	.{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
	134	.{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
	135
	136	.{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
	137	.{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
	138	.{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
	139	.{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
	140	.{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
	141	.{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
	142	.{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
	143	.{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
	144	.{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
	145	.{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
	146	.{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
	147	.{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
	148	.{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
	149	.{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
	150	.{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
	151	.{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
	152	.{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
	153	.{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
	154	.{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
	155	.{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
	156	.{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
	157	.{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
	158	.{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
	159	.{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
	160	.{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
	161	.{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
	162	.{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
	163	.{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
	164	.{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
	165	.{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
	166	.{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
	167	.{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
	168	.{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
	169	.{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
	170	.{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
	171	.{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
	172	.{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
	173	.{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
	174	.{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
	175	.{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
	176	.{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
	177	.{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
	178	.{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
	179	.{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
	180	.{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
	181	.{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
	182	.{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
	183	.{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
	184	.{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
	185	.{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
	186	.{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
	187	.{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
	188	.{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
	189	.{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
	190	.{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
	191	.{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
	192	.{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
	193	.{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
	194	.{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
	195	.{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
	196	.{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .none },
	197	.{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
	198	.{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },
	199	.{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
	200	.{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
	201	.{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
	202	.{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .none },
	203	.{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
	204	.{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },
	205	.{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
	206	.{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
	207	.{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
	208	.{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .none },
	209	.{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
	210	.{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },
	211	.{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
	212	.{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
	213	.{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
	214	.{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
	215	.{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
	216	.{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
	217	.{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
	218	.{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
	219	.{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
	220	.{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .none },
	221	.{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
	222	.{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },
	223	.{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
	224	.{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
	225	.{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
	226
	227	.{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none, .none },
	228	.{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .short, .none },
	229	.{ .cmp, .zi, &.{ .eax, .imm32 }, &.{ 0x3d }, 0, .none, .none },
	230	.{ .cmp, .zi, &.{ .rax, .imm32s }, &.{ 0x3d }, 0, .long, .none },
	231	.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .none, .none },
	232	.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .rex, .none },
	233	.{ .cmp, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 7, .short, .none },
	234	.{ .cmp, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 7, .none, .none },
	235	.{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long, .none },
	236	.{ .cmp, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 7, .short, .none },
	237	.{ .cmp, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 7, .none, .none },
	238	.{ .cmp, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 7, .long, .none },
	239	.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .none, .none },
	240	.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .rex, .none },
	241	.{ .cmp, .mr, &.{ .rm16, .r16 }, &.{ 0x39 }, 0, .short, .none },
	242	.{ .cmp, .mr, &.{ .rm32, .r32 }, &.{ 0x39 }, 0, .none, .none },
	243	.{ .cmp, .mr, &.{ .rm64, .r64 }, &.{ 0x39 }, 0, .long, .none },
	244	.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .none, .none },
	245	.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .rex, .none },
	246	.{ .cmp, .rm, &.{ .r16, .rm16 }, &.{ 0x3b }, 0, .short, .none },
	247	.{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none, .none },
	248	.{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long, .none },
	249
	250	.{ .cmps, .np, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
	251	.{ .cmps, .np, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
	252	.{ .cmps, .np, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
	253	.{ .cmps, .np, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
	254
	255	.{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none, .none },
	256	.{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short, .none },
	257	.{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none, .none },
	258	.{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long, .none },
	259
	260	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
	261	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex, .none },
	262	.{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .short, .none },
	263	.{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none, .none },
	264	.{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long, .none },
	265
	266	.{ .cmpxchg8b, .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
	267	.{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
	268
	269	.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .none, .none },
	270	.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .rex, .none },
	271	.{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .short, .none },
	272	.{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none, .none },
	273	.{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long, .none },
	274
	275	.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
	276	.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
	277	.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
	278
	279	.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
	280	.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
	281	.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
	282
	283	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .none, .none },
	284	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .rex, .none },
	285	.{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .short, .none },
	286	.{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none, .none },
	287	.{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long, .none },
	288
	289	.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .none, .none },
	290	.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .rex, .none },
	291	.{ .imul, .m, &.{ .rm16, }, &.{ 0xf7 }, 5, .short, .none },
	292	.{ .imul, .m, &.{ .rm32, }, &.{ 0xf7 }, 5, .none, .none },
	293	.{ .imul, .m, &.{ .rm64, }, &.{ 0xf7 }, 5, .long, .none },
	294	.{ .imul, .rm, &.{ .r16, .rm16, }, &.{ 0x0f, 0xaf }, 0, .short, .none },
	295	.{ .imul, .rm, &.{ .r32, .rm32, }, &.{ 0x0f, 0xaf }, 0, .none, .none },
	296	.{ .imul, .rm, &.{ .r64, .rm64, }, &.{ 0x0f, 0xaf }, 0, .long, .none },
	297	.{ .imul, .rmi, &.{ .r16, .rm16, .imm8s }, &.{ 0x6b }, 0, .short, .none },
	298	.{ .imul, .rmi, &.{ .r32, .rm32, .imm8s }, &.{ 0x6b }, 0, .none, .none },
	299	.{ .imul, .rmi, &.{ .r64, .rm64, .imm8s }, &.{ 0x6b }, 0, .long, .none },
	300	.{ .imul, .rmi, &.{ .r16, .rm16, .imm16 }, &.{ 0x69 }, 0, .short, .none },
	301	.{ .imul, .rmi, &.{ .r32, .rm32, .imm32 }, &.{ 0x69 }, 0, .none, .none },
	302	.{ .imul, .rmi, &.{ .r64, .rm64, .imm32 }, &.{ 0x69 }, 0, .long, .none },
	303
	304	.{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none, .none },
	305
	306	.{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
	307	.{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
	308	.{ .jb, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
	309	.{ .jbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none },
	310	.{ .jc, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
	311	.{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .none, .none },
	312	.{ .je, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none },
	313	.{ .jg, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none },
	314	.{ .jge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none },
	315	.{ .jl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none, .none },
	316	.{ .jle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none, .none },
	317	.{ .jna, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none },
	318	.{ .jnae, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
	319	.{ .jnb, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
	320	.{ .jnbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
	321	.{ .jnc, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
	322	.{ .jne, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none, .none },
	323	.{ .jng, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none, .none },
	324	.{ .jnge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none, .none },
	325	.{ .jnl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none },
	326	.{ .jnle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none },
	327	.{ .jno, .d, &.{ .rel32 }, &.{ 0x0f, 0x81 }, 0, .none, .none },
	328	.{ .jnp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none, .none },
	329	.{ .jns, .d, &.{ .rel32 }, &.{ 0x0f, 0x89 }, 0, .none, .none },
	330	.{ .jnz, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none, .none },
	331	.{ .jo, .d, &.{ .rel32 }, &.{ 0x0f, 0x80 }, 0, .none, .none },
	332	.{ .jp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none, .none },
	333	.{ .jpe, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none, .none },
	334	.{ .jpo, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none, .none },
	335	.{ .js, .d, &.{ .rel32 }, &.{ 0x0f, 0x88 }, 0, .none, .none },
	336	.{ .jz, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none },
	337
	338	.{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none, .none },
	339	.{ .jmp, .m, &.{ .rm64 }, &.{ 0xff }, 4, .none, .none },
	340
	341	.{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .short, .none },
	342	.{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
	343	.{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
	344
	345	.{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
	346
	347	.{ .lods, .np, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
	348	.{ .lods, .np, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
	349	.{ .lods, .np, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
	350	.{ .lods, .np, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
	351
	352	.{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none, .none },
	353	.{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short, .none },
	354	.{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none, .none },
	355	.{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long, .none },
	356
	357	.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .none },
	358	.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
	359	.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .none },
	360
	361	.{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
	362
	363	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none },
	364	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex, .none },
	365	.{ .mov, .mr, &.{ .rm16, .r16 }, &.{ 0x89 }, 0, .short, .none },
	366	.{ .mov, .mr, &.{ .rm32, .r32 }, &.{ 0x89 }, 0, .none, .none },
	367	.{ .mov, .mr, &.{ .rm64, .r64 }, &.{ 0x89 }, 0, .long, .none },
	368	.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .none, .none },
	369	.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .rex, .none },
	370	.{ .mov, .rm, &.{ .r16, .rm16 }, &.{ 0x8b }, 0, .short, .none },
	371	.{ .mov, .rm, &.{ .r32, .rm32 }, &.{ 0x8b }, 0, .none, .none },
	372	.{ .mov, .rm, &.{ .r64, .rm64 }, &.{ 0x8b }, 0, .long, .none },
	373	.{ .mov, .mr, &.{ .rm16, .sreg }, &.{ 0x8c }, 0, .short, .none },
	374	.{ .mov, .mr, &.{ .r32_m16, .sreg }, &.{ 0x8c }, 0, .none, .none },
	375	.{ .mov, .mr, &.{ .r64_m16, .sreg }, &.{ 0x8c }, 0, .long, .none },
	376	.{ .mov, .rm, &.{ .sreg, .rm16 }, &.{ 0x8e }, 0, .short, .none },
	377	.{ .mov, .rm, &.{ .sreg, .r32_m16 }, &.{ 0x8e }, 0, .none, .none },
	378	.{ .mov, .rm, &.{ .sreg, .r64_m16 }, &.{ 0x8e }, 0, .long, .none },
	379	.{ .mov, .fd, &.{ .al, .moffs }, &.{ 0xa0 }, 0, .none, .none },
	380	.{ .mov, .fd, &.{ .ax, .moffs }, &.{ 0xa1 }, 0, .none, .none },
	381	.{ .mov, .fd, &.{ .eax, .moffs }, &.{ 0xa1 }, 0, .none, .none },
	382	.{ .mov, .fd, &.{ .rax, .moffs }, &.{ 0xa1 }, 0, .long, .none },
	383	.{ .mov, .td, &.{ .moffs, .al }, &.{ 0xa2 }, 0, .none, .none },
	384	.{ .mov, .td, &.{ .moffs, .ax }, &.{ 0xa3 }, 0, .none, .none },
	385	.{ .mov, .td, &.{ .moffs, .eax }, &.{ 0xa3 }, 0, .none, .none },
	386	.{ .mov, .td, &.{ .moffs, .rax }, &.{ 0xa3 }, 0, .long, .none },
	387	.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .none, .none },
	388	.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .rex, .none },
	389	.{ .mov, .oi, &.{ .r16, .imm16 }, &.{ 0xb8 }, 0, .short, .none },
	390	.{ .mov, .oi, &.{ .r32, .imm32 }, &.{ 0xb8 }, 0, .none, .none },
	391	.{ .mov, .oi, &.{ .r64, .imm64 }, &.{ 0xb8 }, 0, .long, .none },
	392	.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .none, .none },
	393	.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .rex, .none },
	394	.{ .mov, .mi, &.{ .rm16, .imm16 }, &.{ 0xc7 }, 0, .short, .none },
	395	.{ .mov, .mi, &.{ .rm32, .imm32 }, &.{ 0xc7 }, 0, .none, .none },
	396	.{ .mov, .mi, &.{ .rm64, .imm32s }, &.{ 0xc7 }, 0, .long, .none },
	397
	398	.{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .short, .none },
	399	.{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
	400	.{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long, .none },
	401	.{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .short, .none },
	402	.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
	403	.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .none },
	404
	405	.{ .movs, .np, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
	406	.{ .movs, .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
	407	.{ .movs, .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
	408	.{ .movs, .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
	409
	410	.{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none, .none },
	411	.{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short, .none },
	412	.{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none, .none },
	413	.{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long, .none },
	414
	415	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .short, .none },
	416	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },
	417	.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .none, .none },
	418	.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex, .none },
	419	.{ .movsx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xbe }, 0, .long, .none },
	420	.{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none, .none },
	421	.{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .rex, .none },
	422	.{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long, .none },
419	423
420	424	// This instruction is discouraged.
421		.{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none },
422		.{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long },
423
424		.{ .movzx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .none },
425		.{ .movzx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .none },
426		.{ .movzx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .long },
427		.{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none },
428		.{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long },
429
430		.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .none },
431		.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .rex },
432		.{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .none },
433		.{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none },
434		.{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long },
435
436		.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .none },
437		.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .rex },
438		.{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .none },
439		.{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none },
440		.{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long },
441
442		.{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none },
443
444		.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none },
445		.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex },
446		.{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .none },
447		.{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none },
448		.{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long },
449
450		.{ .@"or", .zi, &.{ .al, .imm8 }, &.{ 0x0c }, 0, .none },
451		.{ .@"or", .zi, &.{ .ax, .imm16 }, &.{ 0x0d }, 0, .none },
452		.{ .@"or", .zi, &.{ .eax, .imm32 }, &.{ 0x0d }, 0, .none },
453		.{ .@"or", .zi, &.{ .rax, .imm32s }, &.{ 0x0d }, 0, .long },
454		.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .none },
455		.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .rex },
456		.{ .@"or", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 1, .none },
457		.{ .@"or", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 1, .none },
458		.{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long },
459		.{ .@"or", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 1, .none },
460		.{ .@"or", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 1, .none },
461		.{ .@"or", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 1, .long },
462		.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .none },
463		.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .rex },
464		.{ .@"or", .mr, &.{ .rm16, .r16 }, &.{ 0x09 }, 0, .none },
465		.{ .@"or", .mr, &.{ .rm32, .r32 }, &.{ 0x09 }, 0, .none },
466		.{ .@"or", .mr, &.{ .rm64, .r64 }, &.{ 0x09 }, 0, .long },
467		.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .none },
468		.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .rex },
469		.{ .@"or", .rm, &.{ .r16, .rm16 }, &.{ 0x0b }, 0, .none },
470		.{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none },
471		.{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long },
472
473		.{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .none },
474		.{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none },
475		.{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .none },
476		.{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none },
477
478		.{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
479		.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
480		.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },
481
482		.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .none },
483		.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none },
484		.{ .push, .m, &.{ .rm16 }, &.{ 0xff }, 6, .none },
485		.{ .push, .m, &.{ .rm64 }, &.{ 0xff }, 6, .none },
486		.{ .push, .i, &.{ .imm8 }, &.{ 0x6a }, 0, .none },
487		.{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .none },
488		.{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none },
489
490		.{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none },
491
492		.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none },
493		.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex },
494		.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .none },
495		.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .rex },
496		.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .none },
497		.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .rex },
498		.{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .none },
499		.{ .rcl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 2, .none },
500		.{ .rcl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 2, .none },
501		.{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none },
502		.{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long },
503		.{ .rcl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 2, .none },
504		.{ .rcl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 2, .long },
505		.{ .rcl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 2, .none },
506		.{ .rcl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 2, .long },
507
508		.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .none },
509		.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .rex },
510		.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .none },
511		.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .rex },
512		.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .none },
513		.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .rex },
514		.{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .none },
515		.{ .rcr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 3, .none },
516		.{ .rcr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 3, .none },
517		.{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none },
518		.{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long },
519		.{ .rcr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 3, .none },
520		.{ .rcr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 3, .long },
521		.{ .rcr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 3, .none },
522		.{ .rcr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 3, .long },
523
524		.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .none },
525		.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .rex },
526		.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .none },
527		.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .rex },
528		.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .none },
529		.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .rex },
530		.{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .none },
531		.{ .rol, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 0, .none },
532		.{ .rol, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 0, .none },
533		.{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none },
534		.{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long },
535		.{ .rol, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 0, .none },
536		.{ .rol, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 0, .long },
537		.{ .rol, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 0, .none },
538		.{ .rol, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 0, .long },
539
540		.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .none },
541		.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .rex },
542		.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .none },
543		.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .rex },
544		.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .none },
545		.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .rex },
546		.{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .none },
547		.{ .ror, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 1, .none },
548		.{ .ror, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 1, .none },
549		.{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none },
550		.{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long },
551		.{ .ror, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 1, .none },
552		.{ .ror, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 1, .long },
553		.{ .ror, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 1, .none },
554		.{ .ror, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 1, .long },
555
556		.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none },
557		.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex },
558		.{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none },
559		.{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none },
560		.{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long },
561		.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none },
562		.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex },
563		.{ .sal, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .none },
564		.{ .sal, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none },
565		.{ .sal, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long },
566		.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none },
567		.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex },
568		.{ .sal, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .none },
569		.{ .sal, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none },
570		.{ .sal, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long },
571
572		.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .none },
573		.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .rex },
574		.{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .none },
575		.{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none },
576		.{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long },
577		.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .none },
578		.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .rex },
579		.{ .sar, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 7, .none },
580		.{ .sar, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 7, .none },
581		.{ .sar, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 7, .long },
582		.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .none },
583		.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .rex },
584		.{ .sar, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 7, .none },
585		.{ .sar, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 7, .none },
586		.{ .sar, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 7, .long },
587
588		.{ .sbb, .zi, &.{ .al, .imm8 }, &.{ 0x1c }, 0, .none },
589		.{ .sbb, .zi, &.{ .ax, .imm16 }, &.{ 0x1d }, 0, .none },
590		.{ .sbb, .zi, &.{ .eax, .imm32 }, &.{ 0x1d }, 0, .none },
591		.{ .sbb, .zi, &.{ .rax, .imm32s }, &.{ 0x1d }, 0, .long },
592		.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .none },
593		.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .rex },
594		.{ .sbb, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 3, .none },
595		.{ .sbb, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 3, .none },
596		.{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long },
597		.{ .sbb, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 3, .none },
598		.{ .sbb, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 3, .none },
599		.{ .sbb, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 3, .long },
600		.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .none },
601		.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .rex },
602		.{ .sbb, .mr, &.{ .rm16, .r16 }, &.{ 0x19 }, 0, .none },
603		.{ .sbb, .mr, &.{ .rm32, .r32 }, &.{ 0x19 }, 0, .none },
604		.{ .sbb, .mr, &.{ .rm64, .r64 }, &.{ 0x19 }, 0, .long },
605		.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .none },
606		.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .rex },
607		.{ .sbb, .rm, &.{ .r16, .rm16 }, &.{ 0x1b }, 0, .none },
608		.{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none },
609		.{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long },
610
611		.{ .scas, .np, &.{ .m8 }, &.{ 0xae }, 0, .none },
612		.{ .scas, .np, &.{ .m16 }, &.{ 0xaf }, 0, .none },
613		.{ .scas, .np, &.{ .m32 }, &.{ 0xaf }, 0, .none },
614		.{ .scas, .np, &.{ .m64 }, &.{ 0xaf }, 0, .long },
615
616		.{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none },
617		.{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short },
618		.{ .scasd, .np, &.{}, &.{ 0xaf }, 0, .none },
619		.{ .scasq, .np, &.{}, &.{ 0xaf }, 0, .long },
620
621		.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none },
622		.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex },
623		.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },
624		.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex },
625		.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },
626		.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex },
627		.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none },
628		.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex },
629		.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },
630		.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex },
631		.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none },
632		.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex },
633		.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none },
634		.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex },
635		.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none },
636		.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex },
637		.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none },
638		.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex },
639		.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none },
640		.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex },
641		.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none },
642		.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex },
643		.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },
644		.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex },
645		.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },
646		.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex },
647		.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none },
648		.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex },
649		.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },
650		.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex },
651		.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none },
652		.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex },
653		.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none },
654		.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex },
655		.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none },
656		.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex },
657		.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none },
658		.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex },
659		.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none },
660		.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex },
661		.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .none },
662		.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .rex },
663		.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none },
664		.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex },
665		.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .none },
666		.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .rex },
667		.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none },
668		.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex },
669		.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .none },
670		.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .rex },
671		.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none },
672		.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex },
673		.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none },
674		.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex },
675		.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none },
676		.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex },
677		.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .none },
678		.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .rex },
679		.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none },
680		.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex },
681
682		.{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none },
683
684		.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none },
685		.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex },
686		.{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none },
687		.{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none },
688		.{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long },
689		.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none },
690		.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex },
691		.{ .shl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .none },
692		.{ .shl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none },
693		.{ .shl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long },
694		.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none },
695		.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex },
696		.{ .shl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .none },
697		.{ .shl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none },
698		.{ .shl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long },
699
700		.{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none },
701		.{ .shld, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xa5 }, 0, .none },
702		.{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none },
703		.{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long },
704		.{ .shld, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xa5 }, 0, .none },
705		.{ .shld, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xa5 }, 0, .long },
706
707		.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .none },
708		.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .rex },
709		.{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .none },
710		.{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none },
711		.{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long },
712		.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .none },
713		.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .rex },
714		.{ .shr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 5, .none },
715		.{ .shr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 5, .none },
716		.{ .shr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 5, .long },
717		.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .none },
718		.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .rex },
719		.{ .shr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 5, .none },
720		.{ .shr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 5, .none },
721		.{ .shr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 5, .long },
722
723		.{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .none },
724		.{ .shrd, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xad }, 0, .none },
725		.{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none },
726		.{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long },
727		.{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none },
728		.{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long },
729
730		.{ .stos, .np, &.{ .m8 }, &.{ 0xaa }, 0, .none },
731		.{ .stos, .np, &.{ .m16 }, &.{ 0xab }, 0, .none },
732		.{ .stos, .np, &.{ .m32 }, &.{ 0xab }, 0, .none },
733		.{ .stos, .np, &.{ .m64 }, &.{ 0xab }, 0, .long },
734
735		.{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none },
736		.{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short },
737		.{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none },
738		.{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long },
739
740		.{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none },
741		.{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .none },
742		.{ .sub, .zi, &.{ .eax, .imm32 }, &.{ 0x2d }, 0, .none },
743		.{ .sub, .zi, &.{ .rax, .imm32s }, &.{ 0x2d }, 0, .long },
744		.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .none },
745		.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .rex },
746		.{ .sub, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 5, .none },
747		.{ .sub, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 5, .none },
748		.{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long },
749		.{ .sub, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 5, .none },
750		.{ .sub, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 5, .none },
751		.{ .sub, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 5, .long },
752		.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .none },
753		.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .rex },
754		.{ .sub, .mr, &.{ .rm16, .r16 }, &.{ 0x29 }, 0, .none },
755		.{ .sub, .mr, &.{ .rm32, .r32 }, &.{ 0x29 }, 0, .none },
756		.{ .sub, .mr, &.{ .rm64, .r64 }, &.{ 0x29 }, 0, .long },
757		.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .none },
758		.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .rex },
759		.{ .sub, .rm, &.{ .r16, .rm16 }, &.{ 0x2b }, 0, .none },
760		.{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none },
761		.{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long },
762
763		.{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none }
764		,
765		.{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none },
766		.{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .none },
767		.{ .@"test", .zi, &.{ .eax, .imm32 }, &.{ 0xa9 }, 0, .none },
768		.{ .@"test", .zi, &.{ .rax, .imm32s }, &.{ 0xa9 }, 0, .long },
769		.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .none },
770		.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .rex },
771		.{ .@"test", .mi, &.{ .rm16, .imm16 }, &.{ 0xf7 }, 0, .none },
772		.{ .@"test", .mi, &.{ .rm32, .imm32 }, &.{ 0xf7 }, 0, .none },
773		.{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long },
774		.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .none },
775		.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .rex },
776		.{ .@"test", .mr, &.{ .rm16, .r16 }, &.{ 0x85 }, 0, .none },
777		.{ .@"test", .mr, &.{ .rm32, .r32 }, &.{ 0x85 }, 0, .none },
778		.{ .@"test", .mr, &.{ .rm64, .r64 }, &.{ 0x85 }, 0, .long },
779
780		.{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
781		.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
782		.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },
783
784		.{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none },
785
786		.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none },
787		.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex },
788		.{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .none },
789		.{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none },
790		.{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long },
791
792		.{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .none },
793		.{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .none },
794		.{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none },
795		.{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long },
796		.{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none },
797		.{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long },
798		.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none },
799		.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex },
800		.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none },
801		.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .rex },
802		.{ .xchg, .mr, &.{ .rm16, .r16 }, &.{ 0x87 }, 0, .none },
803		.{ .xchg, .rm, &.{ .r16, .rm16 }, &.{ 0x87 }, 0, .none },
804		.{ .xchg, .mr, &.{ .rm32, .r32 }, &.{ 0x87 }, 0, .none },
805		.{ .xchg, .mr, &.{ .rm64, .r64 }, &.{ 0x87 }, 0, .long },
806		.{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none },
807		.{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long },
808
809		.{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none },
810		.{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .none },
811		.{ .xor, .zi, &.{ .eax, .imm32 }, &.{ 0x35 }, 0, .none },
812		.{ .xor, .zi, &.{ .rax, .imm32s }, &.{ 0x35 }, 0, .long },
813		.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .none },
814		.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .rex },
815		.{ .xor, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 6, .none },
816		.{ .xor, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 6, .none },
817		.{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long },
818		.{ .xor, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 6, .none },
819		.{ .xor, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 6, .none },
820		.{ .xor, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 6, .long },
821		.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .none },
822		.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .rex },
823		.{ .xor, .mr, &.{ .rm16, .r16 }, &.{ 0x31 }, 0, .none },
824		.{ .xor, .mr, &.{ .rm32, .r32 }, &.{ 0x31 }, 0, .none },
825		.{ .xor, .mr, &.{ .rm64, .r64 }, &.{ 0x31 }, 0, .long },
826		.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .none },
827		.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .rex },
828		.{ .xor, .rm, &.{ .r16, .rm16 }, &.{ 0x33 }, 0, .none },
829		.{ .xor, .rm, &.{ .r32, .rm32 }, &.{ 0x33 }, 0, .none },
830		.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long },
	425	.{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none, .none },
	426	.{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long, .none },
	427
	428	.{ .movzx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .short, .none },
	429	.{ .movzx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .rex_short, .none },
	430	.{ .movzx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .none, .none },
	431	.{ .movzx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .rex, .none },
	432	.{ .movzx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .long, .none },
	433	.{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none, .none },
	434	.{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .rex, .none },
	435	.{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long, .none },
	436
	437	.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .none, .none },
	438	.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .rex, .none },
	439	.{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .short, .none },
	440	.{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none, .none },
	441	.{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long, .none },
	442
	443	.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .none, .none },
	444	.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .rex, .none },
	445	.{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .short, .none },
	446	.{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
	447	.{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
	448
	449	.{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none, .none },
	450
	451	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none, .none },
	452	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex, .none },
	453	.{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .short, .none },
	454	.{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none, .none },
	455	.{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long, .none },
	456
	457	.{ .@"or", .zi, &.{ .al, .imm8 }, &.{ 0x0c }, 0, .none, .none },
	458	.{ .@"or", .zi, &.{ .ax, .imm16 }, &.{ 0x0d }, 0, .short, .none },
	459	.{ .@"or", .zi, &.{ .eax, .imm32 }, &.{ 0x0d }, 0, .none, .none },
	460	.{ .@"or", .zi, &.{ .rax, .imm32s }, &.{ 0x0d }, 0, .long, .none },
	461	.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .none, .none },
	462	.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .rex, .none },
	463	.{ .@"or", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 1, .short, .none },
	464	.{ .@"or", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 1, .none, .none },
	465	.{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long, .none },
	466	.{ .@"or", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 1, .short, .none },
	467	.{ .@"or", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 1, .none, .none },
	468	.{ .@"or", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 1, .long, .none },
	469	.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .none, .none },
	470	.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .rex, .none },
	471	.{ .@"or", .mr, &.{ .rm16, .r16 }, &.{ 0x09 }, 0, .short, .none },
	472	.{ .@"or", .mr, &.{ .rm32, .r32 }, &.{ 0x09 }, 0, .none, .none },
	473	.{ .@"or", .mr, &.{ .rm64, .r64 }, &.{ 0x09 }, 0, .long, .none },
	474	.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .none, .none },
	475	.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .rex, .none },
	476	.{ .@"or", .rm, &.{ .r16, .rm16 }, &.{ 0x0b }, 0, .short, .none },
	477	.{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none, .none },
	478	.{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long, .none },
	479
	480	.{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .short, .none },
	481	.{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none, .none },
	482	.{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .short, .none },
	483	.{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none, .none },
	484
	485	.{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .none },
	486	.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
	487	.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .none },
	488
	489	.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },
	490	.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },
	491	.{ .push, .m, &.{ .rm16 }, &.{ 0xff }, 6, .short, .none },
	492	.{ .push, .m, &.{ .rm64 }, &.{ 0xff }, 6, .none, .none },
	493	.{ .push, .i, &.{ .imm8 }, &.{ 0x6a }, 0, .none, .none },
	494	.{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },
	495	.{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
	496
	497	.{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none, .none },
	498
	499	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none, .none },
	500	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex, .none },
	501	.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .none, .none },
	502	.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .rex, .none },
	503	.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .none, .none },
	504	.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .rex, .none },
	505	.{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .short, .none },
	506	.{ .rcl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 2, .short, .none },
	507	.{ .rcl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 2, .short, .none },
	508	.{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none, .none },
	509	.{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long, .none },
	510	.{ .rcl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 2, .none, .none },
	511	.{ .rcl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 2, .long, .none },
	512	.{ .rcl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 2, .none, .none },
	513	.{ .rcl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 2, .long, .none },
	514
	515	.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .none, .none },
	516	.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .rex, .none },
	517	.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .none, .none },
	518	.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .rex, .none },
	519	.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .none, .none },
	520	.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .rex, .none },
	521	.{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .short, .none },
	522	.{ .rcr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 3, .short, .none },
	523	.{ .rcr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 3, .short, .none },
	524	.{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none, .none },
	525	.{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long, .none },
	526	.{ .rcr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 3, .none, .none },
	527	.{ .rcr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 3, .long, .none },
	528	.{ .rcr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 3, .none, .none },
	529	.{ .rcr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 3, .long, .none },
	530
	531	.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .none, .none },
	532	.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .rex, .none },
	533	.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .none, .none },
	534	.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .rex, .none },
	535	.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .none, .none },
	536	.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .rex, .none },
	537	.{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .short, .none },
	538	.{ .rol, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 0, .short, .none },
	539	.{ .rol, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 0, .short, .none },
	540	.{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none, .none },
	541	.{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long, .none },
	542	.{ .rol, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 0, .none, .none },
	543	.{ .rol, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 0, .long, .none },
	544	.{ .rol, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 0, .none, .none },
	545	.{ .rol, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 0, .long, .none },
	546
	547	.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .none, .none },
	548	.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .rex, .none },
	549	.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .none, .none },
	550	.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .rex, .none },
	551	.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .none, .none },
	552	.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .rex, .none },
	553	.{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .short, .none },
	554	.{ .ror, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 1, .short, .none },
	555	.{ .ror, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 1, .short, .none },
	556	.{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none, .none },
	557	.{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long, .none },
	558	.{ .ror, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 1, .none, .none },
	559	.{ .ror, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 1, .long, .none },
	560	.{ .ror, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 1, .none, .none },
	561	.{ .ror, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 1, .long, .none },
	562
	563	.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
	564	.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
	565	.{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .short, .none },
	566	.{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
	567	.{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
	568	.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none, .none },
	569	.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex, .none },
	570	.{ .sal, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .short, .none },
	571	.{ .sal, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none, .none },
	572	.{ .sal, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long, .none },
	573	.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none, .none },
	574	.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex, .none },
	575	.{ .sal, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .short, .none },
	576	.{ .sal, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none, .none },
	577	.{ .sal, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long, .none },
	578
	579	.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .none, .none },
	580	.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .rex, .none },
	581	.{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .short, .none },
	582	.{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none, .none },
	583	.{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long, .none },
	584	.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .none, .none },
	585	.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .rex, .none },
	586	.{ .sar, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 7, .short, .none },
	587	.{ .sar, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 7, .none, .none },
	588	.{ .sar, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 7, .long, .none },
	589	.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .none, .none },
	590	.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .rex, .none },
	591	.{ .sar, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 7, .short, .none },
	592	.{ .sar, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 7, .none, .none },
	593	.{ .sar, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 7, .long, .none },
	594
	595	.{ .sbb, .zi, &.{ .al, .imm8 }, &.{ 0x1c }, 0, .none, .none },
	596	.{ .sbb, .zi, &.{ .ax, .imm16 }, &.{ 0x1d }, 0, .short, .none },
	597	.{ .sbb, .zi, &.{ .eax, .imm32 }, &.{ 0x1d }, 0, .none, .none },
	598	.{ .sbb, .zi, &.{ .rax, .imm32s }, &.{ 0x1d }, 0, .long, .none },
	599	.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .none, .none },
	600	.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .rex, .none },
	601	.{ .sbb, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 3, .short, .none },
	602	.{ .sbb, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 3, .none, .none },
	603	.{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long, .none },
	604	.{ .sbb, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 3, .short, .none },
	605	.{ .sbb, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 3, .none, .none },
	606	.{ .sbb, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 3, .long, .none },
	607	.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .none, .none },
	608	.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .rex, .none },
	609	.{ .sbb, .mr, &.{ .rm16, .r16 }, &.{ 0x19 }, 0, .short, .none },
	610	.{ .sbb, .mr, &.{ .rm32, .r32 }, &.{ 0x19 }, 0, .none, .none },
	611	.{ .sbb, .mr, &.{ .rm64, .r64 }, &.{ 0x19 }, 0, .long, .none },
	612	.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .none, .none },
	613	.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .rex, .none },
	614	.{ .sbb, .rm, &.{ .r16, .rm16 }, &.{ 0x1b }, 0, .short, .none },
	615	.{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none, .none },
	616	.{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long, .none },
	617
	618	.{ .scas, .np, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
	619	.{ .scas, .np, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
	620	.{ .scas, .np, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
	621	.{ .scas, .np, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
	622
	623	.{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none, .none },
	624	.{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short, .none },
	625	.{ .scasd, .np, &.{}, &.{ 0xaf }, 0, .none, .none },
	626	.{ .scasq, .np, &.{}, &.{ 0xaf }, 0, .long, .none },
	627
	628	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
	629	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
	630	.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
	631	.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex, .none },
	632	.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
	633	.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex, .none },
	634	.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none, .none },
	635	.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex, .none },
	636	.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
	637	.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex, .none },
	638	.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
	639	.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
	640	.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none, .none },
	641	.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex, .none },
	642	.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none, .none },
	643	.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex, .none },
	644	.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none, .none },
	645	.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex, .none },
	646	.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none, .none },
	647	.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex, .none },
	648	.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none, .none },
	649	.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex, .none },
	650	.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
	651	.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex, .none },
	652	.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
	653	.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex, .none },
	654	.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
	655	.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
	656	.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
	657	.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex, .none },
	658	.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none, .none },
	659	.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex, .none },
	660	.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none, .none },
	661	.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex, .none },
	662	.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none, .none },
	663	.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex, .none },
	664	.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none, .none },
	665	.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex, .none },
	666	.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none, .none },
	667	.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex, .none },
	668	.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .none, .none },
	669	.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .rex, .none },
	670	.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none, .none },
	671	.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex, .none },
	672	.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .none, .none },
	673	.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .rex, .none },
	674	.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none, .none },
	675	.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex, .none },
	676	.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .none, .none },
	677	.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .rex, .none },
	678	.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none, .none },
	679	.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex, .none },
	680	.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none, .none },
	681	.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex, .none },
	682	.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none, .none },
	683	.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex, .none },
	684	.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .none, .none },
	685	.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .rex, .none },
	686	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
	687	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
	688
	689	.{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
	690
	691	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
	692	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
	693	.{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .short, .none },
	694	.{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
	695	.{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
	696	.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none, .none },
	697	.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex, .none },
	698	.{ .shl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .short, .none },
	699	.{ .shl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none, .none },
	700	.{ .shl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long, .none },
	701	.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none, .none },
	702	.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex, .none },
	703	.{ .shl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .short, .none },
	704	.{ .shl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none, .none },
	705	.{ .shl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long, .none },
	706
	707	.{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .short, .none },
	708	.{ .shld, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xa5 }, 0, .short, .none },
	709	.{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none, .none },
	710	.{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long, .none },
	711	.{ .shld, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xa5 }, 0, .none, .none },
	712	.{ .shld, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xa5 }, 0, .long, .none },
	713
	714	.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .none, .none },
	715	.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .rex, .none },
	716	.{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .short, .none },
	717	.{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none, .none },
	718	.{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long, .none },
	719	.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .none, .none },
	720	.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .rex, .none },
	721	.{ .shr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 5, .short, .none },
	722	.{ .shr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 5, .none, .none },
	723	.{ .shr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 5, .long, .none },
	724	.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .none, .none },
	725	.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .rex, .none },
	726	.{ .shr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 5, .short, .none },
	727	.{ .shr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 5, .none, .none },
	728	.{ .shr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 5, .long, .none },
	729
	730	.{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .short, .none },
	731	.{ .shrd, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xad }, 0, .short, .none },
	732	.{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none, .none },
	733	.{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long, .none },
	734	.{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },
	735	.{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },
	736
	737	.{ .stos, .np, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
	738	.{ .stos, .np, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
	739	.{ .stos, .np, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
	740	.{ .stos, .np, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
	741
	742	.{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none, .none },
	743	.{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short, .none },
	744	.{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none, .none },
	745	.{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long, .none },
	746
	747	.{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none, .none },
	748	.{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .short, .none },
	749	.{ .sub, .zi, &.{ .eax, .imm32 }, &.{ 0x2d }, 0, .none, .none },
	750	.{ .sub, .zi, &.{ .rax, .imm32s }, &.{ 0x2d }, 0, .long, .none },
	751	.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .none, .none },
	752	.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .rex, .none },
	753	.{ .sub, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 5, .short, .none },
	754	.{ .sub, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 5, .none, .none },
	755	.{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long, .none },
	756	.{ .sub, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 5, .short, .none },
	757	.{ .sub, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 5, .none, .none },
	758	.{ .sub, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 5, .long, .none },
	759	.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .none, .none },
	760	.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .rex, .none },
	761	.{ .sub, .mr, &.{ .rm16, .r16 }, &.{ 0x29 }, 0, .short, .none },
	762	.{ .sub, .mr, &.{ .rm32, .r32 }, &.{ 0x29 }, 0, .none, .none },
	763	.{ .sub, .mr, &.{ .rm64, .r64 }, &.{ 0x29 }, 0, .long, .none },
	764	.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .none, .none },
	765	.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .rex, .none },
	766	.{ .sub, .rm, &.{ .r16, .rm16 }, &.{ 0x2b }, 0, .short, .none },
	767	.{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none, .none },
	768	.{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long, .none },
	769
	770	.{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
	771
	772	.{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none, .none },
	773	.{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .short, .none },
	774	.{ .@"test", .zi, &.{ .eax, .imm32 }, &.{ 0xa9 }, 0, .none, .none },
	775	.{ .@"test", .zi, &.{ .rax, .imm32s }, &.{ 0xa9 }, 0, .long, .none },
	776	.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .none, .none },
	777	.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .rex, .none },
	778	.{ .@"test", .mi, &.{ .rm16, .imm16 }, &.{ 0xf7 }, 0, .short, .none },
	779	.{ .@"test", .mi, &.{ .rm32, .imm32 }, &.{ 0xf7 }, 0, .none, .none },
	780	.{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long, .none },
	781	.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .none, .none },
	782	.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .rex, .none },
	783	.{ .@"test", .mr, &.{ .rm16, .r16 }, &.{ 0x85 }, 0, .short, .none },
	784	.{ .@"test", .mr, &.{ .rm32, .r32 }, &.{ 0x85 }, 0, .none, .none },
	785	.{ .@"test", .mr, &.{ .rm64, .r64 }, &.{ 0x85 }, 0, .long, .none },
	786
	787	.{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .none },
	788	.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
	789	.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .none },
	790
	791	.{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
	792
	793	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
	794	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex, .none },
	795	.{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .short, .none },
	796	.{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
	797	.{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
	798
	799	.{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
	800	.{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
	801	.{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
	802	.{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
	803	.{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
	804	.{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
	805	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none, .none },
	806	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex, .none },
	807	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none, .none },
	808	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .rex, .none },
	809	.{ .xchg, .mr, &.{ .rm16, .r16 }, &.{ 0x87 }, 0, .short, .none },
	810	.{ .xchg, .rm, &.{ .r16, .rm16 }, &.{ 0x87 }, 0, .short, .none },
	811	.{ .xchg, .mr, &.{ .rm32, .r32 }, &.{ 0x87 }, 0, .none, .none },
	812	.{ .xchg, .mr, &.{ .rm64, .r64 }, &.{ 0x87 }, 0, .long, .none },
	813	.{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none, .none },
	814	.{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long, .none },
	815
	816	.{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none, .none },
	817	.{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .short, .none },
	818	.{ .xor, .zi, &.{ .eax, .imm32 }, &.{ 0x35 }, 0, .none, .none },
	819	.{ .xor, .zi, &.{ .rax, .imm32s }, &.{ 0x35 }, 0, .long, .none },
	820	.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .none, .none },
	821	.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .rex, .none },
	822	.{ .xor, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 6, .short, .none },
	823	.{ .xor, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 6, .none, .none },
	824	.{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long, .none },
	825	.{ .xor, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 6, .short, .none },
	826	.{ .xor, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 6, .none, .none },
	827	.{ .xor, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 6, .long, .none },
	828	.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .none, .none },
	829	.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .rex, .none },
	830	.{ .xor, .mr, &.{ .rm16, .r16 }, &.{ 0x31 }, 0, .short, .none },
	831	.{ .xor, .mr, &.{ .rm32, .r32 }, &.{ 0x31 }, 0, .none, .none },
	832	.{ .xor, .mr, &.{ .rm64, .r64 }, &.{ 0x31 }, 0, .long, .none },
	833	.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .none, .none },
	834	.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .rex, .none },
	835	.{ .xor, .rm, &.{ .r16, .rm16 }, &.{ 0x33 }, 0, .short, .none },
	836	.{ .xor, .rm, &.{ .r32, .rm32 }, &.{ 0x33 }, 0, .none, .none },
	837	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
831	838
832	839	// SSE
833		.{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse },
	840	.{ .addps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .none, .sse },
834	841
835		.{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .sse },
	842	.{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse },
836	843
837		.{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .sse },
	844	.{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .none, .sse },
838	845
839		.{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },
	846	.{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .none, .sse },
840	847
841		.{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse },
842		.{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse_long },
	848	.{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse },
843	849
844		.{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse },
	850	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse },
	851	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse },
845	852
846		.{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse },
	853	.{ .divps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .none, .sse },
847	854
848		.{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse },
	855	.{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse },
849	856
850		.{ .movaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .sse },
851		.{ .movaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .sse },
	857	.{ .maxps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5f }, 0, .none, .sse },
852	858
853		.{ .movss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse },
854		.{ .movss, .mr, &.{ .xmm_m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse },
	859	.{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .none, .sse },
855	860
856		.{ .movups, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .sse },
857		.{ .movups, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x11 }, 0, .sse },
	861	.{ .minps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5d }, 0, .none, .sse },
858	862
859		.{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },
	863	.{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .none, .sse },
860	864
861		.{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .sse },
	865	.{ .movaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .none, .sse },
	866	.{ .movaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .none, .sse },
862	867
863		.{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },
	868	.{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
864	869
865		.{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .sse },
866		.{ .sqrtss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .sse },
	870	.{ .movss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse },
	871	.{ .movss, .mr, &.{ .xmm_m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse },
867	872
868		.{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .sse },
	873	.{ .movups, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .none, .sse },
	874	.{ .movups, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x11 }, 0, .none, .sse },
869	875
870		.{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .sse },
	876	.{ .mulps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x59 }, 0, .none, .sse },
	877
	878	.{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .none, .sse },
	879
	880	.{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .none, .sse },
	881
	882	.{ .subps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .none, .sse },
	883
	884	.{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
	885
	886	.{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },
	887
	888	.{ .sqrtss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse },
	889
	890	.{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .none, .sse },
	891
	892	.{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .none, .sse },
871	893
872	894	// SSE2
873		.{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },
	895	.{ .addpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .none, .sse2 },
	896
	897	.{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .none, .sse2 },
	898
	899	.{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .none, .sse2 },
	900
	901	.{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 },
	902
	903	.{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 },
	904
	905	.{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .none, .sse2 },
	906
	907	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .none, .sse2 },
	908	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .long, .sse2 },
	909
	910	.{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 },
	911
	912	.{ .divpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .none, .sse2 },
	913
	914	.{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 },
	915
	916	.{ .maxpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .none, .sse2 },
	917
	918	.{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .none, .sse2 },
	919
	920	.{ .minpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5d }, 0, .none, .sse2 },
	921
	922	.{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .none, .sse2 },
	923
	924	.{ .movapd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .none, .sse2 },
	925	.{ .movapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .none, .sse2 },
	926
	927	.{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .none, .sse2 },
	928	.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 },
874	929
875		.{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .sse2 },
	930	.{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .long, .sse2 },
	931	.{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .long, .sse2 },
876	932
877		.{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .sse2 },
	933	.{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .none, .sse2 },
	934	.{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .none, .sse2 },
878	935
879		.{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },
	936	.{ .movupd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .none, .sse2 },
	937	.{ .movupd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x11 }, 0, .none, .sse2 },
880	938
881		.{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .sse2 },
	939	.{ .mulpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x59 }, 0, .none, .sse2 },
882	940
883		.{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2 },
884		.{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2_long },
	941	.{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .none, .sse2 },
885	942
886		.{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .sse2 },
	943	.{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
887	944
888		.{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 },
	945	.{ .pextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
889	946
890		.{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 },
	947	.{ .pinsrw, .rmi, &.{ .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
891	948
892		.{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
	949	.{ .pshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .none, .sse2 },
893	950
894		.{ .movapd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .sse2 },
895		.{ .movapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .sse2 },
	951	.{ .pshuflw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 },
896	952
897		.{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2 },
898		.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2 },
	953	.{ .psrlw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 },
	954	.{ .psrlw, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 },
	955	.{ .psrld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .none, .sse2 },
	956	.{ .psrld, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x72 }, 2, .none, .sse2 },
	957	.{ .psrlq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 },
	958	.{ .psrlq, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 },
899	959
900		.{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2_long },
901		.{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2_long },
	960	.{ .punpckhbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 },
	961	.{ .punpckhwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 },
	962	.{ .punpckhdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 },
	963	.{ .punpckhqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .none, .sse2 },
902	964
903		.{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
904		.{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
	965	.{ .punpcklbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 },
	966	.{ .punpcklwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 },
	967	.{ .punpckldq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 },
	968	.{ .punpcklqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 },
905	969
906		.{ .movupd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .sse2 },
907		.{ .movupd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x11 }, 0, .sse2 },
	970	.{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
908	971
909		.{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },
	972	.{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
910	973
911		.{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .sse2 },
	974	.{ .subpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5c }, 0, .none, .sse2 },
912	975
913		.{ .pextrw, .mri, &.{ .r16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .sse2 },
	976	.{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 },
914	977
915		.{ .pinsrw, .rmi, &.{ .xmm, .rm16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .sse2 },
	978	.{ .movsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .none, .sse2 },
	979	.{ .movsd, .mr, &.{ .xmm_m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .none, .sse2 },
916	980
917		.{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .sse2 },
918		.{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .sse2 },
	981	.{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .none, .sse2 },
919	982
920		.{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },
	983	.{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .none, .sse2 },
921	984
922		.{ .movsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },
923		.{ .movsd, .mr, &.{ .xmm_m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .sse2 },
	985	// SSE3
	986	.{ .movddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .none, .sse3 },
924	987
925		.{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 },
	988	.{ .movshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .none, .sse3 },
926	989
927		.{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .sse2 },
	990	.{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
928	991
929	992	// SSE4.1
930		.{ .pextrw, .mri, &.{ .rm16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .sse4_1 },
	993	.{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
	994	.{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },
	995	.{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },
	996
	997	.{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
	998
	999	.{ .pinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .none, .sse4_1 },
	1000	.{ .pinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 },
	1001	.{ .pinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 },
	1002
	1003	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
	1004
	1005	.{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
	1006
	1007	.{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
	1008
	1009	.{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
	1010
	1011	// AVX
	1012	.{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
	1013	.{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
	1014
	1015	.{ .vaddps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
	1016	.{ .vaddps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
	1017
	1018	.{ .vaddsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
	1019
	1020	.{ .vaddss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
	1021
	1022	.{ .vcvtsd2ss, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
	1023
	1024	.{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
	1025	.{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
	1026
	1027	.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
	1028	.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
	1029
	1030	.{ .vcvtss2sd, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
	1031
	1032	.{ .vdivpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
	1033	.{ .vdivpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
	1034
	1035	.{ .vdivps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
	1036	.{ .vdivps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
	1037
	1038	.{ .vdivsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
	1039
	1040	.{ .vdivss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
	1041
	1042	.{ .vmaxpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
	1043	.{ .vmaxpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
	1044
	1045	.{ .vmaxps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
	1046	.{ .vmaxps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
	1047
	1048	.{ .vmaxsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx },
	1049
	1050	.{ .vmaxss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx },
	1051
	1052	.{ .vminpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5d }, 0, .vex_128_wig, .avx },
	1053	.{ .vminpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5d }, 0, .vex_256_wig, .avx },
	1054
	1055	.{ .vminps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5d }, 0, .vex_128_wig, .avx },
	1056	.{ .vminps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5d }, 0, .vex_256_wig, .avx },
	1057
	1058	.{ .vminsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx },
	1059
	1060	.{ .vminss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx },
	1061
	1062	.{ .vmovapd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
	1063	.{ .vmovapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
	1064	.{ .vmovapd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
	1065	.{ .vmovapd, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
	1066
	1067	.{ .vmovaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
	1068	.{ .vmovaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
	1069	.{ .vmovaps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
	1070	.{ .vmovaps, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
	1071
	1072	.{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
	1073	.{ .vmovddup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
	1074
	1075	.{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
	1076
	1077	.{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
	1078	.{ .vmovsd, .rm, &.{ .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
	1079	.{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
	1080	.{ .vmovsd, .mr, &.{ .m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
	1081
	1082	.{ .vmovshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
	1083	.{ .vmovshdup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_256_wig, .avx },
	1084
	1085	.{ .vmovsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
	1086	.{ .vmovsldup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
	1087
	1088	.{ .vmovss, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
	1089	.{ .vmovss, .rm, &.{ .xmm, .m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
	1090	.{ .vmovss, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
	1091	.{ .vmovss, .mr, &.{ .m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
	1092
	1093	.{ .vmovupd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_128_wig, .avx },
	1094	.{ .vmovupd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_128_wig, .avx },
	1095	.{ .vmovupd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
	1096	.{ .vmovupd, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
	1097
	1098	.{ .vmovups, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .vex_128_wig, .avx },
	1099	.{ .vmovups, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x11 }, 0, .vex_128_wig, .avx },
	1100	.{ .vmovups, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
	1101	.{ .vmovups, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
	1102
	1103	.{ .vmulpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x59 }, 0, .vex_128_wig, .avx },
	1104	.{ .vmulpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x59 }, 0, .vex_256_wig, .avx },
	1105
	1106	.{ .vmulps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x59 }, 0, .vex_128_wig, .avx },
	1107	.{ .vmulps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x59 }, 0, .vex_256_wig, .avx },
	1108
	1109	.{ .vmulsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
	1110
	1111	.{ .vmulss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
	1112
	1113	.{ .vpextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx },
	1114	.{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
	1115	.{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
	1116
	1117	.{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx },
	1118	.{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
	1119
	1120	.{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
	1121	.{ .vpinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
	1122	.{ .vpinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
	1123
	1124	.{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
	1125
	1126	.{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx },
	1127	.{ .vpsrlw, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_128_wig, .avx },
	1128	.{ .vpsrld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_128_wig, .avx },
	1129	.{ .vpsrld, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_128_wig, .avx },
	1130	.{ .vpsrlq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_128_wig, .avx },
	1131	.{ .vpsrlq, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_128_wig, .avx },
	1132
	1133	.{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },
	1134	.{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },
	1135	.{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },
	1136	.{ .vpunpckhqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_128_wig, .avx },
	1137
	1138	.{ .vpunpcklbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_128_wig, .avx },
	1139	.{ .vpunpcklwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_128_wig, .avx },
	1140	.{ .vpunpckldq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx },
	1141	.{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx },
	1142
	1143	.{ .vroundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_128_wig, .avx },
	1144	.{ .vroundpd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_256_wig, .avx },
	1145
	1146	.{ .vroundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_128_wig, .avx },
	1147	.{ .vroundps, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_256_wig, .avx },
	1148
	1149	.{ .vroundsd, .rvmi, &.{ .xmm, .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .vex_lig_wig, .avx },
	1150
	1151	.{ .vroundss, .rvmi, &.{ .xmm, .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .vex_lig_wig, .avx },
	1152
	1153	.{ .vsqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
	1154	.{ .vsqrtpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
	1155
	1156	.{ .vsqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
	1157	.{ .vsqrtps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
	1158
	1159	.{ .vsqrtsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
	1160
	1161	.{ .vsqrtss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
	1162
	1163	.{ .vsubpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5c }, 0, .vex_128_wig, .avx },
	1164	.{ .vsubpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5c }, 0, .vex_256_wig, .avx },
	1165
	1166	.{ .vsubps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .vex_128_wig, .avx },
	1167	.{ .vsubps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5c }, 0, .vex_256_wig, .avx },
	1168
	1169	.{ .vsubsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
	1170
	1171	.{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
	1172
	1173	// F16C
	1174	.{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
	1175	.{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c },
	1176
	1177	.{ .vcvtps2ph, .mri, &.{ .xmm_m64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128_w0, .f16c },
	1178	.{ .vcvtps2ph, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_256_w0, .f16c },
	1179
	1180	// FMA
	1181	.{ .vfmadd132pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w1, .fma },
	1182	.{ .vfmadd213pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w1, .fma },
	1183	.{ .vfmadd231pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w1, .fma },
	1184	.{ .vfmadd132pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w1, .fma },
	1185	.{ .vfmadd213pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w1, .fma },
	1186	.{ .vfmadd231pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w1, .fma },
	1187
	1188	.{ .vfmadd132ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w0, .fma },
	1189	.{ .vfmadd213ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w0, .fma },
	1190	.{ .vfmadd231ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w0, .fma },
	1191	.{ .vfmadd132ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w0, .fma },
	1192	.{ .vfmadd213ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w0, .fma },
	1193	.{ .vfmadd231ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w0, .fma },
	1194
	1195	.{ .vfmadd132sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w1, .fma },
	1196	.{ .vfmadd213sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w1, .fma },
	1197	.{ .vfmadd231sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w1, .fma },
	1198
	1199	.{ .vfmadd132ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w0, .fma },
	1200	.{ .vfmadd213ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w0, .fma },
	1201	.{ .vfmadd231ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w0, .fma },
	1202
	1203	// AVX2
	1204	.{ .vpsrlw, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_256_wig, .avx2 },
	1205	.{ .vpsrlw, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_256_wig, .avx2 },
	1206	.{ .vpsrld, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_256_wig, .avx2 },
	1207	.{ .vpsrld, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_256_wig, .avx2 },
	1208	.{ .vpsrlq, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_256_wig, .avx2 },
	1209	.{ .vpsrlq, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_256_wig, .avx2 },
	1210
	1211	.{ .vpunpckhbw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_256_wig, .avx2 },
	1212	.{ .vpunpckhwd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_256_wig, .avx2 },
	1213	.{ .vpunpckhdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_256_wig, .avx2 },
	1214	.{ .vpunpckhqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_256_wig, .avx2 },
931	1215
932		.{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },
933		.{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },
	1216	.{ .vpunpcklbw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_256_wig, .avx2 },
	1217	.{ .vpunpcklwd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_256_wig, .avx2 },
	1218	.{ .vpunpckldq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_256_wig, .avx2 },
	1219	.{ .vpunpcklqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_256_wig, .avx2 },
934	1220	};
935	1221	// zig fmt: on

src/link/Dwarf.zig+29-25

...	...	@@ -526,7 +526,7 @@ pub const DeclState = struct {
526	526	.ErrorUnion => {
527	527	const error_ty = ty.errorUnionSet();
528	528	const payload_ty = ty.errorUnionPayload();
529		const payload_align = payload_ty.abiAlignment(target);
	529	const payload_align = if (payload_ty.isNoReturn()) 0 else payload_ty.abiAlignment(target);
530	530	const error_align = Type.anyerror.abiAlignment(target);
531	531	const abi_size = ty.abiSize(target);
532	532	const payload_off = if (error_align >= payload_align) Type.anyerror.abiSize(target) else 0;
...	...	@@ -540,31 +540,35 @@ pub const DeclState = struct {
540	540	const name = try ty.nameAllocArena(arena, module);
541	541	try dbg_info_buffer.writer().print("{s}\x00", .{name});
542	542
543		// DW.AT.member
544		try dbg_info_buffer.ensureUnusedCapacity(7);
545		dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
546		// DW.AT.name, DW.FORM.string
547		dbg_info_buffer.appendSliceAssumeCapacity("value");
548		dbg_info_buffer.appendAssumeCapacity(0);
549		// DW.AT.type, DW.FORM.ref4
550		var index = dbg_info_buffer.items.len;
551		try dbg_info_buffer.resize(index + 4);
552		try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
553		// DW.AT.data_member_location, DW.FORM.sdata
554		try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);
	543	if (!payload_ty.isNoReturn()) {
	544	// DW.AT.member
	545	try dbg_info_buffer.ensureUnusedCapacity(7);
	546	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
	547	// DW.AT.name, DW.FORM.string
	548	dbg_info_buffer.appendSliceAssumeCapacity("value");
	549	dbg_info_buffer.appendAssumeCapacity(0);
	550	// DW.AT.type, DW.FORM.ref4
	551	const index = dbg_info_buffer.items.len;
	552	try dbg_info_buffer.resize(index + 4);
	553	try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
	554	// DW.AT.data_member_location, DW.FORM.sdata
	555	try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);
	556	}
555	557
556		// DW.AT.member
557		try dbg_info_buffer.ensureUnusedCapacity(5);
558		dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
559		// DW.AT.name, DW.FORM.string
560		dbg_info_buffer.appendSliceAssumeCapacity("err");
561		dbg_info_buffer.appendAssumeCapacity(0);
562		// DW.AT.type, DW.FORM.ref4
563		index = dbg_info_buffer.items.len;
564		try dbg_info_buffer.resize(index + 4);
565		try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));
566		// DW.AT.data_member_location, DW.FORM.sdata
567		try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);
	558	{
	559	// DW.AT.member
	560	try dbg_info_buffer.ensureUnusedCapacity(5);
	561	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
	562	// DW.AT.name, DW.FORM.string
	563	dbg_info_buffer.appendSliceAssumeCapacity("err");
	564	dbg_info_buffer.appendAssumeCapacity(0);
	565	// DW.AT.type, DW.FORM.ref4
	566	const index = dbg_info_buffer.items.len;
	567	try dbg_info_buffer.resize(index + 4);
	568	try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));
	569	// DW.AT.data_member_location, DW.FORM.sdata
	570	try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);
	571	}
568	572
569	573	// DW.AT.structure_type delimit children
570	574	try dbg_info_buffer.append(0);

test/behavior/bugs/12891.zig-6

...	...	@@ -29,7 +29,6 @@ test "inf >= 1" {
29	29	test "isNan(nan * 1)" {
30	30	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
31	31	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
32		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
33	32
34	33	const nan_times_one = comptime std.math.nan(f64) * 1;
35	34	try std.testing.expect(std.math.isNan(nan_times_one));
...	...	@@ -37,7 +36,6 @@ test "isNan(nan * 1)" {
37	36	test "runtime isNan(nan * 1)" {
38	37	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
39	38	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
40		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
41	39
42	40	const nan_times_one = std.math.nan(f64) * 1;
43	41	try std.testing.expect(std.math.isNan(nan_times_one));
...	...	@@ -45,7 +43,6 @@ test "runtime isNan(nan * 1)" {
45	43	test "isNan(nan * 0)" {
46	44	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
47	45	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
48		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
49	46
50	47	const nan_times_zero = comptime std.math.nan(f64) * 0;
51	48	try std.testing.expect(std.math.isNan(nan_times_zero));
...	...	@@ -55,7 +52,6 @@ test "isNan(nan * 0)" {
55	52	test "isNan(inf * 0)" {
56	53	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
57	54	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
58		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
59	55
60	56	const inf_times_zero = comptime std.math.inf(f64) * 0;
61	57	try std.testing.expect(std.math.isNan(inf_times_zero));
...	...	@@ -65,7 +61,6 @@ test "isNan(inf * 0)" {
65	61	test "runtime isNan(nan * 0)" {
66	62	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
67	63	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
68		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
69	64
70	65	const nan_times_zero = std.math.nan(f64) * 0;
71	66	try std.testing.expect(std.math.isNan(nan_times_zero));
...	...	@@ -75,7 +70,6 @@ test "runtime isNan(nan * 0)" {
75	70	test "runtime isNan(inf * 0)" {
76	71	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
77	72	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
78		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
79	73
80	74	const inf_times_zero = std.math.inf(f64) * 0;
81	75	try std.testing.expect(std.math.isNan(inf_times_zero));

test/behavior/bugs/2114.zig+2-1

...	...	@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {
9	9
10	10	test "fixed" {
11	11	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
12		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	12	if (builtin.zig_backend == .stage2_x86_64 and
	13	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .bmi)) return error.SkipZigTest; // TODO
13	14	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
14	15	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15	16	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/error.zig-3

...	...	@@ -757,7 +757,6 @@ test "error union of noreturn used with if" {
757	757	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
758	758	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
759	759	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
760		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
761	760	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
762	761
763	762	NoReturn.a = 64;
...	...	@@ -772,7 +771,6 @@ test "error union of noreturn used with try" {
772	771	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
773	772	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
774	773	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
775		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
776	774	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
777	775
778	776	NoReturn.a = 64;
...	...	@@ -784,7 +782,6 @@ test "error union of noreturn used with catch" {
784	782	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
785	783	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
786	784	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
787		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
788	785	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
789	786
790	787	NoReturn.a = 64;

test/behavior/field_parent_ptr.zig-1

...	...	@@ -2,7 +2,6 @@ const expect = @import("std").testing.expect;
2	2	const builtin = @import("builtin");
3	3
4	4	test "@fieldParentPtr non-first field" {
5		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
6	5	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
7	6	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
8	7	try testParentFieldPtr(&foo.c);

test/behavior/floatop.zig+17-14

...	...	@@ -8,6 +8,8 @@ const has_f80_rt = switch (builtin.cpu.arch) {
8	8	.x86_64, .x86 => true,
9	9	else => false,
10	10	};
	11	const no_x86_64_hardware_f16_support = builtin.zig_backend == .stage2_x86_64 and
	12	!std.Target.x86.featureSetHas(builtin.cpu.features, .f16c);
11	13
12	14	const epsilon_16 = 0.001;
13	15	const epsilon = 0.000001;
...	...	@@ -52,7 +54,7 @@ fn testFloatComparisons() !void {
52	54	}
53	55
54	56	test "different sized float comparisons" {
55		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	57	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
56	58	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
57	59	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
58	60	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -134,7 +136,6 @@ fn testSqrt() !void {
134	136
135	137	test "@sqrt with vectors" {
136	138	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
137		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
138	139	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
139	140	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
140	141
...	...	@@ -152,7 +153,7 @@ fn testSqrtWithVectors() !void {
152	153	}
153	154
154	155	test "more @sqrt f16 tests" {
155		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	156	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
156	157	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
157	158	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
158	159	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -202,7 +203,7 @@ fn testSqrtLegacy(comptime T: type, x: T) !void {
202	203	}
203	204
204	205	test "@sin" {
205		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	206	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
206	207	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
207	208	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
208	209	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -241,7 +242,7 @@ fn testSinWithVectors() !void {
241	242	}
242	243
243	244	test "@cos" {
244		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	245	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
245	246	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
246	247	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
247	248	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -280,7 +281,7 @@ fn testCosWithVectors() !void {
280	281	}
281	282
282	283	test "@exp" {
283		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	284	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
284	285	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
285	286	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
286	287	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -318,7 +319,7 @@ fn testExpWithVectors() !void {
318	319	}
319	320
320	321	test "@exp2" {
321		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	322	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
322	323	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
323	324	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
324	325	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -403,7 +404,7 @@ test "@log with @vectors" {
403	404	}
404	405
405	406	test "@log2" {
406		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	407	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
407	408	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
408	409	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
409	410	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -445,7 +446,7 @@ fn testLog2WithVectors() !void {
445	446	}
446	447
447	448	test "@log10" {
448		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	449	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
449	450	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
450	451	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
451	452	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -617,7 +618,8 @@ fn testFloor() !void {
617	618
618	619	test "@floor with vectors" {
619	620	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
620		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	621	if (builtin.zig_backend == .stage2_x86_64 and
	622	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
621	623	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
622	624	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
623	625
...	...	@@ -707,7 +709,8 @@ fn testCeil() !void {
707	709
708	710	test "@ceil with vectors" {
709	711	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
710		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	712	if (builtin.zig_backend == .stage2_x86_64 and
	713	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
711	714	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
712	715	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
713	716
...	...	@@ -797,7 +800,8 @@ fn testTrunc() !void {
797	800
798	801	test "@trunc with vectors" {
799	802	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
800		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	803	if (builtin.zig_backend == .stage2_x86_64 and
	804	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
801	805	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
802	806	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
803	807
...	...	@@ -878,7 +882,7 @@ fn testTruncLegacy(comptime T: type, x: T) !void {
878	882	}
879	883
880	884	test "negation f16" {
881		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	885	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
882	886	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
883	887	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
884	888	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1037,7 +1041,6 @@ test "comptime_float zero divided by zero produces zero" {
1037	1041	}
1038	1042
1039	1043	test "nan negation f16" {
1040		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1041	1044	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1042	1045	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1043	1046	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/math.zig+4-2

...	...	@@ -77,7 +77,8 @@ fn testClz() !void {
77	77	}
78	78
79	79	test "@clz big ints" {
80		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	80	if (builtin.zig_backend == .stage2_x86_64 and
	81	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .lzcnt)) return error.SkipZigTest; // TODO
81	82	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
82	83	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
83	84	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -398,7 +399,8 @@ fn testBinaryNot128(comptime Type: type, x: Type) !void {
398	399
399	400	test "division" {
400	401	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
401		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	402	if (builtin.zig_backend == .stage2_x86_64 and
	403	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
402	404	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
403	405	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
404	406	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/muladd.zig+8-4

...	...	@@ -1,8 +1,12 @@
	1	const std = @import("std");
1	2	const builtin = @import("builtin");
2		const expect = @import("std").testing.expect;
	3	const expect = std.testing.expect;
	4
	5	const no_x86_64_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
	6	!std.Target.x86.featureSetHas(builtin.cpu.features, .fma);
3	7
4	8	test "@mulAdd" {
5		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	9	if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
6	10	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
7	11	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8	12	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -116,7 +120,7 @@ fn vector32() !void {
116	120
117	121	test "vector f32" {
118	122	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
119		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	123	if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
120	124	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
121	125	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
122	126	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -139,7 +143,7 @@ fn vector64() !void {
139	143
140	144	test "vector f64" {
141	145	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
142		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	146	if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
143	147	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
144	148	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
145	149	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/vector.zig+2-1

...	...	@@ -168,7 +168,8 @@ test "array to vector" {
168	168
169	169	test "array to vector with element type coercion" {
170	170	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
171		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
	171	if (builtin.zig_backend == .stage2_x86_64 and
	172	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest; // TODO
172	173	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
173	174	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
174	175	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO