Commit 5d4288c5f6c69bdd4cbd9b3580016828e38f087d | zig

author	Jacob Young <jacobly@ziglang.org> 2023-05-06 06:00:22-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-05-08 07:36:19-04:00
log	5d4288c5f6c69bdd4cbd9b3580016828e38f087d
tree	c5d1804ee4759fdf4cdd8cb8f649ad3f2fe16c00
parent	ae588a09f2c2146ada0f914c7d279f69a0d79396

x86_64: fix unordered float equality

8 files changed, 599 insertions(+), 363 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig
test
- behavior
  - bugs
    12891.zig
  - field_parent_ptr.zig

src/arch/x86_64/CodeGen.zig+261-154

...	...	@@ -205,16 +205,7 @@ pub const MCValue = union(enum) {
205	205
206	206	fn isMemory(mcv: MCValue) bool {
207	207	return switch (mcv) {
208		.memory,
209		.load_direct,
210		.lea_direct,
211		.load_got,
212		.lea_got,
213		.load_tlv,
214		.lea_tlv,
215		.load_frame,
216		.lea_frame,
217		=> true,
	208	.memory, .indirect, .load_frame => true,
218	209	else => false,
219	210	};
220	211	}
...	...	@@ -937,7 +928,7 @@ fn formatWipMir(
937	928	.target = data.self.target,
938	929	.src_loc = data.self.src_loc,
939	930	};
940		for (lower.lowerMir(data.self.mir_instructions.get(data.inst)) catch \|err\| switch (err) {
	931	for ((lower.lowerMir(data.inst) catch \|err\| switch (err) {
941	932	error.LowerFail => {
942	933	defer {
943	934	lower.err_msg.?.deinit(data.self.gpa);
...	...	@@ -955,7 +946,7 @@ fn formatWipMir(
955	946	return;
956	947	},
957	948	else => \|e\| return e,
958		}) \|lower_inst\| try writer.print(" \| {}", .{lower_inst});
	949	}).insts) \|lowered_inst\| try writer.print(" \| {}", .{lowered_inst});
959	950	}
960	951	fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
961	952	return .{ .data = .{ .self = self, .inst = inst } };
...	...	@@ -1016,7 +1007,14 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1016	1007	_ = try self.addInst(.{
1017	1008	.tag = .setcc,
1018	1009	.ops = .r_cc,
1019		.data = .{ .r_cc = .{ .r = reg, .cc = cc } },
	1010	.data = .{ .r_cc = .{
	1011	.r = reg,
	1012	.scratch = if (cc == .z_and_np or cc == .nz_or_p)
	1013	(try self.register_manager.allocReg(null, gp)).to8()
	1014	else
	1015	.none,
	1016	.cc = cc,
	1017	} },
1020	1018	});
1021	1019	}
1022	1020
...	...	@@ -1028,23 +1026,36 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
1028	1026	.rip => .m_rip_cc,
1029	1027	else => unreachable,
1030	1028	},
1031		.data = .{ .x_cc = .{ .cc = cc, .payload = switch (m) {
1032		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1033		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1034		else => unreachable,
1035		} } },
	1029	.data = .{ .x_cc = .{
	1030	.scratch = if (cc == .z_and_np or cc == .nz_or_p)
	1031	(try self.register_manager.allocReg(null, gp)).to8()
	1032	else
	1033	.none,
	1034	.cc = cc,
	1035	.payload = switch (m) {
	1036	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1037	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1038	else => unreachable,
	1039	},
	1040	} },
1036	1041	});
1037	1042	}
1038	1043
	1044	/// A `cc` of `.z_and_np` clobbers `reg2`!
1039	1045	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1040	1046	_ = try self.addInst(.{
1041	1047	.tag = .cmovcc,
1042	1048	.ops = .rr_cc,
1043		.data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc } },
	1049	.data = .{ .rr_cc = .{
	1050	.r1 = reg1,
	1051	.r2 = reg2,
	1052	.cc = cc,
	1053	} },
1044	1054	});
1045	1055	}
1046	1056
1047	1057	fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
	1058	assert(cc != .z_and_np); // not supported
1048	1059	_ = try self.addInst(.{
1049	1060	.tag = .cmovcc,
1050	1061	.ops = switch (m) {
...	...	@@ -1052,11 +1063,15 @@ fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condi
1052	1063	.rip => .rm_rip_cc,
1053	1064	else => unreachable,
1054	1065	},
1055		.data = .{ .rx_cc = .{ .r = reg, .cc = cc, .payload = switch (m) {
1056		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1057		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1058		else => unreachable,
1059		} } },
	1066	.data = .{ .rx_cc = .{
	1067	.r = reg,
	1068	.cc = cc,
	1069	.payload = switch (m) {
	1070	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1071	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1072	else => unreachable,
	1073	},
	1074	} },
1060	1075	});
1061	1076	}
1062	1077
...	...	@@ -1131,10 +1146,13 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1131	1146	.tag = tag,
1132	1147	.ops = ops,
1133	1148	.data = switch (ops) {
1134		.ri_s, .ri_u => .{ .ri = .{ .r = reg, .i = switch (imm) {
1135		.signed => \|s\| @bitCast(u32, s),
1136		.unsigned => \|u\| @intCast(u32, u),
1137		} } },
	1149	.ri_s, .ri_u => .{ .ri = .{
	1150	.r = reg,
	1151	.i = switch (imm) {
	1152	.signed => \|s\| @bitCast(u32, s),
	1153	.unsigned => \|u\| @intCast(u32, u),
	1154	},
	1155	} },
1138	1156	.ri64 => .{ .rx = .{
1139	1157	.r = reg,
1140	1158	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
...	...	@@ -1171,10 +1189,14 @@ fn asmRegisterRegisterImmediate(
1171	1189	.signed => .rri_s,
1172	1190	.unsigned => .rri_u,
1173	1191	},
1174		.data = .{ .rri = .{ .r1 = reg1, .r2 = reg2, .i = switch (imm) {
1175		.signed => \|s\| @bitCast(u32, s),
1176		.unsigned => \|u\| @intCast(u32, u),
1177		} } },
	1192	.data = .{ .rri = .{
	1193	.r1 = reg1,
	1194	.r2 = reg2,
	1195	.i = switch (imm) {
	1196	.signed => \|s\| @bitCast(u32, s),
	1197	.unsigned => \|u\| @intCast(u32, u),
	1198	},
	1199	} },
1178	1200	});
1179	1201	}
1180	1202
...	...	@@ -1202,11 +1224,14 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
1202	1224	.rip => .rm_rip,
1203	1225	else => unreachable,
1204	1226	},
1205		.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
1206		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1207		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1208		else => unreachable,
1209		} } },
	1227	.data = .{ .rx = .{
	1228	.r = reg,
	1229	.payload = switch (m) {
	1230	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1231	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1232	else => unreachable,
	1233	},
	1234	} },
1210	1235	});
1211	1236	}
1212	1237
...	...	@@ -1224,11 +1249,43 @@ fn asmRegisterMemoryImmediate(
1224	1249	.rip => .rmi_rip,
1225	1250	else => unreachable,
1226	1251	},
1227		.data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
1228		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1229		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1252	.data = .{ .rix = .{
	1253	.r = reg,
	1254	.i = @intCast(u8, imm.unsigned),
	1255	.payload = switch (m) {
	1256	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1257	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1258	else => unreachable,
	1259	},
	1260	} },
	1261	});
	1262	}
	1263
	1264	fn asmRegisterRegisterMemoryImmediate(
	1265	self: *Self,
	1266	tag: Mir.Inst.Tag,
	1267	reg1: Register,
	1268	reg2: Register,
	1269	m: Memory,
	1270	imm: Immediate,
	1271	) !void {
	1272	_ = try self.addInst(.{
	1273	.tag = tag,
	1274	.ops = switch (m) {
	1275	.sib => .rrmi_sib,
	1276	.rip => .rrmi_rip,
1230	1277	else => unreachable,
1231		} } },
	1278	},
	1279	.data = .{ .rrix = .{
	1280	.r1 = reg1,
	1281	.r2 = reg2,
	1282	.i = @intCast(u8, imm.unsigned),
	1283	.payload = switch (m) {
	1284	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1285	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1286	else => unreachable,
	1287	},
	1288	} },
1232	1289	});
1233	1290	}
1234	1291
...	...	@@ -1240,11 +1297,14 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
1240	1297	.rip => .mr_rip,
1241	1298	else => unreachable,
1242	1299	},
1243		.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
1244		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1245		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1246		else => unreachable,
1247		} } },
	1300	.data = .{ .rx = .{
	1301	.r = reg,
	1302	.payload = switch (m) {
	1303	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1304	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1305	else => unreachable,
	1306	},
	1307	} },
1248	1308	});
1249	1309	}
1250	1310
...	...	@@ -1262,14 +1322,17 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
1262	1322	},
1263	1323	else => unreachable,
1264	1324	},
1265		.data = .{ .ix = .{ .i = switch (imm) {
1266		.signed => \|s\| @bitCast(u32, s),
1267		.unsigned => \|u\| @intCast(u32, u),
1268		}, .payload = switch (m) {
1269		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1270		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1271		else => unreachable,
1272		} } },
	1325	.data = .{ .ix = .{
	1326	.i = switch (imm) {
	1327	.signed => \|s\| @bitCast(u32, s),
	1328	.unsigned => \|u\| @intCast(u32, u),
	1329	},
	1330	.payload = switch (m) {
	1331	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	1332	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	1333	else => unreachable,
	1334	},
	1335	} },
1273	1336	});
1274	1337	}
1275	1338
...	...	@@ -6612,11 +6675,13 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6612	6675	_ = try self.addInst(.{
6613	6676	.tag = .mov_linker,
6614	6677	.ops = .import_reloc,
6615		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
6616		.reg = @enumToInt(Register.rax),
6617		.atom_index = atom_index,
6618		.sym_index = sym_index,
6619		}) },
	6678	.data = .{ .rx = .{
	6679	.r = .rax,
	6680	.payload = try self.addExtra(Mir.Reloc{
	6681	.atom_index = atom_index,
	6682	.sym_index = sym_index,
	6683	}),
	6684	} },
6620	6685	});
6621	6686	try self.asmRegister(.call, .rax);
6622	6687	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
...	...	@@ -6695,8 +6760,6 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
6695	6760	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
6696	6761	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
6697	6762	const ty = self.air.typeOf(bin_op.lhs);
6698		const ty_abi_size = ty.abiSize(self.target.*);
6699		const can_reuse = ty_abi_size <= 8;
6700	6763
6701	6764	try self.spillEflagsIfOccupied();
6702	6765	self.eflags_inst = inst;
...	...	@@ -6715,69 +6778,93 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
6715	6778	};
6716	6779	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
6717	6780
6718		const dst_mem_ok = !ty.isRuntimeFloat();
6719		var flipped = false;
6720		const dst_mcv: MCValue = if (can_reuse and !lhs_mcv.isImmediate() and
6721		(dst_mem_ok or lhs_mcv.isRegister()) and self.liveness.operandDies(inst, 0))
6722		lhs_mcv
6723		else if (can_reuse and !rhs_mcv.isImmediate() and
6724		(dst_mem_ok or rhs_mcv.isRegister()) and self.liveness.operandDies(inst, 1))
6725		dst: {
6726		flipped = true;
6727		break :dst rhs_mcv;
6728		} else if (dst_mem_ok) dst: {
6729		const dst_mcv = try self.allocTempRegOrMem(ty, true);
6730		try self.genCopy(ty, dst_mcv, lhs_mcv);
6731		break :dst dst_mcv;
6732		} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
6733		const dst_lock = switch (dst_mcv) {
6734		.register => \|reg\| self.register_manager.lockReg(reg),
6735		else => null,
6736		};
6737		defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
6738
6739		const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
6740		switch (ty.zigTypeTag()) {
6741		else => try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv),
6742		.Float => switch (ty.floatBits(self.target.*)) {
6743		16 => if (self.hasFeature(.f16c)) {
6744		const dst_reg = dst_mcv.getReg().?.to128();
	6781	const result = MCValue{
	6782	.eflags = switch (ty.zigTypeTag()) {
	6783	else => result: {
	6784	var flipped = false;
	6785	const dst_mcv: MCValue = if (lhs_mcv.isRegister() or lhs_mcv.isMemory())
	6786	lhs_mcv
	6787	else if (rhs_mcv.isRegister() or rhs_mcv.isMemory()) dst: {
	6788	flipped = true;
	6789	break :dst rhs_mcv;
	6790	} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
	6791	const dst_lock = switch (dst_mcv) {
	6792	.register => \|reg\| self.register_manager.lockReg(reg),
	6793	else => null,
	6794	};
	6795	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	6796	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
6745	6797
6746		const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
6747		const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6748		defer self.register_manager.unlockReg(tmp_lock);
	6798	try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
	6799	break :result Condition.fromCompareOperator(
	6800	if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
	6801	if (flipped) op.reverse() else op,
	6802	);
	6803	},
	6804	.Float => result: {
	6805	const flipped = switch (op) {
	6806	.lt, .lte => true,
	6807	.eq, .gte, .gt, .neq => false,
	6808	};
6749	6809
6750		if (src_mcv.isRegister())
6751		try self.asmRegisterRegisterRegister(
6752		.vpunpcklwd,
6753		dst_reg,
6754		dst_reg,
6755		src_mcv.getReg().?.to128(),
6756		)
	6810	const dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
	6811	const dst_reg = if (dst_mcv.isRegister())
	6812	dst_mcv.getReg().?
6757	6813	else
6758		try self.asmRegisterMemoryImmediate(
6759		.vpinsrw,
6760		dst_reg,
6761		src_mcv.mem(.word),
6762		Immediate.u(1),
6763		);
6764		try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
6765		try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
6766		try self.genBinOpMir(.ucomiss, ty, dst_mcv, .{ .register = tmp_reg });
6767		} else return self.fail("TODO implement airCmp for {}", .{
6768		ty.fmt(self.bin_file.options.module.?),
6769		}),
6770		32 => try self.genBinOpMir(.ucomiss, ty, dst_mcv, src_mcv),
6771		64 => try self.genBinOpMir(.ucomisd, ty, dst_mcv, src_mcv),
6772		else => return self.fail("TODO implement airCmp for {}", .{
6773		ty.fmt(self.bin_file.options.module.?),
6774		}),
6775		},
6776		}
	6814	try self.copyToTmpRegister(ty, dst_mcv);
	6815	const dst_lock = self.register_manager.lockReg(dst_reg);
	6816	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	6817	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
	6818
	6819	switch (ty.floatBits(self.target.*)) {
	6820	16 => if (self.hasFeature(.f16c)) {
	6821	const tmp1_reg = (try self.register_manager.allocReg(null, sse)).to128();
	6822	const tmp1_mcv = MCValue{ .register = tmp1_reg };
	6823	const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg);
	6824	defer self.register_manager.unlockReg(tmp1_lock);
	6825
	6826	const tmp2_reg = (try self.register_manager.allocReg(null, sse)).to128();
	6827	const tmp2_mcv = MCValue{ .register = tmp2_reg };
	6828	const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
	6829	defer self.register_manager.unlockReg(tmp2_lock);
	6830
	6831	if (src_mcv.isRegister())
	6832	try self.asmRegisterRegisterRegister(
	6833	.vpunpcklwd,
	6834	tmp1_reg,
	6835	dst_reg.to128(),
	6836	src_mcv.getReg().?.to128(),
	6837	)
	6838	else
	6839	try self.asmRegisterRegisterMemoryImmediate(
	6840	.vpinsrw,
	6841	tmp1_reg,
	6842	dst_reg.to128(),
	6843	src_mcv.mem(.word),
	6844	Immediate.u(1),
	6845	);
	6846	try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);
	6847	try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);
	6848	try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);
	6849	} else return self.fail("TODO implement airCmp for {}", .{
	6850	ty.fmt(self.bin_file.options.module.?),
	6851	}),
	6852	32 => try self.genBinOpMir(.ucomiss, ty, .{ .register = dst_reg }, src_mcv),
	6853	64 => try self.genBinOpMir(.ucomisd, ty, .{ .register = dst_reg }, src_mcv),
	6854	else => return self.fail("TODO implement airCmp for {}", .{
	6855	ty.fmt(self.bin_file.options.module.?),
	6856	}),
	6857	}
6777	6858
6778		const signedness = if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
6779		const result = MCValue{
6780		.eflags = Condition.fromCompareOperator(signedness, if (flipped) op.reverse() else op),
	6859	break :result switch (if (flipped) op.reverse() else op) {
	6860	.lt, .lte => unreachable, // required to have been canonicalized to gt(e)
	6861	.gt => .a,
	6862	.gte => .ae,
	6863	.eq => .z_and_np,
	6864	.neq => .nz_or_p,
	6865	};
	6866	},
	6867	},
6781	6868	};
6782	6869	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6783	6870	}
...	...	@@ -7929,11 +8016,13 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7929	8016	_ = try self.addInst(.{
7930	8017	.tag = .mov_linker,
7931	8018	.ops = .direct_reloc,
7932		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7933		.reg = @enumToInt(dst_reg.to64()),
7934		.atom_index = atom_index,
7935		.sym_index = sym_index,
7936		}) },
	8019	.data = .{ .rx = .{
	8020	.r = dst_reg.to64(),
	8021	.payload = try self.addExtra(Mir.Reloc{
	8022	.atom_index = atom_index,
	8023	.sym_index = sym_index,
	8024	}),
	8025	} },
7937	8026	});
7938	8027	return;
7939	8028	},
...	...	@@ -7975,11 +8064,13 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7975	8064	.lea_got => .got_reloc,
7976	8065	else => unreachable,
7977	8066	},
7978		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7979		.reg = @enumToInt(dst_reg.to64()),
7980		.atom_index = atom_index,
7981		.sym_index = sym_index,
7982		}) },
	8067	.data = .{ .rx = .{
	8068	.r = dst_reg.to64(),
	8069	.payload = try self.addExtra(Mir.Reloc{
	8070	.atom_index = atom_index,
	8071	.sym_index = sym_index,
	8072	}),
	8073	} },
7983	8074	});
7984	8075	},
7985	8076	.lea_tlv => \|sym_index\| {
...	...	@@ -7988,11 +8079,13 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7988	8079	_ = try self.addInst(.{
7989	8080	.tag = .lea_linker,
7990	8081	.ops = .tlv_reloc,
7991		.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7992		.reg = @enumToInt(Register.rdi),
7993		.atom_index = atom_index,
7994		.sym_index = sym_index,
7995		}) },
	8082	.data = .{ .rx = .{
	8083	.r = .rdi,
	8084	.payload = try self.addExtra(Mir.Reloc{
	8085	.atom_index = atom_index,
	8086	.sym_index = sym_index,
	8087	}),
	8088	} },
7996	8089	});
7997	8090	// TODO: spill registers before calling
7998	8091	try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
...	...	@@ -8463,14 +8556,20 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
8463	8556
8464	8557	try self.spillEflagsIfOccupied();
8465	8558	if (val_abi_size <= 8) {
8466		_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
8467		.r = registerAlias(new_reg.?, val_abi_size),
8468		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8469		} } });
	8559	_ = try self.addInst(.{
	8560	.tag = .cmpxchg,
	8561	.ops = .lock_mr_sib,
	8562	.data = .{ .rx = .{
	8563	.r = registerAlias(new_reg.?, val_abi_size),
	8564	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	8565	} },
	8566	});
8470	8567	} else {
8471		_ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
8472		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8473		} });
	8568	_ = try self.addInst(.{
	8569	.tag = .cmpxchgb,
	8570	.ops = .lock_m_sib,
	8571	.data = .{ .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)) },
	8572	});
8474	8573	}
8475	8574
8476	8575	const result: MCValue = result: {
...	...	@@ -8571,14 +8670,18 @@ fn atomicOp(
8571	8670	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
8572	8671	try self.genUnOpMir(.neg, val_ty, dst_mcv);
8573	8672	}
8574		_ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
8575		.mov, .xchg => .mr_sib,
8576		.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
8577		else => unreachable,
8578		}, .data = .{ .rx = .{
8579		.r = registerAlias(dst_reg, val_abi_size),
8580		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8581		} } });
	8673	_ = try self.addInst(.{
	8674	.tag = tag,
	8675	.ops = switch (tag) {
	8676	.mov, .xchg => .mr_sib,
	8677	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
	8678	else => unreachable,
	8679	},
	8680	.data = .{ .rx = .{
	8681	.r = registerAlias(dst_reg, val_abi_size),
	8682	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	8683	} },
	8684	});
8582	8685
8583	8686	return if (unused) .unreach else dst_mcv;
8584	8687	},
...	...	@@ -8645,10 +8748,14 @@ fn atomicOp(
8645	8748	}
8646	8749	},
8647	8750	};
8648		_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
8649		.r = registerAlias(tmp_reg, val_abi_size),
8650		.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8651		} } });
	8751	_ = try self.addInst(.{
	8752	.tag = .cmpxchg,
	8753	.ops = .lock_mr_sib,
	8754	.data = .{ .rx = .{
	8755	.r = registerAlias(tmp_reg, val_abi_size),
	8756	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	8757	} },
	8758	});
8652	8759	_ = try self.asmJccReloc(loop, .ne);
8653	8760	return if (unused) .unreach else .{ .register = .rax };
8654	8761	} else {

src/arch/x86_64/Emit.zig+140-133

...	...	@@ -18,142 +18,149 @@ pub const Error = Lower.Error \|\| error{
18	18	};
19	19
20	20	pub fn emitMir(emit: *Emit) Error!void {
21		for (0..emit.lower.mir.instructions.len) \|i\| {
22		const index = @intCast(Mir.Inst.Index, i);
23		const inst = emit.lower.mir.instructions.get(index);
24
25		const start_offset = @intCast(u32, emit.code.items.len);
26		try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
27		for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer(), .{});
28		const end_offset = @intCast(u32, emit.code.items.len);
29
30		switch (inst.tag) {
31		else => {},
32
33		.jmp_reloc => try emit.relocs.append(emit.lower.allocator, .{
34		.source = start_offset,
35		.target = inst.data.inst,
36		.offset = end_offset - 4,
37		.length = 5,
38		}),
39
40		.call_extern => if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
41		// Add relocation to the decl.
42		const atom_index = macho_file.getAtomIndexForSymbol(
43		.{ .sym_index = inst.data.relocation.atom_index, .file = null },
44		).?;
45		const target = macho_file.getGlobalByIndex(inst.data.relocation.sym_index);
46		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
47		.type = .branch,
	21	for (0..emit.lower.mir.instructions.len) \|mir_i\| {
	22	const mir_index = @intCast(Mir.Inst.Index, mir_i);
	23	try emit.code_offset_mapping.putNoClobber(
	24	emit.lower.allocator,
	25	mir_index,
	26	@intCast(u32, emit.code.items.len),
	27	);
	28	const lowered = try emit.lower.lowerMir(mir_index);
	29	var lowered_relocs = lowered.relocs;
	30	for (lowered.insts, 0..) \|lowered_inst, lowered_index\| {
	31	const start_offset = @intCast(u32, emit.code.items.len);
	32	try lowered_inst.encode(emit.code.writer(), .{});
	33	const end_offset = @intCast(u32, emit.code.items.len);
	34	while (lowered_relocs.len > 0 and
	35	lowered_relocs[0].lowered_inst_index == lowered_index) : ({
	36	lowered_relocs = lowered_relocs[1..];
	37	}) switch (lowered_relocs[0].target) {
	38	.inst => \|target\| try emit.relocs.append(emit.lower.allocator, .{
	39	.source = start_offset,
48	40	.target = target,
49	41	.offset = end_offset - 4,
50		.addend = 0,
51		.pcrel = true,
52		.length = 2,
53		});
54		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
55		// Add relocation to the decl.
56		const atom_index = coff_file.getAtomIndexForSymbol(
57		.{ .sym_index = inst.data.relocation.atom_index, .file = null },
58		).?;
59		const target = coff_file.getGlobalByIndex(inst.data.relocation.sym_index);
60		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
61		.type = .direct,
62		.target = target,
63		.offset = end_offset - 4,
64		.addend = 0,
65		.pcrel = true,
66		.length = 2,
67		});
68		} else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),
69
70		.mov_linker, .lea_linker => if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
71		const metadata =
72		emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
73		const atom_index = macho_file.getAtomIndexForSymbol(.{
74		.sym_index = metadata.atom_index,
75		.file = null,
76		}).?;
77		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
78		.type = switch (inst.ops) {
79		.got_reloc => .got,
80		.direct_reloc => .signed,
81		.tlv_reloc => .tlv,
82		else => unreachable,
83		},
84		.target = .{ .sym_index = metadata.sym_index, .file = null },
85		.offset = @intCast(u32, end_offset - 4),
86		.addend = 0,
87		.pcrel = true,
88		.length = 2,
89		});
90		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
91		const metadata =
92		emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
93		const atom_index = coff_file.getAtomIndexForSymbol(.{
94		.sym_index = metadata.atom_index,
95		.file = null,
96		}).?;
97		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
98		.type = switch (inst.ops) {
99		.got_reloc => .got,
100		.direct_reloc => .direct,
101		.import_reloc => .import,
102		else => unreachable,
103		},
104		.target = switch (inst.ops) {
105		.got_reloc,
106		.direct_reloc,
107		=> .{ .sym_index = metadata.sym_index, .file = null },
108		.import_reloc => coff_file.getGlobalByIndex(metadata.sym_index),
109		else => unreachable,
110		},
111		.offset = @intCast(u32, end_offset - 4),
112		.addend = 0,
113		.pcrel = true,
114		.length = 2,
115		});
116		} else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),
117
118		.jcc => try emit.relocs.append(emit.lower.allocator, .{
119		.source = start_offset,
120		.target = inst.data.inst_cc.inst,
121		.offset = end_offset - 4,
122		.length = 6,
123		}),
124
125		.dbg_line => try emit.dbgAdvancePCAndLine(
126		inst.data.line_column.line,
127		inst.data.line_column.column,
128		),
129
130		.dbg_prologue_end => {
131		switch (emit.debug_output) {
132		.dwarf => \|dw\| {
133		try dw.setPrologueEnd();
134		log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
135		emit.prev_di_line, emit.prev_di_column,
136		});
137		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
138		},
139		.plan9 => {},
140		.none => {},
141		}
142		},
	42	.length = @intCast(u5, end_offset - start_offset),
	43	}),
	44	.@"extern" => \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	45	// Add relocation to the decl.
	46	const atom_index = macho_file.getAtomIndexForSymbol(
	47	.{ .sym_index = symbol.atom_index, .file = null },
	48	).?;
	49	const target = macho_file.getGlobalByIndex(symbol.sym_index);
	50	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	51	.type = .branch,
	52	.target = target,
	53	.offset = end_offset - 4,
	54	.addend = 0,
	55	.pcrel = true,
	56	.length = 2,
	57	});
	58	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	59	// Add relocation to the decl.
	60	const atom_index = coff_file.getAtomIndexForSymbol(
	61	.{ .sym_index = symbol.atom_index, .file = null },
	62	).?;
	63	const target = coff_file.getGlobalByIndex(symbol.sym_index);
	64	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	65	.type = .direct,
	66	.target = target,
	67	.offset = end_offset - 4,
	68	.addend = 0,
	69	.pcrel = true,
	70	.length = 2,
	71	});
	72	} else return emit.fail("TODO implement extern reloc for {s}", .{
	73	@tagName(emit.bin_file.tag),
	74	}),
	75	.linker_got,
	76	.linker_direct,
	77	.linker_import,
	78	.linker_tlv,
	79	=> \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	80	const atom_index = macho_file.getAtomIndexForSymbol(.{
	81	.sym_index = symbol.atom_index,
	82	.file = null,
	83	}).?;
	84	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	85	.type = switch (lowered_relocs[0].target) {
	86	.linker_got => .got,
	87	.linker_direct => .signed,
	88	.linker_tlv => .tlv,
	89	else => unreachable,
	90	},
	91	.target = .{ .sym_index = symbol.sym_index, .file = null },
	92	.offset = @intCast(u32, end_offset - 4),
	93	.addend = 0,
	94	.pcrel = true,
	95	.length = 2,
	96	});
	97	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	98	const atom_index = coff_file.getAtomIndexForSymbol(.{
	99	.sym_index = symbol.atom_index,
	100	.file = null,
	101	}).?;
	102	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	103	.type = switch (lowered_relocs[0].target) {
	104	.linker_got => .got,
	105	.linker_direct => .direct,
	106	.linker_import => .import,
	107	else => unreachable,
	108	},
	109	.target = switch (lowered_relocs[0].target) {
	110	.linker_got,
	111	.linker_direct,
	112	=> .{ .sym_index = symbol.sym_index, .file = null },
	113	.linker_import => coff_file.getGlobalByIndex(symbol.sym_index),
	114	else => unreachable,
	115	},
	116	.offset = @intCast(u32, end_offset - 4),
	117	.addend = 0,
	118	.pcrel = true,
	119	.length = 2,
	120	});
	121	} else return emit.fail("TODO implement linker reloc for {s}", .{
	122	@tagName(emit.bin_file.tag),
	123	}),
	124	};
	125	}
	126	std.debug.assert(lowered_relocs.len == 0);
143	127
144		.dbg_epilogue_begin => {
145		switch (emit.debug_output) {
146		.dwarf => \|dw\| {
147		try dw.setEpilogueBegin();
148		log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
149		emit.prev_di_line, emit.prev_di_column,
150		});
151		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
152		},
153		.plan9 => {},
154		.none => {},
155		}
156		},
	128	if (lowered.insts.len == 0) {
	129	const mir_inst = emit.lower.mir.instructions.get(mir_index);
	130	switch (mir_inst.tag) {
	131	else => unreachable,
	132	.dead => {},
	133	.dbg_line => try emit.dbgAdvancePCAndLine(
	134	mir_inst.data.line_column.line,
	135	mir_inst.data.line_column.column,
	136	),
	137	.dbg_prologue_end => {
	138	switch (emit.debug_output) {
	139	.dwarf => \|dw\| {
	140	try dw.setPrologueEnd();
	141	log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
	142	emit.prev_di_line, emit.prev_di_column,
	143	});
	144	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	145	},
	146	.plan9 => {},
	147	.none => {},
	148	}
	149	},
	150	.dbg_epilogue_begin => {
	151	switch (emit.debug_output) {
	152	.dwarf => \|dw\| {
	153	try dw.setEpilogueBegin();
	154	log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
	155	emit.prev_di_line, emit.prev_di_column,
	156	});
	157	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	158	},
	159	.plan9 => {},
	160	.none => {},
	161	}
	162	},
	163	}
157	164	}
158	165	}
159	166	try emit.fixupRelocs();

src/arch/x86_64/Lower.zig+166-54

...	...	@@ -5,13 +5,22 @@ mir: Mir,
5	5	target: *const std.Target,
6	6	err_msg: ?*ErrorMsg = null,
7	7	src_loc: Module.SrcLoc,
8		result: [
	8	result_insts_len: u8 = undefined,
	9	result_relocs_len: u8 = undefined,
	10	result_insts: [
9	11	std.mem.max(usize, &.{
10		abi.Win64.callee_preserved_regs.len,
11		abi.SysV.callee_preserved_regs.len,
	12	2, // cmovcc: cmovcc \ cmovcc
	13	3, // setcc: setcc \ setcc \ logicop
	14	2, // jcc: jcc \ jcc
	15	abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
	16	abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
12	17	})
13	18	]Instruction = undefined,
14		result_len: usize = undefined,
	19	result_relocs: [
	20	std.mem.max(usize, &.{
	21	2, // jcc: jcc \ jcc
	22	})
	23	]Reloc = undefined,
15	24
16	25	pub const Error = error{
17	26	OutOfMemory,
...	...	@@ -20,13 +29,35 @@ pub const Error = error{
20	29	CannotEncode,
21	30	};
22	31
23		/// The returned slice is overwritten by the next call to lowerMir.
24		pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
25		lower.result = undefined;
26		errdefer lower.result = undefined;
27		lower.result_len = 0;
28		defer lower.result_len = undefined;
	32	pub const Reloc = struct {
	33	lowered_inst_index: u8,
	34	target: Target,
	35
	36	const Target = union(enum) {
	37	inst: Mir.Inst.Index,
	38	@"extern": Mir.Reloc,
	39	linker_got: Mir.Reloc,
	40	linker_direct: Mir.Reloc,
	41	linker_import: Mir.Reloc,
	42	linker_tlv: Mir.Reloc,
	43	};
	44	};
29	45
	46	/// The returned slice is overwritten by the next call to lowerMir.
	47	pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
	48	insts: []const Instruction,
	49	relocs: []const Reloc,
	50	} {
	51	lower.result_insts = undefined;
	52	lower.result_relocs = undefined;
	53	errdefer lower.result_insts = undefined;
	54	errdefer lower.result_relocs = undefined;
	55	lower.result_insts_len = 0;
	56	lower.result_relocs_len = 0;
	57	defer lower.result_insts_len = undefined;
	58	defer lower.result_relocs_len = undefined;
	59
	60	const inst = lower.mir.instructions.get(index);
30	61	switch (inst.tag) {
31	62	.adc,
32	63	.add,
...	...	@@ -185,22 +216,26 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
185	216
186	217	.cmpxchgb => try lower.mirCmpxchgBytes(inst),
187	218
188		.jmp_reloc => try lower.emit(.none, .jmp, &.{.{ .imm = Immediate.s(0) }}),
	219	.jmp_reloc => try lower.emitInstWithReloc(.none, .jmp, &.{
	220	.{ .imm = Immediate.s(0) },
	221	}, .{ .inst = inst.data.inst }),
189	222
190		.call_extern => try lower.emit(.none, .call, &.{.{ .imm = Immediate.s(0) }}),
	223	.call_extern => try lower.emitInstWithReloc(.none, .call, &.{
	224	.{ .imm = Immediate.s(0) },
	225	}, .{ .@"extern" = inst.data.relocation }),
191	226
192		.lea_linker => try lower.mirLeaLinker(inst),
193		.mov_linker => try lower.mirMovLinker(inst),
	227	.lea_linker => try lower.mirLinker(.lea, inst),
	228	.mov_linker => try lower.mirLinker(.mov, inst),
194	229
195	230	.mov_moffs => try lower.mirMovMoffs(inst),
196	231
197	232	.movsx => try lower.mirMovsx(inst),
198	233	.cmovcc => try lower.mirCmovcc(inst),
199	234	.setcc => try lower.mirSetcc(inst),
200		.jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
	235	.jcc => try lower.mirJcc(index, inst),
201	236
202		.push_regs => try lower.mirPushPopRegisterList(inst, .push),
203		.pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
	237	.push_regs => try lower.mirRegisterList(.push, inst),
	238	.pop_regs => try lower.mirRegisterList(.pop, inst),
204	239
205	240	.dbg_line,
206	241	.dbg_prologue_end,
...	...	@@ -209,7 +244,10 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
209	244	=> {},
210	245	}
211	246
212		return lower.result[0..lower.result_len];
	247	return .{
	248	.insts = lower.result_insts[0..lower.result_insts_len],
	249	.relocs = lower.result_relocs[0..lower.result_relocs_len],
	250	};
213	251	}
214	252
215	253	pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
...	...	@@ -221,7 +259,10 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
221	259
222	260	fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
223	261	return switch (cc) {
224		inline else => \|c\| @field(Mnemonic, @tagName(base) ++ @tagName(c)),
	262	inline else => \|c\| if (@hasField(Mnemonic, @tagName(base) ++ @tagName(c)))
	263	@field(Mnemonic, @tagName(base) ++ @tagName(c))
	264	else
	265	unreachable,
225	266	};
226	267	}
227	268
...	...	@@ -247,6 +288,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
247	288	.rmi_rip,
248	289	.mri_sib,
249	290	.mri_rip,
	291	.rrmi_sib,
	292	.rrmi_rip,
250	293	=> Immediate.u(i),
251	294
252	295	.ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
...	...	@@ -267,6 +310,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
267	310	.mr_sib,
268	311	.mrr_sib,
269	312	.mri_sib,
	313	.rrmi_sib,
270	314	.lock_m_sib,
271	315	.lock_mi_sib_u,
272	316	.lock_mi_sib_s,
...	...	@@ -283,6 +327,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
283	327	.mr_rip,
284	328	.mrr_rip,
285	329	.mri_rip,
	330	.rrmi_rip,
286	331	.lock_m_rip,
287	332	.lock_mi_rip_u,
288	333	.lock_mi_rip_s,
...	...	@@ -298,13 +343,28 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
298	343	});
299	344	}
300	345
301		fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
302		lower.result[lower.result_len] = try Instruction.new(prefix, mnemonic, ops);
303		lower.result_len += 1;
	346	fn emitInst(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
	347	lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
	348	lower.result_insts_len += 1;
	349	}
	350
	351	fn emitInstWithReloc(
	352	lower: *Lower,
	353	prefix: Prefix,
	354	mnemonic: Mnemonic,
	355	ops: []const Operand,
	356	target: Reloc.Target,
	357	) Error!void {
	358	lower.result_relocs[lower.result_relocs_len] = .{
	359	.lowered_inst_index = lower.result_insts_len,
	360	.target = target,
	361	};
	362	lower.result_relocs_len += 1;
	363	try lower.emitInst(prefix, mnemonic, ops);
304	364	}
305	365
306	366	fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
307		try lower.emit(switch (inst.ops) {
	367	try lower.emitInst(switch (inst.ops) {
308	368	else => .none,
309	369	.lock_m_sib,
310	370	.lock_m_rip,
...	...	@@ -389,13 +449,19 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
389	449	.{ .reg = inst.data.rix.r },
390	450	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
391	451	},
	452	.rrmi_sib, .rrmi_rip => &.{
	453	.{ .reg = inst.data.rrix.r1 },
	454	.{ .reg = inst.data.rrix.r2 },
	455	.{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
	456	.{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
	457	},
392	458	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
393	459	});
394	460	}
395	461
396	462	fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {
397	463	switch (inst.ops) {
398		.string => try lower.emit(switch (inst.data.string.repeat) {
	464	.string => try lower.emitInst(switch (inst.data.string.repeat) {
399	465	inline else => \|repeat\| @field(Prefix, @tagName(repeat)),
400	466	}, switch (inst.tag) {
401	467	inline .cmps, .lods, .movs, .scas, .stos => \|tag\| switch (inst.data.string.width) {
...	...	@@ -414,7 +480,7 @@ fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
414	480	},
415	481	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
416	482	};
417		try lower.emit(switch (inst.ops) {
	483	try lower.emitInst(switch (inst.ops) {
418	484	.m_sib, .m_rip => .none,
419	485	.lock_m_sib, .lock_m_rip => .lock,
420	486	else => unreachable,
...	...	@@ -426,7 +492,7 @@ fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
426	492	}
427	493
428	494	fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
429		try lower.emit(switch (inst.ops) {
	495	try lower.emitInst(switch (inst.ops) {
430	496	.rax_moffs, .moffs_rax => .none,
431	497	.lock_moffs_rax => .lock,
432	498	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
...	...	@@ -455,7 +521,7 @@ fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
455	521	},
456	522	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
457	523	};
458		try lower.emit(.none, switch (ops[0].bitSize()) {
	524	try lower.emitInst(.none, switch (ops[0].bitSize()) {
459	525	32, 64 => switch (ops[1].bitSize()) {
460	526	32 => .movsxd,
461	527	else => .movsx,
...	...	@@ -465,32 +531,82 @@ fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
465	531	}
466	532
467	533	fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
468		switch (inst.ops) {
469		.rr_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rr_cc.cc), &.{
	534	const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
	535	.rr_cc => .{ .cc = inst.data.rr_cc.cc, .ops = .{
470	536	.{ .reg = inst.data.rr_cc.r1 },
471	537	.{ .reg = inst.data.rr_cc.r2 },
472		}),
473		.rm_sib_cc, .rm_rip_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rx_cc.cc), &.{
	538	} },
	539	.rm_sib_cc, .rm_rip_cc => .{ .cc = inst.data.rx_cc.cc, .ops = .{
474	540	.{ .reg = inst.data.rx_cc.r },
475	541	.{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
476		}),
	542	} },
477	543	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
	544	};
	545	switch (data.cc) {
	546	else => \|cc\| try lower.emitInst(.none, mnem_cc(.cmov, cc), &data.ops),
	547	.z_and_np => {
	548	try lower.emitInst(.none, mnem_cc(.cmov, .nz), &.{ data.ops[1], data.ops[0] });
	549	try lower.emitInst(.none, mnem_cc(.cmov, .np), &data.ops);
	550	},
	551	.nz_or_p => {
	552	try lower.emitInst(.none, mnem_cc(.cmov, .nz), &data.ops);
	553	try lower.emitInst(.none, mnem_cc(.cmov, .p), &data.ops);
	554	},
478	555	}
479	556	}
480	557
481	558	fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
482		switch (inst.ops) {
483		.r_cc => try lower.emit(.none, mnem_cc(.set, inst.data.r_cc.cc), &.{
	559	const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
	560	.r_cc => .{ .cc = inst.data.r_cc.cc, .ops = .{
484	561	.{ .reg = inst.data.r_cc.r },
485		}),
486		.m_sib_cc, .m_rip_cc => try lower.emit(.none, mnem_cc(.set, inst.data.x_cc.cc), &.{
	562	.{ .reg = inst.data.r_cc.scratch },
	563	} },
	564	.m_sib_cc, .m_rip_cc => .{ .cc = inst.data.x_cc.cc, .ops = .{
487	565	.{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
488		}),
	566	.{ .reg = inst.data.x_cc.scratch },
	567	} },
489	568	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
	569	};
	570	switch (data.cc) {
	571	else => \|cc\| try lower.emitInst(.none, mnem_cc(.set, cc), data.ops[0..1]),
	572	.z_and_np => {
	573	try lower.emitInst(.none, mnem_cc(.set, .z), data.ops[0..1]);
	574	try lower.emitInst(.none, mnem_cc(.set, .np), data.ops[1..2]);
	575	try lower.emitInst(.none, .@"and", data.ops[0..2]);
	576	},
	577	.nz_or_p => {
	578	try lower.emitInst(.none, mnem_cc(.set, .nz), data.ops[0..1]);
	579	try lower.emitInst(.none, mnem_cc(.set, .p), data.ops[1..2]);
	580	try lower.emitInst(.none, .@"or", data.ops[0..2]);
	581	},
490	582	}
491	583	}
492	584
493		fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
	585	fn mirJcc(lower: *Lower, index: Mir.Inst.Index, inst: Mir.Inst) Error!void {
	586	switch (inst.data.inst_cc.cc) {
	587	else => \|cc\| try lower.emitInstWithReloc(.none, mnem_cc(.j, cc), &.{
	588	.{ .imm = Immediate.s(0) },
	589	}, .{ .inst = inst.data.inst_cc.inst }),
	590	.z_and_np => {
	591	try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
	592	.{ .imm = Immediate.s(0) },
	593	}, .{ .inst = index + 1 });
	594	try lower.emitInstWithReloc(.none, mnem_cc(.j, .np), &.{
	595	.{ .imm = Immediate.s(0) },
	596	}, .{ .inst = inst.data.inst_cc.inst });
	597	},
	598	.nz_or_p => {
	599	try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
	600	.{ .imm = Immediate.s(0) },
	601	}, .{ .inst = inst.data.inst_cc.inst });
	602	try lower.emitInstWithReloc(.none, mnem_cc(.j, .p), &.{
	603	.{ .imm = Immediate.s(0) },
	604	}, .{ .inst = inst.data.inst_cc.inst });
	605	},
	606	}
	607	}
	608
	609	fn mirRegisterList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
494	610	const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
495	611	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
496	612	var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
...	...	@@ -498,24 +614,20 @@ fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnem
498	614	.pop => .forward,
499	615	else => unreachable,
500	616	} });
501		while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
	617	while (it.next()) \|i\| try lower.emitInst(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
502	618	}
503	619
504		fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {
505		const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
506		const reg = @intToEnum(Register, metadata.reg);
507		try lower.emit(.none, .lea, &.{
508		.{ .reg = reg },
509		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
510		});
511		}
512
513		fn mirMovLinker(lower: *Lower, inst: Mir.Inst) Error!void {
514		const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
515		const reg = @intToEnum(Register, metadata.reg);
516		try lower.emit(.none, .mov, &.{
517		.{ .reg = reg },
518		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
	620	fn mirLinker(lower: *Lower, mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
	621	const reloc = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
	622	try lower.emitInstWithReloc(.none, mnemonic, &.{
	623	.{ .reg = inst.data.rx.r },
	624	.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(inst.data.rx.r.bitSize()), 0) },
	625	}, switch (inst.ops) {
	626	.got_reloc => .{ .linker_got = reloc },
	627	.direct_reloc => .{ .linker_direct = reloc },
	628	.import_reloc => .{ .linker_import = reloc },
	629	.tlv_reloc => .{ .linker_tlv = reloc },
	630	else => unreachable,
519	631	});
520	632	}
521	633

src/arch/x86_64/Mir.zig+21-13

...	...	@@ -434,6 +434,12 @@ pub const Inst = struct {
434	434	/// Register, memory (SIB), immediate (byte) operands.
435	435	/// Uses `rix` payload with extra data of type `MemorySib`.
436	436	rmi_sib,
	437	/// Register, register, memory (RIP), immediate (byte) operands.
	438	/// Uses `rrix` payload with extra data of type `MemoryRip`.
	439	rrmi_rip,
	440	/// Register, register, memory (SIB), immediate (byte) operands.
	441	/// Uses `rrix` payload with extra data of type `MemorySib`.
	442	rrmi_sib,
437	443	/// Register, memory (RIP), immediate (byte) operands.
438	444	/// Uses `rix` payload with extra data of type `MemoryRip`.
439	445	rmi_rip,
...	...	@@ -524,16 +530,16 @@ pub const Inst = struct {
524	530	/// Uses `reloc` payload.
525	531	reloc,
526	532	/// Linker relocation - GOT indirection.
527		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	533	/// Uses `rx` payload with extra data of type `Reloc`.
528	534	got_reloc,
529	535	/// Linker relocation - direct reference.
530		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	536	/// Uses `rx` payload with extra data of type `Reloc`.
531	537	direct_reloc,
532	538	/// Linker relocation - imports table indirection (binding).
533		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	539	/// Uses `rx` payload with extra data of type `Reloc`.
534	540	import_reloc,
535	541	/// Linker relocation - threadlocal variable via GOT indirection.
536		/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
	542	/// Uses `rx` payload with extra data of type `Reloc`.
537	543	tlv_reloc,
538	544	};
539	545
...	...	@@ -567,12 +573,14 @@ pub const Inst = struct {
567	573	},
568	574	/// Condition code (CC), followed by custom payload found in extra.
569	575	x_cc: struct {
	576	scratch: Register,
570	577	cc: bits.Condition,
571	578	payload: u32,
572	579	},
573	580	/// Register with condition code (CC).
574	581	r_cc: struct {
575	582	r: Register,
	583	scratch: Register,
576	584	cc: bits.Condition,
577	585	},
578	586	/// Register, register with condition code (CC).
...	...	@@ -614,6 +622,13 @@ pub const Inst = struct {
614	622	i: u8,
615	623	payload: u32,
616	624	},
	625	/// Register, register, byte immediate, followed by Custom payload found in extra.
	626	rrix: struct {
	627	r1: Register,
	628	r2: Register,
	629	i: u8,
	630	payload: u32,
	631	},
617	632	/// String instruction prefix and width.
618	633	string: struct {
619	634	repeat: bits.StringRepeat,
...	...	@@ -622,12 +637,7 @@ pub const Inst = struct {
622	637	/// Relocation for the linker where:
623	638	/// * `atom_index` is the index of the source
624	639	/// * `sym_index` is the index of the target
625		relocation: struct {
626		/// Index of the containing atom.
627		atom_index: u32,
628		/// Index into the linker's symbol table.
629		sym_index: u32,
630		},
	640	relocation: Reloc,
631	641	/// Debug line and column position
632	642	line_column: struct {
633	643	line: u32,
...	...	@@ -646,9 +656,7 @@ pub const Inst = struct {
646	656	}
647	657	};
648	658
649		pub const LeaRegisterReloc = struct {
650		/// Destination register.
651		reg: u32,
	659	pub const Reloc = struct {
652	660	/// Index of the containing atom.
653	661	atom_index: u32,
654	662	/// Index into the linker's symbol table.

src/arch/x86_64/bits.zig+9

...	...	@@ -72,6 +72,12 @@ pub const Condition = enum(u5) {
72	72	/// zero
73	73	z,
74	74
	75	// Pseudo conditions
	76	/// zero and not parity
	77	z_and_np,
	78	/// not zero or parity
	79	nz_or_p,
	80
75	81	/// Converts a std.math.CompareOperator into a condition flag,
76	82	/// i.e. returns the condition that is true iff the result of the
77	83	/// comparison is true. Assumes signed comparison
...	...	@@ -143,6 +149,9 @@ pub const Condition = enum(u5) {
143	149	.po => .pe,
144	150	.s => .ns,
145	151	.z => .nz,
	152
	153	.z_and_np => .nz_or_p,
	154	.nz_or_p => .z_and_np,
146	155	};
147	156	}
148	157	};

src/arch/x86_64/encoder.zig+2-2

...	...	@@ -245,9 +245,9 @@ pub const Instruction = struct {
245	245	},
246	246	.mem => \|mem\| {
247	247	const op = switch (data.op_en) {
248		.m, .mi, .m1, .mc => .none,
	248	.m, .mi, .m1, .mc, .vmi => .none,
249	249	.mr, .mri, .mrc => inst.ops[1],
250		.rm, .rmi => inst.ops[0],
	250	.rm, .rmi, .rvm, .rvmi => inst.ops[0],
251	251	else => unreachable,
252	252	};
253	253	try encodeMemory(enc, mem, op, encoder);

test/behavior/bugs/12891.zig-6

...	...	@@ -29,7 +29,6 @@ test "inf >= 1" {
29	29	test "isNan(nan * 1)" {
30	30	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
31	31	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
32		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
33	32
34	33	const nan_times_one = comptime std.math.nan(f64) * 1;
35	34	try std.testing.expect(std.math.isNan(nan_times_one));
...	...	@@ -37,7 +36,6 @@ test "isNan(nan * 1)" {
37	36	test "runtime isNan(nan * 1)" {
38	37	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
39	38	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
40		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
41	39
42	40	const nan_times_one = std.math.nan(f64) * 1;
43	41	try std.testing.expect(std.math.isNan(nan_times_one));
...	...	@@ -45,7 +43,6 @@ test "runtime isNan(nan * 1)" {
45	43	test "isNan(nan * 0)" {
46	44	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
47	45	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
48		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
49	46
50	47	const nan_times_zero = comptime std.math.nan(f64) * 0;
51	48	try std.testing.expect(std.math.isNan(nan_times_zero));
...	...	@@ -55,7 +52,6 @@ test "isNan(nan * 0)" {
55	52	test "isNan(inf * 0)" {
56	53	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
57	54	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
58		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
59	55
60	56	const inf_times_zero = comptime std.math.inf(f64) * 0;
61	57	try std.testing.expect(std.math.isNan(inf_times_zero));
...	...	@@ -65,7 +61,6 @@ test "isNan(inf * 0)" {
65	61	test "runtime isNan(nan * 0)" {
66	62	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
67	63	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
68		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
69	64
70	65	const nan_times_zero = std.math.nan(f64) * 0;
71	66	try std.testing.expect(std.math.isNan(nan_times_zero));
...	...	@@ -75,7 +70,6 @@ test "runtime isNan(nan * 0)" {
75	70	test "runtime isNan(inf * 0)" {
76	71	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
77	72	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
78		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
79	73
80	74	const inf_times_zero = std.math.inf(f64) * 0;
81	75	try std.testing.expect(std.math.isNan(inf_times_zero));

test/behavior/field_parent_ptr.zig-1

...	...	@@ -2,7 +2,6 @@ const expect = @import("std").testing.expect;
2	2	const builtin = @import("builtin");
3	3
4	4	test "@fieldParentPtr non-first field" {
5		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
6	5	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
7	6	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
8	7	try testParentFieldPtr(&foo.c);