Commit 1fecf86ebfa352927e2e3aedfd16b0755482e96e | zig

author	Jacob Young <jacobly@ziglang.org> 2023-10-28 20:06:32-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-10-29 05:17:43-04:00
log	1fecf86ebfa352927e2e3aedfd16b0755482e96e
tree	dfc2eab0453bcad4cf2a9f80acecaddbbb3ce3fb
parent	20ade4ce7f4e738344dbf370d5df769d2a979908

x86_64: refactor codegen memory repr

Also refactor linker reloc lowering.

8 files changed, 1246 insertions(+), 1198 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Disassembler.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig

src/arch/x86_64/CodeGen.zig+776-624

...	...	@@ -115,6 +115,7 @@ const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged
115	115
116	116	const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
117	117	const RegisterOffset = struct { reg: Register, off: i32 = 0 };
	118	const SymbolOffset = struct { sym: u32, off: i32 = 0 };
118	119
119	120	const Owner = union(enum) {
120	121	func_index: InternPool.Index,
...	...	@@ -195,9 +196,9 @@ pub const MCValue = union(enum) {
195	196	memory: u64,
196	197	/// The value is in memory at an address not-yet-allocated by the linker.
197	198	/// This traditionally corresponds to a relocation emitted in a relocatable object file.
198		load_symbol: u32,
	199	load_symbol: SymbolOffset,
199	200	/// The address of the memory location not-yet-allocated by the linker.
200		lea_symbol: u32,
	201	lea_symbol: SymbolOffset,
201	202	/// The value is in memory at a constant offset from the address in a register.
202	203	indirect: RegisterOffset,
203	204	/// The value is in memory.
...	...	@@ -315,7 +316,7 @@ pub const MCValue = union(enum) {
315	316	.load_got => \|sym_index\| .{ .lea_got = sym_index },
316	317	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
317	318	.load_frame => \|frame_addr\| .{ .lea_frame = frame_addr },
318		.load_symbol => \|sym_index\| .{ .lea_symbol = sym_index },
	319	.load_symbol => \|sym_off\| .{ .lea_symbol = sym_off },
319	320	};
320	321	}
321	322
...	...	@@ -387,7 +388,7 @@ pub const MCValue = union(enum) {
387	388	};
388	389	}
389	390
390		fn mem(mcv: MCValue, ptr_size: Memory.PtrSize) Memory {
	391	fn mem(mcv: MCValue, size: Memory.Size) Memory {
391	392	return switch (mcv) {
392	393	.none,
393	394	.unreach,
...	...	@@ -411,18 +412,27 @@ pub const MCValue = union(enum) {
411	412	.load_symbol,
412	413	.lea_symbol,
413	414	=> unreachable,
414		.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
415		Memory.sib(ptr_size, .{ .base = .{ .reg = .ds }, .disp = small_addr })
416		else
417		Memory.moffs(.ds, addr),
418		.indirect => \|reg_off\| Memory.sib(ptr_size, .{
	415	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\| .{
	416	.base = .{ .reg = .ds },
	417	.mod = .{ .rm = .{
	418	.size = size,
	419	.disp = small_addr,
	420	} },
	421	} else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
	422	.indirect => \|reg_off\| .{
419	423	.base = .{ .reg = reg_off.reg },
420		.disp = reg_off.off,
421		}),
422		.load_frame => \|frame_addr\| Memory.sib(ptr_size, .{
	424	.mod = .{ .rm = .{
	425	.size = size,
	426	.disp = reg_off.off,
	427	} },
	428	},
	429	.load_frame => \|frame_addr\| .{
423	430	.base = .{ .frame = frame_addr.index },
424		.disp = frame_addr.off,
425		}),
	431	.mod = .{ .rm = .{
	432	.size = size,
	433	.disp = frame_addr.off,
	434	} },
	435	},
426	436	};
427	437	}
428	438
...	...	@@ -442,6 +452,8 @@ pub const MCValue = union(enum) {
442	452	.register_overflow => \|pl\| try writer.print("{s}:{s}", .{
443	453	@tagName(pl.eflags), @tagName(pl.reg),
444	454	}),
	455	.load_symbol => \|pl\| try writer.print("[{} + 0x{x}]", .{ pl.sym, pl.off }),
	456	.lea_symbol => \|pl\| try writer.print("{} + 0x{x}", .{ pl.sym, pl.off }),
445	457	.indirect => \|pl\| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
446	458	.load_direct => \|pl\| try writer.print("[direct:{d}]", .{pl}),
447	459	.lea_direct => \|pl\| try writer.print("direct:{d}", .{pl}),
...	...	@@ -453,8 +465,6 @@ pub const MCValue = union(enum) {
453	465	.lea_frame => \|pl\| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
454	466	.reserved_frame => \|pl\| try writer.print("(dead:{})", .{pl}),
455	467	.air_ref => \|pl\| try writer.print("(air:0x{x})", .{@intFromEnum(pl)}),
456		.load_symbol => \|pl\| try writer.print("[symbol:{d}]", .{pl}),
457		.lea_symbol => \|pl\| try writer.print("symbol:{d}", .{pl}),
458	468	}
459	469	}
460	470	};
...	...	@@ -888,7 +898,6 @@ pub fn generate(
888	898	.cc = cc,
889	899	.src_loc = src_loc,
890	900	},
891		.bin_file = bin_file,
892	901	.debug_output = debug_output,
893	902	.code = code,
894	903	.prev_di_pc = 0,
...	...	@@ -976,7 +985,6 @@ pub fn generateLazy(
976	985	.cc = abi.resolveCallingConvention(.Unspecified, function.target.*),
977	986	.src_loc = src_loc,
978	987	},
979		.bin_file = bin_file,
980	988	.debug_output = debug_output,
981	989	.code = code,
982	990	.prev_di_pc = undefined, // no debug info yet
...	...	@@ -1139,7 +1147,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1139	1147	inline for (fields) \|field\| {
1140	1148	self.mir_extra.appendAssumeCapacity(switch (field.type) {
1141	1149	u32 => @field(extra, field.name),
1142		i32 => @bitCast(@field(extra, field.name)),
	1150	i32, Mir.Memory.Info => @bitCast(@field(extra, field.name)),
1143	1151	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
1144	1152	});
1145	1153	}
...	...	@@ -1178,17 +1186,9 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1178	1186	.nz_or_p => .pseudo,
1179	1187	},
1180	1188	.ops = switch (cc) {
1181		else => switch (m) {
1182		.sib => .rm_sib,
1183		.rip => .rm_rip,
1184		else => unreachable,
1185		},
	1189	else => .rm,
1186	1190	.z_and_np => unreachable,
1187		.nz_or_p => switch (m) {
1188		.sib => .pseudo_cmov_nz_or_p_rm_sib,
1189		.rip => .pseudo_cmov_nz_or_p_rm_rip,
1190		else => unreachable,
1191		},
	1191	.nz_or_p => .pseudo_cmov_nz_or_p_rm,
1192	1192	},
1193	1193	.data = .{ .rx = .{
1194	1194	.fixes = switch (cc) {
...	...	@@ -1197,11 +1197,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1197	1197	.nz_or_p => ._,
1198	1198	},
1199	1199	.r1 = reg,
1200		.payload = switch (m) {
1201		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1202		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1203		else => unreachable,
1204		},
	1200	.payload = try self.addExtra(Mir.Memory.encode(m)),
1205	1201	} },
1206	1202	});
1207	1203	}
...	...	@@ -1231,32 +1227,16 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1231	1227	}
1232	1228
1233	1229	fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1234		const payload = switch (m) {
1235		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1236		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1237		else => unreachable,
1238		};
	1230	const payload = try self.addExtra(Mir.Memory.encode(m));
1239	1231	_ = try self.addInst(.{
1240	1232	.tag = switch (cc) {
1241	1233	else => .set,
1242	1234	.z_and_np, .nz_or_p => .pseudo,
1243	1235	},
1244	1236	.ops = switch (cc) {
1245		else => switch (m) {
1246		.sib => .m_sib,
1247		.rip => .m_rip,
1248		else => unreachable,
1249		},
1250		.z_and_np => switch (m) {
1251		.sib => .pseudo_set_z_and_np_m_sib,
1252		.rip => .pseudo_set_z_and_np_m_rip,
1253		else => unreachable,
1254		},
1255		.nz_or_p => switch (m) {
1256		.sib => .pseudo_set_nz_or_p_m_sib,
1257		.rip => .pseudo_set_nz_or_p_m_rip,
1258		else => unreachable,
1259		},
	1237	else => .m,
	1238	.z_and_np => .pseudo_set_z_and_np_m,
	1239	.nz_or_p => .pseudo_set_nz_or_p_m,
1260	1240	},
1261	1241	.data = switch (cc) {
1262	1242	else => .{ .x = .{
...	...	@@ -1504,20 +1484,12 @@ fn asmRegisterRegisterMemory(
1504	1484	) !void {
1505	1485	_ = try self.addInst(.{
1506	1486	.tag = tag[1],
1507		.ops = switch (m) {
1508		.sib => .rrm_sib,
1509		.rip => .rrm_rip,
1510		else => unreachable,
1511		},
	1487	.ops = .rrm,
1512	1488	.data = .{ .rrx = .{
1513	1489	.fixes = tag[0],
1514	1490	.r1 = reg1,
1515	1491	.r2 = reg2,
1516		.payload = switch (m) {
1517		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1518		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1519		else => unreachable,
1520		},
	1492	.payload = try self.addExtra(Mir.Memory.encode(m)),
1521	1493	} },
1522	1494	});
1523	1495	}
...	...	@@ -1525,18 +1497,10 @@ fn asmRegisterRegisterMemory(
1525	1497	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1526	1498	_ = try self.addInst(.{
1527	1499	.tag = tag[1],
1528		.ops = switch (m) {
1529		.sib => .m_sib,
1530		.rip => .m_rip,
1531		else => unreachable,
1532		},
	1500	.ops = .m,
1533	1501	.data = .{ .x = .{
1534	1502	.fixes = tag[0],
1535		.payload = switch (m) {
1536		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1537		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1538		else => unreachable,
1539		},
	1503	.payload = try self.addExtra(Mir.Memory.encode(m)),
1540	1504	} },
1541	1505	});
1542	1506	}
...	...	@@ -1544,19 +1508,11 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1544	1508	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1545	1509	_ = try self.addInst(.{
1546	1510	.tag = tag[1],
1547		.ops = switch (m) {
1548		.sib => .rm_sib,
1549		.rip => .rm_rip,
1550		else => unreachable,
1551		},
	1511	.ops = .rm,
1552	1512	.data = .{ .rx = .{
1553	1513	.fixes = tag[0],
1554	1514	.r1 = reg,
1555		.payload = switch (m) {
1556		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1557		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1558		else => unreachable,
1559		},
	1515	.payload = try self.addExtra(Mir.Memory.encode(m)),
1560	1516	} },
1561	1517	});
1562	1518	}
...	...	@@ -1574,20 +1530,12 @@ fn asmRegisterMemoryImmediate(
1574	1530	}) \|small_imm\| {
1575	1531	_ = try self.addInst(.{
1576	1532	.tag = tag[1],
1577		.ops = switch (m) {
1578		.sib => .rmi_sib,
1579		.rip => .rmi_rip,
1580		else => unreachable,
1581		},
	1533	.ops = .rmi,
1582	1534	.data = .{ .rix = .{
1583	1535	.fixes = tag[0],
1584	1536	.r1 = reg,
1585	1537	.i = small_imm,
1586		.payload = switch (m) {
1587		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1588		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1589		else => unreachable,
1590		},
	1538	.payload = try self.addExtra(Mir.Memory.encode(m)),
1591	1539	} },
1592	1540	});
1593	1541	} else {
...	...	@@ -1595,23 +1543,12 @@ fn asmRegisterMemoryImmediate(
1595	1543	.signed => \|s\| @bitCast(s),
1596	1544	.unsigned => unreachable,
1597	1545	} });
1598		assert(payload + 1 == switch (m) {
1599		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1600		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1601		else => unreachable,
1602		});
	1546	assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));
1603	1547	_ = try self.addInst(.{
1604	1548	.tag = tag[1],
1605		.ops = switch (m) {
1606		.sib => switch (imm) {
1607		.signed => .rmi_sib_s,
1608		.unsigned => .rmi_sib_u,
1609		},
1610		.rip => switch (imm) {
1611		.signed => .rmi_rip_s,
1612		.unsigned => .rmi_rip_u,
1613		},
1614		else => unreachable,
	1549	.ops = switch (imm) {
	1550	.signed => .rmi_s,
	1551	.unsigned => .rmi_u,
1615	1552	},
1616	1553	.data = .{ .rx = .{
1617	1554	.fixes = tag[0],
...	...	@@ -1632,21 +1569,13 @@ fn asmRegisterRegisterMemoryImmediate(
1632	1569	) !void {
1633	1570	_ = try self.addInst(.{
1634	1571	.tag = tag[1],
1635		.ops = switch (m) {
1636		.sib => .rrmi_sib,
1637		.rip => .rrmi_rip,
1638		else => unreachable,
1639		},
	1572	.ops = .rrmi,
1640	1573	.data = .{ .rrix = .{
1641	1574	.fixes = tag[0],
1642	1575	.r1 = reg1,
1643	1576	.r2 = reg2,
1644	1577	.i = @intCast(imm.unsigned),
1645		.payload = switch (m) {
1646		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1647		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1648		else => unreachable,
1649		},
	1578	.payload = try self.addExtra(Mir.Memory.encode(m)),
1650	1579	} },
1651	1580	});
1652	1581	}
...	...	@@ -1654,19 +1583,11 @@ fn asmRegisterRegisterMemoryImmediate(
1654	1583	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1655	1584	_ = try self.addInst(.{
1656	1585	.tag = tag[1],
1657		.ops = switch (m) {
1658		.sib => .mr_sib,
1659		.rip => .mr_rip,
1660		else => unreachable,
1661		},
	1586	.ops = .mr,
1662	1587	.data = .{ .rx = .{
1663	1588	.fixes = tag[0],
1664	1589	.r1 = reg,
1665		.payload = switch (m) {
1666		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1667		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1668		else => unreachable,
1669		},
	1590	.payload = try self.addExtra(Mir.Memory.encode(m)),
1670	1591	} },
1671	1592	});
1672	1593	}
...	...	@@ -1676,23 +1597,12 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
1676	1597	.signed => \|s\| @bitCast(s),
1677	1598	.unsigned => \|u\| @intCast(u),
1678	1599	} });
1679		assert(payload + 1 == switch (m) {
1680		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1681		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1682		else => unreachable,
1683		});
	1600	assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));
1684	1601	_ = try self.addInst(.{
1685	1602	.tag = tag[1],
1686		.ops = switch (m) {
1687		.sib => switch (imm) {
1688		.signed => .mi_sib_s,
1689		.unsigned => .mi_sib_u,
1690		},
1691		.rip => switch (imm) {
1692		.signed => .mi_rip_s,
1693		.unsigned => .mi_rip_u,
1694		},
1695		else => unreachable,
	1603	.ops = switch (imm) {
	1604	.signed => .mi_s,
	1605	.unsigned => .mi_u,
1696	1606	},
1697	1607	.data = .{ .x = .{
1698	1608	.fixes = tag[0],
...	...	@@ -1710,20 +1620,12 @@ fn asmMemoryRegisterRegister(
1710	1620	) !void {
1711	1621	_ = try self.addInst(.{
1712	1622	.tag = tag[1],
1713		.ops = switch (m) {
1714		.sib => .mrr_sib,
1715		.rip => .mrr_rip,
1716		else => unreachable,
1717		},
	1623	.ops = .mrr,
1718	1624	.data = .{ .rrx = .{
1719	1625	.fixes = tag[0],
1720	1626	.r1 = reg1,
1721	1627	.r2 = reg2,
1722		.payload = switch (m) {
1723		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1724		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1725		else => unreachable,
1726		},
	1628	.payload = try self.addExtra(Mir.Memory.encode(m)),
1727	1629	} },
1728	1630	});
1729	1631	}
...	...	@@ -1737,20 +1639,12 @@ fn asmMemoryRegisterImmediate(
1737	1639	) !void {
1738	1640	_ = try self.addInst(.{
1739	1641	.tag = tag[1],
1740		.ops = switch (m) {
1741		.sib => .mri_sib,
1742		.rip => .mri_rip,
1743		else => unreachable,
1744		},
	1642	.ops = .mri,
1745	1643	.data = .{ .rix = .{
1746	1644	.fixes = tag[0],
1747	1645	.r1 = reg,
1748	1646	.i = @intCast(imm.unsigned),
1749		.payload = switch (m) {
1750		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1751		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1752		else => unreachable,
1753		},
	1647	.payload = try self.addExtra(Mir.Memory.encode(m)),
1754	1648	} },
1755	1649	});
1756	1650	}
...	...	@@ -1916,13 +1810,16 @@ fn gen(self: *Self) InnerError!void {
1916	1810	if (need_frame_align or need_stack_adjust) {
1917	1811	self.mir_instructions.set(backpatch_stack_dealloc, .{
1918	1812	.tag = .lea,
1919		.ops = .rm_sib,
	1813	.ops = .rm,
1920	1814	.data = .{ .rx = .{
1921	1815	.r1 = .rsp,
1922		.payload = try self.addExtra(Mir.MemorySib.encode(Memory.sib(.qword, .{
	1816	.payload = try self.addExtra(Mir.Memory.encode(.{
1923	1817	.base = .{ .reg = .rbp },
1924		.disp = -frame_layout.save_reg_list.size(),
1925		}))),
	1818	.mod = .{ .rm = .{
	1819	.size = .qword,
	1820	.disp = -frame_layout.save_reg_list.size(),
	1821	} },
	1822	})),
1926	1823	} },
1927	1824	});
1928	1825	}
...	...	@@ -3098,14 +2995,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3098	2995	.{ .vp_, .@"and" },
3099	2996	dst_reg,
3100	2997	dst_reg,
3101		splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(splat_abi_size)),
	2998	splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
3102	2999	);
3103	3000	try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3104	3001	} else {
3105	3002	try self.asmRegisterMemory(
3106	3003	.{ .p_, .@"and" },
3107	3004	dst_reg,
3108		splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(splat_abi_size)),
	3005	splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),
3109	3006	);
3110	3007	try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3111	3008	}
...	...	@@ -3254,7 +3151,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3254	3151	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(Type.usize, mod));
3255	3152	try self.asmMemoryImmediate(
3256	3153	.{ ._, .mov },
3257		Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
	3154	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
3258	3155	Immediate.u(0),
3259	3156	);
3260	3157
...	...	@@ -3338,9 +3235,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3338	3235	call_mcv.register_pair[0],
3339	3236	call_mcv.register_pair[1],
3340	3237	);
3341		try self.asmSetccMemory(.nz, Memory.sib(.byte, .{
	3238	try self.asmSetccMemory(.nz, .{
3342	3239	.base = .{ .frame = signed_div_floor_state.frame_index },
3343		}));
	3240	.mod = .{ .rm = .{ .size = .byte } },
	3241	});
3344	3242	try self.performReloc(signed_div_floor_state.reloc);
3345	3243	const dst_mcv = try self.genCall(
3346	3244	.{ .lib = .{
...	...	@@ -3356,9 +3254,10 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3356	3254	try self.asmRegisterMemory(
3357	3255	.{ ._, .sub },
3358	3256	dst_mcv.register_pair[0],
3359		Memory.sib(.qword, .{
	3257	.{
3360	3258	.base = .{ .frame = signed_div_floor_state.frame_index },
3361		}),
	3259	.mod = .{ .rm = .{ .size = .qword } },
	3260	},
3362	3261	);
3363	3262	try self.asmRegisterImmediate(
3364	3263	.{ ._, .sbb },
...	...	@@ -3891,7 +3790,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3891	3790	);
3892	3791	try self.asmMemoryImmediate(
3893	3792	.{ ._, .cmp },
3894		overflow.mem(self.memPtrSize(Type.c_int)),
	3793	overflow.mem(self.memSize(Type.c_int)),
3895	3794	Immediate.s(0),
3896	3795	);
3897	3796	try self.genSetMem(
...	...	@@ -4123,7 +4022,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4123	4022	.register => \|reg\| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4124	4023	.memory, .indirect, .load_frame => try self.asmMemory(
4125	4024	tag,
4126		mat_rhs.mem(Memory.PtrSize.fromSize(abi_size)),
	4025	mat_rhs.mem(Memory.Size.fromSize(abi_size)),
4127	4026	),
4128	4027	else => unreachable,
4129	4028	}
...	...	@@ -4360,10 +4259,13 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4360	4259	try self.asmRegisterMemory(
4361	4260	.{ ._, .mov },
4362	4261	registerAlias(dst_reg, err_abi_size),
4363		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	4262	.{
4364	4263	.base = .{ .reg = src_reg },
4365		.disp = err_off,
4366		}),
	4264	.mod = .{ .rm = .{
	4265	.size = Memory.Size.fromSize(err_abi_size),
	4266	.disp = err_off,
	4267	} },
	4268	},
4367	4269	);
4368	4270
4369	4271	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -4398,10 +4300,13 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
4398	4300	const err_abi_size: u32 = @intCast(err_ty.abiSize(mod));
4399	4301	try self.asmMemoryImmediate(
4400	4302	.{ ._, .mov },
4401		Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
	4303	.{
4402	4304	.base = .{ .reg = src_reg },
4403		.disp = err_off,
4404		}),
	4305	.mod = .{ .rm = .{
	4306	.size = Memory.Size.fromSize(err_abi_size),
	4307	.disp = err_off,
	4308	} },
	4309	},
4405	4310	Immediate.u(0),
4406	4311	);
4407	4312
...	...	@@ -4420,7 +4325,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
4420	4325	try self.asmRegisterMemory(
4421	4326	.{ ._, .lea },
4422	4327	registerAlias(dst_reg, dst_abi_size),
4423		Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
	4328	.{
	4329	.base = .{ .reg = src_reg },
	4330	.mod = .{ .rm = .{ .size = .qword, .disp = pl_off } },
	4331	},
4424	4332	);
4425	4333	break :result .{ .register = dst_reg };
4426	4334	};
...	...	@@ -4542,10 +4450,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
4542	4450
4543	4451	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
4544	4452	.{ ._, .mov },
4545		Memory.sib(.byte, .{
	4453	.{
4546	4454	.base = .{ .frame = frame_addr.index },
4547		.disp = frame_addr.off + pl_abi_size,
4548		}),
	4455	.mod = .{ .rm = .{
	4456	.size = .byte,
	4457	.disp = frame_addr.off + pl_abi_size,
	4458	} },
	4459	},
4549	4460	Immediate.u(1),
4550	4461	),
4551	4462	}
...	...	@@ -4658,10 +4569,10 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
4658	4569	try self.asmRegisterMemory(
4659	4570	.{ ._, .lea },
4660	4571	registerAlias(dst_reg, dst_abi_size),
4661		Memory.sib(.qword, .{
	4572	.{
4662	4573	.base = .{ .reg = src_reg },
4663		.disp = @divExact(self.target.ptrBitWidth(), 8),
4664		}),
	4574	.mod = .{ .rm = .{ .size = .qword, .disp = 8 } },
	4575	},
4665	4576	);
4666	4577
4667	4578	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...	...	@@ -4791,13 +4702,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4791	4702	try self.asmRegisterMemory(
4792	4703	.{ ._, .lea },
4793	4704	addr_reg,
4794		Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
	4705	.{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
4795	4706	);
4796	4707	},
4797	4708	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
4798	4709	.{ ._, .lea },
4799	4710	addr_reg,
4800		Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
	4711	.{
	4712	.base = .{ .frame = frame_addr.index },
	4713	.mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } },
	4714	},
4801	4715	),
4802	4716	.memory,
4803	4717	.load_symbol,
...	...	@@ -5610,10 +5524,14 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
5610	5524	try self.asmRegisterMemory(
5611	5525	.{ ._, .lea },
5612	5526	if (limb_abi_size > 4) tmp.to64() else tmp.to32(),
5613		Memory.sib(.qword, .{
	5527	.{
5614	5528	.base = .{ .reg = dst.to64() },
5615		.scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
5616		}),
	5529	.mod = .{ .rm = .{
	5530	.size = .qword,
	5531	.index = tmp.to64(),
	5532	.scale = .@"4",
	5533	} },
	5534	},
5617	5535	);
5618	5536	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
5619	5537	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
...	...	@@ -5633,10 +5551,14 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
5633	5551	try self.asmRegisterMemory(
5634	5552	.{ ._, .lea },
5635	5553	if (limb_abi_size > 4) dst.to64() else dst.to32(),
5636		Memory.sib(.qword, .{
	5554	.{
5637	5555	.base = .{ .reg = tmp.to64() },
5638		.scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
5639		}),
	5556	.mod = .{ .rm = .{
	5557	.size = .qword,
	5558	.index = dst.to64(),
	5559	.scale = .@"2",
	5560	} },
	5561	},
5640	5562	);
5641	5563	// dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1)
5642	5564	}
...	...	@@ -5713,12 +5635,13 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5713	5635	.abs => try vec_ty.maxInt(mod, vec_ty),
5714	5636	else => unreachable,
5715	5637	} });
5716		const sign_mem = if (sign_mcv.isMemory())
5717		sign_mcv.mem(Memory.PtrSize.fromSize(abi_size))
	5638	const sign_mem: Memory = if (sign_mcv.isMemory())
	5639	sign_mcv.mem(Memory.Size.fromSize(abi_size))
5718	5640	else
5719		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	5641	.{
5720	5642	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
5721		});
	5643	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },
	5644	};
5722	5645
5723	5646	if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
5724	5647	switch (scalar_bits) {
...	...	@@ -5896,7 +5819,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5896	5819	mir_tag,
5897	5820	dst_alias,
5898	5821	dst_alias,
5899		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	5822	src_mcv.mem(Memory.Size.fromSize(abi_size)),
5900	5823	Immediate.u(@as(u5, @bitCast(mode))),
5901	5824	) else try self.asmRegisterRegisterRegisterImmediate(
5902	5825	mir_tag,
...	...	@@ -5911,7 +5834,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5911	5834	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
5912	5835	mir_tag,
5913	5836	dst_alias,
5914		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	5837	src_mcv.mem(Memory.Size.fromSize(abi_size)),
5915	5838	Immediate.u(@as(u5, @bitCast(mode))),
5916	5839	) else try self.asmRegisterRegisterImmediate(
5917	5840	mir_tag,
...	...	@@ -5951,7 +5874,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
5951	5874	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
5952	5875	.l,
5953	5876	registerAlias(dst_mcv.register, cmov_abi_size),
5954		src_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
	5877	src_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
5955	5878	),
5956	5879	else => {
5957	5880	const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...	...	@@ -6051,7 +5974,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6051	5974	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6052	5975	mir_tag,
6053	5976	dst_alias,
6054		src_mcv.mem(self.memPtrSize(ty)),
	5977	src_mcv.mem(self.memSize(ty)),
6055	5978	) else try self.asmRegisterRegister(
6056	5979	mir_tag,
6057	5980	dst_alias,
...	...	@@ -6157,7 +6080,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6157	6080	if (src_mcv.isMemory()) try self.asmRegisterMemory(
6158	6081	.{ .v_ps, .cvtph2 },
6159	6082	wide_reg,
6160		src_mcv.mem(Memory.PtrSize.fromSize(
	6083	src_mcv.mem(Memory.Size.fromSize(
6161	6084	@intCast(@divExact(wide_reg.bitSize(), 16)),
6162	6085	)),
6163	6086	) else try self.asmRegisterRegister(
...	...	@@ -6205,7 +6128,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6205	6128	mir_tag,
6206	6129	dst_reg,
6207	6130	dst_reg,
6208		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6131	src_mcv.mem(Memory.Size.fromSize(abi_size)),
6209	6132	) else try self.asmRegisterRegisterRegister(
6210	6133	mir_tag,
6211	6134	dst_reg,
...	...	@@ -6218,7 +6141,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6218	6141	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
6219	6142	mir_tag,
6220	6143	dst_reg,
6221		src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6144	src_mcv.mem(Memory.Size.fromSize(abi_size)),
6222	6145	) else try self.asmRegisterRegister(
6223	6146	mir_tag,
6224	6147	dst_reg,
...	...	@@ -6337,14 +6260,13 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6337	6260	if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
6338	6261	if (load_abi_size <= 8) {
6339	6262	const load_reg = registerAlias(dst_reg, load_abi_size);
6340		try self.asmRegisterMemory(
6341		.{ ._, .mov },
6342		load_reg,
6343		Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
6344		.base = .{ .reg = ptr_reg },
	6263	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
	6264	.base = .{ .reg = ptr_reg },
	6265	.mod = .{ .rm = .{
	6266	.size = Memory.Size.fromSize(load_abi_size),
6345	6267	.disp = val_byte_off,
6346		}),
6347		);
	6268	} },
	6269	});
6348	6270	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
6349	6271	} else {
6350	6272	const tmp_reg =
...	...	@@ -6353,22 +6275,20 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6353	6275	defer self.register_manager.unlockReg(tmp_lock);
6354	6276
6355	6277	const dst_alias = registerAlias(dst_reg, val_abi_size);
6356		try self.asmRegisterMemory(
6357		.{ ._, .mov },
6358		dst_alias,
6359		Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
6360		.base = .{ .reg = ptr_reg },
	6278	try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{
	6279	.base = .{ .reg = ptr_reg },
	6280	.mod = .{ .rm = .{
	6281	.size = Memory.Size.fromSize(val_abi_size),
6361	6282	.disp = val_byte_off,
6362		}),
6363		);
6364		try self.asmRegisterMemory(
6365		.{ ._, .mov },
6366		tmp_reg,
6367		Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
6368		.base = .{ .reg = ptr_reg },
	6283	} },
	6284	});
	6285	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
	6286	.base = .{ .reg = ptr_reg },
	6287	.mod = .{ .rm = .{
	6288	.size = Memory.Size.fromSize(val_abi_size),
6369	6289	.disp = val_byte_off + 1,
6370		}),
6371		);
	6290	} },
	6291	});
6372	6292	try self.asmRegisterRegisterImmediate(
6373	6293	.{ ._rd, .sh },
6374	6294	dst_alias,
...	...	@@ -6480,10 +6400,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6480	6400	const part_bit_off = if (limb_i == 0) src_bit_off else 0;
6481	6401	const part_bit_size =
6482	6402	@min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
6483		const limb_mem = Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), .{
	6403	const limb_mem: Memory = .{
6484	6404	.base = .{ .reg = ptr_reg },
6485		.disp = src_byte_off + limb_i * limb_abi_bits,
6486		});
	6405	.mod = .{ .rm = .{
	6406	.size = Memory.Size.fromSize(limb_abi_size),
	6407	.disp = src_byte_off + limb_i * limb_abi_bits,
	6408	} },
	6409	};
6487	6410
6488	6411	const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) <<
6489	6412	@intCast(part_bit_off);
...	...	@@ -6814,7 +6737,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6814	6737	break :result dst_mcv;
6815	6738	}
6816	6739
6817		const limb_abi_size: u32 = @min(field_abi_size, 8);
	6740	const limb_abi_size: u31 = @min(field_abi_size, 8);
6818	6741	const limb_abi_bits = limb_abi_size * 8;
6819	6742	const field_byte_off: i32 = @intCast(field_off / limb_abi_bits * limb_abi_size);
6820	6743	const field_bit_off = field_off % limb_abi_bits;
...	...	@@ -6832,14 +6755,13 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6832	6755	if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
6833	6756	if (load_abi_size <= 8) {
6834	6757	const load_reg = registerAlias(dst_reg, load_abi_size);
6835		try self.asmRegisterMemory(
6836		.{ ._, .mov },
6837		load_reg,
6838		Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
6839		.base = .{ .frame = frame_addr.index },
	6758	try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{
	6759	.base = .{ .frame = frame_addr.index },
	6760	.mod = .{ .rm = .{
	6761	.size = Memory.Size.fromSize(load_abi_size),
6840	6762	.disp = frame_addr.off + field_byte_off,
6841		}),
6842		);
	6763	} },
	6764	});
6843	6765	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
6844	6766	} else {
6845	6767	const tmp_reg = registerAlias(
...	...	@@ -6853,19 +6775,21 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6853	6775	try self.asmRegisterMemory(
6854	6776	.{ ._, .mov },
6855	6777	dst_alias,
6856		Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
	6778	.{
6857	6779	.base = .{ .frame = frame_addr.index },
6858		.disp = frame_addr.off + field_byte_off,
6859		}),
6860		);
6861		try self.asmRegisterMemory(
6862		.{ ._, .mov },
6863		tmp_reg,
6864		Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
6865		.base = .{ .frame = frame_addr.index },
6866		.disp = frame_addr.off + field_byte_off + @as(i32, @intCast(limb_abi_size)),
6867		}),
	6780	.mod = .{ .rm = .{
	6781	.size = Memory.Size.fromSize(field_abi_size),
	6782	.disp = frame_addr.off + field_byte_off,
	6783	} },
	6784	},
6868	6785	);
	6786	try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{
	6787	.base = .{ .frame = frame_addr.index },
	6788	.mod = .{ .rm = .{
	6789	.size = Memory.Size.fromSize(field_abi_size),
	6790	.disp = frame_addr.off + field_byte_off + limb_abi_size,
	6791	} },
	6792	});
6869	6793	try self.asmRegisterRegisterImmediate(
6870	6794	.{ ._rd, .sh },
6871	6795	dst_alias,
...	...	@@ -6998,14 +6922,13 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
6998	6922	defer self.register_manager.unlockReg(addr_reg_lock);
6999	6923
7000	6924	try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
7001		try self.asmMemory(
7002		mir_tag,
7003		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
7004		);
	6925	try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
	6926	.size = Memory.Size.fromSize(abi_size),
	6927	} } });
7005	6928	},
7006	6929	.indirect, .load_frame => try self.asmMemory(
7007	6930	mir_tag,
7008		dst_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	6931	dst_mcv.mem(Memory.Size.fromSize(abi_size)),
7009	6932	),
7010	6933	}
7011	6934	}
...	...	@@ -7053,25 +6976,34 @@ fn genShiftBinOpMir(
7053	6976	}),
7054	6977	},
7055	6978	.memory, .indirect, .load_frame => {
7056		const lhs_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (lhs_mcv) {
	6979	const lhs_mem: Memory = switch (lhs_mcv) {
7057	6980	.memory => \|addr\| .{
7058	6981	.base = .{ .reg = .ds },
7059		.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
7060		return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
7061		@tagName(lhs_mcv),
7062		@tagName(rhs_mcv),
7063		}),
	6982	.mod = .{ .rm = .{
	6983	.size = Memory.Size.fromSize(abi_size),
	6984	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
	6985	return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	6986	@tagName(lhs_mcv),
	6987	@tagName(rhs_mcv),
	6988	}),
	6989	} },
7064	6990	},
7065	6991	.indirect => \|reg_off\| .{
7066	6992	.base = .{ .reg = reg_off.reg },
7067		.disp = reg_off.off,
	6993	.mod = .{ .rm = .{
	6994	.size = Memory.Size.fromSize(abi_size),
	6995	.disp = reg_off.off,
	6996	} },
7068	6997	},
7069	6998	.load_frame => \|frame_addr\| .{
7070	6999	.base = .{ .frame = frame_addr.index },
7071		.disp = frame_addr.off,
	7000	.mod = .{ .rm = .{
	7001	.size = Memory.Size.fromSize(abi_size),
	7002	.disp = frame_addr.off,
	7003	} },
7072	7004	},
7073	7005	else => unreachable,
7074		});
	7006	};
7075	7007	switch (rhs_mcv) {
7076	7008	.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
7077	7009	tag,
...	...	@@ -7187,26 +7119,35 @@ fn genShiftBinOpMir(
7187	7119	try self.asmRegisterMemory(
7188	7120	.{ ._, .mov },
7189	7121	tmp_reg,
7190		Memory.sib(.qword, .{
	7122	.{
7191	7123	.base = .{ .frame = dst_frame_addr.index },
7192		.disp = dst_frame_addr.off + info.indices[0] * 8,
7193		}),
	7124	.mod = .{ .rm = .{
	7125	.size = .qword,
	7126	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7127	} },
	7128	},
7194	7129	);
7195	7130	try self.asmMemoryRegisterImmediate(
7196	7131	info.double_tag,
7197		Memory.sib(.qword, .{
	7132	.{
7198	7133	.base = .{ .frame = dst_frame_addr.index },
7199		.disp = dst_frame_addr.off + info.indices[1] * 8,
7200		}),
	7134	.mod = .{ .rm = .{
	7135	.size = .qword,
	7136	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7137	} },
	7138	},
7201	7139	tmp_reg,
7202	7140	Immediate.u(rhs_imm),
7203	7141	);
7204	7142	try self.asmMemoryImmediate(
7205	7143	tag,
7206		Memory.sib(.qword, .{
	7144	.{
7207	7145	.base = .{ .frame = dst_frame_addr.index },
7208		.disp = dst_frame_addr.off + info.indices[0] * 8,
7209		}),
	7146	.mod = .{ .rm = .{
	7147	.size = .qword,
	7148	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7149	} },
	7150	},
7210	7151	Immediate.u(rhs_imm),
7211	7152	);
7212	7153	} else {
...	...	@@ -7214,10 +7155,13 @@ fn genShiftBinOpMir(
7214	7155	try self.asmRegisterMemory(
7215	7156	.{ ._, .mov },
7216	7157	tmp_reg,
7217		Memory.sib(.qword, .{
	7158	.{
7218	7159	.base = .{ .frame = dst_frame_addr.index },
7219		.disp = dst_frame_addr.off + info.indices[0] * 8,
7220		}),
	7160	.mod = .{ .rm = .{
	7161	.size = .qword,
	7162	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7163	} },
	7164	},
7221	7165	);
7222	7166	if (rhs_imm > 64) try self.asmRegisterImmediate(
7223	7167	tag,
...	...	@@ -7226,27 +7170,36 @@ fn genShiftBinOpMir(
7226	7170	);
7227	7171	try self.asmMemoryRegister(
7228	7172	.{ ._, .mov },
7229		Memory.sib(.qword, .{
	7173	.{
7230	7174	.base = .{ .frame = dst_frame_addr.index },
7231		.disp = dst_frame_addr.off + info.indices[1] * 8,
7232		}),
	7175	.mod = .{ .rm = .{
	7176	.size = .qword,
	7177	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7178	} },
	7179	},
7233	7180	tmp_reg,
7234	7181	);
7235	7182	if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
7236	7183	tag,
7237		Memory.sib(.qword, .{
	7184	.{
7238	7185	.base = .{ .frame = dst_frame_addr.index },
7239		.disp = dst_frame_addr.off + info.indices[0] * 8,
7240		}),
	7186	.mod = .{ .rm = .{
	7187	.size = .qword,
	7188	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7189	} },
	7190	},
7241	7191	Immediate.u(63),
7242	7192	) else {
7243	7193	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
7244	7194	try self.asmMemoryRegister(
7245	7195	.{ ._, .mov },
7246		Memory.sib(.qword, .{
	7196	.{
7247	7197	.base = .{ .frame = dst_frame_addr.index },
7248		.disp = dst_frame_addr.off + info.indices[0] * 8,
7249		}),
	7198	.mod = .{ .rm = .{
	7199	.size = .qword,
	7200	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7201	} },
	7202	},
7250	7203	tmp_reg,
7251	7204	);
7252	7205	}
...	...	@@ -7265,18 +7218,24 @@ fn genShiftBinOpMir(
7265	7218	try self.asmRegisterMemory(
7266	7219	.{ ._, .mov },
7267	7220	first_reg,
7268		Memory.sib(.qword, .{
	7221	.{
7269	7222	.base = .{ .frame = dst_frame_addr.index },
7270		.disp = dst_frame_addr.off + info.indices[0] * 8,
7271		}),
	7223	.mod = .{ .rm = .{
	7224	.size = .qword,
	7225	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7226	} },
	7227	},
7272	7228	);
7273	7229	try self.asmRegisterMemory(
7274	7230	.{ ._, .mov },
7275	7231	second_reg,
7276		Memory.sib(.qword, .{
	7232	.{
7277	7233	.base = .{ .frame = dst_frame_addr.index },
7278		.disp = dst_frame_addr.off + info.indices[1] * 8,
7279		}),
	7234	.mod = .{ .rm = .{
	7235	.size = .qword,
	7236	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7237	} },
	7238	},
7280	7239	);
7281	7240	if (tag[0] == ._r and tag[1] == .sa) {
7282	7241	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
...	...	@@ -7302,18 +7261,24 @@ fn genShiftBinOpMir(
7302	7261	try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg);
7303	7262	try self.asmMemoryRegister(
7304	7263	.{ ._, .mov },
7305		Memory.sib(.qword, .{
	7264	.{
7306	7265	.base = .{ .frame = dst_frame_addr.index },
7307		.disp = dst_frame_addr.off + info.indices[1] * 8,
7308		}),
	7266	.mod = .{ .rm = .{
	7267	.size = .qword,
	7268	.disp = dst_frame_addr.off + info.indices[1] * 8,
	7269	} },
	7270	},
7309	7271	second_reg,
7310	7272	);
7311	7273	try self.asmMemoryRegister(
7312	7274	.{ ._, .mov },
7313		Memory.sib(.qword, .{
	7275	.{
7314	7276	.base = .{ .frame = dst_frame_addr.index },
7315		.disp = dst_frame_addr.off + info.indices[0] * 8,
7316		}),
	7277	.mod = .{ .rm = .{
	7278	.size = .qword,
	7279	.disp = dst_frame_addr.off + info.indices[0] * 8,
	7280	} },
	7281	},
7317	7282	first_reg,
7318	7283	);
7319	7284	},
...	...	@@ -7541,22 +7506,20 @@ fn genMulDivBinOp(
7541	7506	}, dst_abi_size) };
7542	7507
7543	7508	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
7544		try self.asmMemoryRegister(
7545		.{ ._, .mov },
7546		Memory.sib(.qword, .{
7547		.base = .{ .frame = dst_mcv.load_frame.index },
	7509	try self.asmMemoryRegister(.{ ._, .mov }, .{
	7510	.base = .{ .frame = dst_mcv.load_frame.index },
	7511	.mod = .{ .rm = .{
	7512	.size = .qword,
7548	7513	.disp = dst_mcv.load_frame.off,
7549		}),
7550		.rax,
7551		);
7552		try self.asmMemoryRegister(
7553		.{ ._, .mov },
7554		Memory.sib(.qword, .{
7555		.base = .{ .frame = dst_mcv.load_frame.index },
	7514	} },
	7515	}, .rax);
	7516	try self.asmMemoryRegister(.{ ._, .mov }, .{
	7517	.base = .{ .frame = dst_mcv.load_frame.index },
	7518	.mod = .{ .rm = .{
	7519	.size = .qword,
7556	7520	.disp = dst_mcv.load_frame.off + 8,
7557		}),
7558		.rdx,
7559		);
	7521	} },
	7522	}, .rdx);
7560	7523	return dst_mcv;
7561	7524	},
7562	7525
...	...	@@ -7768,7 +7731,7 @@ fn genBinOp(
7768	7731	mir_tag,
7769	7732	dst_reg,
7770	7733	dst_reg,
7771		src_mcv.mem(Memory.PtrSize.fromBitSize(float_bits)),
	7734	src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
7772	7735	) else try self.asmRegisterRegisterRegister(
7773	7736	mir_tag,
7774	7737	dst_reg,
...	...	@@ -7787,7 +7750,7 @@ fn genBinOp(
7787	7750	if (src_mcv.isMemory()) try self.asmRegisterMemory(
7788	7751	mir_tag,
7789	7752	dst_reg,
7790		src_mcv.mem(Memory.PtrSize.fromBitSize(float_bits)),
	7753	src_mcv.mem(Memory.Size.fromBitSize(float_bits)),
7791	7754	) else try self.asmRegisterRegister(
7792	7755	mir_tag,
7793	7756	dst_reg,
...	...	@@ -8149,21 +8112,30 @@ fn genBinOp(
8149	8112	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
8150	8113	cc,
8151	8114	registerAlias(tmp_reg, cmov_abi_size),
8152		Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
	8115	switch (mat_src_mcv) {
8153	8116	.memory => \|addr\| .{
8154	8117	.base = .{ .reg = .ds },
8155		.disp = @intCast(@as(i64, @bitCast(addr))),
	8118	.mod = .{ .rm = .{
	8119	.size = Memory.Size.fromSize(cmov_abi_size),
	8120	.disp = @intCast(@as(i64, @bitCast(addr))),
	8121	} },
8156	8122	},
8157	8123	.indirect => \|reg_off\| .{
8158	8124	.base = .{ .reg = reg_off.reg },
8159		.disp = reg_off.off,
	8125	.mod = .{ .rm = .{
	8126	.size = Memory.Size.fromSize(cmov_abi_size),
	8127	.disp = reg_off.off,
	8128	} },
8160	8129	},
8161	8130	.load_frame => \|frame_addr\| .{
8162	8131	.base = .{ .frame = frame_addr.index },
8163		.disp = frame_addr.off,
	8132	.mod = .{ .rm = .{
	8133	.size = Memory.Size.fromSize(cmov_abi_size),
	8134	.disp = frame_addr.off,
	8135	} },
8164	8136	},
8165	8137	else => unreachable,
8166		}),
	8138	},
8167	8139	),
8168	8140	}
8169	8141	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
...	...	@@ -8912,8 +8884,8 @@ fn genBinOp(
8912	8884	dst_reg,
8913	8885	lhs_reg,
8914	8886	src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
8915		else => Memory.PtrSize.fromSize(abi_size),
8916		.Vector => Memory.PtrSize.fromBitSize(dst_reg.bitSize()),
	8887	else => Memory.Size.fromSize(abi_size),
	8888	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
8917	8889	}),
8918	8890	) else try self.asmRegisterRegisterRegister(
8919	8891	mir_tag,
...	...	@@ -8930,8 +8902,8 @@ fn genBinOp(
8930	8902	mir_tag,
8931	8903	dst_reg,
8932	8904	src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {
8933		else => Memory.PtrSize.fromSize(abi_size),
8934		.Vector => Memory.PtrSize.fromBitSize(dst_reg.bitSize()),
	8905	else => Memory.Size.fromSize(abi_size),
	8906	.Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
8935	8907	}),
8936	8908	) else try self.asmRegisterRegister(
8937	8909	mir_tag,
...	...	@@ -9202,12 +9174,12 @@ fn genBinOp(
9202	9174	.ty = lhs_ty,
9203	9175	.val = try unsigned_ty.maxInt(mod, unsigned_ty),
9204	9176	});
9205		const not_mem = if (not_mcv.isMemory())
9206		not_mcv.mem(Memory.PtrSize.fromSize(abi_size))
	9177	const not_mem: Memory = if (not_mcv.isMemory())
	9178	not_mcv.mem(Memory.Size.fromSize(abi_size))
9207	9179	else
9208		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{
	9180	.{ .base = .{
9209	9181	.reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
9210		} });
	9182	}, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
9211	9183	switch (mir_tag[0]) {
9212	9184	.vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
9213	9185	.{ .vp_, .xor },
...	...	@@ -9359,25 +9331,30 @@ fn genBinOpMir(
9359	9331	.lea_frame,
9360	9332	=> {
9361	9333	blk: {
9362		return self.asmRegisterMemory(
9363		mir_limb_tag,
9364		dst_alias,
9365		Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), switch (src_mcv) {
9366		.memory => \|addr\| .{
9367		.base = .{ .reg = .ds },
	9334	return self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
	9335	.memory => \|addr\| .{
	9336	.base = .{ .reg = .ds },
	9337	.mod = .{ .rm = .{
	9338	.size = Memory.Size.fromSize(limb_abi_size),
9368	9339	.disp = math.cast(i32, addr + off) orelse break :blk,
9369		},
9370		.indirect => \|reg_off\| .{
9371		.base = .{ .reg = reg_off.reg },
	9340	} },
	9341	},
	9342	.indirect => \|reg_off\| .{
	9343	.base = .{ .reg = reg_off.reg },
	9344	.mod = .{ .rm = .{
	9345	.size = Memory.Size.fromSize(limb_abi_size),
9372	9346	.disp = reg_off.off + off,
9373		},
9374		.load_frame => \|frame_addr\| .{
9375		.base = .{ .frame = frame_addr.index },
	9347	} },
	9348	},
	9349	.load_frame => \|frame_addr\| .{
	9350	.base = .{ .frame = frame_addr.index },
	9351	.mod = .{ .rm = .{
	9352	.size = Memory.Size.fromSize(limb_abi_size),
9376	9353	.disp = frame_addr.off + off,
9377		},
9378		else => break :blk,
9379		}),
9380		);
	9354	} },
	9355	},
	9356	else => break :blk,
	9357	});
9381	9358	}
9382	9359
9383	9360	switch (src_mcv) {
...	...	@@ -9510,26 +9487,35 @@ fn genBinOpMir(
9510	9487	}),
9511	9488	},
9512	9489	};
9513		const dst_limb_mem = Memory.sib(
9514		Memory.PtrSize.fromSize(limb_abi_size),
9515		switch (dst_mcv) {
9516		.memory,
9517		.load_symbol,
9518		.load_got,
9519		.load_direct,
9520		.load_tlv,
9521		=> .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off },
9522		.indirect => \|reg_off\| .{
9523		.base = .{ .reg = reg_off.reg },
	9490	const dst_limb_mem: Memory = switch (dst_mcv) {
	9491	.memory,
	9492	.load_symbol,
	9493	.load_got,
	9494	.load_direct,
	9495	.load_tlv,
	9496	=> .{
	9497	.base = .{ .reg = dst_info.?.addr_reg },
	9498	.mod = .{ .rm = .{
	9499	.size = Memory.Size.fromSize(limb_abi_size),
	9500	.disp = off,
	9501	} },
	9502	},
	9503	.indirect => \|reg_off\| .{
	9504	.base = .{ .reg = reg_off.reg },
	9505	.mod = .{ .rm = .{
	9506	.size = Memory.Size.fromSize(limb_abi_size),
9524	9507	.disp = reg_off.off + off,
9525		},
9526		.load_frame => \|frame_addr\| .{
9527		.base = .{ .frame = frame_addr.index },
	9508	} },
	9509	},
	9510	.load_frame => \|frame_addr\| .{
	9511	.base = .{ .frame = frame_addr.index },
	9512	.mod = .{ .rm = .{
	9513	.size = Memory.Size.fromSize(limb_abi_size),
9528	9514	.disp = frame_addr.off + off,
9529		},
9530		else => unreachable,
	9515	} },
9531	9516	},
9532		);
	9517	else => unreachable,
	9518	};
9533	9519	switch (resolved_src_mcv) {
9534	9520	.none,
9535	9521	.unreach,
...	...	@@ -9730,29 +9716,38 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
9730	9716	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
9731	9717	.{ .i_, .mul },
9732	9718	dst_alias,
9733		Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (resolved_src_mcv) {
	9719	switch (resolved_src_mcv) {
9734	9720	.memory => \|addr\| .{
9735	9721	.base = .{ .reg = .ds },
9736		.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
9737		return self.asmRegisterRegister(
9738		.{ .i_, .mul },
9739		dst_alias,
9740		registerAlias(
9741		try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
9742		abi_size,
	9722	.mod = .{ .rm = .{
	9723	.size = Memory.Size.fromSize(abi_size),
	9724	.disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse
	9725	return self.asmRegisterRegister(
	9726	.{ .i_, .mul },
	9727	dst_alias,
	9728	registerAlias(
	9729	try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
	9730	abi_size,
	9731	),
9743	9732	),
9744		),
	9733	} },
9745	9734	},
9746	9735	.indirect => \|reg_off\| .{
9747	9736	.base = .{ .reg = reg_off.reg },
9748		.disp = reg_off.off,
	9737	.mod = .{ .rm = .{
	9738	.size = Memory.Size.fromSize(abi_size),
	9739	.disp = reg_off.off,
	9740	} },
9749	9741	},
9750	9742	.load_frame => \|frame_addr\| .{
9751	9743	.base = .{ .frame = frame_addr.index },
9752		.disp = frame_addr.off,
	9744	.mod = .{ .rm = .{
	9745	.size = Memory.Size.fromSize(abi_size),
	9746	.disp = frame_addr.off,
	9747	} },
9753	9748	},
9754	9749	else => unreachable,
9755		}),
	9750	},
9756	9751	),
9757	9752	}
9758	9753	},
...	...	@@ -9857,9 +9852,14 @@ fn genVarDbgInfo(
9857	9852	// .offset = -off,
9858	9853	//} },
9859	9854	.memory => \|address\| .{ .memory = address },
9860		.load_symbol => \|sym_index\| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } }, // TODO
	9855	.load_symbol => \|sym_off\| loc: {
	9856	assert(sym_off.off == 0);
	9857	break :loc .{ .linker_load = .{ .type = .direct, .sym_index = sym_off.sym } };
	9858	}, // TODO
9861	9859	.load_got => \|sym_index\| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
9862		.load_direct => \|sym_index\| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
	9860	.load_direct => \|sym_index\| .{
	9861	.linker_load = .{ .type = .direct, .sym_index = sym_index },
	9862	},
9863	9863	.immediate => \|x\| .{ .immediate = x },
9864	9864	.undef => .undef,
9865	9865	.none => .none,
...	...	@@ -10123,19 +10123,16 @@ fn genCall(self: *Self, info: union(enum) {
10123	10123	try self.genSetReg(
10124	10124	callee_reg,
10125	10125	Type.usize,
10126		.{ .load_symbol = sym.esym_index },
	10126	.{ .load_symbol = .{ .sym = sym.esym_index } },
10127	10127	);
10128	10128	try self.asmRegister(.{ ._, .call }, callee_reg);
10129		} else {
10130		_ = try self.addInst(.{
10131		.tag = .call,
10132		.ops = .linker_reloc,
10133		.data = .{ .reloc = .{
10134		.atom_index = try self.owner.getSymbolIndex(self),
10135		.sym_index = sym.esym_index,
10136		} },
10137		});
10138		}
	10129	} else try self.asmMemory(.{ ._, .call }, .{
	10130	.base = .{ .reloc = .{
	10131	.atom_index = try self.owner.getSymbolIndex(self),
	10132	.sym_index = sym.esym_index,
	10133	} },
	10134	.mod = .{ .rm = .{ .size = .qword } },
	10135	});
10139	10136	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
10140	10137	const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
10141	10138	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
...	...	@@ -10149,10 +10146,13 @@ fn genCall(self: *Self, info: union(enum) {
10149	10146	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
10150	10147	const atom_index = try p9.seeDecl(func.owner_decl);
10151	10148	const atom = p9.getAtom(atom_index);
10152		try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
	10149	try self.asmMemory(.{ ._, .call }, .{
10153	10150	.base = .{ .reg = .ds },
10154		.disp = @intCast(atom.getOffsetTableAddress(p9)),
10155		}));
	10151	.mod = .{ .rm = .{
	10152	.size = .qword,
	10153	.disp = @intCast(atom.getOffsetTableAddress(p9)),
	10154	} },
	10155	});
10156	10156	} else unreachable;
10157	10157	},
10158	10158	.extern_func => \|extern_func\| {
...	...	@@ -10637,7 +10637,10 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
10637	10637	try self.asmRegisterMemory(
10638	10638	.{ ._, .cmp },
10639	10639	registerAlias(dst_reg, op_abi_size),
10640		Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
	10640	.{
	10641	.base = .{ .reg = addr_reg },
	10642	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(op_abi_size) } },
	10643	},
10641	10644	);
10642	10645
10643	10646	self.eflags_inst = inst;
...	...	@@ -10891,10 +10894,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
10891	10894	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
10892	10895	try self.asmMemoryImmediate(
10893	10896	.{ ._, .cmp },
10894		Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	10897	.{
10895	10898	.base = .{ .reg = addr_reg },
10896		.disp = some_info.off,
10897		}),
	10899	.mod = .{ .rm = .{
	10900	.size = Memory.Size.fromSize(some_abi_size),
	10901	.disp = some_info.off,
	10902	} },
	10903	},
10898	10904	Immediate.u(0),
10899	10905	);
10900	10906	return .{ .eflags = .e };
...	...	@@ -10904,17 +10910,23 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
10904	10910	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
10905	10911	try self.asmMemoryImmediate(
10906	10912	.{ ._, .cmp },
10907		Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
	10913	switch (opt_mcv) {
10908	10914	.indirect => \|reg_off\| .{
10909	10915	.base = .{ .reg = reg_off.reg },
10910		.disp = reg_off.off + some_info.off,
	10916	.mod = .{ .rm = .{
	10917	.size = Memory.Size.fromSize(some_abi_size),
	10918	.disp = reg_off.off + some_info.off,
	10919	} },
10911	10920	},
10912	10921	.load_frame => \|frame_addr\| .{
10913	10922	.base = .{ .frame = frame_addr.index },
10914		.disp = frame_addr.off + some_info.off,
	10923	.mod = .{ .rm = .{
	10924	.size = Memory.Size.fromSize(some_abi_size),
	10925	.disp = frame_addr.off + some_info.off,
	10926	} },
10915	10927	},
10916	10928	else => unreachable,
10917		}),
	10929	},
10918	10930	Immediate.u(0),
10919	10931	);
10920	10932	return .{ .eflags = .e };
...	...	@@ -10944,10 +10956,13 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
10944	10956	const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
10945	10957	try self.asmMemoryImmediate(
10946	10958	.{ ._, .cmp },
10947		Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
	10959	.{
10948	10960	.base = .{ .reg = ptr_reg },
10949		.disp = some_info.off,
10950		}),
	10961	.mod = .{ .rm = .{
	10962	.size = Memory.Size.fromSize(some_abi_size),
	10963	.disp = some_info.off,
	10964	} },
	10965	},
10951	10966	Immediate.u(0),
10952	10967	);
10953	10968
...	...	@@ -11020,10 +11035,13 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
11020	11035	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(mod), mod));
11021	11036	try self.asmMemoryImmediate(
11022	11037	.{ ._, .cmp },
11023		Memory.sib(self.memPtrSize(Type.anyerror), .{
	11038	.{
11024	11039	.base = .{ .reg = ptr_reg },
11025		.disp = err_off,
11026		}),
	11040	.mod = .{ .rm = .{
	11041	.size = self.memSize(Type.anyerror),
	11042	.disp = err_off,
	11043	} },
	11044	},
11027	11045	Immediate.u(0),
11028	11046	);
11029	11047
...	...	@@ -11601,7 +11619,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11601	11619	label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
11602	11620	} else continue;
11603	11621
11604		var mnem_size: ?Memory.PtrSize = null;
	11622	var mnem_size: ?Memory.Size = null;
11605	11623	const mnem_tag = mnem: {
11606	11624	mnem_size = if (mem.endsWith(u8, mnem_str, "b"))
11607	11625	.byte
...	...	@@ -11620,7 +11638,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11620	11638	mnem_size = null;
11621	11639	break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);
11622	11640	} orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
11623		if (@as(?Memory.PtrSize, switch (mnem_tag) {
	11641	if (@as(?Memory.Size, switch (mnem_tag) {
11624	11642	.fldenv, .fnstenv, .fstenv => .none,
11625	11643	.ldmxcsr, .stmxcsr, .vldmxcsr, .vstmxcsr => .dword,
11626	11644	else => null,
...	...	@@ -11685,10 +11703,13 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11685	11703	if (colon) \|colon_pos\| {
11686	11704	const disp = std.fmt.parseInt(i32, op_str[colon_pos + ":".len ..], 0) catch
11687	11705	return self.fail("invalid displacement: '{s}'", .{op_str});
11688		op.* = .{ .mem = Memory.sib(
11689		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11690		.{ .base = .{ .reg = reg }, .disp = disp },
11691		) };
	11706	op.* = .{ .mem = .{
	11707	.base = .{ .reg = reg },
	11708	.mod = .{ .rm = .{
	11709	.size = mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
	11710	.disp = disp,
	11711	} },
	11712	} };
11692	11713	} else {
11693	11714	if (mnem_size) \|size\| if (reg.bitSize() != size.bitSize())
11694	11715	return self.fail("invalid register size: '{s}'", .{op_str});
...	...	@@ -11712,34 +11733,45 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11712	11733	.{ .reg = reg }
11713	11734	else
11714	11735	return self.fail("invalid modifier: '{s}'", .{modifier}),
11715		.memory => \|addr\| if (mem.eql(u8, modifier, "") or
11716		mem.eql(u8, modifier, "P"))
11717		.{ .mem = Memory.sib(
11718		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11719		.{ .base = .{ .reg = .ds }, .disp = @intCast(@as(i64, @bitCast(addr))) },
11720		) }
	11736	.memory => \|addr\| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P"))
	11737	.{ .mem = .{
	11738	.base = .{ .reg = .ds },
	11739	.mod = .{ .rm = .{
	11740	.size = mnem_size orelse
	11741	return self.fail("unknown size: '{s}'", .{op_str}),
	11742	.disp = @intCast(@as(i64, @bitCast(addr))),
	11743	} },
	11744	} }
11721	11745	else
11722	11746	return self.fail("invalid modifier: '{s}'", .{modifier}),
11723	11747	.indirect => \|reg_off\| if (mem.eql(u8, modifier, ""))
11724		.{ .mem = Memory.sib(
11725		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11726		.{ .base = .{ .reg = reg_off.reg }, .disp = reg_off.off },
11727		) }
	11748	.{ .mem = .{
	11749	.base = .{ .reg = reg_off.reg },
	11750	.mod = .{ .rm = .{
	11751	.size = mnem_size orelse
	11752	return self.fail("unknown size: '{s}'", .{op_str}),
	11753	.disp = reg_off.off,
	11754	} },
	11755	} }
11728	11756	else
11729	11757	return self.fail("invalid modifier: '{s}'", .{modifier}),
11730	11758	.load_frame => \|frame_addr\| if (mem.eql(u8, modifier, ""))
11731		.{ .mem = Memory.sib(
11732		mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
11733		.{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off },
11734		) }
	11759	.{ .mem = .{
	11760	.base = .{ .frame = frame_addr.index },
	11761	.mod = .{ .rm = .{
	11762	.size = mnem_size orelse
	11763	return self.fail("unknown size: '{s}'", .{op_str}),
	11764	.disp = frame_addr.off,
	11765	} },
	11766	} }
11735	11767	else
11736	11768	return self.fail("invalid modifier: '{s}'", .{modifier}),
11737	11769	.lea_got => \|sym_index\| if (mem.eql(u8, modifier, "P"))
11738	11770	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }
11739	11771	else
11740	11772	return self.fail("invalid modifier: '{s}'", .{modifier}),
11741		.load_symbol => \|sym_index\| if (mem.eql(u8, modifier, "P"))
11742		.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .load_symbol = sym_index }) }
	11773	.load_symbol => \|sym_off\| if (mem.eql(u8, modifier, "P"))
	11774	.{ .reg = try self.copyToTmpRegister(Type.usize, .{ .load_symbol = sym_off }) }
11743	11775	else
11744	11776	return self.fail("invalid modifier: '{s}'", .{modifier}),
11745	11777	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
...	...	@@ -11774,47 +11806,61 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
11774	11806	const scale_str = sib_it.next() orelse "";
11775	11807	if (index_str.len == 0 and scale_str.len > 0)
11776	11808	return self.fail("invalid memory operand: '{s}'", .{op_str});
11777		const scale = if (scale_str.len > 0) switch (std.fmt.parseInt(u4, scale_str, 10) catch
11778		return self.fail("invalid scale: '{s}'", .{op_str})) {
11779		1, 2, 4, 8 => \|scale\| scale,
11780		else => return self.fail("invalid scale: '{s}'", .{op_str}),
11781		} else 1;
	11809	const scale: Memory.Scale = if (scale_str.len > 0)
	11810	switch (std.fmt.parseInt(u4, scale_str, 10) catch
	11811	return self.fail("invalid scale: '{s}'", .{op_str})) {
	11812	1 => .@"1",
	11813	2 => .@"2",
	11814	4 => .@"4",
	11815	8 => .@"8",
	11816	else => return self.fail("invalid scale: '{s}'", .{op_str}),
	11817	}
	11818	else
	11819	.@"1";
11782	11820	if (sib_it.next()) \|_\| return self.fail("invalid memory operand: '{s}'", .{op_str});
11783		op.* = .{ .mem = Memory.sib(mnem_size orelse
11784		return self.fail("unknown size: '{s}'", .{op_str}), .{
11785		.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and
11786		mem.endsWith(u8, op_str[0..open], "]"))
11787		disp: {
11788		const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
11789		const modifier = if (colon) \|colon_pos\|
11790		op_str[colon_pos + ":".len .. open - "]".len]
	11821	op.* = .{
	11822	.mem = .{
	11823	.base = if (base_str.len > 0)
	11824	.{ .reg = parseRegName(base_str["%%".len..]) orelse
	11825	return self.fail("invalid base register: '{s}'", .{base_str}) }
11791	11826	else
11792		"";
11793		break :disp switch (args.items[
11794		arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
11795		return self.fail("no matching constraint: '{s}'", .{op_str})
11796		]) {
11797		.immediate => \|imm\| if (mem.eql(u8, modifier, "") or
11798		mem.eql(u8, modifier, "c"))
11799		math.cast(i32, @as(i64, @bitCast(imm))) orelse
	11827	.none,
	11828	.mod = .{ .rm = .{
	11829	.size = mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
	11830	.index = if (index_str.len > 0)
	11831	parseRegName(index_str["%%".len..]) orelse
	11832	return self.fail("invalid index register: '{s}'", .{op_str})
	11833	else
	11834	.none,
	11835	.scale = scale,
	11836	.disp = if (mem.startsWith(u8, op_str[0..open], "%[") and
	11837	mem.endsWith(u8, op_str[0..open], "]"))
	11838	disp: {
	11839	const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':');
	11840	const modifier = if (colon) \|colon_pos\|
	11841	op_str[colon_pos + ":".len .. open - "]".len]
	11842	else
	11843	"";
	11844	break :disp switch (args.items[
	11845	arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse
	11846	return self.fail("no matching constraint: '{s}'", .{op_str})
	11847	]) {
	11848	.immediate => \|imm\| if (mem.eql(u8, modifier, "") or
	11849	mem.eql(u8, modifier, "c"))
	11850	math.cast(i32, @as(i64, @bitCast(imm))) orelse
	11851	return self.fail("invalid displacement: '{s}'", .{op_str})
	11852	else
	11853	return self.fail("invalid modifier: '{s}'", .{modifier}),
	11854	else => return self.fail("invalid constraint: '{s}'", .{op_str}),
	11855	};
	11856	} else if (open > 0)
	11857	std.fmt.parseInt(i32, op_str[0..open], 0) catch
11800	11858	return self.fail("invalid displacement: '{s}'", .{op_str})
11801	11859	else
11802		return self.fail("invalid modifier: '{s}'", .{modifier}),
11803		else => return self.fail("invalid constraint: '{s}'", .{op_str}),
11804		};
11805		} else if (open > 0)
11806		std.fmt.parseInt(i32, op_str[0..open], 0) catch
11807		return self.fail("invalid displacement: '{s}'", .{op_str})
11808		else
11809		0,
11810		.base = if (base_str.len > 0) .{ .reg = parseRegName(base_str["%%".len..]) orelse
11811		return self.fail("invalid base register: '{s}'", .{base_str}) } else .none,
11812		.scale_index = if (index_str.len > 0) .{
11813		.index = parseRegName(index_str["%%".len..]) orelse
11814		return self.fail("invalid index register: '{s}'", .{op_str}),
11815		.scale = scale,
11816		} else null,
11817		}) };
	11860	0,
	11861	} },
	11862	},
	11863	};
11818	11864	} else if (Label.isValid(.reference, op_str)) {
11819	11865	const anon = std.ascii.isDigit(op_str[0]);
11820	11866	const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]);
...	...	@@ -12534,20 +12580,23 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12534	12580	),
12535	12581	.lea_frame => .{ .move = .{ ._, .lea } },
12536	12582	else => unreachable,
12537		}).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
12538		self.memPtrSize(ty),
12539		switch (src_mcv) {
12540		.register_offset, .indirect => \|reg_off\| .{
12541		.base = .{ .reg = reg_off.reg },
	12583	}).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) {
	12584	.register_offset, .indirect => \|reg_off\| .{
	12585	.base = .{ .reg = reg_off.reg },
	12586	.mod = .{ .rm = .{
	12587	.size = self.memSize(ty),
12542	12588	.disp = reg_off.off,
12543		},
12544		.load_frame, .lea_frame => \|frame_addr\| .{
12545		.base = .{ .frame = frame_addr.index },
	12589	} },
	12590	},
	12591	.load_frame, .lea_frame => \|frame_addr\| .{
	12592	.base = .{ .frame = frame_addr.index },
	12593	.mod = .{ .rm = .{
	12594	.size = self.memSize(ty),
12546	12595	.disp = frame_addr.off,
12547		},
12548		else => unreachable,
	12596	} },
12549	12597	},
12550		)),
	12598	else => unreachable,
	12599	}),
12551	12600	.memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
12552	12601	switch (src_mcv) {
12553	12602	.memory => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|small_addr\|
...	...	@@ -12555,20 +12604,40 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12555	12604	ty,
12556	12605	dst_reg.class(),
12557	12606	ty.abiAlignment(mod).check(@as(u32, @bitCast(small_addr))),
12558		)).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
12559		self.memPtrSize(ty),
12560		.{ .base = .{ .reg = .ds }, .disp = small_addr },
12561		)),
	12607	)).read(self, registerAlias(dst_reg, abi_size), .{
	12608	.base = .{ .reg = .ds },
	12609	.mod = .{ .rm = .{
	12610	.size = self.memSize(ty),
	12611	.disp = small_addr,
	12612	} },
	12613	}),
	12614	.load_symbol => \|sym_off\| switch (dst_reg.class()) {
	12615	.general_purpose => {
	12616	assert(sym_off.off == 0);
	12617	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
	12618	.base = .{ .reloc = .{
	12619	.atom_index = try self.owner.getSymbolIndex(self),
	12620	.sym_index = sym_off.sym,
	12621	} },
	12622	.mod = .{ .rm = .{
	12623	.size = self.memSize(ty),
	12624	.disp = sym_off.off,
	12625	} },
	12626	});
	12627	return;
	12628	},
	12629	.segment, .mmx => unreachable,
	12630	.x87, .sse => {},
	12631	},
12562	12632	.load_direct => \|sym_index\| switch (dst_reg.class()) {
12563	12633	.general_purpose => {
12564		const atom_index = try self.owner.getSymbolIndex(self);
12565	12634	_ = try self.addInst(.{
12566	12635	.tag = .mov,
12567	12636	.ops = .direct_reloc,
12568	12637	.data = .{ .rx = .{
12569		.r1 = dst_reg.to64(),
12570		.payload = try self.addExtra(Mir.Reloc{
12571		.atom_index = atom_index,
	12638	.r1 = registerAlias(dst_reg, abi_size),
	12639	.payload = try self.addExtra(bits.Symbol{
	12640	.atom_index = try self.owner.getSymbolIndex(self),
12572	12641	.sym_index = sym_index,
12573	12642	}),
12574	12643	} },
...	...	@@ -12578,7 +12647,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12578	12647	.segment, .mmx => unreachable,
12579	12648	.x87, .sse => {},
12580	12649	},
12581		.load_symbol, .load_got, .load_tlv => {},
	12650	.load_got, .load_tlv => {},
12582	12651	else => unreachable,
12583	12652	}
12584	12653
...	...	@@ -12589,22 +12658,42 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12589	12658	try (try self.moveStrategy(ty, dst_reg.class(), false)).read(
12590	12659	self,
12591	12660	registerAlias(dst_reg, abi_size),
12592		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
	12661	.{
	12662	.base = .{ .reg = addr_reg },
	12663	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } },
	12664	},
12593	12665	);
12594	12666	},
12595		.lea_symbol, .lea_direct, .lea_got => \|sym_index\| {
	12667	.lea_symbol => \|sym_index\| {
12596	12668	const atom_index = try self.owner.getSymbolIndex(self);
12597	12669	if (self.bin_file.cast(link.File.Elf)) \|_\| {
12598		_ = try self.addInst(.{
12599		.tag = .lea,
12600		.ops = .linker_reloc,
12601		.data = .{ .rx = .{
12602		.r1 = dst_reg.to64(),
12603		.payload = try self.addExtra(Mir.Reloc{
	12670	try self.asmRegisterMemory(
	12671	.{ ._, .lea },
	12672	dst_reg.to64(),
	12673	.{
	12674	.base = .{ .reloc = .{
12604	12675	.atom_index = atom_index,
12605		.sym_index = sym_index,
12606		}),
	12676	.sym_index = sym_index.sym,
	12677	} },
	12678	.mod = .{ .rm = .{
	12679	.size = .qword,
	12680	.disp = sym_index.off,
	12681	} },
	12682	},
	12683	);
	12684	} else return self.fail("TODO emit symbol sequence on {s}", .{
	12685	@tagName(self.bin_file.tag),
	12686	});
	12687	},
	12688	.lea_direct, .lea_got => \|sym_index\| {
	12689	const atom_index = try self.owner.getSymbolIndex(self);
	12690	if (self.bin_file.cast(link.File.Elf)) \|_\| {
	12691	try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{
	12692	.base = .{ .reloc = .{
	12693	.atom_index = atom_index,
	12694	.sym_index = sym_index,
12607	12695	} },
	12696	.mod = .{ .rm = .{ .size = .qword } },
12608	12697	});
12609	12698	} else {
12610	12699	_ = try self.addInst(.{
...	...	@@ -12620,7 +12709,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12620	12709	},
12621	12710	.data = .{ .rx = .{
12622	12711	.r1 = dst_reg.to64(),
12623		.payload = try self.addExtra(Mir.Reloc{
	12712	.payload = try self.addExtra(bits.Symbol{
12624	12713	.atom_index = atom_index,
12625	12714	.sym_index = sym_index,
12626	12715	}),
...	...	@@ -12636,14 +12725,17 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12636	12725	.ops = .tlv_reloc,
12637	12726	.data = .{ .rx = .{
12638	12727	.r1 = .rdi,
12639		.payload = try self.addExtra(Mir.Reloc{
	12728	.payload = try self.addExtra(bits.Symbol{
12640	12729	.atom_index = atom_index,
12641	12730	.sym_index = sym_index,
12642	12731	}),
12643	12732	} },
12644	12733	});
12645	12734	// TODO: spill registers before calling
12646		try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
	12735	try self.asmMemory(.{ ._, .call }, .{
	12736	.base = .{ .reg = .rdi },
	12737	.mod = .{ .rm = .{ .size = .qword } },
	12738	});
12647	12739	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
12648	12740	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
12649	12741	@tagName(self.bin_file.tag),
...	...	@@ -12660,6 +12752,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12660	12752	.none => .{ .immediate = @bitCast(@as(i64, disp)) },
12661	12753	.reg => \|base_reg\| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
12662	12754	.frame => \|base_frame_index\| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
	12755	.reloc => \|base_symbol\| .{ .lea_symbol = .{ .sym = base_symbol.sym_index, .off = disp } },
12663	12756	};
12664	12757	switch (src_mcv) {
12665	12758	.none, .unreach, .dead, .reserved_frame => unreachable,
...	...	@@ -12675,7 +12768,10 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12675	12768	};
12676	12769	try self.asmMemoryImmediate(
12677	12770	.{ ._, .mov },
12678		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	12771	.{ .base = base, .mod = .{ .rm = .{
	12772	.size = Memory.Size.fromSize(abi_size),
	12773	.disp = disp,
	12774	} } },
12679	12775	immediate,
12680	12776	);
12681	12777	},
...	...	@@ -12683,27 +12779,31 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12683	12779	else => if (math.cast(i32, @as(i64, @bitCast(imm)))) \|small\| {
12684	12780	try self.asmMemoryImmediate(
12685	12781	.{ ._, .mov },
12686		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
	12782	.{ .base = base, .mod = .{ .rm = .{
	12783	.size = Memory.Size.fromSize(abi_size),
	12784	.disp = disp,
	12785	} } },
12687	12786	Immediate.s(small),
12688	12787	);
12689	12788	} else {
12690	12789	var offset: i32 = 0;
12691	12790	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
12692	12791	.{ ._, .mov },
12693		Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
12694		if (ty.isSignedInt(mod))
12695		Immediate.s(
12696		@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
12697		)
12698		else
12699		Immediate.u(@as(
12700		u32,
12701		@truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0),
12702		)),
	12792	.{ .base = base, .mod = .{ .rm = .{
	12793	.size = .dword,
	12794	.disp = disp + offset,
	12795	} } },
	12796	if (ty.isSignedInt(mod)) Immediate.s(
	12797	@truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)),
	12798	) else Immediate.u(
	12799	@as(u32, @truncate(if (math.cast(u6, offset * 8)) \|shift\| imm >> shift else 0)),
	12800	),
12703	12801	);
12704	12802	},
12705	12803	},
12706		.eflags => \|cc\| try self.asmSetccMemory(cc, Memory.sib(.byte, .{ .base = base, .disp = disp })),
	12804	.eflags => \|cc\| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{
	12805	.rm = .{ .size = .byte, .disp = disp },
	12806	} }),
12707	12807	.register => \|src_reg\| try (try self.moveStrategy(ty, src_reg.class(), switch (base) {
12708	12808	.none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
12709	12809	.reg => \|reg\| switch (reg) {
...	...	@@ -12713,9 +12813,13 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12713	12813	.frame => \|frame_index\| self.getFrameAddrAlignment(
12714	12814	.{ .index = frame_index, .off = disp },
12715	12815	).compare(.gte, ty.abiAlignment(mod)),
	12816	.reloc => false,
12716	12817	})).write(
12717	12818	self,
12718		Memory.sib(self.memPtrSize(ty), .{ .base = base, .disp = disp }),
	12819	.{ .base = base, .mod = .{ .rm = .{
	12820	.size = self.memSize(ty),
	12821	.disp = disp,
	12822	} } },
12719	12823	registerAlias(src_reg, abi_size),
12720	12824	),
12721	12825	.register_pair => \|src_regs\| for (src_regs, 0..) \|src_reg, src_reg_i\| {
...	...	@@ -12732,11 +12836,12 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12732	12836	.frame => \|frame_index\| self.getFrameAddrAlignment(
12733	12837	.{ .index = frame_index, .off = disp },
12734	12838	).compare(.gte, ty.abiAlignment(mod)),
	12839	.reloc => false,
12735	12840	},
12736		)).write(self, Memory.sib(
12737		Memory.PtrSize.fromSize(part_size),
12738		.{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) },
12739		), registerAlias(src_reg, part_size));
	12841	)).write(self, .{ .base = base, .mod = .{ .rm = .{
	12842	.size = Memory.Size.fromSize(part_size),
	12843	.disp = disp + @as(i32, @intCast(src_reg_i * 8)),
	12844	} } }, registerAlias(src_reg, part_size));
12740	12845	},
12741	12846	.register_overflow => \|ro\| switch (ty.zigTypeTag(mod)) {
12742	12847	.Struct => {
...	...	@@ -12834,7 +12939,7 @@ fn genExternSymbolRef(
12834	12939	.ops = .import_reloc,
12835	12940	.data = .{ .rx = .{
12836	12941	.r1 = .rax,
12837		.payload = try self.addExtra(Mir.Reloc{
	12942	.payload = try self.addExtra(bits.Symbol{
12838	12943	.atom_index = atom_index,
12839	12944	.sym_index = try coff_file.getGlobalSymbol(callee, lib),
12840	12945	}),
...	...	@@ -12871,8 +12976,12 @@ fn genLazySymbolRef(
12871	12976
12872	12977	if (self.bin_file.options.pic) {
12873	12978	switch (tag) {
12874		.lea, .call => try self.genSetReg(reg, Type.usize, .{ .load_symbol = sym.esym_index }),
12875		.mov => try self.genSetReg(reg, Type.usize, .{ .load_symbol = sym.esym_index }),
	12979	.lea, .call => try self.genSetReg(reg, Type.usize, .{
	12980	.load_symbol = .{ .sym = sym.esym_index },
	12981	}),
	12982	.mov => try self.genSetReg(reg, Type.usize, .{
	12983	.load_symbol = .{ .sym = sym.esym_index },
	12984	}),
12876	12985	else => unreachable,
12877	12986	}
12878	12987	switch (tag) {
...	...	@@ -12881,23 +12990,18 @@ fn genLazySymbolRef(
12881	12990	else => unreachable,
12882	12991	}
12883	12992	} else {
12884		const reloc = Mir.Reloc{
	12993	const reloc = bits.Symbol{
12885	12994	.atom_index = try self.owner.getSymbolIndex(self),
12886	12995	.sym_index = sym.esym_index,
12887	12996	};
12888	12997	switch (tag) {
12889		.lea, .mov => _ = try self.addInst(.{
12890		.tag = .mov,
12891		.ops = .linker_reloc,
12892		.data = .{ .rx = .{
12893		.r1 = reg.to64(),
12894		.payload = try self.addExtra(reloc),
12895		} },
	12998	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), .{
	12999	.base = .{ .reloc = reloc },
	13000	.mod = .{ .rm = .{ .size = .qword } },
12896	13001	}),
12897		.call => _ = try self.addInst(.{
12898		.tag = .call,
12899		.ops = .linker_reloc,
12900		.data = .{ .reloc = reloc },
	13002	.call => try self.asmMemory(.{ ._, .call }, .{
	13003	.base = .{ .reloc = reloc },
	13004	.mod = .{ .rm = .{ .size = .qword } },
12901	13005	}),
12902	13006	else => unreachable,
12903	13007	}
...	...	@@ -12908,8 +13012,13 @@ fn genLazySymbolRef(
12908	13012	var atom = p9_file.getAtom(atom_index);
12909	13013	_ = atom.getOrCreateOffsetTableEntry(p9_file);
12910	13014	const got_addr = atom.getOffsetTableAddress(p9_file);
12911		const got_mem =
12912		Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(got_addr) });
	13015	const got_mem: Memory = .{
	13016	.base = .{ .reg = .ds },
	13017	.mod = .{ .rm = .{
	13018	.size = .qword,
	13019	.disp = @intCast(got_addr),
	13020	} },
	13021	};
12913	13022	switch (tag) {
12914	13023	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
12915	13024	.call => try self.asmMemory(.{ ._, .call }, got_mem),
...	...	@@ -13255,19 +13364,20 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
13255	13364	defer if (new_lock) \|lock\| self.register_manager.unlockReg(lock);
13256	13365
13257	13366	const ptr_mcv = try self.resolveInst(extra.ptr);
13258		const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
13259		const ptr_mem = switch (ptr_mcv) {
13260		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
13261		else => Memory.sib(ptr_size, .{
	13367	const mem_size = Memory.Size.fromSize(val_abi_size);
	13368	const ptr_mem: Memory = switch (ptr_mcv) {
	13369	.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	13370	else => .{
13262	13371	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13263		}),
	13372	.mod = .{ .rm = .{ .size = mem_size } },
	13373	},
13264	13374	};
13265		switch (ptr_mem) {
13266		.sib, .rip => {},
13267		.moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
	13375	switch (ptr_mem.mod) {
	13376	.rm => {},
	13377	.off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
13268	13378	}
13269		const ptr_lock = switch (ptr_mem.base()) {
13270		.none, .frame => null,
	13379	const ptr_lock = switch (ptr_mem.base) {
	13380	.none, .frame, .reloc => null,
13271	13381	.reg => \|reg\| self.register_manager.lockReg(reg),
13272	13382	};
13273	13383	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -13320,19 +13430,20 @@ fn atomicOp(
13320	13430	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);
13321	13431
13322	13432	const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
13323		const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
13324		const ptr_mem = switch (ptr_mcv) {
13325		.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
13326		else => Memory.sib(ptr_size, .{
	13433	const mem_size = Memory.Size.fromSize(val_abi_size);
	13434	const ptr_mem: Memory = switch (ptr_mcv) {
	13435	.immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),
	13436	else => .{
13327	13437	.base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13328		}),
	13438	.mod = .{ .rm = .{ .size = mem_size } },
	13439	},
13329	13440	};
13330		switch (ptr_mem) {
13331		.sib, .rip => {},
13332		.moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
	13441	switch (ptr_mem.mod) {
	13442	.rm => {},
	13443	.off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
13333	13444	}
13334		const mem_lock = switch (ptr_mem.base()) {
13335		.none, .frame => null,
	13445	const mem_lock = switch (ptr_mem.base) {
	13446	.none, .frame, .reloc => null,
13336	13447	.reg => \|reg\| self.register_manager.lockReg(reg),
13337	13448	};
13338	13449	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -13440,7 +13551,7 @@ fn atomicOp(
13440	13551	.memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
13441	13552	cc,
13442	13553	registerAlias(tmp_reg, cmov_abi_size),
13443		val_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
	13554	val_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
13444	13555	),
13445	13556	else => {
13446	13557	const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
...	...	@@ -13461,16 +13572,24 @@ fn atomicOp(
13461	13572	_ = try self.asmJccReloc(.ne, loop);
13462	13573	return if (unused) .unreach else .{ .register = .rax };
13463	13574	} else {
13464		try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
13465		.base = ptr_mem.sib.base,
13466		.scale_index = ptr_mem.scaleIndex(),
13467		.disp = ptr_mem.sib.disp + 0,
13468		}));
13469		try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
13470		.base = ptr_mem.sib.base,
13471		.scale_index = ptr_mem.scaleIndex(),
13472		.disp = ptr_mem.sib.disp + 8,
13473		}));
	13575	try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{
	13576	.base = ptr_mem.base,
	13577	.mod = .{ .rm = .{
	13578	.size = .qword,
	13579	.index = ptr_mem.mod.rm.index,
	13580	.scale = ptr_mem.mod.rm.scale,
	13581	.disp = ptr_mem.mod.rm.disp + 0,
	13582	} },
	13583	});
	13584	try self.asmRegisterMemory(.{ ._, .mov }, .rdx, .{
	13585	.base = ptr_mem.base,
	13586	.mod = .{ .rm = .{
	13587	.size = .qword,
	13588	.index = ptr_mem.mod.rm.index,
	13589	.scale = ptr_mem.mod.rm.scale,
	13590	.disp = ptr_mem.mod.rm.disp + 8,
	13591	} },
	13592	});
13474	13593	const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
13475	13594	const val_mem_mcv: MCValue = switch (val_mcv) {
13476	13595	.memory, .indirect, .load_frame => val_mcv,
...	...	@@ -13524,22 +13643,20 @@ fn atomicOp(
13524	13643
13525	13644	if (unused) return .unreach;
13526	13645	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
13527		try self.asmMemoryRegister(
13528		.{ ._, .mov },
13529		Memory.sib(.qword, .{
13530		.base = .{ .frame = dst_mcv.load_frame.index },
	13646	try self.asmMemoryRegister(.{ ._, .mov }, .{
	13647	.base = .{ .frame = dst_mcv.load_frame.index },
	13648	.mod = .{ .rm = .{
	13649	.size = .qword,
13531	13650	.disp = dst_mcv.load_frame.off + 0,
13532		}),
13533		.rax,
13534		);
13535		try self.asmMemoryRegister(
13536		.{ ._, .mov },
13537		Memory.sib(.qword, .{
13538		.base = .{ .frame = dst_mcv.load_frame.index },
	13651	} },
	13652	}, .rax);
	13653	try self.asmMemoryRegister(.{ ._, .mov }, .{
	13654	.base = .{ .frame = dst_mcv.load_frame.index },
	13655	.mod = .{ .rm = .{
	13656	.size = .qword,
13539	13657	.disp = dst_mcv.load_frame.off + 8,
13540		}),
13541		.rdx,
13542		);
	13658	} },
	13659	}, .rdx);
13543	13660	return dst_mcv;
13544	13661	},
13545	13662	.libcall => return self.fail("TODO implement x86 atomic libcall", .{}),
...	...	@@ -13845,55 +13962,74 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
13845	13962	try self.asmRegisterMemory(
13846	13963	.{ ._, .mov },
13847	13964	start_reg.to32(),
13848		Memory.sib(.dword, .{
	13965	.{
13849	13966	.base = .{ .reg = addr_reg.to64() },
13850		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
13851		.disp = 4,
13852		}),
	13967	.mod = .{ .rm = .{
	13968	.size = .dword,
	13969	.index = err_reg.to64(),
	13970	.scale = .@"4",
	13971	.disp = 4,
	13972	} },
	13973	},
13853	13974	);
13854	13975	try self.asmRegisterMemory(
13855	13976	.{ ._, .mov },
13856	13977	end_reg.to32(),
13857		Memory.sib(.dword, .{
	13978	.{
13858	13979	.base = .{ .reg = addr_reg.to64() },
13859		.scale_index = .{ .scale = 4, .index = err_reg.to64() },
13860		.disp = 8,
13861		}),
	13980	.mod = .{ .rm = .{
	13981	.size = .dword,
	13982	.index = err_reg.to64(),
	13983	.scale = .@"4",
	13984	.disp = 8,
	13985	} },
	13986	},
13862	13987	);
13863	13988	try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
13864	13989	try self.asmRegisterMemory(
13865	13990	.{ ._, .lea },
13866	13991	start_reg.to64(),
13867		Memory.sib(.byte, .{
	13992	.{
13868	13993	.base = .{ .reg = addr_reg.to64() },
13869		.scale_index = .{ .scale = 1, .index = start_reg.to64() },
13870		.disp = 0,
13871		}),
	13994	.mod = .{ .rm = .{
	13995	.size = .dword,
	13996	.index = start_reg.to64(),
	13997	} },
	13998	},
13872	13999	);
13873	14000	try self.asmRegisterMemory(
13874	14001	.{ ._, .lea },
13875	14002	end_reg.to32(),
13876		Memory.sib(.byte, .{
	14003	.{
13877	14004	.base = .{ .reg = end_reg.to64() },
13878		.disp = -1,
13879		}),
	14005	.mod = .{ .rm = .{
	14006	.size = .byte,
	14007	.disp = -1,
	14008	} },
	14009	},
13880	14010	);
13881	14011
13882	14012	const dst_mcv = try self.allocRegOrMem(inst, false);
13883	14013	try self.asmMemoryRegister(
13884	14014	.{ ._, .mov },
13885		Memory.sib(.qword, .{
	14015	.{
13886	14016	.base = .{ .frame = dst_mcv.load_frame.index },
13887		.disp = dst_mcv.load_frame.off,
13888		}),
	14017	.mod = .{ .rm = .{
	14018	.size = .qword,
	14019	.disp = dst_mcv.load_frame.off,
	14020	} },
	14021	},
13889	14022	start_reg.to64(),
13890	14023	);
13891	14024	try self.asmMemoryRegister(
13892	14025	.{ ._, .mov },
13893		Memory.sib(.qword, .{
	14026	.{
13894	14027	.base = .{ .frame = dst_mcv.load_frame.index },
13895		.disp = dst_mcv.load_frame.off + 8,
13896		}),
	14028	.mod = .{ .rm = .{
	14029	.size = .qword,
	14030	.disp = dst_mcv.load_frame.off + 8,
	14031	} },
	14032	},
13897	14033	end_reg.to64(),
13898	14034	);
13899	14035
...	...	@@ -14130,7 +14266,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14130	14266	.Or => {
14131	14267	if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
14132	14268	.{ ._, .@"test" },
14133		operand_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
	14269	operand_mcv.mem(Memory.Size.fromSize(abi_size)),
14134	14270	Immediate.u(mask),
14135	14271	) else {
14136	14272	const operand_reg = registerAlias(if (operand_mcv.isRegister())
...	...	@@ -14522,7 +14658,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
14522	14658	mir_tag,
14523	14659	mop1_reg,
14524	14660	mop2_reg,
14525		mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
	14661	mops[2].mem(Memory.Size.fromSize(abi_size)),
14526	14662	);
14527	14663	break :result mops[0];
14528	14664	};
...	...	@@ -14633,24 +14769,32 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
14633	14769	const mem_reloc = try self.asmJccReloc(.ae, undefined);
14634	14770
14635	14771	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
14636		if (!unused)
14637		try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, Memory.sib(.qword, .{
14638		.base = .{ .reg = addr_reg },
14639		.scale_index = .{ .scale = 1, .index = offset_reg.to64() },
14640		}));
14641		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, Memory.sib(.qword, .{
	14772	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
	14773	.base = .{ .reg = addr_reg },
	14774	.mod = .{ .rm = .{
	14775	.size = .qword,
	14776	.index = offset_reg.to64(),
	14777	} },
	14778	});
	14779	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
14642	14780	.base = .{ .reg = offset_reg.to64() },
14643		.disp = 8,
14644		}));
	14781	.mod = .{ .rm = .{
	14782	.size = .qword,
	14783	.disp = 8,
	14784	} },
	14785	});
14645	14786	try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg });
14646	14787	const done_reloc = try self.asmJmpReloc(undefined);
14647	14788
14648	14789	try self.performReloc(mem_reloc);
14649	14790	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
14650		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), Memory.sib(.qword, .{
	14791	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
14651	14792	.base = .{ .reg = addr_reg },
14652		.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
14653		}));
	14793	.mod = .{ .rm = .{
	14794	.size = .qword,
	14795	.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
	14796	} },
	14797	});
14654	14798	try self.genCopy(
14655	14799	ptr_anyopaque_ty,
14656	14800	overflow_arg_area,
...	...	@@ -14672,24 +14816,32 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
14672	14816	const mem_reloc = try self.asmJccReloc(.ae, undefined);
14673	14817
14674	14818	try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
14675		if (!unused)
14676		try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, Memory.sib(.qword, .{
14677		.base = .{ .reg = addr_reg },
14678		.scale_index = .{ .scale = 1, .index = offset_reg.to64() },
14679		}));
14680		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, Memory.sib(.qword, .{
	14819	if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
	14820	.base = .{ .reg = addr_reg },
	14821	.mod = .{ .rm = .{
	14822	.size = .qword,
	14823	.index = offset_reg.to64(),
	14824	} },
	14825	});
	14826	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{
14681	14827	.base = .{ .reg = offset_reg.to64() },
14682		.disp = 16,
14683		}));
	14828	.mod = .{ .rm = .{
	14829	.size = .qword,
	14830	.disp = 16,
	14831	} },
	14832	});
14684	14833	try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg });
14685	14834	const done_reloc = try self.asmJmpReloc(undefined);
14686	14835
14687	14836	try self.performReloc(mem_reloc);
14688	14837	try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
14689		try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), Memory.sib(.qword, .{
	14838	try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
14690	14839	.base = .{ .reg = addr_reg },
14691		.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
14692		}));
	14840	.mod = .{ .rm = .{
	14841	.size = .qword,
	14842	.disp = @intCast(@max(promote_ty.abiSize(mod), 8)),
	14843	} },
	14844	});
14693	14845	try self.genCopy(
14694	14846	ptr_anyopaque_ty,
14695	14847	overflow_arg_area,
...	...	@@ -14838,7 +14990,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
14838	14990	.undef => .undef,
14839	14991	.immediate => \|imm\| .{ .immediate = imm },
14840	14992	.memory => \|addr\| .{ .memory = addr },
14841		.load_symbol => \|sym_index\| .{ .load_symbol = sym_index },
	14993	.load_symbol => \|sym_index\| .{ .load_symbol = .{ .sym = sym_index } },
14842	14994	.load_direct => \|sym_index\| .{ .load_direct = sym_index },
14843	14995	.load_got => \|sym_index\| .{ .lea_got = sym_index },
14844	14996	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
...	...	@@ -15182,11 +15334,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
15182	15334	};
15183	15335	}
15184	15336
15185		fn memPtrSize(self: *Self, ty: Type) Memory.PtrSize {
	15337	fn memSize(self: *Self, ty: Type) Memory.Size {
15186	15338	const mod = self.bin_file.options.module.?;
15187	15339	return switch (ty.zigTypeTag(mod)) {
15188		.Float => Memory.PtrSize.fromBitSize(ty.floatBits(self.target.*)),
15189		else => Memory.PtrSize.fromSize(@intCast(ty.abiSize(mod))),
	15340	.Float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
	15341	else => Memory.Size.fromSize(@intCast(ty.abiSize(mod))),
15190	15342	};
15191	15343	}
15192	15344

src/arch/x86_64/Disassembler.zig+1-1

...	...	@@ -11,7 +11,7 @@ const Encoding = @import("Encoding.zig");
11	11	const Immediate = bits.Immediate;
12	12	const Instruction = encoder.Instruction;
13	13	const LegacyPrefixes = encoder.LegacyPrefixes;
14		const Memory = bits.Memory;
	14	const Memory = Instruction.Memory;
15	15	const Register = bits.Register;
16	16	const Rex = encoder.Rex;
17	17

src/arch/x86_64/Emit.zig+15-14

...	...	@@ -1,7 +1,6 @@
1	1	//! This file contains the functionality for emitting x86_64 MIR as machine code
2	2
3	3	lower: Lower,
4		bin_file: *link.File,
5	4	debug_output: DebugInfoOutput,
6	5	code: *std.ArrayList(u8),
7	6
...	...	@@ -41,7 +40,7 @@ pub fn emitMir(emit: *Emit) Error!void {
41	40	.offset = end_offset - 4,
42	41	.length = @intCast(end_offset - start_offset),
43	42	}),
44		.linker_extern_fn => \|symbol\| if (emit.bin_file.cast(link.File.Elf)) \|elf_file\| {
	43	.linker_extern_fn => \|symbol\| if (emit.lower.bin_file.cast(link.File.Elf)) \|elf_file\| {
45	44	// Add relocation to the decl.
46	45	const atom_ptr = elf_file.symbol(symbol.atom_index).atom(elf_file).?;
47	46	try atom_ptr.addReloc(elf_file, .{
...	...	@@ -49,9 +48,10 @@ pub fn emitMir(emit: *Emit) Error!void {
49	48	.r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) \| std.elf.R_X86_64_PLT32,
50	49	.r_addend = -4,
51	50	});
52		} else if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	51	} else if (emit.lower.bin_file.cast(link.File.MachO)) \|macho_file\| {
53	52	// Add relocation to the decl.
54		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
	53	const atom_index =
	54	macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
55	55	const target = macho_file.getGlobalByIndex(symbol.sym_index);
56	56	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
57	57	.type = .branch,
...	...	@@ -61,7 +61,7 @@ pub fn emitMir(emit: *Emit) Error!void {
61	61	.pcrel = true,
62	62	.length = 2,
63	63	});
64		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	64	} else if (emit.lower.bin_file.cast(link.File.Coff)) \|coff_file\| {
65	65	// Add relocation to the decl.
66	66	const atom_index = coff_file.getAtomIndexForSymbol(
67	67	.{ .sym_index = symbol.atom_index, .file = null },
...	...	@@ -76,12 +76,12 @@ pub fn emitMir(emit: *Emit) Error!void {
76	76	.length = 2,
77	77	});
78	78	} else return emit.fail("TODO implement extern reloc for {s}", .{
79		@tagName(emit.bin_file.tag),
	79	@tagName(emit.lower.bin_file.tag),
80	80	}),
81		.linker_reloc => \|data\| if (emit.bin_file.cast(link.File.Elf)) \|elf_file\| {
	81	.linker_reloc => \|data\| if (emit.lower.bin_file.cast(link.File.Elf)) \|elf_file\| {
82	82	const atom = elf_file.symbol(data.atom_index).atom(elf_file).?;
83	83	const sym = elf_file.symbol(elf_file.zigModulePtr().symbol(data.sym_index));
84		if (emit.bin_file.options.pic) {
	84	if (emit.lower.bin_file.options.pic) {
85	85	const r_type: u32 = if (sym.flags.has_zig_got)
86	86	link.File.Elf.R_X86_64_ZIG_GOTPCREL
87	87	else if (sym.flags.needs_got)
...	...	@@ -111,10 +111,11 @@ pub fn emitMir(emit: *Emit) Error!void {
111	111	.linker_direct,
112	112	.linker_import,
113	113	.linker_tlv,
114		=> \|symbol\| if (emit.bin_file.cast(link.File.Elf)) \|_\| {
	114	=> \|symbol\| if (emit.lower.bin_file.cast(link.File.Elf)) \|_\| {
115	115	unreachable;
116		} else if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
117		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
	116	} else if (emit.lower.bin_file.cast(link.File.MachO)) \|macho_file\| {
	117	const atom_index =
	118	macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?;
118	119	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
119	120	.type = switch (lowered_relocs[0].target) {
120	121	.linker_got => .got,
...	...	@@ -128,7 +129,7 @@ pub fn emitMir(emit: *Emit) Error!void {
128	129	.pcrel = true,
129	130	.length = 2,
130	131	});
131		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	132	} else if (emit.lower.bin_file.cast(link.File.Coff)) \|coff_file\| {
132	133	const atom_index = coff_file.getAtomIndexForSymbol(.{
133	134	.sym_index = symbol.atom_index,
134	135	.file = null,
...	...	@@ -152,7 +153,7 @@ pub fn emitMir(emit: *Emit) Error!void {
152	153	.pcrel = true,
153	154	.length = 2,
154	155	});
155		} else if (emit.bin_file.cast(link.File.Plan9)) \|p9_file\| {
	156	} else if (emit.lower.bin_file.cast(link.File.Plan9)) \|p9_file\| {
156	157	const atom_index = symbol.atom_index;
157	158	try p9_file.addReloc(atom_index, .{ // TODO we may need to add a .type field to the relocs if they are .linker_got instead of just .linker_direct
158	159	.target = symbol.sym_index, // we set sym_index to just be the atom index
...	...	@@ -161,7 +162,7 @@ pub fn emitMir(emit: *Emit) Error!void {
161	162	.type = .pcrel,
162	163	});
163	164	} else return emit.fail("TODO implement linker reloc for {s}", .{
164		@tagName(emit.bin_file.tag),
	165	@tagName(emit.lower.bin_file.tag),
165	166	}),
166	167	};
167	168	}

src/arch/x86_64/Encoding.zig+4-1

...	...	@@ -803,7 +803,10 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
803	803	@memcpy(inst.ops[0..ops.len], ops);
804	804
805	805	var cwriter = std.io.countingWriter(std.io.null_writer);
806		inst.encode(cwriter.writer(), .{ .allow_frame_loc = true }) catch unreachable; // Not allowed to fail here unless OOM.
	806	inst.encode(cwriter.writer(), .{
	807	.allow_frame_locs = true,
	808	.allow_symbols = true,
	809	}) catch unreachable; // Not allowed to fail here unless OOM.
807	810	return @as(usize, @intCast(cwriter.bytes_written));
808	811	}
809	812

src/arch/x86_64/Lower.zig+90-181

...	...	@@ -50,12 +50,12 @@ pub const Reloc = struct {
50	50
51	51	const Target = union(enum) {
52	52	inst: Mir.Inst.Index,
53		linker_reloc: Mir.Reloc,
54		linker_extern_fn: Mir.Reloc,
55		linker_got: Mir.Reloc,
56		linker_direct: Mir.Reloc,
57		linker_import: Mir.Reloc,
58		linker_tlv: Mir.Reloc,
	53	linker_reloc: bits.Symbol,
	54	linker_extern_fn: bits.Symbol,
	55	linker_got: bits.Symbol,
	56	linker_direct: bits.Symbol,
	57	linker_import: bits.Symbol,
	58	linker_tlv: bits.Symbol,
59	59	};
60	60	};
61	61
...	...	@@ -99,17 +99,15 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
99	99	.{ .reg = inst.data.rr.r2 },
100	100	});
101	101	},
102		.pseudo_cmov_nz_or_p_rm_sib,
103		.pseudo_cmov_nz_or_p_rm_rip,
104		=> {
	102	.pseudo_cmov_nz_or_p_rm => {
105	103	assert(inst.data.rx.fixes == ._);
106	104	try lower.emit(.none, .cmovnz, &.{
107	105	.{ .reg = inst.data.rx.r1 },
108		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	106	.{ .mem = lower.mem(inst.data.rx.payload) },
109	107	});
110	108	try lower.emit(.none, .cmovp, &.{
111	109	.{ .reg = inst.data.rx.r1 },
112		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	110	.{ .mem = lower.mem(inst.data.rx.payload) },
113	111	});
114	112	},
115	113	.pseudo_set_z_and_np_r => {
...	...	@@ -125,18 +123,16 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
125	123	.{ .reg = inst.data.rr.r2 },
126	124	});
127	125	},
128		.pseudo_set_z_and_np_m_sib,
129		.pseudo_set_z_and_np_m_rip,
130		=> {
	126	.pseudo_set_z_and_np_m => {
131	127	assert(inst.data.rx.fixes == ._);
132	128	try lower.emit(.none, .setz, &.{
133		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	129	.{ .mem = lower.mem(inst.data.rx.payload) },
134	130	});
135	131	try lower.emit(.none, .setnp, &.{
136	132	.{ .reg = inst.data.rx.r1 },
137	133	});
138	134	try lower.emit(.none, .@"and", &.{
139		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	135	.{ .mem = lower.mem(inst.data.rx.payload) },
140	136	.{ .reg = inst.data.rx.r1 },
141	137	});
142	138	},
...	...	@@ -153,18 +149,16 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
153	149	.{ .reg = inst.data.rr.r2 },
154	150	});
155	151	},
156		.pseudo_set_nz_or_p_m_sib,
157		.pseudo_set_nz_or_p_m_rip,
158		=> {
	152	.pseudo_set_nz_or_p_m => {
159	153	assert(inst.data.rx.fixes == ._);
160	154	try lower.emit(.none, .setnz, &.{
161		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	155	.{ .mem = lower.mem(inst.data.rx.payload) },
162	156	});
163	157	try lower.emit(.none, .setp, &.{
164	158	.{ .reg = inst.data.rx.r1 },
165	159	});
166	160	try lower.emit(.none, .@"or", &.{
167		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	161	.{ .mem = lower.mem(inst.data.rx.payload) },
168	162	.{ .reg = inst.data.rx.r1 },
169	163	});
170	164	},
...	...	@@ -289,28 +283,20 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
289	283	.rri_s,
290	284	.ri_s,
291	285	.i_s,
292		.mi_sib_s,
293		.mi_rip_s,
294		.rmi_sib_s,
295		.rmi_rip_s,
	286	.mi_s,
	287	.rmi_s,
296	288	=> Immediate.s(@bitCast(i)),
297	289
298	290	.rrri,
299	291	.rri_u,
300	292	.ri_u,
301	293	.i_u,
302		.mi_sib_u,
303		.mi_rip_u,
304		.rmi_sib,
305		.rmi_rip,
306		.rmi_sib_u,
307		.rmi_rip_u,
308		.mri_sib,
309		.mri_rip,
310		.rrm_sib,
311		.rrm_rip,
312		.rrmi_sib,
313		.rrmi_rip,
	294	.mi_u,
	295	.rmi,
	296	.rmi_u,
	297	.mri,
	298	.rrm,
	299	.rrmi,
314	300	=> Immediate.u(i),
315	301
316	302	.ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
...	...	@@ -319,50 +305,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
319	305	};
320	306	}
321	307
322		fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
323		return lower.mir.resolveFrameLoc(switch (ops) {
324		.rm_sib,
325		.rmi_sib,
326		.rmi_sib_s,
327		.rmi_sib_u,
328		.m_sib,
329		.mi_sib_u,
330		.mi_sib_s,
331		.mr_sib,
332		.mrr_sib,
333		.mri_sib,
334		.rrm_sib,
335		.rrmi_sib,
336
337		.pseudo_cmov_nz_or_p_rm_sib,
338		.pseudo_set_z_and_np_m_sib,
339		.pseudo_set_nz_or_p_m_sib,
340		=> lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
341
342		.rm_rip,
343		.rmi_rip,
344		.rmi_rip_s,
345		.rmi_rip_u,
346		.m_rip,
347		.mi_rip_u,
348		.mi_rip_s,
349		.mr_rip,
350		.mrr_rip,
351		.mri_rip,
352		.rrm_rip,
353		.rrmi_rip,
354
355		.pseudo_cmov_nz_or_p_rm_rip,
356		.pseudo_set_z_and_np_m_rip,
357		.pseudo_set_nz_or_p_m_rip,
358		=> lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
359
360		.rax_moffs,
361		.moffs_rax,
362		=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
363
364		else => unreachable,
365		});
	308	fn mem(lower: Lower, payload: u32) Memory {
	309	return lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode();
366	310	}
367	311
368	312	fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
...	...	@@ -375,7 +319,42 @@ fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
375	319	}
376	320
377	321	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
378		lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
	322	var emit_prefix = prefix;
	323	var emit_mnemonic = mnemonic;
	324	var emit_ops_storage: [4]Operand = undefined;
	325	const emit_ops = emit_ops_storage[0..ops.len];
	326	for (emit_ops, ops) \|*emit_op, op\| {
	327	emit_op.* = switch (op) {
	328	else => op,
	329	.mem => \|mem_op\| switch (mem_op.base()) {
	330	else => op,
	331	.reloc => \|sym\| op: {
	332	assert(prefix == .none);
	333	assert(mem_op.sib.disp == 0);
	334	assert(mem_op.sib.scale_index.scale == 0);
	335	_ = lower.reloc(.{ .linker_reloc = sym });
	336	break :op if (lower.bin_file.options.pic) switch (mnemonic) {
	337	.mov, .lea => .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) },
	338	else => unreachable,
	339	} else switch (mnemonic) {
	340	.call => .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{
	341	.base = .{ .reg = .ds },
	342	}) },
	343	.lea => {
	344	emit_mnemonic = .mov;
	345	break :op .{ .imm = Immediate.s(0) };
	346	},
	347	.mov => .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{
	348	.base = .{ .reg = .ds },
	349	}) },
	350	else => unreachable,
	351	};
	352	},
	353	},
	354	};
	355	}
	356	lower.result_insts[lower.result_insts_len] =
	357	try Instruction.new(emit_prefix, emit_mnemonic, emit_ops);
379	358	lower.result_insts_len += 1;
380	359	}
381	360
...	...	@@ -391,74 +370,13 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
391	370	.rrri => inst.data.rrri.fixes,
392	371	.rri_s, .rri_u => inst.data.rri.fixes,
393	372	.ri_s, .ri_u => inst.data.ri.fixes,
394		.ri64,
395		.rm_sib,
396		.rm_rip,
397		.rmi_sib_s,
398		.rmi_sib_u,
399		.rmi_rip_s,
400		.rmi_rip_u,
401		.mr_sib,
402		.mr_rip,
403		=> inst.data.rx.fixes,
404		.mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
405		.rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
406		.rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
407		.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
408		.m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
409		.extern_fn_reloc,
410		.got_reloc,
411		.direct_reloc,
412		.import_reloc,
413		.tlv_reloc,
414		=> ._,
415		.linker_reloc => {
416		if (lower.bin_file.options.pic) {
417		assert(inst.data.rx.fixes == ._);
418		const reg = inst.data.rx.r1;
419		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
420		_ = lower.reloc(.{ .linker_reloc = extra });
421		const mnemonic: Mnemonic = switch (inst.tag) {
422		.mov => .mov,
423		.lea => .lea,
424		else => unreachable,
425		};
426		try lower.emit(.none, mnemonic, &.{
427		.{ .reg = reg },
428		.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
429		});
430		} else {
431		switch (inst.tag) {
432		.call => {
433		_ = lower.reloc(.{ .linker_reloc = inst.data.reloc });
434		try lower.emit(.none, .call, &.{
435		.{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0 }) },
436		});
437		},
438		.lea => {
439		assert(inst.data.rx.fixes == ._);
440		const reg = inst.data.rx.r1;
441		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
442		try lower.emit(.none, .mov, &.{
443		.{ .reg = reg },
444		.{ .imm = lower.reloc(.{ .linker_reloc = extra }) },
445		});
446		},
447		.mov => {
448		assert(inst.data.rx.fixes == ._);
449		const reg = inst.data.rx.r1;
450		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
451		_ = lower.reloc(.{ .linker_reloc = extra });
452		try lower.emit(.none, .mov, &.{
453		.{ .reg = reg },
454		.{ .mem = Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0 }) },
455		});
456		},
457		else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
458		}
459		}
460		return;
461		},
	373	.ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,
	374	.mrr, .rrm => inst.data.rrx.fixes,
	375	.rmi, .mri => inst.data.rix.fixes,
	376	.rrmi => inst.data.rrix.fixes,
	377	.mi_u, .mi_s => inst.data.x.fixes,
	378	.m => inst.data.x.fixes,
	379	.extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
462	380	else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
463	381	};
464	382	try lower.emit(switch (fixes) {
...	...	@@ -527,73 +445,64 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
527	445	.{ .reg = inst.data.rri.r2 },
528	446	.{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
529	447	},
530		.m_sib, .m_rip => &.{
531		.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
	448	.m => &.{
	449	.{ .mem = lower.mem(inst.data.x.payload) },
532	450	},
533		.mi_sib_s, .mi_sib_u, .mi_rip_s, .mi_rip_u => &.{
534		.{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
	451	.mi_s, .mi_u => &.{
	452	.{ .mem = lower.mem(inst.data.x.payload + 1) },
535	453	.{ .imm = lower.imm(
536	454	inst.ops,
537	455	lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
538	456	) },
539	457	},
540		.rm_sib, .rm_rip => &.{
	458	.rm => &.{
541	459	.{ .reg = inst.data.rx.r1 },
542		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	460	.{ .mem = lower.mem(inst.data.rx.payload) },
543	461	},
544		.rmi_sib, .rmi_rip => &.{
	462	.rmi => &.{
545	463	.{ .reg = inst.data.rix.r1 },
546		.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
	464	.{ .mem = lower.mem(inst.data.rix.payload) },
547	465	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
548	466	},
549		.rmi_sib_s, .rmi_sib_u, .rmi_rip_s, .rmi_rip_u => &.{
	467	.rmi_s, .rmi_u => &.{
550	468	.{ .reg = inst.data.rx.r1 },
551		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload + 1) },
	469	.{ .mem = lower.mem(inst.data.rx.payload + 1) },
552	470	.{ .imm = lower.imm(
553	471	inst.ops,
554	472	lower.mir.extraData(Mir.Imm32, inst.data.rx.payload).data.imm,
555	473	) },
556	474	},
557		.mr_sib, .mr_rip => &.{
558		.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
	475	.mr => &.{
	476	.{ .mem = lower.mem(inst.data.rx.payload) },
559	477	.{ .reg = inst.data.rx.r1 },
560	478	},
561		.mrr_sib, .mrr_rip => &.{
562		.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
	479	.mrr => &.{
	480	.{ .mem = lower.mem(inst.data.rrx.payload) },
563	481	.{ .reg = inst.data.rrx.r1 },
564	482	.{ .reg = inst.data.rrx.r2 },
565	483	},
566		.mri_sib, .mri_rip => &.{
567		.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
	484	.mri => &.{
	485	.{ .mem = lower.mem(inst.data.rix.payload) },
568	486	.{ .reg = inst.data.rix.r1 },
569	487	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
570	488	},
571		.rrm_sib, .rrm_rip => &.{
	489	.rrm => &.{
572	490	.{ .reg = inst.data.rrx.r1 },
573	491	.{ .reg = inst.data.rrx.r2 },
574		.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
	492	.{ .mem = lower.mem(inst.data.rrx.payload) },
575	493	},
576		.rrmi_sib, .rrmi_rip => &.{
	494	.rrmi => &.{
577	495	.{ .reg = inst.data.rrix.r1 },
578	496	.{ .reg = inst.data.rrix.r2 },
579		.{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
	497	.{ .mem = lower.mem(inst.data.rrix.payload) },
580	498	.{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
581	499	},
582		.rax_moffs => &.{
583		.{ .reg = .rax },
584		.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
585		},
586		.moffs_rax => &.{
587		.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
588		.{ .reg = .rax },
589		},
590	500	.extern_fn_reloc => &.{
591	501	.{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
592	502	},
593		.linker_reloc => unreachable,
594	503	.got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
595	504	const reg = inst.data.rx.r1;
596		const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
	505	const extra = lower.mir.extraData(bits.Symbol, inst.data.rx.payload).data;
597	506	_ = lower.reloc(switch (inst.ops) {
598	507	.got_reloc => .{ .linker_got = extra },
599	508	.direct_reloc => .{ .linker_direct = extra },
...	...	@@ -635,7 +544,7 @@ const ErrorMsg = Module.ErrorMsg;
635	544	const Immediate = bits.Immediate;
636	545	const Instruction = encoder.Instruction;
637	546	const Lower = @This();
638		const Memory = bits.Memory;
	547	const Memory = Instruction.Memory;
639	548	const Mir = @import("Mir.zig");
640	549	const Mnemonic = Instruction.Mnemonic;
641	550	const Module = @import("../../Module.zig");

src/arch/x86_64/Mir.zig+136-203

...	...	@@ -17,7 +17,6 @@ const encoder = @import("encoder.zig");
17	17	const Air = @import("../../Air.zig");
18	18	const CodeGen = @import("CodeGen.zig");
19	19	const IntegerBitSet = std.bit_set.IntegerBitSet;
20		const Memory = bits.Memory;
21	20	const Register = bits.Register;
22	21
23	22	instructions: std.MultiArrayList(Inst).Slice,
...	...	@@ -767,84 +766,42 @@ pub const Inst = struct {
767	766	/// Relative displacement operand.
768	767	/// Uses `imm` payload.
769	768	rel,
770		/// Register, memory (SIB) operands.
	769	/// Register, memory operands.
771	770	/// Uses `rx` payload.
772		rm_sib,
773		/// Register, memory (RIP) operands.
774		/// Uses `rx` payload.
775		rm_rip,
776		/// Register, memory (SIB), immediate (word) operands.
777		/// Uses `rix` payload with extra data of type `MemorySib`.
778		rmi_sib,
779		/// Register, memory (RIP), immediate (word) operands.
780		/// Uses `rix` payload with extra data of type `MemoryRip`.
781		rmi_rip,
782		/// Register, memory (SIB), immediate (signed) operands.
783		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemorySib`.
784		rmi_sib_s,
785		/// Register, memory (SIB), immediate (unsigned) operands.
786		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemorySib`.
787		rmi_sib_u,
788		/// Register, memory (RIP), immediate (signed) operands.
789		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemoryRip`.
790		rmi_rip_s,
791		/// Register, memory (RIP), immediate (unsigned) operands.
792		/// Uses `rx` payload with extra data of type `Imm32` followed by `MemoryRip`.
793		rmi_rip_u,
794		/// Register, register, memory (RIP).
795		/// Uses `rrix` payload with extra data of type `MemoryRip`.
796		rrm_rip,
797		/// Register, register, memory (SIB).
798		/// Uses `rrix` payload with extra data of type `MemorySib`.
799		rrm_sib,
800		/// Register, register, memory (RIP), immediate (byte) operands.
801		/// Uses `rrix` payload with extra data of type `MemoryRip`.
802		rrmi_rip,
803		/// Register, register, memory (SIB), immediate (byte) operands.
804		/// Uses `rrix` payload with extra data of type `MemorySib`.
805		rrmi_sib,
806		/// Single memory (SIB) operand.
807		/// Uses `x` with extra data of type `MemorySib`.
808		m_sib,
809		/// Single memory (RIP) operand.
810		/// Uses `x` with extra data of type `MemoryRip`.
811		m_rip,
812		/// Memory (SIB), immediate (sign-extend) operands.
813		/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
814		mi_sib_s,
815		/// Memory (SIB), immediate (unsigned) operands.
816		/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
817		mi_sib_u,
818		/// Memory (RIP), immediate (sign-extend) operands.
819		/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
820		mi_rip_s,
821		/// Memory (RIP), immediate (unsigned) operands.
822		/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
823		mi_rip_u,
824		/// Memory (SIB), register operands.
825		/// Uses `rx` payload with extra data of type `MemorySib`.
826		mr_sib,
827		/// Memory (RIP), register operands.
828		/// Uses `rx` payload with extra data of type `MemoryRip`.
829		mr_rip,
830		/// Memory (SIB), register, register operands.
831		/// Uses `rrx` payload with extra data of type `MemorySib`.
832		mrr_sib,
833		/// Memory (RIP), register, register operands.
834		/// Uses `rrx` payload with extra data of type `MemoryRip`.
835		mrr_rip,
836		/// Memory (SIB), register, immediate (word) operands.
837		/// Uses `rix` payload with extra data of type `MemorySib`.
838		mri_sib,
839		/// Memory (RIP), register, immediate (word) operands.
840		/// Uses `rix` payload with extra data of type `MemoryRip`.
841		mri_rip,
842		/// Rax, Memory moffs.
843		/// Uses `x` with extra data of type `MemoryMoffs`.
844		rax_moffs,
845		/// Memory moffs, rax.
846		/// Uses `x` with extra data of type `MemoryMoffs`.
847		moffs_rax,
	771	rm,
	772	/// Register, memory, immediate (word) operands.
	773	/// Uses `rix` payload with extra data of type `Memory`.
	774	rmi,
	775	/// Register, memory, immediate (signed) operands.
	776	/// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
	777	rmi_s,
	778	/// Register, memory, immediate (unsigned) operands.
	779	/// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
	780	rmi_u,
	781	/// Register, register, memory.
	782	/// Uses `rrix` payload with extra data of type `Memory`.
	783	rrm,
	784	/// Register, register, memory, immediate (byte) operands.
	785	/// Uses `rrix` payload with extra data of type `Memory`.
	786	rrmi,
	787	/// Single memory operand.
	788	/// Uses `x` with extra data of type `Memory`.
	789	m,
	790	/// Memory, immediate (sign-extend) operands.
	791	/// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
	792	mi_s,
	793	/// Memory, immediate (unsigned) operands.
	794	/// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
	795	mi_u,
	796	/// Memory, register operands.
	797	/// Uses `rx` payload with extra data of type `Memory`.
	798	mr,
	799	/// Memory, register, register operands.
	800	/// Uses `rrx` payload with extra data of type `Memory`.
	801	mrr,
	802	/// Memory, register, immediate (word) operands.
	803	/// Uses `rix` payload with extra data of type `Memory`.
	804	mri,
848	805	/// References another Mir instruction directly.
849	806	/// Uses `inst` payload.
850	807	inst,
...	...	@@ -852,20 +809,17 @@ pub const Inst = struct {
852	809	/// Uses `reloc` payload.
853	810	extern_fn_reloc,
854	811	/// Linker relocation - GOT indirection.
855		/// Uses `rx` payload with extra data of type `Reloc`.
	812	/// Uses `rx` payload with extra data of type `bits.Symbol`.
856	813	got_reloc,
857	814	/// Linker relocation - direct reference.
858		/// Uses `rx` payload with extra data of type `Reloc`.
	815	/// Uses `rx` payload with extra data of type `bits.Symbol`.
859	816	direct_reloc,
860	817	/// Linker relocation - imports table indirection (binding).
861		/// Uses `rx` payload with extra data of type `Reloc`.
	818	/// Uses `rx` payload with extra data of type `bits.Symbol`.
862	819	import_reloc,
863	820	/// Linker relocation - threadlocal variable via GOT indirection.
864		/// Uses `rx` payload with extra data of type `Reloc`.
	821	/// Uses `rx` payload with extra data of type `bits.Symbol`.
865	822	tlv_reloc,
866		/// Linker relocation.
867		/// Uses `rx` payload with extra data of type `Reloc`.
868		linker_reloc,
869	823
870	824	// Pseudo instructions:
871	825
...	...	@@ -878,10 +832,7 @@ pub const Inst = struct {
878	832	pseudo_cmov_nz_or_p_rr,
879	833	/// Conditional move if zero flag not set or parity flag set
880	834	/// Uses `rx` payload.
881		pseudo_cmov_nz_or_p_rm_sib,
882		/// Conditional move if zero flag not set or parity flag set
883		/// Uses `rx` payload.
884		pseudo_cmov_nz_or_p_rm_rip,
	835	pseudo_cmov_nz_or_p_rm,
885	836	/// Set byte if zero flag set and parity flag not set
886	837	/// Requires a scratch register!
887	838	/// Uses `rr` payload.
...	...	@@ -889,11 +840,7 @@ pub const Inst = struct {
889	840	/// Set byte if zero flag set and parity flag not set
890	841	/// Requires a scratch register!
891	842	/// Uses `rx` payload.
892		pseudo_set_z_and_np_m_sib,
893		/// Set byte if zero flag set and parity flag not set
894		/// Requires a scratch register!
895		/// Uses `rx` payload.
896		pseudo_set_z_and_np_m_rip,
	843	pseudo_set_z_and_np_m,
897	844	/// Set byte if zero flag not set or parity flag set
898	845	/// Requires a scratch register!
899	846	/// Uses `rr` payload.
...	...	@@ -901,11 +848,7 @@ pub const Inst = struct {
901	848	/// Set byte if zero flag not set or parity flag set
902	849	/// Requires a scratch register!
903	850	/// Uses `rx` payload.
904		pseudo_set_nz_or_p_m_sib,
905		/// Set byte if zero flag not set or parity flag set
906		/// Requires a scratch register!
907		/// Uses `rx` payload.
908		pseudo_set_nz_or_p_m_rip,
	851	pseudo_set_nz_or_p_m,
909	852	/// Jump if zero flag set and parity flag not set
910	853	/// Uses `inst` payload.
911	854	pseudo_j_z_and_np_inst,
...	...	@@ -1036,7 +979,7 @@ pub const Inst = struct {
1036	979	/// Relocation for the linker where:
1037	980	/// * `atom_index` is the index of the source
1038	981	/// * `sym_index` is the index of the target
1039		reloc: Reloc,
	982	reloc: bits.Symbol,
1040	983	/// Debug line and column position
1041	984	line_column: struct {
1042	985	line: u32,
...	...	@@ -1055,14 +998,6 @@ pub const Inst = struct {
1055	998	}
1056	999	};
1057	1000
1058		/// A linker symbol not yet allocated in VM.
1059		pub const Reloc = struct {
1060		/// Index of the containing atom.
1061		atom_index: u32,
1062		/// Index into the linker's symbol table.
1063		sym_index: u32,
1064		};
1065
1066	1001	/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
1067	1002	pub const RegisterList = struct {
1068	1003	bitset: BitSet = BitSet.initEmpty(),
...	...	@@ -1123,100 +1058,94 @@ pub const Imm64 = struct {
1123	1058	}
1124	1059	};
1125	1060
1126		// TODO this can be further compacted using packed struct
1127		pub const MemorySib = struct {
1128		/// Size of the pointer.
1129		ptr_size: u32,
1130		/// Base register tag of type Memory.Base.Tag
1131		base_tag: u32,
1132		/// Base register of type Register or FrameIndex
	1061	pub const Memory = struct {
	1062	info: Info,
1133	1063	base: u32,
1134		/// Scale starting at bit 0 and index register starting at bit 4.
1135		scale_index: u32,
1136		/// Displacement value.
1137		disp: i32,
	1064	off: u32,
	1065	extra: u32,
	1066
	1067	pub const Info = packed struct(u32) {
	1068	base: @typeInfo(bits.Memory.Base).Union.tag_type.?,
	1069	mod: @typeInfo(bits.Memory.Mod).Union.tag_type.?,
	1070	size: bits.Memory.Size,
	1071	index: Register,
	1072	scale: bits.Memory.Scale,
	1073	_: u16 = undefined,
	1074	};
1138	1075
1139		pub fn encode(mem: Memory) MemorySib {
1140		const sib = mem.sib;
1141		assert(sib.scale_index.scale == 0 or std.math.isPowerOfTwo(sib.scale_index.scale));
	1076	pub fn encode(mem: bits.Memory) Memory {
	1077	assert(mem.base != .reloc or mem.mod != .off);
1142	1078	return .{
1143		.ptr_size = @intFromEnum(sib.ptr_size),
1144		.base_tag = @intFromEnum(sib.base),
1145		.base = switch (sib.base) {
	1079	.info = .{
	1080	.base = mem.base,
	1081	.mod = mem.mod,
	1082	.size = switch (mem.mod) {
	1083	.rm => \|rm\| rm.size,
	1084	.off => undefined,
	1085	},
	1086	.index = switch (mem.mod) {
	1087	.rm => \|rm\| rm.index,
	1088	.off => undefined,
	1089	},
	1090	.scale = switch (mem.mod) {
	1091	.rm => \|rm\| rm.scale,
	1092	.off => undefined,
	1093	},
	1094	},
	1095	.base = switch (mem.base) {
1146	1096	.none => undefined,
1147		.reg => \|r\| @intFromEnum(r),
1148		.frame => \|fi\| @intFromEnum(fi),
	1097	.reg => \|reg\| @intFromEnum(reg),
	1098	.frame => \|frame_index\| @intFromEnum(frame_index),
	1099	.reloc => \|symbol\| symbol.sym_index,
	1100	},
	1101	.off = switch (mem.mod) {
	1102	.rm => \|rm\| @bitCast(rm.disp),
	1103	.off => \|off\| @truncate(off),
1149	1104	},
1150		.scale_index = @as(u32, sib.scale_index.scale) << 0 \|
1151		@as(u32, if (sib.scale_index.scale > 0)
1152		@intFromEnum(sib.scale_index.index)
	1105	.extra = if (mem.base == .reloc)
	1106	mem.base.reloc.atom_index
	1107	else if (mem.mod == .off)
	1108	@intCast(mem.mod.off >> 32)
1153	1109	else
1154		undefined) << 4,
1155		.disp = sib.disp,
	1110	undefined,
1156	1111	};
1157	1112	}
1158	1113
1159		pub fn decode(msib: MemorySib) Memory {
1160		const scale: u4 = @truncate(msib.scale_index);
1161		assert(scale == 0 or std.math.isPowerOfTwo(scale));
1162		return .{ .sib = .{
1163		.ptr_size = @enumFromInt(msib.ptr_size),
1164		.base = switch (@as(Memory.Base.Tag, @enumFromInt(msib.base_tag))) {
1165		.none => .none,
1166		.reg => .{ .reg = @enumFromInt(msib.base) },
1167		.frame => .{ .frame = @enumFromInt(msib.base) },
	1114	pub fn decode(mem: Memory) encoder.Instruction.Memory {
	1115	switch (mem.info.mod) {
	1116	.rm => {
	1117	if (mem.info.base == .reg and @as(Register, @enumFromInt(mem.base)) == .rip) {
	1118	assert(mem.info.index == .none and mem.info.scale == .@"1");
	1119	return encoder.Instruction.Memory.rip(mem.info.size, @bitCast(mem.off));
	1120	}
	1121	return encoder.Instruction.Memory.sib(mem.info.size, .{
	1122	.disp = @bitCast(mem.off),
	1123	.base = switch (mem.info.base) {
	1124	.none => .none,
	1125	.reg => .{ .reg = @enumFromInt(mem.base) },
	1126	.frame => .{ .frame = @enumFromInt(mem.base) },
	1127	.reloc => .{ .reloc = .{ .atom_index = mem.extra, .sym_index = mem.base } },
	1128	},
	1129	.scale_index = switch (mem.info.index) {
	1130	.none => null,
	1131	else => \|index\| .{ .scale = switch (mem.info.scale) {
	1132	inline else => \|scale\| comptime std.fmt.parseInt(
	1133	u4,
	1134	@tagName(scale),
	1135	10,
	1136	) catch unreachable,
	1137	}, .index = index },
	1138	},
	1139	});
1168	1140	},
1169		.scale_index = .{
1170		.scale = scale,
1171		.index = if (scale > 0) @enumFromInt(msib.scale_index >> 4) else undefined,
	1141	.off => {
	1142	assert(mem.info.base == .reg);
	1143	return encoder.Instruction.Memory.moffs(
	1144	@enumFromInt(mem.base),
	1145	@as(u64, mem.extra) << 32 \| mem.off,
	1146	);
1172	1147	},
1173		.disp = msib.disp,
1174		} };
1175		}
1176		};
1177
1178		pub const MemoryRip = struct {
1179		/// Size of the pointer.
1180		ptr_size: u32,
1181		/// Displacement value.
1182		disp: i32,
1183
1184		pub fn encode(mem: Memory) MemoryRip {
1185		return .{
1186		.ptr_size = @intFromEnum(mem.rip.ptr_size),
1187		.disp = mem.rip.disp,
1188		};
1189		}
1190
1191		pub fn decode(mrip: MemoryRip) Memory {
1192		return .{ .rip = .{
1193		.ptr_size = @enumFromInt(mrip.ptr_size),
1194		.disp = mrip.disp,
1195		} };
1196		}
1197		};
1198
1199		pub const MemoryMoffs = struct {
1200		/// Segment register.
1201		seg: u32,
1202		/// Absolute offset wrt to the segment register split between MSB and LSB parts much like
1203		/// `Imm64` payload.
1204		msb: u32,
1205		lsb: u32,
1206
1207		pub fn encode(seg: Register, offset: u64) MemoryMoffs {
1208		return .{
1209		.seg = @intFromEnum(seg),
1210		.msb = @truncate(offset >> 32),
1211		.lsb = @truncate(offset >> 0),
1212		};
1213		}
1214
1215		pub fn decode(moffs: MemoryMoffs) Memory {
1216		return .{ .moffs = .{
1217		.seg = @enumFromInt(moffs.seg),
1218		.offset = @as(u64, moffs.msb) << 32 \| @as(u64, moffs.lsb) << 0,
1219		} };
	1148	}
1220	1149	}
1221	1150	};
1222	1151
...	...	@@ -1234,8 +1163,8 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:
1234	1163	inline for (fields) \|field\| {
1235	1164	@field(result, field.name) = switch (field.type) {
1236	1165	u32 => mir.extra[i],
1237		i32 => @bitCast(mir.extra[i]),
1238		else => @compileError("bad field type"),
	1166	i32, Memory.Info => @bitCast(mir.extra[i]),
	1167	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
1239	1168	};
1240	1169	i += 1;
1241	1170	}
...	...	@@ -1251,15 +1180,19 @@ pub const FrameLoc = struct {
1251	1180	};
1252	1181
1253	1182	pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
1254		return switch (mem) {
1255		.sib => \|sib\| switch (sib.base) {
1256		.none, .reg => mem,
1257		.frame => \|index\| if (mir.frame_locs.len > 0) Memory.sib(sib.ptr_size, .{
1258		.base = .{ .reg = mir.frame_locs.items(.base)[@intFromEnum(index)] },
1259		.disp = mir.frame_locs.items(.disp)[@intFromEnum(index)] + sib.disp,
1260		.scale_index = mem.scaleIndex(),
1261		}) else mem,
1262		},
1263		.rip, .moffs => mem,
	1183	return switch (mem.info.base) {
	1184	.none, .reg, .reloc => mem,
	1185	.frame => if (mir.frame_locs.len > 0) Memory{
	1186	.info = .{
	1187	.base = .reg,
	1188	.mod = mem.info.mod,
	1189	.size = mem.info.size,
	1190	.index = mem.info.index,
	1191	.scale = mem.info.scale,
	1192	},
	1193	.base = @intFromEnum(mir.frame_locs.items(.base)[mem.base]),
	1194	.off = @bitCast(mir.frame_locs.items(.disp)[mem.base] + @as(i32, @bitCast(mem.off))),
	1195	.extra = mem.extra,
	1196	} else mem,
1264	1197	};
1265	1198	}

src/arch/x86_64/bits.zig+43-89

...	...	@@ -181,6 +181,8 @@ pub const Register = enum(u7) {
181	181
182	182	es, cs, ss, ds, fs, gs,
183	183
	184	rip, eip, ip,
	185
184	186	none,
185	187	// zig fmt: on
186	188
...	...	@@ -442,34 +444,58 @@ pub const FrameIndex = enum(u32) {
442	444	}
443	445	};
444	446
445		pub const Memory = union(enum) {
446		sib: Sib,
447		rip: Rip,
448		moffs: Moffs,
	447	/// A linker symbol not yet allocated in VM.
	448	pub const Symbol = struct {
	449	/// Index of the containing atom.
	450	atom_index: u32,
	451	/// Index into the linker's symbol table.
	452	sym_index: u32,
	453
	454	pub fn format(
	455	sym: Symbol,
	456	comptime fmt: []const u8,
	457	options: std.fmt.FormatOptions,
	458	writer: anytype,
	459	) @TypeOf(writer).Error!void {
	460	try writer.writeAll("Symbol(");
	461	try std.fmt.formatType(sym.atom_index, fmt, options, writer, 0);
	462	try writer.writeAll(", ");
	463	try std.fmt.formatType(sym.sym_index, fmt, options, writer, 0);
	464	try writer.writeByte(')');
	465	}
	466	};
	467
	468	pub const Memory = struct {
	469	base: Base,
	470	mod: Mod,
449	471
450		pub const Base = union(enum) {
	472	pub const Base = union(enum(u2)) {
451	473	none,
452	474	reg: Register,
453	475	frame: FrameIndex,
	476	reloc: Symbol,
454	477
455	478	pub const Tag = @typeInfo(Base).Union.tag_type.?;
456	479
457	480	pub fn isExtended(self: Base) bool {
458	481	return switch (self) {
459		.none, .frame => false, // neither rsp nor rbp are extended
	482	.none, .frame, .reloc => false, // rsp, rbp, and rip are not extended
460	483	.reg => \|reg\| reg.isExtended(),
461	484	};
462	485	}
463	486	};
464	487
465		pub const ScaleIndex = struct {
466		scale: u4,
467		index: Register,
468
469		const none = ScaleIndex{ .scale = 0, .index = undefined };
	488	pub const Mod = union(enum(u1)) {
	489	rm: struct {
	490	size: Size,
	491	index: Register = .none,
	492	scale: Scale = .@"1",
	493	disp: i32 = 0,
	494	},
	495	off: u64,
470	496	};
471	497
472		pub const PtrSize = enum {
	498	pub const Size = enum(u4) {
473	499	none,
474	500	byte,
475	501	word,
...	...	@@ -480,7 +506,7 @@ pub const Memory = union(enum) {
480	506	yword,
481	507	zword,
482	508
483		pub fn fromSize(size: u32) PtrSize {
	509	pub fn fromSize(size: u32) Size {
484	510	return switch (size) {
485	511	1...1 => .byte,
486	512	2...2 => .word,
...	...	@@ -493,7 +519,7 @@ pub const Memory = union(enum) {
493	519	};
494	520	}
495	521
496		pub fn fromBitSize(bit_size: u64) PtrSize {
	522	pub fn fromBitSize(bit_size: u64) Size {
497	523	return switch (bit_size) {
498	524	8 => .byte,
499	525	16 => .word,
...	...	@@ -507,7 +533,7 @@ pub const Memory = union(enum) {
507	533	};
508	534	}
509	535
510		pub fn bitSize(s: PtrSize) u64 {
	536	pub fn bitSize(s: Size) u64 {
511	537	return switch (s) {
512	538	.none => 0,
513	539	.byte => 8,
...	...	@@ -522,7 +548,7 @@ pub const Memory = union(enum) {
522	548	}
523	549
524	550	pub fn format(
525		s: PtrSize,
	551	s: Size,
526	552	comptime _: []const u8,
527	553	_: std.fmt.FormatOptions,
528	554	writer: anytype,
...	...	@@ -533,79 +559,7 @@ pub const Memory = union(enum) {
533	559	}
534	560	};
535	561
536		pub const Sib = struct {
537		ptr_size: PtrSize,
538		base: Base,
539		scale_index: ScaleIndex,
540		disp: i32,
541		};
542
543		pub const Rip = struct {
544		ptr_size: PtrSize,
545		disp: i32,
546		};
547
548		pub const Moffs = struct {
549		seg: Register,
550		offset: u64,
551		};
552
553		pub fn moffs(reg: Register, offset: u64) Memory {
554		assert(reg.class() == .segment);
555		return .{ .moffs = .{ .seg = reg, .offset = offset } };
556		}
557
558		pub fn sib(ptr_size: PtrSize, args: struct {
559		disp: i32 = 0,
560		base: Base = .none,
561		scale_index: ?ScaleIndex = null,
562		}) Memory {
563		if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
564		return .{ .sib = .{
565		.base = args.base,
566		.disp = args.disp,
567		.ptr_size = ptr_size,
568		.scale_index = if (args.scale_index) \|si\| si else ScaleIndex.none,
569		} };
570		}
571
572		pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
573		return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
574		}
575
576		pub fn isSegmentRegister(mem: Memory) bool {
577		return switch (mem) {
578		.moffs => true,
579		.rip => false,
580		.sib => \|s\| switch (s.base) {
581		.none, .frame => false,
582		.reg => \|reg\| reg.class() == .segment,
583		},
584		};
585		}
586
587		pub fn base(mem: Memory) Base {
588		return switch (mem) {
589		.moffs => \|m\| .{ .reg = m.seg },
590		.sib => \|s\| s.base,
591		.rip => .none,
592		};
593		}
594
595		pub fn scaleIndex(mem: Memory) ?ScaleIndex {
596		return switch (mem) {
597		.moffs, .rip => null,
598		.sib => \|s\| if (s.scale_index.scale > 0) s.scale_index else null,
599		};
600		}
601
602		pub fn bitSize(mem: Memory) u64 {
603		return switch (mem) {
604		.rip => \|r\| r.ptr_size.bitSize(),
605		.sib => \|s\| s.ptr_size.bitSize(),
606		.moffs => 64,
607		};
608		}
	562	pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" };
609	563	};
610	564
611	565	pub const Immediate = union(enum) {

src/arch/x86_64/encoder.zig+181-85

...	...	@@ -6,9 +6,10 @@ const testing = std.testing;
6	6
7	7	const bits = @import("bits.zig");
8	8	const Encoding = @import("Encoding.zig");
	9	const FrameIndex = bits.FrameIndex;
9	10	const Immediate = bits.Immediate;
10		const Memory = bits.Memory;
11	11	const Register = bits.Register;
	12	const Symbol = bits.Symbol;
12	13
13	14	pub const Instruction = struct {
14	15	prefix: Prefix = .none,
...	...	@@ -27,6 +28,97 @@ pub const Instruction = struct {
27	28	repnz,
28	29	};
29	30
	31	pub const Memory = union(enum) {
	32	sib: Sib,
	33	rip: Rip,
	34	moffs: Moffs,
	35
	36	pub const Base = bits.Memory.Base;
	37
	38	pub const ScaleIndex = struct {
	39	scale: u4,
	40	index: Register,
	41
	42	const none = ScaleIndex{ .scale = 0, .index = undefined };
	43	};
	44
	45	pub const PtrSize = bits.Memory.Size;
	46
	47	pub const Sib = struct {
	48	ptr_size: PtrSize,
	49	base: Base,
	50	scale_index: ScaleIndex,
	51	disp: i32,
	52	};
	53
	54	pub const Rip = struct {
	55	ptr_size: PtrSize,
	56	disp: i32,
	57	};
	58
	59	pub const Moffs = struct {
	60	seg: Register,
	61	offset: u64,
	62	};
	63
	64	pub fn moffs(reg: Register, offset: u64) Memory {
	65	assert(reg.class() == .segment);
	66	return .{ .moffs = .{ .seg = reg, .offset = offset } };
	67	}
	68
	69	pub fn sib(ptr_size: PtrSize, args: struct {
	70	disp: i32 = 0,
	71	base: Base = .none,
	72	scale_index: ?ScaleIndex = null,
	73	}) Memory {
	74	if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
	75	return .{ .sib = .{
	76	.base = args.base,
	77	.disp = args.disp,
	78	.ptr_size = ptr_size,
	79	.scale_index = if (args.scale_index) \|si\| si else ScaleIndex.none,
	80	} };
	81	}
	82
	83	pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
	84	return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
	85	}
	86
	87	pub fn isSegmentRegister(mem: Memory) bool {
	88	return switch (mem) {
	89	.moffs => true,
	90	.rip => false,
	91	.sib => \|s\| switch (s.base) {
	92	.none, .frame, .reloc => false,
	93	.reg => \|reg\| reg.class() == .segment,
	94	},
	95	};
	96	}
	97
	98	pub fn base(mem: Memory) Base {
	99	return switch (mem) {
	100	.moffs => \|m\| .{ .reg = m.seg },
	101	.sib => \|s\| s.base,
	102	.rip => .none,
	103	};
	104	}
	105
	106	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
	107	return switch (mem) {
	108	.moffs, .rip => null,
	109	.sib => \|s\| if (s.scale_index.scale > 0) s.scale_index else null,
	110	};
	111	}
	112
	113	pub fn bitSize(mem: Memory) u64 {
	114	return switch (mem) {
	115	.rip => \|r\| r.ptr_size.bitSize(),
	116	.sib => \|s\| s.ptr_size.bitSize(),
	117	.moffs => 64,
	118	};
	119	}
	120	};
	121
30	122	pub const Operand = union(enum) {
31	123	none,
32	124	reg: Register,
...	...	@@ -125,8 +217,8 @@ pub const Instruction = struct {
125	217	try writer.print("{s}", .{@tagName(reg)});
126	218	any = true;
127	219	},
128		.frame => \|frame\| {
129		try writer.print("{}", .{frame});
	220	inline .frame, .reloc => \|payload\| {
	221	try writer.print("{}", .{payload});
130	222	any = true;
131	223	},
132	224	}
...	...	@@ -498,7 +590,11 @@ pub const Instruction = struct {
498	590	}
499	591	}
500	592	},
501		.frame => if (@TypeOf(encoder).options.allow_frame_loc) {
	593	.frame => if (@TypeOf(encoder).options.allow_frame_locs) {
	594	try encoder.modRm_indirectDisp32(operand_enc, undefined);
	595	try encoder.disp32(undefined);
	596	} else return error.CannotEncode,
	597	.reloc => if (@TypeOf(encoder).options.allow_symbols) {
502	598	try encoder.modRm_indirectDisp32(operand_enc, undefined);
503	599	try encoder.disp32(undefined);
504	600	} else return error.CannotEncode,
...	...	@@ -570,7 +666,7 @@ pub const LegacyPrefixes = packed struct {
570	666	}
571	667	};
572	668
573		pub const Options = struct { allow_frame_loc: bool = false };
	669	pub const Options = struct { allow_frame_locs: bool = false, allow_symbols: bool = false };
574	670
575	671	fn Encoder(comptime T: type, comptime opts: Options) type {
576	672	return struct {
...	...	@@ -1085,7 +1181,7 @@ test "lower MI encoding" {
1085	1181	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
1086	1182
1087	1183	try enc.encode(.mov, &.{
1088		.{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
	1184	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .r12 }) },
1089	1185	.{ .imm = Immediate.u(0x10) },
1090	1186	});
1091	1187	try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
...	...	@@ -1109,13 +1205,13 @@ test "lower MI encoding" {
1109	1205	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
1110	1206
1111	1207	try enc.encode(.mov, &.{
1112		.{ .mem = Memory.sib(.dword, .{ .base = .r11 }) },
	1208	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .r11 }) },
1113	1209	.{ .imm = Immediate.u(0x10) },
1114	1210	});
1115	1211	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
1116	1212
1117	1213	try enc.encode(.mov, &.{
1118		.{ .mem = Memory.rip(.qword, 0x10) },
	1214	.{ .mem = Instruction.Memory.rip(.qword, 0x10) },
1119	1215	.{ .imm = Immediate.u(0x10) },
1120	1216	});
1121	1217	try expectEqualHexStrings(
...	...	@@ -1125,25 +1221,25 @@ test "lower MI encoding" {
1125	1221	);
1126	1222
1127	1223	try enc.encode(.mov, &.{
1128		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -8 }) },
	1224	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -8 }) },
1129	1225	.{ .imm = Immediate.u(0x10) },
1130	1226	});
1131	1227	try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10");
1132	1228
1133	1229	try enc.encode(.mov, &.{
1134		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -2 }) },
	1230	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -2 }) },
1135	1231	.{ .imm = Immediate.s(-16) },
1136	1232	});
1137	1233	try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16");
1138	1234
1139	1235	try enc.encode(.mov, &.{
1140		.{ .mem = Memory.sib(.byte, .{ .base = .rbp, .disp = -1 }) },
	1236	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .rbp, .disp = -1 }) },
1141	1237	.{ .imm = Immediate.u(0x10) },
1142	1238	});
1143	1239	try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10");
1144	1240
1145	1241	try enc.encode(.mov, &.{
1146		.{ .mem = Memory.sib(.qword, .{
	1242	.{ .mem = Instruction.Memory.sib(.qword, .{
1147	1243	.base = .ds,
1148	1244	.disp = 0x10000000,
1149	1245	.scale_index = .{ .scale = 2, .index = .rcx },
...	...	@@ -1157,13 +1253,13 @@ test "lower MI encoding" {
1157	1253	);
1158	1254
1159	1255	try enc.encode(.adc, &.{
1160		.{ .mem = Memory.sib(.byte, .{ .base = .rbp, .disp = -0x10 }) },
	1256	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .rbp, .disp = -0x10 }) },
1161	1257	.{ .imm = Immediate.u(0x10) },
1162	1258	});
1163	1259	try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10");
1164	1260
1165	1261	try enc.encode(.adc, &.{
1166		.{ .mem = Memory.rip(.qword, 0) },
	1262	.{ .mem = Instruction.Memory.rip(.qword, 0) },
1167	1263	.{ .imm = Immediate.u(0x10) },
1168	1264	});
1169	1265	try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10");
...	...	@@ -1175,7 +1271,7 @@ test "lower MI encoding" {
1175	1271	try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10");
1176	1272
1177	1273	try enc.encode(.add, &.{
1178		.{ .mem = Memory.sib(.dword, .{ .base = .rdx, .disp = -8 }) },
	1274	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .rdx, .disp = -8 }) },
1179	1275	.{ .imm = Immediate.u(0x10) },
1180	1276	});
1181	1277	try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10");
...	...	@@ -1187,13 +1283,13 @@ test "lower MI encoding" {
1187	1283	try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
1188	1284
1189	1285	try enc.encode(.add, &.{
1190		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -0x10 }) },
	1286	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -0x10 }) },
1191	1287	.{ .imm = Immediate.s(-0x10) },
1192	1288	});
1193	1289	try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10");
1194	1290
1195	1291	try enc.encode(.@"and", &.{
1196		.{ .mem = Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
	1292	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
1197	1293	.{ .imm = Immediate.u(0x10) },
1198	1294	});
1199	1295	try expectEqualHexStrings(
...	...	@@ -1203,7 +1299,7 @@ test "lower MI encoding" {
1203	1299	);
1204	1300
1205	1301	try enc.encode(.@"and", &.{
1206		.{ .mem = Memory.sib(.dword, .{ .base = .es, .disp = 0x10000000 }) },
	1302	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .es, .disp = 0x10000000 }) },
1207	1303	.{ .imm = Immediate.u(0x10) },
1208	1304	});
1209	1305	try expectEqualHexStrings(
...	...	@@ -1213,7 +1309,7 @@ test "lower MI encoding" {
1213	1309	);
1214	1310
1215	1311	try enc.encode(.@"and", &.{
1216		.{ .mem = Memory.sib(.dword, .{ .base = .r12, .disp = 0x10000000 }) },
	1312	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .r12, .disp = 0x10000000 }) },
1217	1313	.{ .imm = Immediate.u(0x10) },
1218	1314	});
1219	1315	try expectEqualHexStrings(
...	...	@@ -1223,7 +1319,7 @@ test "lower MI encoding" {
1223	1319	);
1224	1320
1225	1321	try enc.encode(.sub, &.{
1226		.{ .mem = Memory.sib(.dword, .{ .base = .r11, .disp = 0x10000000 }) },
	1322	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .r11, .disp = 0x10000000 }) },
1227	1323	.{ .imm = Immediate.u(0x10) },
1228	1324	});
1229	1325	try expectEqualHexStrings(
...	...	@@ -1238,25 +1334,25 @@ test "lower RM encoding" {
1238	1334
1239	1335	try enc.encode(.mov, &.{
1240	1336	.{ .reg = .rax },
1241		.{ .mem = Memory.sib(.qword, .{ .base = .r11 }) },
	1337	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r11 }) },
1242	1338	});
1243	1339	try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
1244	1340
1245	1341	try enc.encode(.mov, &.{
1246	1342	.{ .reg = .rbx },
1247		.{ .mem = Memory.sib(.qword, .{ .base = .ds, .disp = 0x10 }) },
	1343	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .ds, .disp = 0x10 }) },
1248	1344	});
1249	1345	try expectEqualHexStrings("\x48\x8B\x1C\x25\x10\x00\x00\x00", enc.code(), "mov rbx, QWORD PTR ds:0x10");
1250	1346
1251	1347	try enc.encode(.mov, &.{
1252	1348	.{ .reg = .rax },
1253		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
	1349	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
1254	1350	});
1255	1351	try expectEqualHexStrings("\x48\x8B\x45\xFC", enc.code(), "mov rax, QWORD PTR [rbp - 4]");
1256	1352
1257	1353	try enc.encode(.mov, &.{
1258	1354	.{ .reg = .rax },
1259		.{ .mem = Memory.sib(.qword, .{
	1355	.{ .mem = Instruction.Memory.sib(.qword, .{
1260	1356	.base = .rbp,
1261	1357	.scale_index = .{ .scale = 1, .index = .rcx },
1262	1358	.disp = -8,
...	...	@@ -1266,7 +1362,7 @@ test "lower RM encoding" {
1266	1362
1267	1363	try enc.encode(.mov, &.{
1268	1364	.{ .reg = .eax },
1269		.{ .mem = Memory.sib(.dword, .{
	1365	.{ .mem = Instruction.Memory.sib(.dword, .{
1270	1366	.base = .rbp,
1271	1367	.scale_index = .{ .scale = 4, .index = .rdx },
1272	1368	.disp = -4,
...	...	@@ -1276,7 +1372,7 @@ test "lower RM encoding" {
1276	1372
1277	1373	try enc.encode(.mov, &.{
1278	1374	.{ .reg = .rax },
1279		.{ .mem = Memory.sib(.qword, .{
	1375	.{ .mem = Instruction.Memory.sib(.qword, .{
1280	1376	.base = .rbp,
1281	1377	.scale_index = .{ .scale = 8, .index = .rcx },
1282	1378	.disp = -8,
...	...	@@ -1286,7 +1382,7 @@ test "lower RM encoding" {
1286	1382
1287	1383	try enc.encode(.mov, &.{
1288	1384	.{ .reg = .r8b },
1289		.{ .mem = Memory.sib(.byte, .{
	1385	.{ .mem = Instruction.Memory.sib(.byte, .{
1290	1386	.base = .rsi,
1291	1387	.scale_index = .{ .scale = 1, .index = .rcx },
1292	1388	.disp = -24,
...	...	@@ -1302,7 +1398,7 @@ test "lower RM encoding" {
1302	1398	try expectEqualHexStrings("\x48\x8C\xC8", enc.code(), "mov rax, cs");
1303	1399
1304	1400	try enc.encode(.mov, &.{
1305		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -16 }) },
	1401	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -16 }) },
1306	1402	.{ .reg = .fs },
1307	1403	});
1308	1404	try expectEqualHexStrings("\x48\x8C\x65\xF0", enc.code(), "mov QWORD PTR [rbp - 16], fs");
...	...	@@ -1314,7 +1410,7 @@ test "lower RM encoding" {
1314	1410	try expectEqualHexStrings("\x66\x41\x8C\xCC", enc.code(), "mov r12w, cs");
1315	1411
1316	1412	try enc.encode(.mov, &.{
1317		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
	1413	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
1318	1414	.{ .reg = .fs },
1319	1415	});
1320	1416	try expectEqualHexStrings("\x66\x8C\x65\xF0", enc.code(), "mov WORD PTR [rbp - 16], fs");
...	...	@@ -1339,19 +1435,19 @@ test "lower RM encoding" {
1339	1435
1340	1436	try enc.encode(.movsx, &.{
1341	1437	.{ .reg = .eax },
1342		.{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
	1438	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp }) },
1343	1439	});
1344	1440	try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
1345	1441
1346	1442	try enc.encode(.movsx, &.{
1347	1443	.{ .reg = .eax },
1348		.{ .mem = Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
	1444	.{ .mem = Instruction.Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
1349	1445	});
1350	1446	try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
1351	1447
1352	1448	try enc.encode(.movsx, &.{
1353	1449	.{ .reg = .ax },
1354		.{ .mem = Memory.rip(.byte, 0x10) },
	1450	.{ .mem = Instruction.Memory.rip(.byte, 0x10) },
1355	1451	});
1356	1452	try expectEqualHexStrings("\x66\x0F\xBE\x05\x10\x00\x00\x00", enc.code(), "movsx ax, BYTE PTR [rip + 0x10]");
1357	1453
...	...	@@ -1369,37 +1465,37 @@ test "lower RM encoding" {
1369	1465
1370	1466	try enc.encode(.lea, &.{
1371	1467	.{ .reg = .rax },
1372		.{ .mem = Memory.rip(.qword, 0x10) },
	1468	.{ .mem = Instruction.Memory.rip(.qword, 0x10) },
1373	1469	});
1374	1470	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, QWORD PTR [rip + 0x10]");
1375	1471
1376	1472	try enc.encode(.lea, &.{
1377	1473	.{ .reg = .rax },
1378		.{ .mem = Memory.rip(.dword, 0x10) },
	1474	.{ .mem = Instruction.Memory.rip(.dword, 0x10) },
1379	1475	});
1380	1476	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, DWORD PTR [rip + 0x10]");
1381	1477
1382	1478	try enc.encode(.lea, &.{
1383	1479	.{ .reg = .eax },
1384		.{ .mem = Memory.rip(.dword, 0x10) },
	1480	.{ .mem = Instruction.Memory.rip(.dword, 0x10) },
1385	1481	});
1386	1482	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, DWORD PTR [rip + 0x10]");
1387	1483
1388	1484	try enc.encode(.lea, &.{
1389	1485	.{ .reg = .eax },
1390		.{ .mem = Memory.rip(.word, 0x10) },
	1486	.{ .mem = Instruction.Memory.rip(.word, 0x10) },
1391	1487	});
1392	1488	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, WORD PTR [rip + 0x10]");
1393	1489
1394	1490	try enc.encode(.lea, &.{
1395	1491	.{ .reg = .ax },
1396		.{ .mem = Memory.rip(.byte, 0x10) },
	1492	.{ .mem = Instruction.Memory.rip(.byte, 0x10) },
1397	1493	});
1398	1494	try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
1399	1495
1400	1496	try enc.encode(.lea, &.{
1401	1497	.{ .reg = .rsi },
1402		.{ .mem = Memory.sib(.qword, .{
	1498	.{ .mem = Instruction.Memory.sib(.qword, .{
1403	1499	.base = .rbp,
1404	1500	.scale_index = .{ .scale = 1, .index = .rcx },
1405	1501	}) },
...	...	@@ -1408,31 +1504,31 @@ test "lower RM encoding" {
1408	1504
1409	1505	try enc.encode(.add, &.{
1410	1506	.{ .reg = .r11 },
1411		.{ .mem = Memory.sib(.qword, .{ .base = .ds, .disp = 0x10000000 }) },
	1507	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .ds, .disp = 0x10000000 }) },
1412	1508	});
1413	1509	try expectEqualHexStrings("\x4C\x03\x1C\x25\x00\x00\x00\x10", enc.code(), "add r11, QWORD PTR ds:0x10000000");
1414	1510
1415	1511	try enc.encode(.add, &.{
1416	1512	.{ .reg = .r12b },
1417		.{ .mem = Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
	1513	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
1418	1514	});
1419	1515	try expectEqualHexStrings("\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR ds:0x10000000");
1420	1516
1421	1517	try enc.encode(.add, &.{
1422	1518	.{ .reg = .r12b },
1423		.{ .mem = Memory.sib(.byte, .{ .base = .fs, .disp = 0x10000000 }) },
	1519	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .fs, .disp = 0x10000000 }) },
1424	1520	});
1425	1521	try expectEqualHexStrings("\x64\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR fs:0x10000000");
1426	1522
1427	1523	try enc.encode(.sub, &.{
1428	1524	.{ .reg = .r11 },
1429		.{ .mem = Memory.sib(.qword, .{ .base = .r13, .disp = 0x10000000 }) },
	1525	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r13, .disp = 0x10000000 }) },
1430	1526	});
1431	1527	try expectEqualHexStrings("\x4D\x2B\x9D\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r13 + 0x10000000]");
1432	1528
1433	1529	try enc.encode(.sub, &.{
1434	1530	.{ .reg = .r11 },
1435		.{ .mem = Memory.sib(.qword, .{ .base = .r12, .disp = 0x10000000 }) },
	1531	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r12, .disp = 0x10000000 }) },
1436	1532	});
1437	1533	try expectEqualHexStrings("\x4D\x2B\x9C\x24\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r12 + 0x10000000]");
1438	1534
...	...	@@ -1455,7 +1551,7 @@ test "lower RMI encoding" {
1455	1551
1456	1552	try enc.encode(.imul, &.{
1457	1553	.{ .reg = .r11 },
1458		.{ .mem = Memory.rip(.qword, -16) },
	1554	.{ .mem = Instruction.Memory.rip(.qword, -16) },
1459	1555	.{ .imm = Immediate.s(-1024) },
1460	1556	});
1461	1557	try expectEqualHexStrings(
...	...	@@ -1466,7 +1562,7 @@ test "lower RMI encoding" {
1466	1562
1467	1563	try enc.encode(.imul, &.{
1468	1564	.{ .reg = .bx },
1469		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
	1565	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
1470	1566	.{ .imm = Immediate.s(-1024) },
1471	1567	});
1472	1568	try expectEqualHexStrings(
...	...	@@ -1477,7 +1573,7 @@ test "lower RMI encoding" {
1477	1573
1478	1574	try enc.encode(.imul, &.{
1479	1575	.{ .reg = .bx },
1480		.{ .mem = Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
	1576	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp, .disp = -16 }) },
1481	1577	.{ .imm = Immediate.u(1024) },
1482	1578	});
1483	1579	try expectEqualHexStrings(
...	...	@@ -1497,19 +1593,19 @@ test "lower MR encoding" {
1497	1593	try expectEqualHexStrings("\x48\x89\xD8", enc.code(), "mov rax, rbx");
1498	1594
1499	1595	try enc.encode(.mov, &.{
1500		.{ .mem = Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
	1596	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp, .disp = -4 }) },
1501	1597	.{ .reg = .r11 },
1502	1598	});
1503	1599	try expectEqualHexStrings("\x4c\x89\x5d\xfc", enc.code(), "mov QWORD PTR [rbp - 4], r11");
1504	1600
1505	1601	try enc.encode(.mov, &.{
1506		.{ .mem = Memory.rip(.qword, 0x10) },
	1602	.{ .mem = Instruction.Memory.rip(.qword, 0x10) },
1507	1603	.{ .reg = .r12 },
1508	1604	});
1509	1605	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rip + 0x10], r12");
1510	1606
1511	1607	try enc.encode(.mov, &.{
1512		.{ .mem = Memory.sib(.qword, .{
	1608	.{ .mem = Instruction.Memory.sib(.qword, .{
1513	1609	.base = .r11,
1514	1610	.scale_index = .{ .scale = 2, .index = .r12 },
1515	1611	.disp = 0x10,
...	...	@@ -1519,13 +1615,13 @@ test "lower MR encoding" {
1519	1615	try expectEqualHexStrings("\x4F\x89\x6C\x63\x10", enc.code(), "mov QWORD PTR [r11 + 2 * r12 + 0x10], r13");
1520	1616
1521	1617	try enc.encode(.mov, &.{
1522		.{ .mem = Memory.rip(.word, -0x10) },
	1618	.{ .mem = Instruction.Memory.rip(.word, -0x10) },
1523	1619	.{ .reg = .r12w },
1524	1620	});
1525	1621	try expectEqualHexStrings("\x66\x44\x89\x25\xF0\xFF\xFF\xFF", enc.code(), "mov WORD PTR [rip - 0x10], r12w");
1526	1622
1527	1623	try enc.encode(.mov, &.{
1528		.{ .mem = Memory.sib(.byte, .{
	1624	.{ .mem = Instruction.Memory.sib(.byte, .{
1529	1625	.base = .r11,
1530	1626	.scale_index = .{ .scale = 2, .index = .r12 },
1531	1627	.disp = 0x10,
...	...	@@ -1535,25 +1631,25 @@ test "lower MR encoding" {
1535	1631	try expectEqualHexStrings("\x47\x88\x6C\x63\x10", enc.code(), "mov BYTE PTR [r11 + 2 * r12 + 0x10], r13b");
1536	1632
1537	1633	try enc.encode(.add, &.{
1538		.{ .mem = Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
	1634	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .ds, .disp = 0x10000000 }) },
1539	1635	.{ .reg = .r12b },
1540	1636	});
1541	1637	try expectEqualHexStrings("\x44\x00\x24\x25\x00\x00\x00\x10", enc.code(), "add BYTE PTR ds:0x10000000, r12b");
1542	1638
1543	1639	try enc.encode(.add, &.{
1544		.{ .mem = Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
	1640	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .ds, .disp = 0x10000000 }) },
1545	1641	.{ .reg = .r12d },
1546	1642	});
1547	1643	try expectEqualHexStrings("\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [ds:0x10000000], r12d");
1548	1644
1549	1645	try enc.encode(.add, &.{
1550		.{ .mem = Memory.sib(.dword, .{ .base = .gs, .disp = 0x10000000 }) },
	1646	.{ .mem = Instruction.Memory.sib(.dword, .{ .base = .gs, .disp = 0x10000000 }) },
1551	1647	.{ .reg = .r12d },
1552	1648	});
1553	1649	try expectEqualHexStrings("\x65\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [gs:0x10000000], r12d");
1554	1650
1555	1651	try enc.encode(.sub, &.{
1556		.{ .mem = Memory.sib(.qword, .{ .base = .r11, .disp = 0x10000000 }) },
	1652	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r11, .disp = 0x10000000 }) },
1557	1653	.{ .reg = .r12 },
1558	1654	});
1559	1655	try expectEqualHexStrings("\x4D\x29\xA3\x00\x00\x00\x10", enc.code(), "sub QWORD PTR [r11 + 0x10000000], r12");
...	...	@@ -1568,12 +1664,12 @@ test "lower M encoding" {
1568	1664	try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
1569	1665
1570	1666	try enc.encode(.call, &.{
1571		.{ .mem = Memory.sib(.qword, .{ .base = .r12 }) },
	1667	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .r12 }) },
1572	1668	});
1573	1669	try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
1574	1670
1575	1671	try enc.encode(.call, &.{
1576		.{ .mem = Memory.sib(.qword, .{
	1672	.{ .mem = Instruction.Memory.sib(.qword, .{
1577	1673	.base = null,
1578	1674	.scale_index = .{ .index = .r11, .scale = 2 },
1579	1675	}) },
...	...	@@ -1581,7 +1677,7 @@ test "lower M encoding" {
1581	1677	try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
1582	1678
1583	1679	try enc.encode(.call, &.{
1584		.{ .mem = Memory.sib(.qword, .{
	1680	.{ .mem = Instruction.Memory.sib(.qword, .{
1585	1681	.base = null,
1586	1682	.scale_index = .{ .index = .r12, .scale = 2 },
1587	1683	}) },
...	...	@@ -1589,7 +1685,7 @@ test "lower M encoding" {
1589	1685	try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
1590	1686
1591	1687	try enc.encode(.call, &.{
1592		.{ .mem = Memory.sib(.qword, .{ .base = .gs }) },
	1688	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .gs }) },
1593	1689	});
1594	1690	try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
1595	1691
...	...	@@ -1599,22 +1695,22 @@ test "lower M encoding" {
1599	1695	try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
1600	1696
1601	1697	try enc.encode(.push, &.{
1602		.{ .mem = Memory.sib(.qword, .{ .base = .rbp }) },
	1698	.{ .mem = Instruction.Memory.sib(.qword, .{ .base = .rbp }) },
1603	1699	});
1604	1700	try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1605	1701
1606	1702	try enc.encode(.push, &.{
1607		.{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
	1703	.{ .mem = Instruction.Memory.sib(.word, .{ .base = .rbp }) },
1608	1704	});
1609	1705	try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1610	1706
1611	1707	try enc.encode(.pop, &.{
1612		.{ .mem = Memory.rip(.qword, 0) },
	1708	.{ .mem = Instruction.Memory.rip(.qword, 0) },
1613	1709	});
1614	1710	try expectEqualHexStrings("\x8F\x05\x00\x00\x00\x00", enc.code(), "pop QWORD PTR [rip]");
1615	1711
1616	1712	try enc.encode(.pop, &.{
1617		.{ .mem = Memory.rip(.word, 0) },
	1713	.{ .mem = Instruction.Memory.rip(.word, 0) },
1618	1714	});
1619	1715	try expectEqualHexStrings("\x66\x8F\x05\x00\x00\x00\x00", enc.code(), "pop WORD PTR [rbp]");
1620	1716
...	...	@@ -1695,48 +1791,48 @@ test "lower FD/TD encoding" {
1695	1791
1696	1792	try enc.encode(.mov, &.{
1697	1793	.{ .reg = .rax },
1698		.{ .mem = Memory.moffs(.cs, 0x10) },
	1794	.{ .mem = Instruction.Memory.moffs(.cs, 0x10) },
1699	1795	});
1700	1796	try expectEqualHexStrings("\x2E\x48\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs rax, cs:0x10");
1701	1797
1702	1798	try enc.encode(.mov, &.{
1703	1799	.{ .reg = .eax },
1704		.{ .mem = Memory.moffs(.fs, 0x10) },
	1800	.{ .mem = Instruction.Memory.moffs(.fs, 0x10) },
1705	1801	});
1706	1802	try expectEqualHexStrings("\x64\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs eax, fs:0x10");
1707	1803
1708	1804	try enc.encode(.mov, &.{
1709	1805	.{ .reg = .ax },
1710		.{ .mem = Memory.moffs(.gs, 0x10) },
	1806	.{ .mem = Instruction.Memory.moffs(.gs, 0x10) },
1711	1807	});
1712	1808	try expectEqualHexStrings("\x65\x66\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ax, gs:0x10");
1713	1809
1714	1810	try enc.encode(.mov, &.{
1715	1811	.{ .reg = .al },
1716		.{ .mem = Memory.moffs(.ds, 0x10) },
	1812	.{ .mem = Instruction.Memory.moffs(.ds, 0x10) },
1717	1813	});
1718	1814	try expectEqualHexStrings("\xA0\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs al, ds:0x10");
1719	1815
1720	1816	try enc.encode(.mov, &.{
1721		.{ .mem = Memory.moffs(.cs, 0x10) },
	1817	.{ .mem = Instruction.Memory.moffs(.cs, 0x10) },
1722	1818	.{ .reg = .rax },
1723	1819	});
1724	1820	try expectEqualHexStrings("\x2E\x48\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs cs:0x10, rax");
1725	1821
1726	1822	try enc.encode(.mov, &.{
1727		.{ .mem = Memory.moffs(.fs, 0x10) },
	1823	.{ .mem = Instruction.Memory.moffs(.fs, 0x10) },
1728	1824	.{ .reg = .eax },
1729	1825	});
1730	1826	try expectEqualHexStrings("\x64\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs fs:0x10, eax");
1731	1827
1732	1828	try enc.encode(.mov, &.{
1733		.{ .mem = Memory.moffs(.gs, 0x10) },
	1829	.{ .mem = Instruction.Memory.moffs(.gs, 0x10) },
1734	1830	.{ .reg = .ax },
1735	1831	});
1736	1832	try expectEqualHexStrings("\x65\x66\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs gs:0x10, ax");
1737	1833
1738	1834	try enc.encode(.mov, &.{
1739		.{ .mem = Memory.moffs(.ds, 0x10) },
	1835	.{ .mem = Instruction.Memory.moffs(.ds, 0x10) },
1740	1836	.{ .reg = .al },
1741	1837	});
1742	1838	try expectEqualHexStrings("\xA2\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ds:0x10, al");
...	...	@@ -1774,16 +1870,16 @@ test "invalid instruction" {
1774	1870	.{ .reg = .al },
1775	1871	});
1776	1872	try invalidInstruction(.call, &.{
1777		.{ .mem = Memory.rip(.dword, 0) },
	1873	.{ .mem = Instruction.Memory.rip(.dword, 0) },
1778	1874	});
1779	1875	try invalidInstruction(.call, &.{
1780		.{ .mem = Memory.rip(.word, 0) },
	1876	.{ .mem = Instruction.Memory.rip(.word, 0) },
1781	1877	});
1782	1878	try invalidInstruction(.call, &.{
1783		.{ .mem = Memory.rip(.byte, 0) },
	1879	.{ .mem = Instruction.Memory.rip(.byte, 0) },
1784	1880	});
1785	1881	try invalidInstruction(.mov, &.{
1786		.{ .mem = Memory.rip(.word, 0x10) },
	1882	.{ .mem = Instruction.Memory.rip(.word, 0x10) },
1787	1883	.{ .reg = .r12 },
1788	1884	});
1789	1885	try invalidInstruction(.lea, &.{
...	...	@@ -1792,7 +1888,7 @@ test "invalid instruction" {
1792	1888	});
1793	1889	try invalidInstruction(.lea, &.{
1794	1890	.{ .reg = .al },
1795		.{ .mem = Memory.rip(.byte, 0) },
	1891	.{ .mem = Instruction.Memory.rip(.byte, 0) },
1796	1892	});
1797	1893	try invalidInstruction(.pop, &.{
1798	1894	.{ .reg = .r12b },
...	...	@@ -1817,7 +1913,7 @@ fn cannotEncode(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand
1817	1913
1818	1914	test "cannot encode" {
1819	1915	try cannotEncode(.@"test", &.{
1820		.{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
	1916	.{ .mem = Instruction.Memory.sib(.byte, .{ .base = .r12 }) },
1821	1917	.{ .reg = .ah },
1822	1918	});
1823	1919	try cannotEncode(.@"test", &.{
...	...	@@ -2149,8 +2245,8 @@ const Assembler = struct {
2149	2245	return null;
2150	2246	}
2151	2247
2152		fn parseMemory(as: *Assembler) ParseError!Memory {
2153		const ptr_size: ?Memory.PtrSize = blk: {
	2248	fn parseMemory(as: *Assembler) ParseError!Instruction.Memory {
	2249	const ptr_size: ?Instruction.Memory.PtrSize = blk: {
2154	2250	const pos = as.it.pos;
2155	2251	const ptr_size = as.parsePtrSize() catch \|err\| switch (err) {
2156	2252	error.UnexpectedToken => {
...	...	@@ -2194,7 +2290,7 @@ const Assembler = struct {
2194	2290	if (res.rip) {
2195	2291	if (res.base != null or res.scale_index != null or res.offset != null)
2196	2292	return error.InvalidMemoryOperand;
2197		return Memory.rip(ptr_size orelse .qword, res.disp orelse 0);
	2293	return Instruction.Memory.rip(ptr_size orelse .qword, res.disp orelse 0);
2198	2294	}
2199	2295	if (res.base) \|base\| {
2200	2296	if (res.rip)
...	...	@@ -2202,9 +2298,9 @@ const Assembler = struct {
2202	2298	if (res.offset) \|offset\| {
2203	2299	if (res.scale_index != null or res.disp != null)
2204	2300	return error.InvalidMemoryOperand;
2205		return Memory.moffs(base, offset);
	2301	return Instruction.Memory.moffs(base, offset);
2206	2302	}
2207		return Memory.sib(ptr_size orelse .qword, .{
	2303	return Instruction.Memory.sib(ptr_size orelse .qword, .{
2208	2304	.base = base,
2209	2305	.scale_index = res.scale_index,
2210	2306	.disp = res.disp orelse 0,
...	...	@@ -2222,12 +2318,12 @@ const Assembler = struct {
2222	2318	const MemoryParseResult = struct {
2223	2319	rip: bool = false,
2224	2320	base: ?Register = null,
2225		scale_index: ?Memory.ScaleIndex = null,
	2321	scale_index: ?Instruction.Memory.ScaleIndex = null,
2226	2322	disp: ?i32 = null,
2227	2323	offset: ?u64 = null,
2228	2324	};
2229	2325
2230		fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!MemoryParseResult {
	2326	fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!Instruction.MemoryParseResult {
2231	2327	var res: MemoryParseResult = .{};
2232	2328	inline for (rule, 0..) \|cond, i\| {
2233	2329	if (@typeInfo(@TypeOf(cond)) != .EnumLiteral) {
...	...	@@ -2294,7 +2390,7 @@ const Assembler = struct {
2294	2390	return res;
2295	2391	}
2296	2392
2297		fn parsePtrSize(as: *Assembler) ParseError!Memory.PtrSize {
	2393	fn parsePtrSize(as: *Assembler) ParseError!Instruction.Memory.PtrSize {
2298	2394	const size = try as.expect(.string);
2299	2395	try as.skip(1, .{.space});
2300	2396	const ptr = try as.expect(.string);