Commit 6e882d730b2ed1f2494e7b28f1fed2726e4a1ac0 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-03-08 23:45:05+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2023-03-11 20:05:49+01:00
log	6e882d730b2ed1f2494e7b28f1fed2726e4a1ac0
tree	52a74722da7a2456be973e2cf702308e38b041f5
parent	5b3770102845b17207f85d3a43a3b6cab551a329

x86_64: introduce assemble() helper which encodes/decodes into MIR -> Instruction

4 files changed, 1748 insertions(+), 2512 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    encoder.zig

src/arch/x86_64/CodeGen.zig+1110-1050

...	...	@@ -374,71 +374,99 @@ pub fn generate(
374	374	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
375	375	const gpa = self.gpa;
376	376	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
377		const result_index = @intCast(Air.Inst.Index, self.mir_instructions.len);
	377	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
378	378	self.mir_instructions.appendAssumeCapacity(inst);
379	379	return result_index;
380	380	}
381	381
382		pub fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
	382	fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
383	383	const fields = std.meta.fields(@TypeOf(extra));
384	384	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
385	385	return self.addExtraAssumeCapacity(extra);
386	386	}
387	387
388		fn extraData(self: *Self, comptime T: type, index: u32) struct { data: T, end: u32 } {
389		const fields = std.meta.fields(T);
390		var i: u32 = index;
391		var result: T = undefined;
392		inline for (fields) \|field\| {
393		@field(result, field.name) = switch (field.type) {
394		u32 => self.mir_extra.items[i],
395		i32 => @bitCast(i32, self.mir_extra.items[i]),
396		else => @compileError("bad field type"),
397		};
398		i += 1;
399		}
400		return .{
401		.data = result,
402		.end = i,
403		};
404		}
405
406		pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
	388	fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
407	389	const fields = std.meta.fields(@TypeOf(extra));
408	390	const result = @intCast(u32, self.mir_extra.items.len);
409	391	inline for (fields) \|field\| {
410	392	self.mir_extra.appendAssumeCapacity(switch (field.type) {
411	393	u32 => @field(extra, field.name),
412	394	i32 => @bitCast(u32, @field(extra, field.name)),
413		else => @compileError("bad field type"),
	395	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
414	396	});
415	397	}
416	398	return result;
417	399	}
418	400
	401	fn assemble(self: *Self, tag: Mir.Inst.Tag, args: struct {
	402	op1: Mir.Operand = .none,
	403	op2: Mir.Operand = .none,
	404	op3: Mir.Operand = .none,
	405	op4: Mir.Operand = .none,
	406	}) !void {
	407	const ops: Mir.Inst.Ops = blk: {
	408	if (args.op1 == .none and args.op2 == .none and args.op3 == .none and args.op4 == .none)
	409	break :blk .none;
	410
	411	if (args.op1 == .reg and args.op2 == .reg)
	412	break :blk .rr;
	413	if (args.op1 == .reg and args.op2 == .imm) switch (args.op2.imm) {
	414	.signed => break :blk .ri_s,
	415	.unsigned => break :blk .ri_u,
	416	};
	417	if (args.op1 == .reg)
	418	break :blk .r;
	419	if (args.op1 == .imm) switch (args.op1.imm) {
	420	.signed => break :blk .imm_s,
	421	.unsigned => break :blk .imm_u, // TODO 64bits
	422	};
	423
	424	unreachable;
	425	};
	426	const data: Mir.Inst.Data = switch (ops) {
	427	.none => undefined,
	428	.imm_s => .{ .imm_s = args.op1.imm.signed },
	429	.imm_u => .{ .imm_u = @intCast(u32, args.op1.imm.unsigned) },
	430	.r => .{ .r = args.op1.reg },
	431	.rr => .{ .rr = .{
	432	.r1 = args.op1.reg,
	433	.r2 = args.op2.reg,
	434	} },
	435	.ri_s => .{ .ri_s = .{
	436	.r1 = args.op1.reg,
	437	.imm = args.op2.imm.signed,
	438	} },
	439	.ri_u => .{ .ri_u = .{
	440	.r1 = args.op1.reg,
	441	.imm = @intCast(u32, args.op2.imm.unsigned),
	442	} },
	443	else => unreachable,
	444	};
	445	_ = try self.addInst(.{
	446	.tag = tag,
	447	.ops = ops,
	448	.data = data,
	449	});
	450	}
	451
419	452	fn gen(self: *Self) InnerError!void {
420	453	const cc = self.fn_type.fnCallingConvention();
421	454	if (cc != .Naked) {
422		_ = try self.addInst(.{
423		.tag = .push,
424		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
425		.data = undefined, // unused for push reg,
	455	try self.assemble(.push, .{
	456	.op1 = .{ .reg = .rbp },
426	457	});
427		_ = try self.addInst(.{
428		.tag = .mov,
429		.ops = Mir.Inst.Ops.encode(.{
430		.reg1 = .rbp,
431		.reg2 = .rsp,
432		}),
433		.data = undefined,
	458	try self.assemble(.mov, .{
	459	.op1 = .{ .reg = .rbp },
	460	.op2 = .{ .reg = .rsp },
434	461	});
	462
435	463	// We want to subtract the aligned stack frame size from rsp here, but we don't
436	464	// yet know how big it will be, so we leave room for a 4-byte stack size.
437	465	// TODO During semantic analysis, check if there are no function calls. If there
438	466	// are none, here we can omit the part where we subtract and then add rsp.
439	467	const backpatch_stack_sub = try self.addInst(.{
440	468	.tag = .nop,
441		.ops = undefined,
	469	.ops = .none,
442	470	.data = undefined,
443	471	});
444	472
...	...	@@ -465,7 +493,7 @@ fn gen(self: *Self) InnerError!void {
465	493	// Push callee-preserved regs that were used actually in use.
466	494	const backpatch_push_callee_preserved_regs = try self.addInst(.{
467	495	.tag = .nop,
468		.ops = undefined,
	496	.ops = .none,
469	497	.data = undefined,
470	498	});
471	499
...	...	@@ -496,7 +524,7 @@ fn gen(self: *Self) InnerError!void {
496	524	// Pop saved callee-preserved regs.
497	525	const backpatch_pop_callee_preserved_regs = try self.addInst(.{
498	526	.tag = .nop,
499		.ops = undefined,
	527	.ops = .none,
500	528	.data = undefined,
501	529	});
502	530
...	...	@@ -509,21 +537,12 @@ fn gen(self: *Self) InnerError!void {
509	537	// Maybe add rsp, x if required. This is backpatched later.
510	538	const backpatch_stack_add = try self.addInst(.{
511	539	.tag = .nop,
512		.ops = undefined,
	540	.ops = .none,
513	541	.data = undefined,
514	542	});
515	543
516		_ = try self.addInst(.{
517		.tag = .pop,
518		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
519		.data = undefined,
520		});
521
522		_ = try self.addInst(.{
523		.tag = .ret,
524		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
525		.data = undefined,
526		});
	544	try self.assemble(.pop, .{ .op1 = .{ .reg = .rbp } });
	545	try self.assemble(.ret, .{});
527	546
528	547	// Adjust the stack
529	548	if (self.max_end_stack > math.maxInt(i32)) {
...	...	@@ -537,27 +556,34 @@ fn gen(self: *Self) InnerError!void {
537	556	if (aligned_stack_end > 0) {
538	557	self.mir_instructions.set(backpatch_stack_sub, .{
539	558	.tag = .sub,
540		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
541		.data = .{ .imm = aligned_stack_end },
	559	.ops = .ri_u,
	560	.data = .{ .ri_u = .{
	561	.r1 = .rsp,
	562	.imm = aligned_stack_end,
	563	} },
542	564	});
543	565	self.mir_instructions.set(backpatch_stack_add, .{
544	566	.tag = .add,
545		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
546		.data = .{ .imm = aligned_stack_end },
	567	.ops = .ri_u,
	568	.data = .{ .ri_u = .{
	569	.r1 = .rsp,
	570	.imm = aligned_stack_end,
	571	} },
547	572	});
548	573
549	574	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
	575	.base_reg = @enumToInt(Register.rbp),
550	576	.register_list = reg_list.asInt(),
551	577	.stack_end = aligned_stack_end,
552	578	});
553	579	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
554	580	.tag = .push_regs,
555		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
	581	.ops = undefined,
556	582	.data = .{ .payload = save_reg_list },
557	583	});
558	584	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
559	585	.tag = .pop_regs,
560		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
	586	.ops = undefined,
561	587	.data = .{ .payload = save_reg_list },
562	588	});
563	589	}
...	...	@@ -1306,14 +1332,15 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1306	1332	.unsigned => .b,
1307	1333	.signed => .l,
1308	1334	};
1309		_ = try self.addInst(.{
1310		.tag = .cond_mov,
1311		.ops = Mir.Inst.Ops.encode(.{
1312		.reg1 = dst_mcv.register,
1313		.reg2 = lhs_reg,
1314		}),
1315		.data = .{ .cc = cc },
1316		});
	1335	_ = cc;
	1336	// _ = try self.addInst(.{
	1337	// .tag = .cond_mov,
	1338	// .ops = Mir.Inst.Ops.encode(.{
	1339	// .reg1 = dst_mcv.register,
	1340	// .reg2 = lhs_reg,
	1341	// }),
	1342	// .data = .{ .cc = cc },
	1343	// });
1317	1344
1318	1345	break :result dst_mcv;
1319	1346	};
...	...	@@ -1513,13 +1540,14 @@ fn genSetStackTruncatedOverflowCompare(
1513	1540	.signed => .o,
1514	1541	.unsigned => .c,
1515	1542	};
1516		_ = try self.addInst(.{
1517		.tag = .cond_set_byte,
1518		.ops = Mir.Inst.Ops.encode(.{
1519		.reg1 = overflow_reg.to8(),
1520		}),
1521		.data = .{ .cc = cc },
1522		});
	1543	_ = cc;
	1544	// _ = try self.addInst(.{
	1545	// .tag = .cond_set_byte,
	1546	// .ops = Mir.Inst.Ops.encode(.{
	1547	// .reg1 = overflow_reg.to8(),
	1548	// }),
	1549	// .data = .{ .cc = cc },
	1550	// });
1523	1551
1524	1552	const scratch_reg = temp_regs[1];
1525	1553	try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });
...	...	@@ -1532,11 +1560,11 @@ fn genSetStackTruncatedOverflowCompare(
1532	1560	);
1533	1561
1534	1562	const eq_reg = temp_regs[2];
1535		_ = try self.addInst(.{
1536		.tag = .cond_set_byte,
1537		.ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
1538		.data = .{ .cc = .ne },
1539		});
	1563	// _ = try self.addInst(.{
	1564	// .tag = .cond_set_byte,
	1565	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
	1566	// .data = .{ .cc = .ne },
	1567	// });
1540	1568
1541	1569	try self.genBinOpMir(
1542	1570	.@"or",
...	...	@@ -1680,25 +1708,26 @@ fn genIntMulDivOpMir(
1680	1708	try self.genSetReg(ty, .rax, lhs);
1681	1709	}
1682	1710
1683		switch (signedness) {
1684		.signed => {
1685		_ = try self.addInst(.{
1686		.tag = .cwd,
1687		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
1688		.data = undefined,
1689		});
1690		},
1691		.unsigned => {
1692		_ = try self.addInst(.{
1693		.tag = .xor,
1694		.ops = Mir.Inst.Ops.encode(.{
1695		.reg1 = .rdx,
1696		.reg2 = .rdx,
1697		}),
1698		.data = undefined,
1699		});
1700		},
1701		}
	1711	_ = signedness;
	1712	// switch (signedness) {
	1713	// .signed => {
	1714	// _ = try self.addInst(.{
	1715	// .tag = .cwd,
	1716	// .ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
	1717	// .data = undefined,
	1718	// });
	1719	// },
	1720	// .unsigned => {
	1721	// _ = try self.addInst(.{
	1722	// .tag = .xor,
	1723	// .ops = Mir.Inst.Ops.encode(.{
	1724	// .reg1 = .rdx,
	1725	// .reg2 = .rdx,
	1726	// }),
	1727	// .data = undefined,
	1728	// });
	1729	// },
	1730	// }
1702	1731
1703	1732	const factor = switch (rhs) {
1704	1733	.register => rhs,
...	...	@@ -1708,33 +1737,35 @@ fn genIntMulDivOpMir(
1708	1737	break :blk MCValue{ .register = reg };
1709	1738	},
1710	1739	};
1711
1712		switch (factor) {
1713		.register => \|reg\| {
1714		_ = try self.addInst(.{
1715		.tag = tag,
1716		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
1717		.data = undefined,
1718		});
1719		},
1720		.stack_offset => \|off\| {
1721		_ = try self.addInst(.{
1722		.tag = tag,
1723		.ops = Mir.Inst.Ops.encode(.{
1724		.reg2 = .rbp,
1725		.flags = switch (abi_size) {
1726		1 => 0b00,
1727		2 => 0b01,
1728		4 => 0b10,
1729		8 => 0b11,
1730		else => unreachable,
1731		},
1732		}),
1733		.data = .{ .disp = -off },
1734		});
1735		},
1736		else => unreachable,
1737		}
	1740	_ = factor;
	1741	_ = tag;
	1742
	1743	// switch (factor) {
	1744	// .register => \|reg\| {
	1745	// _ = try self.addInst(.{
	1746	// .tag = tag,
	1747	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
	1748	// .data = undefined,
	1749	// });
	1750	// },
	1751	// .stack_offset => \|off\| {
	1752	// _ = try self.addInst(.{
	1753	// .tag = tag,
	1754	// .ops = Mir.Inst.Ops.encode(.{
	1755	// .reg2 = .rbp,
	1756	// .flags = switch (abi_size) {
	1757	// 1 => 0b00,
	1758	// 2 => 0b01,
	1759	// 4 => 0b10,
	1760	// 8 => 0b11,
	1761	// else => unreachable,
	1762	// },
	1763	// }),
	1764	// .data = .{ .disp = -off },
	1765	// });
	1766	// },
	1767	// else => unreachable,
	1768	// }
1738	1769	}
1739	1770
1740	1771	/// Always returns a register.
...	...	@@ -1760,38 +1791,38 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
1760	1791	.unsigned => .div,
1761	1792	}, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });
1762	1793
1763		_ = try self.addInst(.{
1764		.tag = .xor,
1765		.ops = Mir.Inst.Ops.encode(.{
1766		.reg1 = divisor.to64(),
1767		.reg2 = dividend.to64(),
1768		}),
1769		.data = undefined,
1770		});
1771		_ = try self.addInst(.{
1772		.tag = .sar,
1773		.ops = Mir.Inst.Ops.encode(.{
1774		.reg1 = divisor.to64(),
1775		.flags = 0b10,
1776		}),
1777		.data = .{ .imm = 63 },
1778		});
1779		_ = try self.addInst(.{
1780		.tag = .@"test",
1781		.ops = Mir.Inst.Ops.encode(.{
1782		.reg1 = .rdx,
1783		.reg2 = .rdx,
1784		}),
1785		.data = undefined,
1786		});
1787		_ = try self.addInst(.{
1788		.tag = .cond_mov,
1789		.ops = Mir.Inst.Ops.encode(.{
1790		.reg1 = divisor.to64(),
1791		.reg2 = .rdx,
1792		}),
1793		.data = .{ .cc = .e },
1794		});
	1794	// _ = try self.addInst(.{
	1795	// .tag = .xor,
	1796	// .ops = Mir.Inst.Ops.encode(.{
	1797	// .reg1 = divisor.to64(),
	1798	// .reg2 = dividend.to64(),
	1799	// }),
	1800	// .data = undefined,
	1801	// });
	1802	// _ = try self.addInst(.{
	1803	// .tag = .sar,
	1804	// .ops = Mir.Inst.Ops.encode(.{
	1805	// .reg1 = divisor.to64(),
	1806	// .flags = 0b10,
	1807	// }),
	1808	// .data = .{ .imm = 63 },
	1809	// });
	1810	// _ = try self.addInst(.{
	1811	// .tag = .@"test",
	1812	// .ops = Mir.Inst.Ops.encode(.{
	1813	// .reg1 = .rdx,
	1814	// .reg2 = .rdx,
	1815	// }),
	1816	// .data = undefined,
	1817	// });
	1818	// _ = try self.addInst(.{
	1819	// .tag = .cond_mov,
	1820	// .ops = Mir.Inst.Ops.encode(.{
	1821	// .reg1 = divisor.to64(),
	1822	// .reg2 = .rdx,
	1823	// }),
	1824	// .data = .{ .cc = .e },
	1825	// });
1795	1826	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
1796	1827	return MCValue{ .register = divisor };
1797	1828	}
...	...	@@ -2226,16 +2257,17 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2226	2257	const addr_reg = try self.register_manager.allocReg(null, gp);
2227	2258	switch (slice_mcv) {
2228	2259	.stack_offset => \|off\| {
	2260	_ = off;
2229	2261	// mov reg, [rbp - 8]
2230		_ = try self.addInst(.{
2231		.tag = .mov,
2232		.ops = Mir.Inst.Ops.encode(.{
2233		.reg1 = addr_reg.to64(),
2234		.reg2 = .rbp,
2235		.flags = 0b01,
2236		}),
2237		.data = .{ .disp = -@intCast(i32, off) },
2238		});
	2262	// _ = try self.addInst(.{
	2263	// .tag = .mov,
	2264	// .ops = Mir.Inst.Ops.encode(.{
	2265	// .reg1 = addr_reg.to64(),
	2266	// .reg2 = .rbp,
	2267	// .flags = 0b01,
	2268	// }),
	2269	// .data = .{ .disp = -@intCast(i32, off) },
	2270	// });
2239	2271	},
2240	2272	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2241	2273	}
...	...	@@ -2312,25 +2344,26 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2312	2344	));
2313	2345	try self.genSetStack(array_ty, off, array, .{});
2314	2346	// lea reg, [rbp]
2315		_ = try self.addInst(.{
2316		.tag = .lea,
2317		.ops = Mir.Inst.Ops.encode(.{
2318		.reg1 = addr_reg.to64(),
2319		.reg2 = .rbp,
2320		}),
2321		.data = .{ .disp = -off },
2322		});
	2347	// _ = try self.addInst(.{
	2348	// .tag = .lea,
	2349	// .ops = Mir.Inst.Ops.encode(.{
	2350	// .reg1 = addr_reg.to64(),
	2351	// .reg2 = .rbp,
	2352	// }),
	2353	// .data = .{ .disp = -off },
	2354	// });
2323	2355	},
2324	2356	.stack_offset => \|off\| {
	2357	_ = off;
2325	2358	// lea reg, [rbp]
2326		_ = try self.addInst(.{
2327		.tag = .lea,
2328		.ops = Mir.Inst.Ops.encode(.{
2329		.reg1 = addr_reg.to64(),
2330		.reg2 = .rbp,
2331		}),
2332		.data = .{ .disp = -off },
2333		});
	2359	// _ = try self.addInst(.{
	2360	// .tag = .lea,
	2361	// .ops = Mir.Inst.Ops.encode(.{
	2362	// .reg1 = addr_reg.to64(),
	2363	// .reg2 = .rbp,
	2364	// }),
	2365	// .data = .{ .disp = -off },
	2366	// });
2334	2367	},
2335	2368	.memory, .linker_load => {
2336	2369	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
...	...	@@ -2388,15 +2421,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2388	2421	return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});
2389	2422	} else {
2390	2423	// mov dst_mcv, [dst_mcv]
2391		_ = try self.addInst(.{
2392		.tag = .mov,
2393		.ops = Mir.Inst.Ops.encode(.{
2394		.reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),
2395		.reg2 = dst_mcv.register,
2396		.flags = 0b01,
2397		}),
2398		.data = .{ .disp = 0 },
2399		});
	2424	// _ = try self.addInst(.{
	2425	// .tag = .mov,
	2426	// .ops = Mir.Inst.Ops.encode(.{
	2427	// .reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),
	2428	// .reg2 = dst_mcv.register,
	2429	// .flags = 0b01,
	2430	// }),
	2431	// .data = .{ .disp = 0 },
	2432	// });
2400	2433	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2401	2434	}
2402	2435	};
...	...	@@ -2650,16 +2683,17 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2650	2683	.undef => unreachable,
2651	2684	.eflags => unreachable,
2652	2685	.register => \|dst_reg\| {
	2686	_ = dst_reg;
2653	2687	// mov dst_reg, [reg]
2654		_ = try self.addInst(.{
2655		.tag = .mov,
2656		.ops = Mir.Inst.Ops.encode(.{
2657		.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
2658		.reg2 = reg,
2659		.flags = 0b01,
2660		}),
2661		.data = .{ .disp = 0 },
2662		});
	2688	// _ = try self.addInst(.{
	2689	// .tag = .mov,
	2690	// .ops = Mir.Inst.Ops.encode(.{
	2691	// .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
	2692	// .reg2 = reg,
	2693	// .flags = 0b01,
	2694	// }),
	2695	// .data = .{ .disp = 0 },
	2696	// });
2663	2697	},
2664	2698	.stack_offset => \|off\| {
2665	2699	if (abi_size <= 8) {
...	...	@@ -2724,19 +2758,22 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2724	2758	.direct => 0b01,
2725	2759	.import => 0b10,
2726	2760	};
2727		_ = try self.addInst(.{
2728		.tag = .lea_pic,
2729		.ops = Mir.Inst.Ops.encode(.{
2730		.reg1 = registerAlias(reg, abi_size),
2731		.flags = flags,
2732		}),
2733		.data = .{
2734		.relocation = .{
2735		.atom_index = atom_index,
2736		.sym_index = load_struct.sym_index,
2737		},
2738		},
2739		});
	2761	_ = abi_size;
	2762	_ = atom_index;
	2763	_ = flags;
	2764	// _ = try self.addInst(.{
	2765	// .tag = .lea_pic,
	2766	// .ops = Mir.Inst.Ops.encode(.{
	2767	// .reg1 = registerAlias(reg, abi_size),
	2768	// .flags = flags,
	2769	// }),
	2770	// .data = .{
	2771	// .relocation = .{
	2772	// .atom_index = atom_index,
	2773	// .sym_index = load_struct.sym_index,
	2774	// },
	2775	// },
	2776	// });
2740	2777	},
2741	2778	.memory => \|addr\| {
2742	2779	// TODO: in case the address fits in an imm32 we can use [ds:imm32]
...	...	@@ -2779,27 +2816,28 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2779	2816	try self.genSetReg(value_ty, reg, value);
2780	2817	},
2781	2818	.immediate => \|imm\| {
	2819	_ = imm;
2782	2820	switch (abi_size) {
2783	2821	1, 2, 4 => {
2784	2822	// TODO this is wasteful!
2785	2823	// introduce new MIR tag specifically for mov [reg + 0], imm
2786		const payload = try self.addExtra(Mir.ImmPair{
2787		.dest_off = 0,
2788		.operand = @truncate(u32, imm),
2789		});
2790		_ = try self.addInst(.{
2791		.tag = .mov_mem_imm,
2792		.ops = Mir.Inst.Ops.encode(.{
2793		.reg1 = reg.to64(),
2794		.flags = switch (abi_size) {
2795		1 => 0b00,
2796		2 => 0b01,
2797		4 => 0b10,
2798		else => unreachable,
2799		},
2800		}),
2801		.data = .{ .payload = payload },
2802		});
	2824	// const payload = try self.addExtra(Mir.ImmPair{
	2825	// .dest_off = 0,
	2826	// .operand = @truncate(u32, imm),
	2827	// });
	2828	// _ = try self.addInst(.{
	2829	// .tag = .mov_mem_imm,
	2830	// .ops = Mir.Inst.Ops.encode(.{
	2831	// .reg1 = reg.to64(),
	2832	// .flags = switch (abi_size) {
	2833	// 1 => 0b00,
	2834	// 2 => 0b01,
	2835	// 4 => 0b10,
	2836	// else => unreachable,
	2837	// },
	2838	// }),
	2839	// .data = .{ .payload = payload },
	2840	// });
2803	2841	},
2804	2842	8 => {
2805	2843	// TODO: optimization: if the imm is only using the lower
...	...	@@ -2829,13 +2867,13 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2829	2867	const overflow_bit_ty = value_ty.structFieldType(1);
2830	2868	const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
2831	2869	const tmp_reg = try self.register_manager.allocReg(null, gp);
2832		_ = try self.addInst(.{
2833		.tag = .cond_set_byte,
2834		.ops = Mir.Inst.Ops.encode(.{
2835		.reg1 = tmp_reg.to8(),
2836		}),
2837		.data = .{ .cc = ro.eflags },
2838		});
	2870	// _ = try self.addInst(.{
	2871	// .tag = .cond_set_byte,
	2872	// .ops = Mir.Inst.Ops.encode(.{
	2873	// .reg1 = tmp_reg.to8(),
	2874	// }),
	2875	// .data = .{ .cc = ro.eflags },
	2876	// });
2839	2877	try self.genInlineMemcpyRegisterRegister(
2840	2878	overflow_bit_ty,
2841	2879	reg,
...	...	@@ -2878,15 +2916,15 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2878	2916
2879	2917	// to get the actual address of the value we want to modify we have to go through the GOT
2880	2918	// mov reg, [reg]
2881		_ = try self.addInst(.{
2882		.tag = .mov,
2883		.ops = Mir.Inst.Ops.encode(.{
2884		.reg1 = addr_reg.to64(),
2885		.reg2 = addr_reg.to64(),
2886		.flags = 0b01,
2887		}),
2888		.data = .{ .disp = 0 },
2889		});
	2919	// _ = try self.addInst(.{
	2920	// .tag = .mov,
	2921	// .ops = Mir.Inst.Ops.encode(.{
	2922	// .reg1 = addr_reg.to64(),
	2923	// .reg2 = addr_reg.to64(),
	2924	// .flags = 0b01,
	2925	// }),
	2926	// .data = .{ .disp = 0 },
	2927	// });
2890	2928
2891	2929	const new_ptr = MCValue{ .register = addr_reg.to64() };
2892	2930
...	...	@@ -2896,11 +2934,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2896	2934	return self.fail("TODO saving imm to memory for abi_size {}", .{abi_size});
2897	2935	}
2898	2936
2899		const payload = try self.addExtra(Mir.ImmPair{
2900		.dest_off = 0,
2901		// TODO check if this logic is correct
2902		.operand = @intCast(u32, imm),
2903		});
	2937	// const payload = try self.addExtra(Mir.ImmPair{
	2938	// .dest_off = 0,
	2939	// // TODO check if this logic is correct
	2940	// .operand = @intCast(u32, imm),
	2941	// });
2904	2942	const flags: u2 = switch (abi_size) {
2905	2943	1 => 0b00,
2906	2944	2 => 0b01,
...	...	@@ -2919,14 +2957,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2919	2957	return self.fail("TODO imm64 would get incorrectly sign extended", .{});
2920	2958	}
2921	2959	}
2922		_ = try self.addInst(.{
2923		.tag = .mov_mem_imm,
2924		.ops = Mir.Inst.Ops.encode(.{
2925		.reg1 = addr_reg.to64(),
2926		.flags = flags,
2927		}),
2928		.data = .{ .payload = payload },
2929		});
	2960	// _ = try self.addInst(.{
	2961	// .tag = .mov_mem_imm,
	2962	// .ops = Mir.Inst.Ops.encode(.{
	2963	// .reg1 = addr_reg.to64(),
	2964	// .flags = flags,
	2965	// }),
	2966	// .data = .{ .payload = payload },
	2967	// });
2930	2968	},
2931	2969	.register => {
2932	2970	return self.store(new_ptr, value, ptr_ty, value_ty);
...	...	@@ -2939,15 +2977,15 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2939	2977
2940	2978	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
2941	2979
2942		_ = try self.addInst(.{
2943		.tag = .mov,
2944		.ops = Mir.Inst.Ops.encode(.{
2945		.reg1 = tmp_reg,
2946		.reg2 = tmp_reg,
2947		.flags = 0b01,
2948		}),
2949		.data = .{ .disp = 0 },
2950		});
	2980	// _ = try self.addInst(.{
	2981	// .tag = .mov,
	2982	// .ops = Mir.Inst.Ops.encode(.{
	2983	// .reg1 = tmp_reg,
	2984	// .reg2 = tmp_reg,
	2985	// .flags = 0b01,
	2986	// }),
	2987	// .data = .{ .disp = 0 },
	2988	// });
2951	2989	return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2952	2990	}
2953	2991
...	...	@@ -3109,14 +3147,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3109	3147	};
3110	3148	const field_size = @intCast(u32, struct_field_ty.abiSize(self.target.*));
3111	3149	if (signedness == .signed and field_size < 8) {
3112		_ = try self.addInst(.{
3113		.tag = .mov_sign_extend,
3114		.ops = Mir.Inst.Ops.encode(.{
3115		.reg1 = dst_mcv.register,
3116		.reg2 = registerAlias(dst_mcv.register, field_size),
3117		}),
3118		.data = undefined,
3119		});
	3150	// _ = try self.addInst(.{
	3151	// .tag = .mov_sign_extend,
	3152	// .ops = Mir.Inst.Ops.encode(.{
	3153	// .reg1 = dst_mcv.register,
	3154	// .reg2 = registerAlias(dst_mcv.register, field_size),
	3155	// }),
	3156	// .data = undefined,
	3157	// });
3120	3158	}
3121	3159
3122	3160	break :result dst_mcv;
...	...	@@ -3133,13 +3171,13 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3133	3171	defer self.register_manager.unlockReg(reg_lock);
3134	3172
3135	3173	const dst_reg = try self.register_manager.allocReg(inst, gp);
3136		_ = try self.addInst(.{
3137		.tag = .cond_set_byte,
3138		.ops = Mir.Inst.Ops.encode(.{
3139		.reg1 = dst_reg.to8(),
3140		}),
3141		.data = .{ .cc = ro.eflags },
3142		});
	3174	// _ = try self.addInst(.{
	3175	// .tag = .cond_set_byte,
	3176	// .ops = Mir.Inst.Ops.encode(.{
	3177	// .reg1 = dst_reg.to8(),
	3178	// }),
	3179	// .data = .{ .cc = ro.eflags },
	3180	// });
3143	3181	break :result MCValue{ .register = dst_reg.to8() };
3144	3182	},
3145	3183	else => unreachable,
...	...	@@ -3176,22 +3214,22 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3176	3214	.immediate => \|imm\| switch (imm) {
3177	3215	0 => return,
3178	3216	1 => {
3179		_ = try self.addInst(.{
3180		.tag = tag,
3181		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
3182		.data = undefined,
3183		});
	3217	// _ = try self.addInst(.{
	3218	// .tag = tag,
	3219	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
	3220	// .data = undefined,
	3221	// });
3184	3222	return;
3185	3223	},
3186	3224	else => {
3187		_ = try self.addInst(.{
3188		.tag = tag,
3189		.ops = Mir.Inst.Ops.encode(.{
3190		.reg1 = registerAlias(reg, abi_size),
3191		.flags = 0b10,
3192		}),
3193		.data = .{ .imm = @intCast(u8, imm) },
3194		});
	3225	// _ = try self.addInst(.{
	3226	// .tag = tag,
	3227	// .ops = Mir.Inst.Ops.encode(.{
	3228	// .reg1 = registerAlias(reg, abi_size),
	3229	// .flags = 0b10,
	3230	// }),
	3231	// .data = .{ .imm = @intCast(u8, imm) },
	3232	// });
3195	3233	return;
3196	3234	},
3197	3235	},
...	...	@@ -3204,15 +3242,16 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3204	3242	try self.register_manager.getReg(.rcx, null);
3205	3243	try self.genSetReg(Type.u8, .rcx, shift);
3206	3244	}
	3245	_ = abi_size;
3207	3246
3208		_ = try self.addInst(.{
3209		.tag = tag,
3210		.ops = Mir.Inst.Ops.encode(.{
3211		.reg1 = registerAlias(reg, abi_size),
3212		.flags = 0b01,
3213		}),
3214		.data = undefined,
3215		});
	3247	// _ = try self.addInst(.{
	3248	// .tag = tag,
	3249	// .ops = Mir.Inst.Ops.encode(.{
	3250	// .reg1 = registerAlias(reg, abi_size),
	3251	// .flags = 0b01,
	3252	// }),
	3253	// .data = undefined,
	3254	// });
3216	3255	}
3217	3256
3218	3257	/// Result is always a register.
...	...	@@ -3583,49 +3622,51 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3583	3622	.register => \|src_reg\| switch (dst_ty.zigTypeTag()) {
3584	3623	.Float => {
3585	3624	if (intrinsicsAllowed(self.target.*, dst_ty)) {
3586		const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3587		.f32 => switch (mir_tag) {
3588		.add => Mir.Inst.Tag.add_f32,
3589		.cmp => Mir.Inst.Tag.cmp_f32,
3590		else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
3591		},
3592		.f64 => switch (mir_tag) {
3593		.add => Mir.Inst.Tag.add_f64,
3594		.cmp => Mir.Inst.Tag.cmp_f64,
3595		else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
3596		},
3597		else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
3598		};
3599		_ = try self.addInst(.{
3600		.tag = actual_tag,
3601		.ops = Mir.Inst.Ops.encode(.{
3602		.reg1 = dst_reg.to128(),
3603		.reg2 = src_reg.to128(),
3604		}),
3605		.data = undefined,
3606		});
	3625	// const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
	3626	// .f32 => switch (mir_tag) {
	3627	// .add => Mir.Inst.Tag.add_f32,
	3628	// .cmp => Mir.Inst.Tag.cmp_f32,
	3629	// else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
	3630	// },
	3631	// .f64 => switch (mir_tag) {
	3632	// .add => Mir.Inst.Tag.add_f64,
	3633	// .cmp => Mir.Inst.Tag.cmp_f64,
	3634	// else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
	3635	// },
	3636	// else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
	3637	// };
	3638	// _ = try self.addInst(.{
	3639	// .tag = actual_tag,
	3640	// .ops = Mir.Inst.Ops.encode(.{
	3641	// .reg1 = dst_reg.to128(),
	3642	// .reg2 = src_reg.to128(),
	3643	// }),
	3644	// .data = undefined,
	3645	// });
3607	3646	return;
3608	3647	}
3609	3648
3610	3649	return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
3611	3650	},
3612	3651	else => {
3613		_ = try self.addInst(.{
3614		.tag = mir_tag,
3615		.ops = Mir.Inst.Ops.encode(.{
3616		.reg1 = registerAlias(dst_reg, abi_size),
3617		.reg2 = registerAlias(src_reg, abi_size),
3618		}),
3619		.data = undefined,
3620		});
	3652	_ = src_reg;
	3653	// _ = try self.addInst(.{
	3654	// .tag = mir_tag,
	3655	// .ops = Mir.Inst.Ops.encode(.{
	3656	// .reg1 = registerAlias(dst_reg, abi_size),
	3657	// .reg2 = registerAlias(src_reg, abi_size),
	3658	// }),
	3659	// .data = undefined,
	3660	// });
3621	3661	},
3622	3662	},
3623	3663	.immediate => \|imm\| {
3624		_ = try self.addInst(.{
3625		.tag = mir_tag,
3626		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),
3627		.data = .{ .imm = @intCast(u32, imm) },
3628		});
	3664	_ = imm;
	3665	// _ = try self.addInst(.{
	3666	// .tag = mir_tag,
	3667	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),
	3668	// .data = .{ .imm = @intCast(u32, imm) },
	3669	// });
3629	3670	},
3630	3671	.memory,
3631	3672	.linker_load,
...	...	@@ -3642,15 +3683,15 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3642	3683	if (off > math.maxInt(i32)) {
3643	3684	return self.fail("stack offset too large", .{});
3644	3685	}
3645		_ = try self.addInst(.{
3646		.tag = mir_tag,
3647		.ops = Mir.Inst.Ops.encode(.{
3648		.reg1 = registerAlias(dst_reg, abi_size),
3649		.reg2 = .rbp,
3650		.flags = 0b01,
3651		}),
3652		.data = .{ .disp = -off },
3653		});
	3686	// _ = try self.addInst(.{
	3687	// .tag = mir_tag,
	3688	// .ops = Mir.Inst.Ops.encode(.{
	3689	// .reg1 = registerAlias(dst_reg, abi_size),
	3690	// .reg2 = .rbp,
	3691	// .flags = 0b01,
	3692	// }),
	3693	// .data = .{ .disp = -off },
	3694	// });
3654	3695	},
3655	3696	}
3656	3697	},
...	...	@@ -3668,26 +3709,28 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3668	3709	.dead, .unreach => unreachable,
3669	3710	.register_overflow => unreachable,
3670	3711	.register => \|src_reg\| {
3671		_ = try self.addInst(.{
3672		.tag = mir_tag,
3673		.ops = Mir.Inst.Ops.encode(.{
3674		.reg1 = .rbp,
3675		.reg2 = registerAlias(src_reg, abi_size),
3676		.flags = 0b10,
3677		}),
3678		.data = .{ .disp = -off },
3679		});
	3712	_ = src_reg;
	3713	// _ = try self.addInst(.{
	3714	// .tag = mir_tag,
	3715	// .ops = Mir.Inst.Ops.encode(.{
	3716	// .reg1 = .rbp,
	3717	// .reg2 = registerAlias(src_reg, abi_size),
	3718	// .flags = 0b10,
	3719	// }),
	3720	// .data = .{ .disp = -off },
	3721	// });
3680	3722	},
3681	3723	.immediate => \|imm\| {
3682		const tag: Mir.Inst.Tag = switch (mir_tag) {
3683		.add => .add_mem_imm,
3684		.@"or" => .or_mem_imm,
3685		.@"and" => .and_mem_imm,
3686		.sub => .sub_mem_imm,
3687		.xor => .xor_mem_imm,
3688		.cmp => .cmp_mem_imm,
3689		else => unreachable,
3690		};
	3724	_ = imm;
	3725	// const tag: Mir.Inst.Tag = switch (mir_tag) {
	3726	// .add => .add_mem_imm,
	3727	// .@"or" => .or_mem_imm,
	3728	// .@"and" => .and_mem_imm,
	3729	// .sub => .sub_mem_imm,
	3730	// .xor => .xor_mem_imm,
	3731	// .cmp => .cmp_mem_imm,
	3732	// else => unreachable,
	3733	// };
3691	3734	const flags: u2 = switch (abi_size) {
3692	3735	1 => 0b00,
3693	3736	2 => 0b01,
...	...	@@ -3695,18 +3738,19 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3695	3738	8 => 0b11,
3696	3739	else => unreachable,
3697	3740	};
3698		const payload = try self.addExtra(Mir.ImmPair{
3699		.dest_off = -off,
3700		.operand = @intCast(u32, imm),
3701		});
3702		_ = try self.addInst(.{
3703		.tag = tag,
3704		.ops = Mir.Inst.Ops.encode(.{
3705		.reg1 = .rbp,
3706		.flags = flags,
3707		}),
3708		.data = .{ .payload = payload },
3709		});
	3741	// const payload = try self.addExtra(Mir.ImmPair{
	3742	// .dest_off = -off,
	3743	// .operand = @intCast(u32, imm),
	3744	// });
	3745	_ = flags;
	3746	// _ = try self.addInst(.{
	3747	// .tag = tag,
	3748	// .ops = Mir.Inst.Ops.encode(.{
	3749	// .reg1 = .rbp,
	3750	// .flags = flags,
	3751	// }),
	3752	// .data = .{ .payload = payload },
	3753	// });
3710	3754	},
3711	3755	.memory,
3712	3756	.stack_offset,
...	...	@@ -3735,6 +3779,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3735	3779	/// Does not support byte-size operands.
3736	3780	fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3737	3781	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	3782	_ = abi_size;
3738	3783	switch (dst_mcv) {
3739	3784	.none => unreachable,
3740	3785	.undef => unreachable,
...	...	@@ -3750,29 +3795,30 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3750	3795	.ptr_stack_offset => unreachable,
3751	3796	.register_overflow => unreachable,
3752	3797	.register => \|src_reg\| {
	3798	_ = src_reg;
3753	3799	// register, register
3754		_ = try self.addInst(.{
3755		.tag = .imul_complex,
3756		.ops = Mir.Inst.Ops.encode(.{
3757		.reg1 = registerAlias(dst_reg, abi_size),
3758		.reg2 = registerAlias(src_reg, abi_size),
3759		}),
3760		.data = undefined,
3761		});
	3800	// _ = try self.addInst(.{
	3801	// .tag = .imul_complex,
	3802	// .ops = Mir.Inst.Ops.encode(.{
	3803	// .reg1 = registerAlias(dst_reg, abi_size),
	3804	// .reg2 = registerAlias(src_reg, abi_size),
	3805	// }),
	3806	// .data = undefined,
	3807	// });
3762	3808	},
3763	3809	.immediate => \|imm\| {
3764	3810	// TODO take into account the type's ABI size when selecting the register alias
3765	3811	// register, immediate
3766	3812	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
3767		_ = try self.addInst(.{
3768		.tag = .imul_complex,
3769		.ops = Mir.Inst.Ops.encode(.{
3770		.reg1 = dst_reg.to32(),
3771		.reg2 = dst_reg.to32(),
3772		.flags = 0b10,
3773		}),
3774		.data = .{ .imm = @intCast(u32, imm) },
3775		});
	3813	// _ = try self.addInst(.{
	3814	// .tag = .imul_complex,
	3815	// .ops = Mir.Inst.Ops.encode(.{
	3816	// .reg1 = dst_reg.to32(),
	3817	// .reg2 = dst_reg.to32(),
	3818	// .flags = 0b10,
	3819	// }),
	3820	// .data = .{ .imm = @intCast(u32, imm) },
	3821	// });
3776	3822	} else {
3777	3823	// TODO verify we don't spill and assign to the same register as dst_mcv
3778	3824	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
...	...	@@ -3780,15 +3826,16 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3780	3826	}
3781	3827	},
3782	3828	.stack_offset => \|off\| {
3783		_ = try self.addInst(.{
3784		.tag = .imul_complex,
3785		.ops = Mir.Inst.Ops.encode(.{
3786		.reg1 = registerAlias(dst_reg, abi_size),
3787		.reg2 = .rbp,
3788		.flags = 0b01,
3789		}),
3790		.data = .{ .disp = -off },
3791		});
	3829	_ = off;
	3830	// _ = try self.addInst(.{
	3831	// .tag = .imul_complex,
	3832	// .ops = Mir.Inst.Ops.encode(.{
	3833	// .reg1 = registerAlias(dst_reg, abi_size),
	3834	// .reg2 = .rbp,
	3835	// .flags = 0b01,
	3836	// }),
	3837	// .data = .{ .disp = -off },
	3838	// });
3792	3839	},
3793	3840	.memory => {
3794	3841	return self.fail("TODO implement x86 multiply source memory", .{});
...	...	@@ -3811,16 +3858,17 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3811	3858	.register => \|src_reg\| {
3812	3859	// copy dst to a register
3813	3860	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
	3861	_ = src_reg;
3814	3862	// multiply into dst_reg
3815	3863	// register, register
3816		_ = try self.addInst(.{
3817		.tag = .imul_complex,
3818		.ops = Mir.Inst.Ops.encode(.{
3819		.reg1 = registerAlias(dst_reg, abi_size),
3820		.reg2 = registerAlias(src_reg, abi_size),
3821		}),
3822		.data = undefined,
3823		});
	3864	// _ = try self.addInst(.{
	3865	// .tag = .imul_complex,
	3866	// .ops = Mir.Inst.Ops.encode(.{
	3867	// .reg1 = registerAlias(dst_reg, abi_size),
	3868	// .reg2 = registerAlias(src_reg, abi_size),
	3869	// }),
	3870	// .data = undefined,
	3871	// });
3824	3872	// copy dst_reg back out
3825	3873	return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
3826	3874	},
...	...	@@ -3946,20 +3994,20 @@ fn genVarDbgInfo(
3946	3994	}
3947	3995
3948	3996	fn airTrap(self: *Self) !void {
3949		_ = try self.addInst(.{
3950		.tag = .ud,
3951		.ops = Mir.Inst.Ops.encode(.{}),
3952		.data = undefined,
3953		});
	3997	// _ = try self.addInst(.{
	3998	// .tag = .ud,
	3999	// .ops = Mir.Inst.Ops.encode(.{}),
	4000	// .data = undefined,
	4001	// });
3954	4002	return self.finishAirBookkeeping();
3955	4003	}
3956	4004
3957	4005	fn airBreakpoint(self: *Self) !void {
3958		_ = try self.addInst(.{
3959		.tag = .interrupt,
3960		.ops = Mir.Inst.Ops.encode(.{}),
3961		.data = undefined,
3962		});
	4006	// _ = try self.addInst(.{
	4007	// .tag = .interrupt,
	4008	// .ops = Mir.Inst.Ops.encode(.{}),
	4009	// .data = undefined,
	4010	// });
3963	4011	return self.finishAirBookkeeping();
3964	4012	}
3965	4013
...	...	@@ -4054,11 +4102,11 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4054	4102
4055	4103	if (info.stack_byte_count > 0) {
4056	4104	// Adjust the stack
4057		_ = try self.addInst(.{
4058		.tag = .sub,
4059		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4060		.data = .{ .imm = info.stack_byte_count },
4061		});
	4105	// _ = try self.addInst(.{
	4106	// .tag = .sub,
	4107	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
	4108	// .data = .{ .imm = info.stack_byte_count },
	4109	// });
4062	4110	}
4063	4111
4064	4112	// Due to incremental compilation, how function calls are generated depends
...	...	@@ -4072,11 +4120,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4072	4120	const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
4073	4121	const atom = elf_file.getAtom(atom_index);
4074	4122	const got_addr = @intCast(i32, atom.getOffsetTableAddress(elf_file));
4075		_ = try self.addInst(.{
4076		.tag = .call,
4077		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4078		.data = .{ .disp = got_addr },
4079		});
	4123	_ = got_addr;
	4124	// _ = try self.addInst(.{
	4125	// .tag = .call,
	4126	// .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
	4127	// .data = .{ .disp = got_addr },
	4128	// });
4080	4129	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
4081	4130	const atom_index = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
4082	4131	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
...	...	@@ -4086,14 +4135,14 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4086	4135	.sym_index = sym_index,
4087	4136	},
4088	4137	});
4089		_ = try self.addInst(.{
4090		.tag = .call,
4091		.ops = Mir.Inst.Ops.encode(.{
4092		.reg1 = .rax,
4093		.flags = 0b01,
4094		}),
4095		.data = undefined,
4096		});
	4138	// _ = try self.addInst(.{
	4139	// .tag = .call,
	4140	// .ops = Mir.Inst.Ops.encode(.{
	4141	// .reg1 = .rax,
	4142	// .flags = 0b01,
	4143	// }),
	4144	// .data = undefined,
	4145	// });
4097	4146	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4098	4147	const atom_index = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
4099	4148	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
...	...	@@ -4103,14 +4152,14 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4103	4152	.sym_index = sym_index,
4104	4153	},
4105	4154	});
4106		_ = try self.addInst(.{
4107		.tag = .call,
4108		.ops = Mir.Inst.Ops.encode(.{
4109		.reg1 = .rax,
4110		.flags = 0b01,
4111		}),
4112		.data = undefined,
4113		});
	4155	// _ = try self.addInst(.{
	4156	// .tag = .call,
	4157	// .ops = Mir.Inst.Ops.encode(.{
	4158	// .reg1 = .rax,
	4159	// .flags = 0b01,
	4160	// }),
	4161	// .data = undefined,
	4162	// });
4114	4163	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
4115	4164	const decl_block_index = try p9.seeDecl(func.owner_decl);
4116	4165	const decl_block = p9.getDeclBlock(decl_block_index);
...	...	@@ -4119,11 +4168,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4119	4168	const got_addr = p9.bases.data;
4120	4169	const got_index = decl_block.got_index.?;
4121	4170	const fn_got_addr = got_addr + got_index * ptr_bytes;
4122		_ = try self.addInst(.{
4123		.tag = .call,
4124		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
4125		.data = .{ .disp = @intCast(i32, fn_got_addr) },
4126		});
	4171	_ = fn_got_addr;
	4172	// _ = try self.addInst(.{
	4173	// .tag = .call,
	4174	// .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),
	4175	// .data = .{ .disp = @intCast(i32, fn_got_addr) },
	4176	// });
4127	4177	} else unreachable;
4128	4178	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
4129	4179	const extern_fn = func_payload.data;
...	...	@@ -4143,26 +4193,28 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4143	4193	.sym_index = sym_index,
4144	4194	},
4145	4195	});
4146		_ = try self.addInst(.{
4147		.tag = .call,
4148		.ops = Mir.Inst.Ops.encode(.{
4149		.reg1 = .rax,
4150		.flags = 0b01,
4151		}),
4152		.data = undefined,
4153		});
	4196	// _ = try self.addInst(.{
	4197	// .tag = .call,
	4198	// .ops = Mir.Inst.Ops.encode(.{
	4199	// .reg1 = .rax,
	4200	// .flags = 0b01,
	4201	// }),
	4202	// .data = undefined,
	4203	// });
4154	4204	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4155	4205	const sym_index = try macho_file.getGlobalSymbol(mem.sliceTo(decl_name, 0));
4156	4206	const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);
4157	4207	const atom_index = macho_file.getAtom(atom).getSymbolIndex().?;
4158		_ = try self.addInst(.{
4159		.tag = .call_extern,
4160		.ops = undefined,
4161		.data = .{ .relocation = .{
4162		.atom_index = atom_index,
4163		.sym_index = sym_index,
4164		} },
4165		});
	4208	_ = sym_index;
	4209	_ = atom_index;
	4210	// _ = try self.addInst(.{
	4211	// .tag = .call_extern,
	4212	// .ops = undefined,
	4213	// .data = .{ .relocation = .{
	4214	// .atom_index = atom_index,
	4215	// .sym_index = sym_index,
	4216	// } },
	4217	// });
4166	4218	} else {
4167	4219	return self.fail("TODO implement calling extern functions", .{});
4168	4220	}
...	...	@@ -4173,23 +4225,23 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4173	4225	assert(ty.zigTypeTag() == .Pointer);
4174	4226	const mcv = try self.resolveInst(callee);
4175	4227	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4176		_ = try self.addInst(.{
4177		.tag = .call,
4178		.ops = Mir.Inst.Ops.encode(.{
4179		.reg1 = .rax,
4180		.flags = 0b01,
4181		}),
4182		.data = undefined,
4183		});
	4228	// _ = try self.addInst(.{
	4229	// .tag = .call,
	4230	// .ops = Mir.Inst.Ops.encode(.{
	4231	// .reg1 = .rax,
	4232	// .flags = 0b01,
	4233	// }),
	4234	// .data = undefined,
	4235	// });
4184	4236	}
4185	4237
4186	4238	if (info.stack_byte_count > 0) {
4187	4239	// Readjust the stack
4188		_ = try self.addInst(.{
4189		.tag = .add,
4190		.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4191		.data = .{ .imm = info.stack_byte_count },
4192		});
	4240	// _ = try self.addInst(.{
	4241	// .tag = .add,
	4242	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
	4243	// .data = .{ .imm = info.stack_byte_count },
	4244	// });
4193	4245	}
4194	4246
4195	4247	const result: MCValue = result: {
...	...	@@ -4246,12 +4298,12 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
4246	4298	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
4247	4299	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
4248	4300	// which is available if the jump is 127 bytes or less forward.
4249		const jmp_reloc = try self.addInst(.{
4250		.tag = .jmp,
4251		.ops = Mir.Inst.Ops.encode(.{}),
4252		.data = .{ .inst = undefined },
4253		});
4254		try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
	4301	// const jmp_reloc = try self.addInst(.{
	4302	// .tag = .jmp,
	4303	// .ops = Mir.Inst.Ops.encode(.{}),
	4304	// .data = .{ .inst = undefined },
	4305	// });
	4306	// try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
4255	4307	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4256	4308	}
4257	4309
...	...	@@ -4282,12 +4334,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
4282	4334	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
4283	4335	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
4284	4336	// which is available if the jump is 127 bytes or less forward.
4285		const jmp_reloc = try self.addInst(.{
4286		.tag = .jmp,
4287		.ops = Mir.Inst.Ops.encode(.{}),
4288		.data = .{ .inst = undefined },
4289		});
4290		try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
	4337	// const jmp_reloc = try self.addInst(.{
	4338	// .tag = .jmp,
	4339	// .ops = Mir.Inst.Ops.encode(.{}),
	4340	// .data = .{ .inst = undefined },
	4341	// });
	4342	// try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
4291	4343	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4292	4344	}
4293	4345
...	...	@@ -4461,33 +4513,35 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4461	4513	const abi_size = ty.abiSize(self.target.*);
4462	4514	switch (mcv) {
4463	4515	.eflags => \|cc\| {
4464		return self.addInst(.{
4465		.tag = .cond_jmp,
4466		.ops = Mir.Inst.Ops.encode(.{}),
4467		.data = .{
4468		.inst_cc = .{
4469		.inst = undefined,
4470		// Here we map the opposites since the jump is to the false branch.
4471		.cc = cc.negate(),
4472		},
4473		},
4474		});
	4516	_ = cc;
	4517	// return self.addInst(.{
	4518	// .tag = .cond_jmp,
	4519	// .ops = Mir.Inst.Ops.encode(.{}),
	4520	// .data = .{
	4521	// .inst_cc = .{
	4522	// .inst = undefined,
	4523	// // Here we map the opposites since the jump is to the false branch.
	4524	// .cc = cc.negate(),
	4525	// },
	4526	// },
	4527	// });
4475	4528	},
4476	4529	.register => \|reg\| {
	4530	_ = reg;
4477	4531	try self.spillEflagsIfOccupied();
4478		_ = try self.addInst(.{
4479		.tag = .@"test",
4480		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
4481		.data = .{ .imm = 1 },
4482		});
4483		return self.addInst(.{
4484		.tag = .cond_jmp,
4485		.ops = Mir.Inst.Ops.encode(.{}),
4486		.data = .{ .inst_cc = .{
4487		.inst = undefined,
4488		.cc = .e,
4489		} },
4490		});
	4532	// _ = try self.addInst(.{
	4533	// .tag = .@"test",
	4534	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
	4535	// .data = .{ .imm = 1 },
	4536	// });
	4537	// return self.addInst(.{
	4538	// .tag = .cond_jmp,
	4539	// .ops = Mir.Inst.Ops.encode(.{}),
	4540	// .data = .{ .inst_cc = .{
	4541	// .inst = undefined,
	4542	// .cc = .e,
	4543	// } },
	4544	// });
4491	4545	},
4492	4546	.immediate,
4493	4547	.stack_offset,
...	...	@@ -4501,6 +4555,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4501	4555	},
4502	4556	else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}),
4503	4557	}
	4558	return 0; // TODO
4504	4559	}
4505	4560
4506	4561	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -4825,12 +4880,13 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4825	4880	const loop = self.air.extraData(Air.Block, ty_pl.payload);
4826	4881	const body = self.air.extra[loop.end..][0..loop.data.body_len];
4827	4882	const jmp_target = @intCast(u32, self.mir_instructions.len);
	4883	_ = jmp_target;
4828	4884	try self.genBody(body);
4829		_ = try self.addInst(.{
4830		.tag = .jmp,
4831		.ops = Mir.Inst.Ops.encode(.{}),
4832		.data = .{ .inst = jmp_target },
4833		});
	4885	// _ = try self.addInst(.{
	4886	// .tag = .jmp,
	4887	// .ops = Mir.Inst.Ops.encode(.{}),
	4888	// .data = .{ .inst = jmp_target },
	4889	// });
4834	4890	return self.finishAirBookkeeping();
4835	4891	}
4836	4892
...	...	@@ -4876,21 +4932,23 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4876	4932	.undef => unreachable,
4877	4933	.dead, .unreach => unreachable,
4878	4934	.immediate => \|imm\| {
4879		_ = try self.addInst(.{
4880		.tag = .xor,
4881		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),
4882		.data = .{ .imm = @intCast(u32, imm) },
4883		});
	4935	_ = imm;
	4936	// _ = try self.addInst(.{
	4937	// .tag = .xor,
	4938	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),
	4939	// .data = .{ .imm = @intCast(u32, imm) },
	4940	// });
4884	4941	},
4885	4942	.register => \|reg\| {
4886		_ = try self.addInst(.{
4887		.tag = .xor,
4888		.ops = Mir.Inst.Ops.encode(.{
4889		.reg1 = registerAlias(cond_reg, abi_size),
4890		.reg2 = registerAlias(reg, abi_size),
4891		}),
4892		.data = undefined,
4893		});
	4943	_ = reg;
	4944	// _ = try self.addInst(.{
	4945	// .tag = .xor,
	4946	// .ops = Mir.Inst.Ops.encode(.{
	4947	// .reg1 = registerAlias(cond_reg, abi_size),
	4948	// .reg2 = registerAlias(reg, abi_size),
	4949	// }),
	4950	// .data = undefined,
	4951	// });
4894	4952	},
4895	4953	.stack_offset => {
4896	4954	if (abi_size <= 8) {
...	...	@@ -4905,22 +4963,22 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4905	4963	},
4906	4964	}
4907	4965
4908		_ = try self.addInst(.{
4909		.tag = .@"test",
4910		.ops = Mir.Inst.Ops.encode(.{
4911		.reg1 = registerAlias(cond_reg, abi_size),
4912		.reg2 = registerAlias(cond_reg, abi_size),
4913		}),
4914		.data = undefined,
4915		});
4916		return self.addInst(.{
4917		.tag = .cond_jmp,
4918		.ops = Mir.Inst.Ops.encode(.{}),
4919		.data = .{ .inst_cc = .{
4920		.inst = undefined,
4921		.cc = .ne,
4922		} },
4923		});
	4966	// _ = try self.addInst(.{
	4967	// .tag = .@"test",
	4968	// .ops = Mir.Inst.Ops.encode(.{
	4969	// .reg1 = registerAlias(cond_reg, abi_size),
	4970	// .reg2 = registerAlias(cond_reg, abi_size),
	4971	// }),
	4972	// .data = undefined,
	4973	// });
	4974	// return self.addInst(.{
	4975	// .tag = .cond_jmp,
	4976	// .ops = Mir.Inst.Ops.encode(.{}),
	4977	// .data = .{ .inst_cc = .{
	4978	// .inst = undefined,
	4979	// .cc = .ne,
	4980	// } },
	4981	// });
4924	4982	},
4925	4983	.stack_offset => {
4926	4984	try self.spillEflagsIfOccupied();
...	...	@@ -4938,6 +4996,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4938	4996	return self.fail("TODO implemenent switch mir when condition is {}", .{condition});
4939	4997	},
4940	4998	}
	4999	return 0; // TODO
4941	5000	}
4942	5001
4943	5002	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5132,9 +5191,9 @@ fn canonicaliseBranches(self: Self, parent_branch: Branch, canon_branch: *Bran
5132	5191	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
5133	5192	const next_inst = @intCast(u32, self.mir_instructions.len);
5134	5193	switch (self.mir_instructions.items(.tag)[reloc]) {
5135		.cond_jmp => {
5136		self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
5137		},
	5194	// .cond_jmp => {
	5195	// self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
	5196	// },
5138	5197	.jmp => {
5139	5198	self.mir_instructions.items(.data)[reloc].inst = next_inst;
5140	5199	},
...	...	@@ -5177,12 +5236,12 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5177	5236	// Emit a jump with a relocation. It will be patched up after the block ends.
5178	5237	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
5179	5238	// Leave the jump offset undefined
5180		const jmp_reloc = try self.addInst(.{
5181		.tag = .jmp,
5182		.ops = Mir.Inst.Ops.encode(.{}),
5183		.data = .{ .inst = undefined },
5184		});
5185		block_data.relocs.appendAssumeCapacity(jmp_reloc);
	5239	// const jmp_reloc = try self.addInst(.{
	5240	// .tag = .jmp,
	5241	// .ops = Mir.Inst.Ops.encode(.{}),
	5242	// .data = .{ .inst = undefined },
	5243	// });
	5244	// block_data.relocs.appendAssumeCapacity(jmp_reloc);
5186	5245	}
5187	5246
5188	5247	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -5254,30 +5313,22 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5254	5313	var iter = std.mem.tokenize(u8, asm_source, "\n\r");
5255	5314	while (iter.next()) \|ins\| {
5256	5315	if (mem.eql(u8, ins, "syscall")) {
5257		_ = try self.addInst(.{
5258		.tag = .syscall,
5259		.ops = undefined,
5260		.data = undefined,
5261		});
	5316	try self.assemble(.syscall, .{});
5262	5317	} else if (mem.indexOf(u8, ins, "push")) \|_\| {
5263	5318	const arg = ins[4..];
5264	5319	if (mem.indexOf(u8, arg, "$")) \|l\| {
5265	5320	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch {
5266	5321	return self.fail("TODO implement more inline asm int parsing", .{});
5267	5322	};
5268		_ = try self.addInst(.{
5269		.tag = .push,
5270		.ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }),
5271		.data = .{ .imm = n },
	5323	try self.assemble(.push, .{
	5324	.op1 = .{ .imm = Mir.Operand.Immediate.u(n) },
5272	5325	});
5273	5326	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
5274	5327	const reg_name = ins[4 + l + 2 ..];
5275	5328	const reg = parseRegName(reg_name) orelse
5276	5329	return self.fail("unrecognized register: '{s}'", .{reg_name});
5277		_ = try self.addInst(.{
5278		.tag = .push,
5279		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5280		.data = undefined,
	5330	try self.assemble(.push, .{
	5331	.op1 = .{ .reg = reg },
5281	5332	});
5282	5333	} else return self.fail("TODO more push operands", .{});
5283	5334	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
...	...	@@ -5286,10 +5337,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5286	5337	const reg_name = ins[3 + l + 2 ..];
5287	5338	const reg = parseRegName(reg_name) orelse
5288	5339	return self.fail("unrecognized register: '{s}'", .{reg_name});
5289		_ = try self.addInst(.{
5290		.tag = .pop,
5291		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5292		.data = undefined,
	5340	try self.assemble(.pop, .{
	5341	.op1 = .{ .reg = reg },
5293	5342	});
5294	5343	} else return self.fail("TODO more pop operands", .{});
5295	5344	} else {
...	...	@@ -5433,39 +5482,40 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5433	5482	switch (ty.zigTypeTag()) {
5434	5483	.Float => {
5435	5484	if (intrinsicsAllowed(self.target.*, ty)) {
5436		const tag: Mir.Inst.Tag = switch (ty.tag()) {
5437		.f32 => Mir.Inst.Tag.mov_f32,
5438		.f64 => Mir.Inst.Tag.mov_f64,
5439		else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
5440		};
5441		_ = try self.addInst(.{
5442		.tag = tag,
5443		.ops = Mir.Inst.Ops.encode(.{
5444		.reg1 = switch (ty.tag()) {
5445		.f32 => .esp,
5446		.f64 => .rsp,
5447		else => unreachable,
5448		},
5449		.reg2 = reg.to128(),
5450		.flags = 0b01,
5451		}),
5452		.data = .{ .disp = -stack_offset },
5453		});
	5485	// const tag: Mir.Inst.Tag = switch (ty.tag()) {
	5486	// .f32 => Mir.Inst.Tag.mov_f32,
	5487	// .f64 => Mir.Inst.Tag.mov_f64,
	5488	// else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
	5489	// };
	5490	_ = reg;
	5491	// _ = try self.addInst(.{
	5492	// .tag = tag,
	5493	// .ops = Mir.Inst.Ops.encode(.{
	5494	// .reg1 = switch (ty.tag()) {
	5495	// .f32 => .esp,
	5496	// .f64 => .rsp,
	5497	// else => unreachable,
	5498	// },
	5499	// .reg2 = reg.to128(),
	5500	// .flags = 0b01,
	5501	// }),
	5502	// .data = .{ .disp = -stack_offset },
	5503	// });
5454	5504	return;
5455	5505	}
5456	5506
5457	5507	return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
5458	5508	},
5459	5509	else => {
5460		_ = try self.addInst(.{
5461		.tag = .mov,
5462		.ops = Mir.Inst.Ops.encode(.{
5463		.reg1 = .rsp,
5464		.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5465		.flags = 0b10,
5466		}),
5467		.data = .{ .disp = -stack_offset },
5468		});
	5510	// _ = try self.addInst(.{
	5511	// .tag = .mov,
	5512	// .ops = Mir.Inst.Ops.encode(.{
	5513	// .reg1 = .rsp,
	5514	// .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
	5515	// .flags = 0b10,
	5516	// }),
	5517	// .data = .{ .disp = -stack_offset },
	5518	// });
5469	5519	},
5470	5520	}
5471	5521	},
...	...	@@ -5519,13 +5569,13 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5519	5569	const overflow_bit_ty = ty.structFieldType(1);
5520	5570	const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
5521	5571	const tmp_reg = try self.register_manager.allocReg(null, gp);
5522		_ = try self.addInst(.{
5523		.tag = .cond_set_byte,
5524		.ops = Mir.Inst.Ops.encode(.{
5525		.reg1 = tmp_reg.to8(),
5526		}),
5527		.data = .{ .cc = ro.eflags },
5528		});
	5572	// _ = try self.addInst(.{
	5573	// .tag = .cond_set_byte,
	5574	// .ops = Mir.Inst.Ops.encode(.{
	5575	// .reg1 = tmp_reg.to8(),
	5576	// }),
	5577	// .data = .{ .cc = ro.eflags },
	5578	// });
5529	5579
5530	5580	return self.genSetStack(
5531	5581	overflow_bit_ty,
...	...	@@ -5539,72 +5589,74 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5539	5589	return self.genSetStack(ty, stack_offset, .{ .register = reg }, opts);
5540	5590	},
5541	5591	.immediate => \|x_big\| {
	5592	_ = x_big;
5542	5593	const base_reg = opts.dest_stack_base orelse .rbp;
	5594	_ = base_reg;
5543	5595	switch (abi_size) {
5544	5596	0 => {
5545	5597	assert(ty.isError());
5546		const payload = try self.addExtra(Mir.ImmPair{
5547		.dest_off = -stack_offset,
5548		.operand = @truncate(u32, x_big),
5549		});
5550		_ = try self.addInst(.{
5551		.tag = .mov_mem_imm,
5552		.ops = Mir.Inst.Ops.encode(.{
5553		.reg1 = base_reg,
5554		.flags = 0b00,
5555		}),
5556		.data = .{ .payload = payload },
5557		});
	5598	// const payload = try self.addExtra(Mir.ImmPair{
	5599	// .dest_off = -stack_offset,
	5600	// .operand = @truncate(u32, x_big),
	5601	// });
	5602	// _ = try self.addInst(.{
	5603	// .tag = .mov_mem_imm,
	5604	// .ops = Mir.Inst.Ops.encode(.{
	5605	// .reg1 = base_reg,
	5606	// .flags = 0b00,
	5607	// }),
	5608	// .data = .{ .payload = payload },
	5609	// });
5558	5610	},
5559	5611	1, 2, 4 => {
5560		const payload = try self.addExtra(Mir.ImmPair{
5561		.dest_off = -stack_offset,
5562		.operand = @truncate(u32, x_big),
5563		});
5564		_ = try self.addInst(.{
5565		.tag = .mov_mem_imm,
5566		.ops = Mir.Inst.Ops.encode(.{
5567		.reg1 = base_reg,
5568		.flags = switch (abi_size) {
5569		1 => 0b00,
5570		2 => 0b01,
5571		4 => 0b10,
5572		else => unreachable,
5573		},
5574		}),
5575		.data = .{ .payload = payload },
5576		});
	5612	// const payload = try self.addExtra(Mir.ImmPair{
	5613	// .dest_off = -stack_offset,
	5614	// .operand = @truncate(u32, x_big),
	5615	// });
	5616	// _ = try self.addInst(.{
	5617	// .tag = .mov_mem_imm,
	5618	// .ops = Mir.Inst.Ops.encode(.{
	5619	// .reg1 = base_reg,
	5620	// .flags = switch (abi_size) {
	5621	// 1 => 0b00,
	5622	// 2 => 0b01,
	5623	// 4 => 0b10,
	5624	// else => unreachable,
	5625	// },
	5626	// }),
	5627	// .data = .{ .payload = payload },
	5628	// });
5577	5629	},
5578	5630	8 => {
5579	5631	// 64 bit write to memory would take two mov's anyways so we
5580	5632	// insted just use two 32 bit writes to avoid register allocation
5581	5633	{
5582		const payload = try self.addExtra(Mir.ImmPair{
5583		.dest_off = -stack_offset + 4,
5584		.operand = @truncate(u32, x_big >> 32),
5585		});
5586		_ = try self.addInst(.{
5587		.tag = .mov_mem_imm,
5588		.ops = Mir.Inst.Ops.encode(.{
5589		.reg1 = base_reg,
5590		.flags = 0b10,
5591		}),
5592		.data = .{ .payload = payload },
5593		});
	5634	// const payload = try self.addExtra(Mir.ImmPair{
	5635	// .dest_off = -stack_offset + 4,
	5636	// .operand = @truncate(u32, x_big >> 32),
	5637	// });
	5638	// _ = try self.addInst(.{
	5639	// .tag = .mov_mem_imm,
	5640	// .ops = Mir.Inst.Ops.encode(.{
	5641	// .reg1 = base_reg,
	5642	// .flags = 0b10,
	5643	// }),
	5644	// .data = .{ .payload = payload },
	5645	// });
5594	5646	}
5595	5647	{
5596		const payload = try self.addExtra(Mir.ImmPair{
5597		.dest_off = -stack_offset,
5598		.operand = @truncate(u32, x_big),
5599		});
5600		_ = try self.addInst(.{
5601		.tag = .mov_mem_imm,
5602		.ops = Mir.Inst.Ops.encode(.{
5603		.reg1 = base_reg,
5604		.flags = 0b10,
5605		}),
5606		.data = .{ .payload = payload },
5607		});
	5648	// const payload = try self.addExtra(Mir.ImmPair{
	5649	// .dest_off = -stack_offset,
	5650	// .operand = @truncate(u32, x_big),
	5651	// });
	5652	// _ = try self.addInst(.{
	5653	// .tag = .mov_mem_imm,
	5654	// .ops = Mir.Inst.Ops.encode(.{
	5655	// .reg1 = base_reg,
	5656	// .flags = 0b10,
	5657	// }),
	5658	// .data = .{ .payload = payload },
	5659	// });
5608	5660	}
5609	5661	},
5610	5662	else => {
...	...	@@ -5622,24 +5674,24 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5622	5674	switch (ty.zigTypeTag()) {
5623	5675	.Float => {
5624	5676	if (intrinsicsAllowed(self.target.*, ty)) {
5625		const tag: Mir.Inst.Tag = switch (ty.tag()) {
5626		.f32 => Mir.Inst.Tag.mov_f32,
5627		.f64 => Mir.Inst.Tag.mov_f64,
5628		else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
5629		};
5630		_ = try self.addInst(.{
5631		.tag = tag,
5632		.ops = Mir.Inst.Ops.encode(.{
5633		.reg1 = switch (ty.tag()) {
5634		.f32 => base_reg.to32(),
5635		.f64 => base_reg.to64(),
5636		else => unreachable,
5637		},
5638		.reg2 = reg.to128(),
5639		.flags = 0b01,
5640		}),
5641		.data = .{ .disp = -stack_offset },
5642		});
	5677	// const tag: Mir.Inst.Tag = switch (ty.tag()) {
	5678	// .f32 => Mir.Inst.Tag.mov_f32,
	5679	// .f64 => Mir.Inst.Tag.mov_f64,
	5680	// else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
	5681	// };
	5682	// _ = try self.addInst(.{
	5683	// .tag = tag,
	5684	// .ops = Mir.Inst.Ops.encode(.{
	5685	// .reg1 = switch (ty.tag()) {
	5686	// .f32 => base_reg.to32(),
	5687	// .f64 => base_reg.to64(),
	5688	// else => unreachable,
	5689	// },
	5690	// .reg2 = reg.to128(),
	5691	// .flags = 0b01,
	5692	// }),
	5693	// .data = .{ .disp = -stack_offset },
	5694	// });
5643	5695	return;
5644	5696	}
5645	5697
...	...	@@ -5706,15 +5758,15 @@ fn genInlineMemcpyRegisterRegister(
5706	5758	while (remainder > 0) {
5707	5759	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5708	5760
5709		_ = try self.addInst(.{
5710		.tag = .mov,
5711		.ops = Mir.Inst.Ops.encode(.{
5712		.reg1 = dst_reg,
5713		.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5714		.flags = 0b10,
5715		}),
5716		.data = .{ .disp = -next_offset },
5717		});
	5761	// _ = try self.addInst(.{
	5762	// .tag = .mov,
	5763	// .ops = Mir.Inst.Ops.encode(.{
	5764	// .reg1 = dst_reg,
	5765	// .reg2 = registerAlias(tmp_reg, nearest_power_of_two),
	5766	// .flags = 0b10,
	5767	// }),
	5768	// .data = .{ .disp = -next_offset },
	5769	// });
5718	5770
5719	5771	if (nearest_power_of_two > 1) {
5720	5772	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{
...	...	@@ -5726,15 +5778,15 @@ fn genInlineMemcpyRegisterRegister(
5726	5778	next_offset -= nearest_power_of_two;
5727	5779	}
5728	5780	} else {
5729		_ = try self.addInst(.{
5730		.tag = .mov,
5731		.ops = Mir.Inst.Ops.encode(.{
5732		.reg1 = dst_reg,
5733		.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
5734		.flags = 0b10,
5735		}),
5736		.data = .{ .disp = -offset },
5737		});
	5781	// _ = try self.addInst(.{
	5782	// .tag = .mov,
	5783	// .ops = Mir.Inst.Ops.encode(.{
	5784	// .reg1 = dst_reg,
	5785	// .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
	5786	// .flags = 0b10,
	5787	// }),
	5788	// .data = .{ .disp = -offset },
	5789	// });
5738	5790	}
5739	5791	}
5740	5792
...	...	@@ -5768,30 +5820,34 @@ fn genInlineMemcpy(
5768	5820	const index_reg = regs[2].to64();
5769	5821	const count_reg = regs[3].to64();
5770	5822	const tmp_reg = regs[4].to8();
	5823	_ = index_reg;
	5824	_ = tmp_reg;
5771	5825
5772	5826	switch (dst_ptr) {
5773	5827	.memory, .linker_load => {
5774	5828	try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
5775	5829	},
5776	5830	.ptr_stack_offset, .stack_offset => \|off\| {
5777		_ = try self.addInst(.{
5778		.tag = .lea,
5779		.ops = Mir.Inst.Ops.encode(.{
5780		.reg1 = dst_addr_reg.to64(),
5781		.reg2 = opts.dest_stack_base orelse .rbp,
5782		}),
5783		.data = .{ .disp = -off },
5784		});
	5831	_ = off;
	5832	// _ = try self.addInst(.{
	5833	// .tag = .lea,
	5834	// .ops = Mir.Inst.Ops.encode(.{
	5835	// .reg1 = dst_addr_reg.to64(),
	5836	// .reg2 = opts.dest_stack_base orelse .rbp,
	5837	// }),
	5838	// .data = .{ .disp = -off },
	5839	// });
5785	5840	},
5786	5841	.register => \|reg\| {
5787		_ = try self.addInst(.{
5788		.tag = .mov,
5789		.ops = Mir.Inst.Ops.encode(.{
5790		.reg1 = registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5791		.reg2 = reg,
5792		}),
5793		.data = undefined,
5794		});
	5842	_ = reg;
	5843	// _ = try self.addInst(.{
	5844	// .tag = .mov,
	5845	// .ops = Mir.Inst.Ops.encode(.{
	5846	// .reg1 = registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	5847	// .reg2 = reg,
	5848	// }),
	5849	// .data = undefined,
	5850	// });
5795	5851	},
5796	5852	else => {
5797	5853	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
...	...	@@ -5803,24 +5859,26 @@ fn genInlineMemcpy(
5803	5859	try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
5804	5860	},
5805	5861	.ptr_stack_offset, .stack_offset => \|off\| {
5806		_ = try self.addInst(.{
5807		.tag = .lea,
5808		.ops = Mir.Inst.Ops.encode(.{
5809		.reg1 = src_addr_reg.to64(),
5810		.reg2 = opts.source_stack_base orelse .rbp,
5811		}),
5812		.data = .{ .disp = -off },
5813		});
	5862	_ = off;
	5863	// _ = try self.addInst(.{
	5864	// .tag = .lea,
	5865	// .ops = Mir.Inst.Ops.encode(.{
	5866	// .reg1 = src_addr_reg.to64(),
	5867	// .reg2 = opts.source_stack_base orelse .rbp,
	5868	// }),
	5869	// .data = .{ .disp = -off },
	5870	// });
5814	5871	},
5815	5872	.register => \|reg\| {
5816		_ = try self.addInst(.{
5817		.tag = .mov,
5818		.ops = Mir.Inst.Ops.encode(.{
5819		.reg1 = registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5820		.reg2 = reg,
5821		}),
5822		.data = undefined,
5823		});
	5873	_ = reg;
	5874	// _ = try self.addInst(.{
	5875	// .tag = .mov,
	5876	// .ops = Mir.Inst.Ops.encode(.{
	5877	// .reg1 = registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	5878	// .reg2 = reg,
	5879	// }),
	5880	// .data = undefined,
	5881	// });
5824	5882	},
5825	5883	else => {
5826	5884	return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
...	...	@@ -5830,73 +5888,73 @@ fn genInlineMemcpy(
5830	5888	try self.genSetReg(Type.usize, count_reg, len);
5831	5889
5832	5890	// mov index_reg, 0
5833		_ = try self.addInst(.{
5834		.tag = .mov,
5835		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5836		.data = .{ .imm = 0 },
5837		});
	5891	// _ = try self.addInst(.{
	5892	// .tag = .mov,
	5893	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
	5894	// .data = .{ .imm = 0 },
	5895	// });
5838	5896
5839	5897	// loop:
5840	5898	// cmp count, 0
5841		const loop_start = try self.addInst(.{
5842		.tag = .cmp,
5843		.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5844		.data = .{ .imm = 0 },
5845		});
	5899	// const loop_start = try self.addInst(.{
	5900	// .tag = .cmp,
	5901	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
	5902	// .data = .{ .imm = 0 },
	5903	// });
5846	5904
5847	5905	// je end
5848		const loop_reloc = try self.addInst(.{
5849		.tag = .cond_jmp,
5850		.ops = Mir.Inst.Ops.encode(.{}),
5851		.data = .{ .inst_cc = .{
5852		.inst = undefined,
5853		.cc = .e,
5854		} },
5855		});
	5906	// const loop_reloc = try self.addInst(.{
	5907	// .tag = .cond_jmp,
	5908	// .ops = Mir.Inst.Ops.encode(.{}),
	5909	// .data = .{ .inst_cc = .{
	5910	// .inst = undefined,
	5911	// .cc = .e,
	5912	// } },
	5913	// });
5856	5914
5857	5915	// mov tmp, [addr + index_reg]
5858		_ = try self.addInst(.{
5859		.tag = .mov_scale_src,
5860		.ops = Mir.Inst.Ops.encode(.{
5861		.reg1 = tmp_reg.to8(),
5862		.reg2 = src_addr_reg,
5863		}),
5864		.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },
5865		});
	5916	// _ = try self.addInst(.{
	5917	// .tag = .mov_scale_src,
	5918	// .ops = Mir.Inst.Ops.encode(.{
	5919	// .reg1 = tmp_reg.to8(),
	5920	// .reg2 = src_addr_reg,
	5921	// }),
	5922	// .data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },
	5923	// });
5866	5924
5867	5925	// mov [stack_offset + index_reg], tmp
5868		_ = try self.addInst(.{
5869		.tag = .mov_scale_dst,
5870		.ops = Mir.Inst.Ops.encode(.{
5871		.reg1 = dst_addr_reg,
5872		.reg2 = tmp_reg.to8(),
5873		}),
5874		.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },
5875		});
	5926	// _ = try self.addInst(.{
	5927	// .tag = .mov_scale_dst,
	5928	// .ops = Mir.Inst.Ops.encode(.{
	5929	// .reg1 = dst_addr_reg,
	5930	// .reg2 = tmp_reg.to8(),
	5931	// }),
	5932	// .data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },
	5933	// });
5876	5934
5877	5935	// add index_reg, 1
5878		_ = try self.addInst(.{
5879		.tag = .add,
5880		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5881		.data = .{ .imm = 1 },
5882		});
	5936	// _ = try self.addInst(.{
	5937	// .tag = .add,
	5938	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
	5939	// .data = .{ .imm = 1 },
	5940	// });
5883	5941
5884	5942	// sub count, 1
5885		_ = try self.addInst(.{
5886		.tag = .sub,
5887		.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5888		.data = .{ .imm = 1 },
5889		});
	5943	// _ = try self.addInst(.{
	5944	// .tag = .sub,
	5945	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
	5946	// .data = .{ .imm = 1 },
	5947	// });
5890	5948
5891	5949	// jmp loop
5892		_ = try self.addInst(.{
5893		.tag = .jmp,
5894		.ops = Mir.Inst.Ops.encode(.{}),
5895		.data = .{ .inst = loop_start },
5896		});
	5950	// _ = try self.addInst(.{
	5951	// .tag = .jmp,
	5952	// .ops = Mir.Inst.Ops.encode(.{}),
	5953	// .data = .{ .inst = loop_start },
	5954	// });
5897	5955
5898	5956	// end:
5899		try self.performReloc(loop_reloc);
	5957	// try self.performReloc(loop_reloc);
5900	5958	}
5901	5959
5902	5960	fn genInlineMemset(
...	...	@@ -5927,24 +5985,26 @@ fn genInlineMemset(
5927	5985	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
5928	5986	},
5929	5987	.ptr_stack_offset, .stack_offset => \|off\| {
5930		_ = try self.addInst(.{
5931		.tag = .lea,
5932		.ops = Mir.Inst.Ops.encode(.{
5933		.reg1 = addr_reg.to64(),
5934		.reg2 = opts.dest_stack_base orelse .rbp,
5935		}),
5936		.data = .{ .disp = -off },
5937		});
	5988	_ = off;
	5989	// _ = try self.addInst(.{
	5990	// .tag = .lea,
	5991	// .ops = Mir.Inst.Ops.encode(.{
	5992	// .reg1 = addr_reg.to64(),
	5993	// .reg2 = opts.dest_stack_base orelse .rbp,
	5994	// }),
	5995	// .data = .{ .disp = -off },
	5996	// });
5938	5997	},
5939	5998	.register => \|reg\| {
5940		_ = try self.addInst(.{
5941		.tag = .mov,
5942		.ops = Mir.Inst.Ops.encode(.{
5943		.reg1 = registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5944		.reg2 = reg,
5945		}),
5946		.data = undefined,
5947		});
	5999	_ = reg;
	6000	// _ = try self.addInst(.{
	6001	// .tag = .mov,
	6002	// .ops = Mir.Inst.Ops.encode(.{
	6003	// .reg1 = registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	6004	// .reg2 = reg,
	6005	// }),
	6006	// .data = undefined,
	6007	// });
5948	6008	},
5949	6009	else => {
5950	6010	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
...	...	@@ -5956,24 +6016,24 @@ fn genInlineMemset(
5956	6016
5957	6017	// loop:
5958	6018	// cmp index_reg, -1
5959		const loop_start = try self.addInst(.{
5960		.tag = .cmp,
5961		.ops = Mir.Inst.Ops.encode(.{
5962		.reg1 = index_reg,
5963		.flags = 0b11,
5964		}),
5965		.data = .{ .imm_s = -1 },
5966		});
	6019	// const loop_start = try self.addInst(.{
	6020	// .tag = .cmp,
	6021	// .ops = Mir.Inst.Ops.encode(.{
	6022	// .reg1 = index_reg,
	6023	// .flags = 0b11,
	6024	// }),
	6025	// .data = .{ .imm_s = -1 },
	6026	// });
5967	6027
5968	6028	// je end
5969		const loop_reloc = try self.addInst(.{
5970		.tag = .cond_jmp,
5971		.ops = Mir.Inst.Ops.encode(.{}),
5972		.data = .{ .inst_cc = .{
5973		.inst = undefined,
5974		.cc = .e,
5975		} },
5976		});
	6029	// const loop_reloc = try self.addInst(.{
	6030	// .tag = .cond_jmp,
	6031	// .ops = Mir.Inst.Ops.encode(.{}),
	6032	// .data = .{ .inst_cc = .{
	6033	// .inst = undefined,
	6034	// .cc = .e,
	6035	// } },
	6036	// });
5977	6037
5978	6038	switch (value) {
5979	6039	.immediate => \|x\| {
...	...	@@ -5981,37 +6041,37 @@ fn genInlineMemset(
5981	6041	return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
5982	6042	}
5983	6043	// mov byte ptr [rbp + index_reg + stack_offset], imm
5984		_ = try self.addInst(.{
5985		.tag = .mov_mem_index_imm,
5986		.ops = Mir.Inst.Ops.encode(.{
5987		.reg1 = addr_reg,
5988		}),
5989		.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDispImm.encode(
5990		index_reg,
5991		0,
5992		@intCast(u32, x),
5993		)) },
5994		});
	6044	// _ = try self.addInst(.{
	6045	// .tag = .mov_mem_index_imm,
	6046	// .ops = Mir.Inst.Ops.encode(.{
	6047	// .reg1 = addr_reg,
	6048	// }),
	6049	// .data = .{ .payload = try self.addExtra(Mir.IndexRegisterDispImm.encode(
	6050	// index_reg,
	6051	// 0,
	6052	// @intCast(u32, x),
	6053	// )) },
	6054	// });
5995	6055	},
5996	6056	else => return self.fail("TODO inline memset for value of type {}", .{value}),
5997	6057	}
5998	6058
5999	6059	// sub index_reg, 1
6000		_ = try self.addInst(.{
6001		.tag = .sub,
6002		.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
6003		.data = .{ .imm = 1 },
6004		});
	6060	// _ = try self.addInst(.{
	6061	// .tag = .sub,
	6062	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
	6063	// .data = .{ .imm = 1 },
	6064	// });
6005	6065
6006	6066	// jmp loop
6007		_ = try self.addInst(.{
6008		.tag = .jmp,
6009		.ops = Mir.Inst.Ops.encode(.{}),
6010		.data = .{ .inst = loop_start },
6011		});
	6067	// _ = try self.addInst(.{
	6068	// .tag = .jmp,
	6069	// .ops = Mir.Inst.Ops.encode(.{}),
	6070	// .data = .{ .inst = loop_start },
	6071	// });
6012	6072
6013	6073	// end:
6014		try self.performReloc(loop_reloc);
	6074	// try self.performReloc(loop_reloc);
6015	6075	}
6016	6076
6017	6077	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
...	...	@@ -6023,14 +6083,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6023	6083	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
6024	6084	return self.fail("stack offset too large", .{});
6025	6085	}
6026		_ = try self.addInst(.{
6027		.tag = .lea,
6028		.ops = Mir.Inst.Ops.encode(.{
6029		.reg1 = registerAlias(reg, abi_size),
6030		.reg2 = .rbp,
6031		}),
6032		.data = .{ .disp = -off },
6033		});
	6086	// _ = try self.addInst(.{
	6087	// .tag = .lea,
	6088	// .ops = Mir.Inst.Ops.encode(.{
	6089	// .reg1 = registerAlias(reg, abi_size),
	6090	// .reg2 = .rbp,
	6091	// }),
	6092	// .data = .{ .disp = -off },
	6093	// });
6034	6094	},
6035	6095	.unreach, .none => return, // Nothing to do.
6036	6096	.undef => {
...	...	@@ -6046,34 +6106,30 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6046	6106	}
6047	6107	},
6048	6108	.eflags => \|cc\| {
6049		_ = try self.addInst(.{
6050		.tag = .cond_set_byte,
6051		.ops = Mir.Inst.Ops.encode(.{
6052		.reg1 = reg.to8(),
6053		}),
6054		.data = .{ .cc = cc },
6055		});
	6109	_ = cc;
	6110	// _ = try self.addInst(.{
	6111	// .tag = .cond_set_byte,
	6112	// .ops = Mir.Inst.Ops.encode(.{
	6113	// .reg1 = reg.to8(),
	6114	// }),
	6115	// .data = .{ .cc = cc },
	6116	// });
6056	6117	},
6057	6118	.immediate => \|x\| {
6058	6119	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
6059	6120	// register is the fastest way to zero a register.
6060	6121	if (x == 0) {
6061		_ = try self.addInst(.{
6062		.tag = .xor,
6063		.ops = Mir.Inst.Ops.encode(.{
6064		.reg1 = reg.to32(),
6065		.reg2 = reg.to32(),
6066		}),
6067		.data = undefined,
	6122	try self.assemble(.xor, .{
	6123	.op1 = .{ .reg = reg.to32() },
	6124	.op2 = .{ .reg = reg.to32() },
6068	6125	});
6069	6126	return;
6070	6127	}
6071	6128	if (x <= math.maxInt(i32)) {
6072	6129	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
6073		_ = try self.addInst(.{
6074		.tag = .mov,
6075		.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
6076		.data = .{ .imm = @intCast(u32, x) },
	6130	try self.assemble(.mov, .{
	6131	.op1 = .{ .reg = registerAlias(reg, abi_size) },
	6132	.op2 = .{ .imm = Mir.Operand.Immediate.u(@intCast(u32, x)) },
6077	6133	});
6078	6134	return;
6079	6135	}
...	...	@@ -6084,12 +6140,12 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6084	6140	// This encoding is, in fact, the same as the one used for 32-bit loads. The only
6085	6141	// difference is that we set REX.W before the instruction, which extends the load to
6086	6142	// 64-bit and uses the full bit-width of the register.
6087		const payload = try self.addExtra(Mir.Imm64.encode(x));
6088		_ = try self.addInst(.{
6089		.tag = .movabs,
6090		.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
6091		.data = .{ .payload = payload },
6092		});
	6143	// const payload = try self.addExtra(Mir.Imm64.encode(x));
	6144	// _ = try self.addInst(.{
	6145	// .tag = .movabs,
	6146	// .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
	6147	// .data = .{ .payload = payload },
	6148	// });
6093	6149	},
6094	6150	.register => \|src_reg\| {
6095	6151	// If the registers are the same, nothing to do.
...	...	@@ -6100,47 +6156,47 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6100	6156	.Int => switch (ty.intInfo(self.target.*).signedness) {
6101	6157	.signed => {
6102	6158	if (abi_size <= 4) {
6103		_ = try self.addInst(.{
6104		.tag = .mov_sign_extend,
6105		.ops = Mir.Inst.Ops.encode(.{
6106		.reg1 = reg.to64(),
6107		.reg2 = registerAlias(src_reg, abi_size),
6108		}),
6109		.data = undefined,
6110		});
	6159	// _ = try self.addInst(.{
	6160	// .tag = .mov_sign_extend,
	6161	// .ops = Mir.Inst.Ops.encode(.{
	6162	// .reg1 = reg.to64(),
	6163	// .reg2 = registerAlias(src_reg, abi_size),
	6164	// }),
	6165	// .data = undefined,
	6166	// });
6111	6167	return;
6112	6168	}
6113	6169	},
6114	6170	.unsigned => {
6115	6171	if (abi_size <= 2) {
6116		_ = try self.addInst(.{
6117		.tag = .mov_zero_extend,
6118		.ops = Mir.Inst.Ops.encode(.{
6119		.reg1 = reg.to64(),
6120		.reg2 = registerAlias(src_reg, abi_size),
6121		}),
6122		.data = undefined,
6123		});
	6172	// _ = try self.addInst(.{
	6173	// .tag = .mov_zero_extend,
	6174	// .ops = Mir.Inst.Ops.encode(.{
	6175	// .reg1 = reg.to64(),
	6176	// .reg2 = registerAlias(src_reg, abi_size),
	6177	// }),
	6178	// .data = undefined,
	6179	// });
6124	6180	return;
6125	6181	}
6126	6182	},
6127	6183	},
6128	6184	.Float => {
6129	6185	if (intrinsicsAllowed(self.target.*, ty)) {
6130		const tag: Mir.Inst.Tag = switch (ty.tag()) {
6131		.f32 => Mir.Inst.Tag.mov_f32,
6132		.f64 => Mir.Inst.Tag.mov_f64,
6133		else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6134		};
6135		_ = try self.addInst(.{
6136		.tag = tag,
6137		.ops = Mir.Inst.Ops.encode(.{
6138		.reg1 = reg.to128(),
6139		.reg2 = src_reg.to128(),
6140		.flags = 0b10,
6141		}),
6142		.data = undefined,
6143		});
	6186	// const tag: Mir.Inst.Tag = switch (ty.tag()) {
	6187	// .f32 => Mir.Inst.Tag.mov_f32,
	6188	// .f64 => Mir.Inst.Tag.mov_f64,
	6189	// else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
	6190	// };
	6191	// _ = try self.addInst(.{
	6192	// .tag = tag,
	6193	// .ops = Mir.Inst.Ops.encode(.{
	6194	// .reg1 = reg.to128(),
	6195	// .reg2 = src_reg.to128(),
	6196	// .flags = 0b10,
	6197	// }),
	6198	// .data = undefined,
	6199	// });
6144	6200	return;
6145	6201	}
6146	6202
...	...	@@ -6149,14 +6205,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6149	6205	else => {},
6150	6206	}
6151	6207
6152		_ = try self.addInst(.{
6153		.tag = .mov,
6154		.ops = Mir.Inst.Ops.encode(.{
6155		.reg1 = registerAlias(reg, abi_size),
6156		.reg2 = registerAlias(src_reg, abi_size),
6157		}),
6158		.data = undefined,
6159		});
	6208	// _ = try self.addInst(.{
	6209	// .tag = .mov,
	6210	// .ops = Mir.Inst.Ops.encode(.{
	6211	// .reg1 = registerAlias(reg, abi_size),
	6212	// .reg2 = registerAlias(src_reg, abi_size),
	6213	// }),
	6214	// .data = undefined,
	6215	// });
6160	6216	},
6161	6217	.linker_load => {
6162	6218	switch (ty.zigTypeTag()) {
...	...	@@ -6165,24 +6221,24 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6165	6221	try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
6166	6222
6167	6223	if (intrinsicsAllowed(self.target.*, ty)) {
6168		const tag: Mir.Inst.Tag = switch (ty.tag()) {
6169		.f32 => Mir.Inst.Tag.mov_f32,
6170		.f64 => Mir.Inst.Tag.mov_f64,
6171		else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6172		};
6173
6174		_ = try self.addInst(.{
6175		.tag = tag,
6176		.ops = Mir.Inst.Ops.encode(.{
6177		.reg1 = reg.to128(),
6178		.reg2 = switch (ty.tag()) {
6179		.f32 => base_reg.to32(),
6180		.f64 => base_reg.to64(),
6181		else => unreachable,
6182		},
6183		}),
6184		.data = .{ .disp = 0 },
6185		});
	6224	// const tag: Mir.Inst.Tag = switch (ty.tag()) {
	6225	// .f32 => Mir.Inst.Tag.mov_f32,
	6226	// .f64 => Mir.Inst.Tag.mov_f64,
	6227	// else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
	6228	// };
	6229
	6230	// _ = try self.addInst(.{
	6231	// .tag = tag,
	6232	// .ops = Mir.Inst.Ops.encode(.{
	6233	// .reg1 = reg.to128(),
	6234	// .reg2 = switch (ty.tag()) {
	6235	// .f32 => base_reg.to32(),
	6236	// .f64 => base_reg.to64(),
	6237	// else => unreachable,
	6238	// },
	6239	// }),
	6240	// .data = .{ .disp = 0 },
	6241	// });
6186	6242	return;
6187	6243	}
6188	6244
...	...	@@ -6190,15 +6246,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6190	6246	},
6191	6247	else => {
6192	6248	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
6193		_ = try self.addInst(.{
6194		.tag = .mov,
6195		.ops = Mir.Inst.Ops.encode(.{
6196		.reg1 = registerAlias(reg, abi_size),
6197		.reg2 = reg.to64(),
6198		.flags = 0b01,
6199		}),
6200		.data = .{ .disp = 0 },
6201		});
	6249	// _ = try self.addInst(.{
	6250	// .tag = .mov,
	6251	// .ops = Mir.Inst.Ops.encode(.{
	6252	// .reg1 = registerAlias(reg, abi_size),
	6253	// .reg2 = reg.to64(),
	6254	// .flags = 0b01,
	6255	// }),
	6256	// .data = .{ .disp = 0 },
	6257	// });
6202	6258	},
6203	6259	}
6204	6260	},
...	...	@@ -6208,24 +6264,24 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6208	6264	try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
6209	6265
6210	6266	if (intrinsicsAllowed(self.target.*, ty)) {
6211		const tag: Mir.Inst.Tag = switch (ty.tag()) {
6212		.f32 => Mir.Inst.Tag.mov_f32,
6213		.f64 => Mir.Inst.Tag.mov_f64,
6214		else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6215		};
6216
6217		_ = try self.addInst(.{
6218		.tag = tag,
6219		.ops = Mir.Inst.Ops.encode(.{
6220		.reg1 = reg.to128(),
6221		.reg2 = switch (ty.tag()) {
6222		.f32 => base_reg.to32(),
6223		.f64 => base_reg.to64(),
6224		else => unreachable,
6225		},
6226		}),
6227		.data = .{ .disp = 0 },
6228		});
	6267	// const tag: Mir.Inst.Tag = switch (ty.tag()) {
	6268	// .f32 => Mir.Inst.Tag.mov_f32,
	6269	// .f64 => Mir.Inst.Tag.mov_f64,
	6270	// else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
	6271	// };
	6272
	6273	// _ = try self.addInst(.{
	6274	// .tag = tag,
	6275	// .ops = Mir.Inst.Ops.encode(.{
	6276	// .reg1 = reg.to128(),
	6277	// .reg2 = switch (ty.tag()) {
	6278	// .f32 => base_reg.to32(),
	6279	// .f64 => base_reg.to64(),
	6280	// else => unreachable,
	6281	// },
	6282	// }),
	6283	// .data = .{ .disp = 0 },
	6284	// });
6229	6285	return;
6230	6286	}
6231	6287
...	...	@@ -6234,42 +6290,42 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6234	6290	else => {
6235	6291	if (x <= math.maxInt(i32)) {
6236	6292	// mov reg, [ds:imm32]
6237		_ = try self.addInst(.{
6238		.tag = .mov,
6239		.ops = Mir.Inst.Ops.encode(.{
6240		.reg1 = registerAlias(reg, abi_size),
6241		.flags = 0b01,
6242		}),
6243		.data = .{ .disp = @intCast(i32, x) },
6244		});
	6293	// _ = try self.addInst(.{
	6294	// .tag = .mov,
	6295	// .ops = Mir.Inst.Ops.encode(.{
	6296	// .reg1 = registerAlias(reg, abi_size),
	6297	// .flags = 0b01,
	6298	// }),
	6299	// .data = .{ .disp = @intCast(i32, x) },
	6300	// });
6245	6301	} else {
6246	6302	// If this is RAX, we can use a direct load.
6247	6303	// Otherwise, we need to load the address, then indirectly load the value.
6248	6304	if (reg.id() == 0) {
6249	6305	// movabs rax, ds:moffs64
6250		const payload = try self.addExtra(Mir.Imm64.encode(x));
6251		_ = try self.addInst(.{
6252		.tag = .movabs,
6253		.ops = Mir.Inst.Ops.encode(.{
6254		.reg1 = .rax,
6255		.flags = 0b01, // imm64 will become moffs64
6256		}),
6257		.data = .{ .payload = payload },
6258		});
	6306	// const payload = try self.addExtra(Mir.Imm64.encode(x));
	6307	// _ = try self.addInst(.{
	6308	// .tag = .movabs,
	6309	// .ops = Mir.Inst.Ops.encode(.{
	6310	// .reg1 = .rax,
	6311	// .flags = 0b01, // imm64 will become moffs64
	6312	// }),
	6313	// .data = .{ .payload = payload },
	6314	// });
6259	6315	} else {
6260	6316	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
6261	6317	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6262	6318
6263	6319	// mov reg, [reg + 0x0]
6264		_ = try self.addInst(.{
6265		.tag = .mov,
6266		.ops = Mir.Inst.Ops.encode(.{
6267		.reg1 = registerAlias(reg, abi_size),
6268		.reg2 = reg.to64(),
6269		.flags = 0b01,
6270		}),
6271		.data = .{ .disp = 0 },
6272		});
	6320	// _ = try self.addInst(.{
	6321	// .tag = .mov,
	6322	// .ops = Mir.Inst.Ops.encode(.{
	6323	// .reg1 = registerAlias(reg, abi_size),
	6324	// .reg2 = reg.to64(),
	6325	// .flags = 0b01,
	6326	// }),
	6327	// .data = .{ .disp = 0 },
	6328	// });
6273	6329	}
6274	6330	}
6275	6331	},
...	...	@@ -6289,15 +6345,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6289	6345	4 => 0b11,
6290	6346	else => unreachable,
6291	6347	};
6292		_ = try self.addInst(.{
6293		.tag = .mov_sign_extend,
6294		.ops = Mir.Inst.Ops.encode(.{
6295		.reg1 = reg.to64(),
6296		.reg2 = .rbp,
6297		.flags = flags,
6298		}),
6299		.data = .{ .disp = -off },
6300		});
	6348	_ = flags;
	6349	// _ = try self.addInst(.{
	6350	// .tag = .mov_sign_extend,
	6351	// .ops = Mir.Inst.Ops.encode(.{
	6352	// .reg1 = reg.to64(),
	6353	// .reg2 = .rbp,
	6354	// .flags = flags,
	6355	// }),
	6356	// .data = .{ .disp = -off },
	6357	// });
6301	6358	return;
6302	6359	}
6303	6360	},
...	...	@@ -6308,38 +6365,39 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6308	6365	2 => 0b10,
6309	6366	else => unreachable,
6310	6367	};
6311		_ = try self.addInst(.{
6312		.tag = .mov_zero_extend,
6313		.ops = Mir.Inst.Ops.encode(.{
6314		.reg1 = reg.to64(),
6315		.reg2 = .rbp,
6316		.flags = flags,
6317		}),
6318		.data = .{ .disp = -off },
6319		});
	6368	_ = flags;
	6369	// _ = try self.addInst(.{
	6370	// .tag = .mov_zero_extend,
	6371	// .ops = Mir.Inst.Ops.encode(.{
	6372	// .reg1 = reg.to64(),
	6373	// .reg2 = .rbp,
	6374	// .flags = flags,
	6375	// }),
	6376	// .data = .{ .disp = -off },
	6377	// });
6320	6378	return;
6321	6379	}
6322	6380	},
6323	6381	},
6324	6382	.Float => {
6325	6383	if (intrinsicsAllowed(self.target.*, ty)) {
6326		const tag: Mir.Inst.Tag = switch (ty.tag()) {
6327		.f32 => Mir.Inst.Tag.mov_f32,
6328		.f64 => Mir.Inst.Tag.mov_f64,
6329		else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
6330		};
6331		_ = try self.addInst(.{
6332		.tag = tag,
6333		.ops = Mir.Inst.Ops.encode(.{
6334		.reg1 = reg.to128(),
6335		.reg2 = switch (ty.tag()) {
6336		.f32 => .ebp,
6337		.f64 => .rbp,
6338		else => unreachable,
6339		},
6340		}),
6341		.data = .{ .disp = -off },
6342		});
	6384	// const tag: Mir.Inst.Tag = switch (ty.tag()) {
	6385	// .f32 => Mir.Inst.Tag.mov_f32,
	6386	// .f64 => Mir.Inst.Tag.mov_f64,
	6387	// else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
	6388	// };
	6389	// _ = try self.addInst(.{
	6390	// .tag = tag,
	6391	// .ops = Mir.Inst.Ops.encode(.{
	6392	// .reg1 = reg.to128(),
	6393	// .reg2 = switch (ty.tag()) {
	6394	// .f32 => .ebp,
	6395	// .f64 => .rbp,
	6396	// else => unreachable,
	6397	// },
	6398	// }),
	6399	// .data = .{ .disp = -off },
	6400	// });
6343	6401	return;
6344	6402	}
6345	6403	return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
...	...	@@ -6347,15 +6405,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6347	6405	else => {},
6348	6406	}
6349	6407
6350		_ = try self.addInst(.{
6351		.tag = .mov,
6352		.ops = Mir.Inst.Ops.encode(.{
6353		.reg1 = registerAlias(reg, abi_size),
6354		.reg2 = .rbp,
6355		.flags = 0b01,
6356		}),
6357		.data = .{ .disp = -off },
6358		});
	6408	// _ = try self.addInst(.{
	6409	// .tag = .mov,
	6410	// .ops = Mir.Inst.Ops.encode(.{
	6411	// .reg1 = registerAlias(reg, abi_size),
	6412	// .reg2 = .rbp,
	6413	// .flags = 0b01,
	6414	// }),
	6415	// .data = .{ .disp = -off },
	6416	// });
6359	6417	},
6360	6418	}
6361	6419	}
...	...	@@ -6419,6 +6477,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6419	6477	const src_ty = self.air.typeOf(ty_op.operand);
6420	6478	const dst_ty = self.air.typeOfIndex(inst);
6421	6479	const operand = try self.resolveInst(ty_op.operand);
	6480	_ = dst_ty;
6422	6481
6423	6482	// move float src to ST(0)
6424	6483	const stack_offset = switch (operand) {
...	...	@@ -6433,34 +6492,35 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6433	6492	break :blk offset;
6434	6493	},
6435	6494	};
6436		_ = try self.addInst(.{
6437		.tag = .fld,
6438		.ops = Mir.Inst.Ops.encode(.{
6439		.reg1 = .rbp,
6440		.flags = switch (src_ty.abiSize(self.target.*)) {
6441		4 => 0b01,
6442		8 => 0b10,
6443		else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
6444		},
6445		}),
6446		.data = .{ .disp = -stack_offset },
6447		});
	6495	_ = stack_offset;
	6496	// _ = try self.addInst(.{
	6497	// .tag = .fld,
	6498	// .ops = Mir.Inst.Ops.encode(.{
	6499	// .reg1 = .rbp,
	6500	// .flags = switch (src_ty.abiSize(self.target.*)) {
	6501	// 4 => 0b01,
	6502	// 8 => 0b10,
	6503	// else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
	6504	// },
	6505	// }),
	6506	// .data = .{ .disp = -stack_offset },
	6507	// });
6448	6508
6449	6509	// convert
6450	6510	const stack_dst = try self.allocRegOrMem(inst, false);
6451		_ = try self.addInst(.{
6452		.tag = .fisttp,
6453		.ops = Mir.Inst.Ops.encode(.{
6454		.reg1 = .rbp,
6455		.flags = switch (dst_ty.abiSize(self.target.*)) {
6456		1...2 => 0b00,
6457		3...4 => 0b01,
6458		5...8 => 0b10,
6459		else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
6460		},
6461		}),
6462		.data = .{ .disp = -stack_dst.stack_offset },
6463		});
	6511	// _ = try self.addInst(.{
	6512	// .tag = .fisttp,
	6513	// .ops = Mir.Inst.Ops.encode(.{
	6514	// .reg1 = .rbp,
	6515	// .flags = switch (dst_ty.abiSize(self.target.*)) {
	6516	// 1...2 => 0b00,
	6517	// 3...4 => 0b01,
	6518	// 5...8 => 0b10,
	6519	// else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
	6520	// },
	6521	// }),
	6522	// .data = .{ .disp = -stack_dst.stack_offset },
	6523	// });
6464	6524
6465	6525	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
6466	6526	}
...	...	@@ -6551,15 +6611,15 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6551	6611	.linker_load, .memory => {
6552	6612	const reg = try self.register_manager.allocReg(null, gp);
6553	6613	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6554		_ = try self.addInst(.{
6555		.tag = .mov,
6556		.ops = Mir.Inst.Ops.encode(.{
6557		.reg1 = reg,
6558		.reg2 = reg,
6559		.flags = 0b01,
6560		}),
6561		.data = .{ .disp = 0 },
6562		});
	6614	// _ = try self.addInst(.{
	6615	// .tag = .mov,
	6616	// .ops = Mir.Inst.Ops.encode(.{
	6617	// .reg1 = reg,
	6618	// .reg2 = reg,
	6619	// .flags = 0b01,
	6620	// }),
	6621	// .data = .{ .disp = 0 },
	6622	// });
6563	6623	break :blk MCValue{ .register = reg };
6564	6624	},
6565	6625	else => break :blk src_ptr,

src/arch/x86_64/Emit.zig+418-1015

...	...	@@ -71,124 +71,53 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
71	71	const inst = @intCast(u32, index);
72	72	try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
73	73	switch (tag) {
74		// GPR instructions
75		.adc => try emit.mirArith(.adc, inst),
76		.add => try emit.mirArith(.add, inst),
77		.sub => try emit.mirArith(.sub, inst),
78		.xor => try emit.mirArith(.xor, inst),
79		.@"and" => try emit.mirArith(.@"and", inst),
80		.@"or" => try emit.mirArith(.@"or", inst),
81		.sbb => try emit.mirArith(.sbb, inst),
82		.cmp => try emit.mirArith(.cmp, inst),
83		.mov => try emit.mirArith(.mov, inst),
84
85		.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
86		.add_mem_imm => try emit.mirArithMemImm(.add, inst),
87		.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
88		.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
89		.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
90		.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
91		.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
92		.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
93		.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
94
95		.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
96		.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
97		.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
98		.xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
99		.and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
100		.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
101		.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
102		.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
103		.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
104
105		.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
106		.add_scale_dst => try emit.mirArithScaleDst(.add, inst),
107		.sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
108		.xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
109		.and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
110		.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
111		.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
112		.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
113		.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
114
115		.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
116		.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
117		.sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
118		.xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
119		.and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
120		.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
121		.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
122		.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
123		.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
124
125		.adc_mem_index_imm => try emit.mirArithMemIndexImm(.adc, inst),
126		.add_mem_index_imm => try emit.mirArithMemIndexImm(.add, inst),
127		.sub_mem_index_imm => try emit.mirArithMemIndexImm(.sub, inst),
128		.xor_mem_index_imm => try emit.mirArithMemIndexImm(.xor, inst),
129		.and_mem_index_imm => try emit.mirArithMemIndexImm(.@"and", inst),
130		.or_mem_index_imm => try emit.mirArithMemIndexImm(.@"or", inst),
131		.sbb_mem_index_imm => try emit.mirArithMemIndexImm(.sbb, inst),
132		.cmp_mem_index_imm => try emit.mirArithMemIndexImm(.cmp, inst),
133		.mov_mem_index_imm => try emit.mirArithMemIndexImm(.mov, inst),
134
135		.mov_sign_extend => try emit.mirMovSignExtend(inst),
136		.mov_zero_extend => try emit.mirMovZeroExtend(inst),
137
138		.movabs => try emit.mirMovabs(inst),
139
140		.fisttp => try emit.mirFisttp(inst),
141		.fld => try emit.mirFld(inst),
142
143		.lea => try emit.mirLea(inst),
144		.lea_pic => try emit.mirLeaPic(inst),
145
146		.shl => try emit.mirShift(.shl, inst),
147		.sal => try emit.mirShift(.sal, inst),
148		.shr => try emit.mirShift(.shr, inst),
149		.sar => try emit.mirShift(.sar, inst),
150
151		.imul => try emit.mirMulDiv(.imul, inst),
152		.mul => try emit.mirMulDiv(.mul, inst),
153		.idiv => try emit.mirMulDiv(.idiv, inst),
154		.div => try emit.mirMulDiv(.div, inst),
155		.imul_complex => try emit.mirIMulComplex(inst),
156
157		.cwd => try emit.mirCwd(inst),
158
159		.push => try emit.mirPushPop(.push, inst),
160		.pop => try emit.mirPushPop(.pop, inst),
161
162		.jmp => try emit.mirJmpCall(.jmp, inst),
163		.call => try emit.mirJmpCall(.call, inst),
164
165		.cond_jmp => try emit.mirCondJmp(inst),
166		.cond_set_byte => try emit.mirCondSetByte(inst),
167		.cond_mov => try emit.mirCondMov(inst),
168
169		.ret => try emit.mirRet(inst),
170
171		.syscall => try emit.mirSyscall(),
172
173		.@"test" => try emit.mirTest(inst),
174
175		.ud => try emit.mirUndefinedInstruction(),
176		.interrupt => try emit.mirInterrupt(inst),
177		.nop => {}, // just skip it
178
179		// SSE/AVX instructions
180		.mov_f64 => try emit.mirMovFloat(.movsd, inst),
181		.mov_f32 => try emit.mirMovFloat(.movss, inst),
182
183		.add_f64 => try emit.mirAddFloat(.addsd, inst),
184		.add_f32 => try emit.mirAddFloat(.addss, inst),
185
186		.cmp_f64 => try emit.mirCmpFloat(.ucomisd, inst),
187		.cmp_f32 => try emit.mirCmpFloat(.ucomiss, inst),
	74	.adc,
	75	.add,
	76	.@"and",
	77	.cbw,
	78	.cwde,
	79	.cdqe,
	80	.cwd,
	81	.cdq,
	82	.cqo,
	83	.cmp,
	84	.div,
	85	.fisttp,
	86	.fld,
	87	.idiv,
	88	.imul,
	89	.int3,
	90	.mov,
	91	.movsx,
	92	.movzx,
	93	.mul,
	94	.nop,
	95	.@"or",
	96	.pop,
	97	.push,
	98	.ret,
	99	.sal,
	100	.sar,
	101	.sbb,
	102	.shl,
	103	.shr,
	104	.sub,
	105	.syscall,
	106	.@"test",
	107	.ud2,
	108	.xor,
	109
	110	.addss,
	111	.cmpss,
	112	.movss,
	113	.ucomiss,
	114	.addsd,
	115	.cmpsd,
	116	.movsd,
	117	.ucomisd,
	118	=> try emit.mirEncodeGeneric(tag, inst),
188	119
189	120	// Pseudo-instructions
190		.call_extern => try emit.mirCallExtern(inst),
191
192	121	.dbg_line => try emit.mirDbgLine(inst),
193	122	.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
194	123	.dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
...	...	@@ -196,9 +125,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
196	125	.push_regs => try emit.mirPushPopRegisterList(.push, inst),
197	126	.pop_regs => try emit.mirPushPopRegisterList(.pop, inst),
198	127
199		else => {
200		return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {}", .{tag});
201		},
	128	else => return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {}", .{tag}),
202	129	}
203	130	}
204	131
...	...	@@ -246,66 +173,57 @@ fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
246	173	return inst.encode(emit.code.writer());
247	174	}
248	175
249		fn mirUndefinedInstruction(emit: *Emit) InnerError!void {
250		return emit.encode(.ud2, .{});
251		}
252
253		fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
254		const tag = emit.mir.instructions.items(.tag)[inst];
255		assert(tag == .interrupt);
256		const ops = emit.mir.instructions.items(.ops)[inst].decode();
257		switch (ops.flags) {
258		0b00 => return emit.encode(.int3, .{}),
259		else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
260		}
261		}
262
263		fn mirSyscall(emit: *Emit) InnerError!void {
264		return emit.encode(.syscall, .{});
265		}
	176	fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	177	const mnemonic = inline for (@typeInfo(Instruction.Mnemonic).Enum.fields) \|field\| {
	178	if (mem.eql(u8, field.name, @tagName(tag))) break @field(Instruction.Mnemonic, field.name);
	179	} else unreachable;
266	180
267		fn mirPushPop(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
268		const ops = emit.mir.instructions.items(.ops)[inst].decode();
269		switch (ops.flags) {
270		0b00 => {
271		return emit.encode(mnemonic, .{
272		.op1 = .{ .reg = ops.reg1 },
273		});
	181	var operands = [4]Instruction.Operand{ .none, .none, .none, .none };
	182	const ops = emit.mir.instructions.items(.ops)[inst];
	183	const data = emit.mir.instructions.items(.data)[inst];
	184	switch (ops) {
	185	.none => {},
	186	.imm_s => operands[0] = .{ .imm = Immediate.s(data.imm_s) },
	187	.imm_u => operands[0] = .{ .imm = Immediate.u(data.imm_u) },
	188	.r => operands[0] = .{ .reg = data.r },
	189	.rr => operands[0..2].* = .{
	190	.{ .reg = data.rr.r1 },
	191	.{ .reg = data.rr.r2 },
274	192	},
275		0b01 => {
276		const disp = emit.mir.instructions.items(.data)[inst].disp;
277		return emit.encode(mnemonic, .{
278		.op1 = .{ .mem = Memory.sib(.qword, .{
279		.base = ops.reg1,
280		.disp = disp,
281		}) },
282		});
	193	.ri_s => operands[0..2].* = .{
	194	.{ .reg = data.ri_s.r1 },
	195	.{ .imm = Immediate.s(data.ri_s.imm) },
283	196	},
284		0b10 => {
285		const imm = emit.mir.instructions.items(.data)[inst].imm;
286		return emit.encode(.push, .{
287		.op1 = .{ .imm = Immediate.u(imm) },
288		});
	197	.ri_u => operands[0..2].* = .{
	198	.{ .reg = data.ri_u.r1 },
	199	.{ .imm = Immediate.u(data.ri_u.imm) },
289	200	},
290		0b11 => unreachable,
	201	else => unreachable,
291	202	}
	203
	204	return emit.encode(mnemonic, .{
	205	.op1 = operands[0],
	206	.op2 = operands[1],
	207	.op3 = operands[2],
	208	.op4 = operands[3],
	209	});
292	210	}
293	211
294		fn mirPushPopRegisterList(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
295		const ops = emit.mir.instructions.items(.ops)[inst].decode();
	212	fn mirPushPopRegisterList(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
296	213	const payload = emit.mir.instructions.items(.data)[inst].payload;
297	214	const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
	215	const base = @intToEnum(Register, save_reg_list.base_reg);
298	216	var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
299	217	const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
300	218	const callee_preserved_regs = abi.getCalleePreservedRegs(emit.target.*);
301	219	for (callee_preserved_regs) \|reg\| {
302	220	if (reg_list.isSet(callee_preserved_regs, reg)) {
303	221	const op1: Instruction.Operand = .{ .mem = Memory.sib(.qword, .{
304		.base = ops.reg1,
	222	.base = base,
305	223	.disp = disp,
306	224	}) };
307	225	const op2: Instruction.Operand = .{ .reg = reg };
308		switch (mnemonic) {
	226	switch (tag) {
309	227	.push => try emit.encode(.mov, .{
310	228	.op1 = op1,
311	229	.op2 = op2,
...	...	@@ -321,858 +239,345 @@ fn mirPushPopRegisterList(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir
321	239	}
322	240	}
323	241
324		fn mirJmpCall(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
325		const ops = emit.mir.instructions.items(.ops)[inst].decode();
326		switch (ops.flags) {
327		0b00 => {
328		const target = emit.mir.instructions.items(.data)[inst].inst;
329		const source = emit.code.items.len;
330		try emit.encode(mnemonic, .{
331		.op1 = .{ .imm = Immediate.s(0) },
332		});
333		try emit.relocs.append(emit.bin_file.allocator, .{
334		.source = source,
335		.target = target,
336		.offset = emit.code.items.len - 4,
337		.length = 5,
338		});
339		},
340		0b01 => {
341		if (ops.reg1 == .none) {
342		const disp = emit.mir.instructions.items(.data)[inst].disp;
343		return emit.encode(mnemonic, .{
344		.op1 = .{ .mem = Memory.sib(.qword, .{ .disp = disp }) },
345		});
346		}
347		return emit.encode(mnemonic, .{
348		.op1 = .{ .reg = ops.reg1 },
349		});
350		},
351		0b10 => {
352		const disp = emit.mir.instructions.items(.data)[inst].disp;
353		return emit.encode(mnemonic, .{
354		.op1 = .{ .mem = Memory.sib(.qword, .{
355		.base = ops.reg1,
356		.disp = disp,
357		}) },
358		});
359		},
360		0b11 => return emit.fail("TODO unused variant jmp/call 0b11", .{}),
361		}
362		}
363
364		fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
365		const tag = emit.mir.instructions.items(.tag)[inst];
366		assert(tag == .cond_jmp);
367		const inst_cc = emit.mir.instructions.items(.data)[inst].inst_cc;
368		const mnemonic: Instruction.Mnemonic = switch (inst_cc.cc) {
369		.a => .ja,
370		.ae => .jae,
371		.b => .jb,
372		.be => .jbe,
373		.c => .jc,
374		.e => .je,
375		.g => .jg,
376		.ge => .jge,
377		.l => .jl,
378		.le => .jle,
379		.na => .jna,
380		.nae => .jnae,
381		.nb => .jnb,
382		.nbe => .jnbe,
383		.nc => .jnc,
384		.ne => .jne,
385		.ng => .jng,
386		.nge => .jnge,
387		.nl => .jnl,
388		.nle => .jnle,
389		.no => .jno,
390		.np => .jnp,
391		.ns => .jns,
392		.nz => .jnz,
393		.o => .jo,
394		.p => .jp,
395		.pe => .jpe,
396		.po => .jpo,
397		.s => .js,
398		.z => .jz,
399		};
400		const source = emit.code.items.len;
401		try emit.encode(mnemonic, .{
402		.op1 = .{ .imm = Immediate.s(0) },
403		});
404		try emit.relocs.append(emit.bin_file.allocator, .{
405		.source = source,
406		.target = inst_cc.inst,
407		.offset = emit.code.items.len - 4,
408		.length = 6,
409		});
410		}
411
412		fn mirCondSetByte(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
413		const tag = emit.mir.instructions.items(.tag)[inst];
414		assert(tag == .cond_set_byte);
415		const ops = emit.mir.instructions.items(.ops)[inst].decode();
416		const cc = emit.mir.instructions.items(.data)[inst].cc;
417		const mnemonic: Instruction.Mnemonic = switch (cc) {
418		.a => .seta,
419		.ae => .setae,
420		.b => .setb,
421		.be => .setbe,
422		.c => .setc,
423		.e => .sete,
424		.g => .setg,
425		.ge => .setge,
426		.l => .setl,
427		.le => .setle,
428		.na => .setna,
429		.nae => .setnae,
430		.nb => .setnb,
431		.nbe => .setnbe,
432		.nc => .setnc,
433		.ne => .setne,
434		.ng => .setng,
435		.nge => .setnge,
436		.nl => .setnl,
437		.nle => .setnle,
438		.no => .setno,
439		.np => .setnp,
440		.ns => .setns,
441		.nz => .setnz,
442		.o => .seto,
443		.p => .setp,
444		.pe => .setpe,
445		.po => .setpo,
446		.s => .sets,
447		.z => .setz,
448		};
449		return emit.encode(mnemonic, .{ .op1 = .{ .reg = ops.reg1 } });
450		}
451
452		fn mirCondMov(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
453		const tag = emit.mir.instructions.items(.tag)[inst];
454		assert(tag == .cond_mov);
455		const ops = emit.mir.instructions.items(.ops)[inst].decode();
456		const cc = emit.mir.instructions.items(.data)[inst].cc;
457		const mnemonic: Instruction.Mnemonic = switch (cc) {
458		.a => .cmova,
459		.ae => .cmovae,
460		.b => .cmovb,
461		.be => .cmovbe,
462		.c => .cmovc,
463		.e => .cmove,
464		.g => .cmovg,
465		.ge => .cmovge,
466		.l => .cmovl,
467		.le => .cmovle,
468		.na => .cmovna,
469		.nae => .cmovnae,
470		.nb => .cmovnb,
471		.nbe => .cmovnbe,
472		.nc => .cmovnc,
473		.ne => .cmovne,
474		.ng => .cmovng,
475		.nge => .cmovnge,
476		.nl => .cmovnl,
477		.nle => .cmovnle,
478		.no => .cmovno,
479		.np => .cmovnp,
480		.ns => .cmovns,
481		.nz => .cmovnz,
482		.o => .cmovo,
483		.p => .cmovp,
484		.pe => .cmovpe,
485		.po => .cmovpo,
486		.s => .cmovs,
487		.z => .cmovz,
488		};
489		const op1: Instruction.Operand = .{ .reg = ops.reg1 };
490
491		if (ops.flags == 0b00) {
492		return emit.encode(mnemonic, .{
493		.op1 = op1,
494		.op2 = .{ .reg = ops.reg2 },
495		});
496		}
497		const disp = emit.mir.instructions.items(.data)[inst].disp;
498		const ptr_size: Memory.PtrSize = switch (ops.flags) {
499		0b00 => unreachable,
500		0b01 => .word,
501		0b10 => .dword,
502		0b11 => .qword,
503		};
504		return emit.encode(mnemonic, .{
505		.op1 = op1,
506		.op2 = .{ .mem = Memory.sib(ptr_size, .{
507		.base = ops.reg2,
508		.disp = disp,
509		}) },
510		});
511		}
512
513		fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
514		const tag = emit.mir.instructions.items(.tag)[inst];
515		assert(tag == .@"test");
516		const ops = emit.mir.instructions.items(.ops)[inst].decode();
517		switch (ops.flags) {
518		0b00 => {
519		if (ops.reg2 == .none) {
520		const imm = emit.mir.instructions.items(.data)[inst].imm;
521		return emit.encode(.@"test", .{
522		.op1 = .{ .reg = ops.reg1 },
523		.op2 = .{ .imm = Immediate.u(imm) },
524		});
525		}
526		return emit.encode(.@"test", .{
527		.op1 = .{ .reg = ops.reg1 },
528		.op2 = .{ .reg = ops.reg2 },
529		});
530		},
531		else => return emit.fail("TODO more TEST alternatives", .{}),
532		}
533		}
534
535		fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
536		const tag = emit.mir.instructions.items(.tag)[inst];
537		assert(tag == .ret);
538		const ops = emit.mir.instructions.items(.ops)[inst].decode();
539		switch (ops.flags) {
540		0b00 => unreachable,
541		0b01 => unreachable,
542		0b10 => {
543		const imm = emit.mir.instructions.items(.data)[inst].imm;
544		return emit.encode(.ret, .{
545		.op1 = .{ .imm = Immediate.u(imm) },
546		});
547		},
548		0b11 => {
549		return emit.encode(.ret, .{});
550		},
551		}
552		}
553
554		fn mirArith(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
555		const ops = emit.mir.instructions.items(.ops)[inst].decode();
556		switch (ops.flags) {
557		0b00 => {
558		if (ops.reg2 == .none) {
559		const imm = emit.mir.instructions.items(.data)[inst].imm;
560		return emit.encode(mnemonic, .{
561		.op1 = .{ .reg = ops.reg1 },
562		.op2 = .{ .imm = Immediate.u(imm) },
563		});
564		}
565		return emit.encode(mnemonic, .{
566		.op1 = .{ .reg = ops.reg1 },
567		.op2 = .{ .reg = ops.reg2 },
568		});
569		},
570		0b01 => {
571		const disp = emit.mir.instructions.items(.data)[inst].disp;
572		const base: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
573		return emit.encode(mnemonic, .{
574		.op1 = .{ .reg = ops.reg1 },
575		.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
576		.base = base,
577		.disp = disp,
578		}) },
579		});
580		},
581		0b10 => {
582		if (ops.reg2 == .none) {
583		return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
584		}
585		const disp = emit.mir.instructions.items(.data)[inst].disp;
586		return emit.encode(mnemonic, .{
587		.op1 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg2.bitSize()), .{
588		.base = ops.reg1,
589		.disp = disp,
590		}) },
591		.op2 = .{ .reg = ops.reg2 },
592		});
593		},
594		0b11 => {
595		const imm_s = emit.mir.instructions.items(.data)[inst].imm_s;
596		return emit.encode(mnemonic, .{
597		.op1 = .{ .reg = ops.reg1 },
598		.op2 = .{ .imm = Immediate.s(imm_s) },
599		});
600		},
601		}
602		}
603
604		fn mirArithMemImm(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
605		const ops = emit.mir.instructions.items(.ops)[inst].decode();
606		assert(ops.reg2 == .none);
607		const payload = emit.mir.instructions.items(.data)[inst].payload;
608		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
609		const ptr_size: Memory.PtrSize = switch (ops.flags) {
610		0b00 => .byte,
611		0b01 => .word,
612		0b10 => .dword,
613		0b11 => .qword,
614		};
615		const imm = switch (ops.flags) {
616		0b00 => @truncate(u8, imm_pair.operand),
617		0b01 => @truncate(u16, imm_pair.operand),
618		0b10, 0b11 => @truncate(u32, imm_pair.operand),
619		};
620		return emit.encode(mnemonic, .{
621		.op1 = .{ .mem = Memory.sib(ptr_size, .{
622		.disp = imm_pair.dest_off,
623		.base = ops.reg1,
624		}) },
625		.op2 = .{ .imm = Immediate.u(imm) },
626		});
627		}
628
629		fn mirArithScaleSrc(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
630		const ops = emit.mir.instructions.items(.ops)[inst].decode();
631		const scale = ops.flags;
632		const payload = emit.mir.instructions.items(.data)[inst].payload;
633		const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
634		const scale_index = Memory.ScaleIndex{
635		.scale = @as(u4, 1) << scale,
636		.index = index_reg_disp.index,
637		};
638		return emit.encode(mnemonic, .{
639		.op1 = .{ .reg = ops.reg1 },
640		.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
641		.base = ops.reg2,
642		.scale_index = scale_index,
643		.disp = index_reg_disp.disp,
644		}) },
645		});
646		}
647
648		fn mirArithScaleDst(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
649		const ops = emit.mir.instructions.items(.ops)[inst].decode();
650		const scale = ops.flags;
651		const payload = emit.mir.instructions.items(.data)[inst].payload;
652		const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
653		const scale_index = Memory.ScaleIndex{
654		.scale = @as(u4, 1) << scale,
655		.index = index_reg_disp.index,
656		};
657		assert(ops.reg2 != .none);
658		return emit.encode(mnemonic, .{
659		.op1 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg2.bitSize()), .{
660		.base = ops.reg1,
661		.scale_index = scale_index,
662		.disp = index_reg_disp.disp,
663		}) },
664		.op2 = .{ .reg = ops.reg2 },
665		});
666		}
667
668		fn mirArithScaleImm(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
669		const ops = emit.mir.instructions.items(.ops)[inst].decode();
670		const scale = ops.flags;
671		const payload = emit.mir.instructions.items(.data)[inst].payload;
672		const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
673		const scale_index = Memory.ScaleIndex{
674		.scale = @as(u4, 1) << scale,
675		.index = index_reg_disp_imm.index,
676		};
677		return emit.encode(mnemonic, .{
678		.op1 = .{ .mem = Memory.sib(.qword, .{
679		.base = ops.reg1,
680		.disp = index_reg_disp_imm.disp,
681		.scale_index = scale_index,
682		}) },
683		.op2 = .{ .imm = Immediate.u(index_reg_disp_imm.imm) },
684		});
685		}
686
687		fn mirArithMemIndexImm(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
688		const ops = emit.mir.instructions.items(.ops)[inst].decode();
689		assert(ops.reg2 == .none);
690		const payload = emit.mir.instructions.items(.data)[inst].payload;
691		const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
692		const ptr_size: Memory.PtrSize = switch (ops.flags) {
693		0b00 => .byte,
694		0b01 => .word,
695		0b10 => .dword,
696		0b11 => .qword,
697		};
698		const scale_index = Memory.ScaleIndex{
699		.scale = 1,
700		.index = index_reg_disp_imm.index,
701		};
702		return emit.encode(mnemonic, .{
703		.op1 = .{ .mem = Memory.sib(ptr_size, .{
704		.disp = index_reg_disp_imm.disp,
705		.base = ops.reg1,
706		.scale_index = scale_index,
707		}) },
708		.op2 = .{ .imm = Immediate.u(index_reg_disp_imm.imm) },
709		});
710		}
711
712		fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
713		const tag = emit.mir.instructions.items(.tag)[inst];
714		assert(tag == .mov_sign_extend);
715		const ops = emit.mir.instructions.items(.ops)[inst].decode();
716		const disp = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].disp else undefined;
717		switch (ops.flags) {
718		0b00 => {
719		const mnemonic: Instruction.Mnemonic = if (ops.reg2.bitSize() == 32) .movsxd else .movsx;
720		return emit.encode(mnemonic, .{
721		.op1 = .{ .reg = ops.reg1 },
722		.op2 = .{ .reg = ops.reg2 },
723		});
724		},
725		else => {
726		const ptr_size: Memory.PtrSize = switch (ops.flags) {
727		0b01 => .byte,
728		0b10 => .word,
729		0b11 => .dword,
730		else => unreachable,
731		};
732		const mnemonic: Instruction.Mnemonic = if (ops.flags == 0b11) .movsxd else .movsx;
733		return emit.encode(mnemonic, .{
734		.op1 = .{ .reg = ops.reg1 },
735		.op2 = .{ .mem = Memory.sib(ptr_size, .{
736		.base = ops.reg2,
737		.disp = disp,
738		}) },
739		});
740		},
741		}
742		}
743
744		fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
745		const tag = emit.mir.instructions.items(.tag)[inst];
746		assert(tag == .mov_zero_extend);
747		const ops = emit.mir.instructions.items(.ops)[inst].decode();
748		const disp = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].disp else undefined;
749		switch (ops.flags) {
750		0b00 => {
751		return emit.encode(.movzx, .{
752		.op1 = .{ .reg = ops.reg1 },
753		.op2 = .{ .reg = ops.reg2 },
754		});
755		},
756		0b01, 0b10 => {
757		const ptr_size: Memory.PtrSize = switch (ops.flags) {
758		0b01 => .byte,
759		0b10 => .word,
760		else => unreachable,
761		};
762		return emit.encode(.movzx, .{
763		.op1 = .{ .reg = ops.reg1 },
764		.op2 = .{ .mem = Memory.sib(ptr_size, .{
765		.disp = disp,
766		.base = ops.reg2,
767		}) },
768		});
769		},
770		0b11 => {
771		return emit.fail("TODO unused variant: movzx 0b11", .{});
772		},
773		}
774		}
775
776		fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
777		const tag = emit.mir.instructions.items(.tag)[inst];
778		assert(tag == .movabs);
779		const ops = emit.mir.instructions.items(.ops)[inst].decode();
780		switch (ops.flags) {
781		0b00 => {
782		const imm: u64 = if (ops.reg1.bitSize() == 64) blk: {
783		const payload = emit.mir.instructions.items(.data)[inst].payload;
784		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
785		break :blk imm.decode();
786		} else emit.mir.instructions.items(.data)[inst].imm;
787		return emit.encode(.mov, .{
788		.op1 = .{ .reg = ops.reg1 },
789		.op2 = .{ .imm = Immediate.u(imm) },
790		});
791		},
792		0b01 => {
793		if (ops.reg1 == .none) {
794		const imm: u64 = if (ops.reg2.bitSize() == 64) blk: {
795		const payload = emit.mir.instructions.items(.data)[inst].payload;
796		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
797		break :blk imm.decode();
798		} else emit.mir.instructions.items(.data)[inst].imm;
799		return emit.encode(.mov, .{
800		.op1 = .{ .mem = Memory.moffs(ops.reg2, imm) },
801		.op2 = .{ .reg = .rax },
802		});
803		}
804		const imm: u64 = if (ops.reg1.bitSize() == 64) blk: {
805		const payload = emit.mir.instructions.items(.data)[inst].payload;
806		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
807		break :blk imm.decode();
808		} else emit.mir.instructions.items(.data)[inst].imm;
809		return emit.encode(.mov, .{
810		.op1 = .{ .reg = .rax },
811		.op2 = .{ .mem = Memory.moffs(ops.reg1, imm) },
812		});
813		},
814		else => return emit.fail("TODO unused movabs variant", .{}),
815		}
816		}
817
818		fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
819		const tag = emit.mir.instructions.items(.tag)[inst];
820		assert(tag == .fisttp);
821		const ops = emit.mir.instructions.items(.ops)[inst].decode();
822		const ptr_size: Memory.PtrSize = switch (ops.flags) {
823		0b00 => .word,
824		0b01 => .dword,
825		0b10 => .qword,
826		else => unreachable,
827		};
828		return emit.encode(.fisttp, .{
829		.op1 = .{ .mem = Memory.sib(ptr_size, .{
830		.base = ops.reg1,
831		.disp = emit.mir.instructions.items(.data)[inst].disp,
832		}) },
833		});
834		}
835
836		fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
837		const tag = emit.mir.instructions.items(.tag)[inst];
838		assert(tag == .fld);
839		const ops = emit.mir.instructions.items(.ops)[inst].decode();
840		const ptr_size: Memory.PtrSize = switch (ops.flags) {
841		0b01 => .dword,
842		0b10 => .qword,
843		else => unreachable,
844		};
845		return emit.encode(.fld, .{
846		.op1 = .{ .mem = Memory.sib(ptr_size, .{
847		.base = ops.reg1,
848		.disp = emit.mir.instructions.items(.data)[inst].disp,
849		}) },
850		});
851		}
852
853		fn mirShift(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
854		const ops = emit.mir.instructions.items(.ops)[inst].decode();
855		switch (ops.flags) {
856		0b00 => {
857		return emit.encode(mnemonic, .{
858		.op1 = .{ .reg = ops.reg1 },
859		.op2 = .{ .imm = Immediate.u(1) },
860		});
861		},
862		0b01 => {
863		return emit.encode(mnemonic, .{
864		.op1 = .{ .reg = ops.reg1 },
865		.op2 = .{ .reg = .cl },
866		});
867		},
868		0b10 => {
869		const imm = @truncate(u8, emit.mir.instructions.items(.data)[inst].imm);
870		return emit.encode(mnemonic, .{
871		.op1 = .{ .reg = ops.reg1 },
872		.op2 = .{ .imm = Immediate.u(imm) },
873		});
874		},
875		0b11 => {
876		return emit.fail("TODO unused variant: SHIFT reg1, 0b11", .{});
877		},
878		}
879		}
880
881		fn mirMulDiv(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
882		const ops = emit.mir.instructions.items(.ops)[inst].decode();
883		if (ops.reg1 != .none) {
884		assert(ops.reg2 == .none);
885		return emit.encode(mnemonic, .{
886		.op1 = .{ .reg = ops.reg1 },
887		});
888		}
889		assert(ops.reg2 != .none);
890		const disp = emit.mir.instructions.items(.data)[inst].disp;
891		const ptr_size: Memory.PtrSize = switch (ops.flags) {
892		0b00 => .byte,
893		0b01 => .word,
894		0b10 => .dword,
895		0b11 => .qword,
896		};
897		return emit.encode(mnemonic, .{
898		.op1 = .{ .mem = Memory.sib(ptr_size, .{
899		.base = ops.reg2,
900		.disp = disp,
901		}) },
902		});
903		}
904
905		fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
906		const tag = emit.mir.instructions.items(.tag)[inst];
907		assert(tag == .imul_complex);
908		const ops = emit.mir.instructions.items(.ops)[inst].decode();
909		switch (ops.flags) {
910		0b00 => {
911		return emit.encode(.imul, .{
912		.op1 = .{ .reg = ops.reg1 },
913		.op2 = .{ .reg = ops.reg2 },
914		});
915		},
916		0b01 => {
917		const disp = emit.mir.instructions.items(.data)[inst].disp;
918		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
919		return emit.encode(.imul, .{
920		.op1 = .{ .reg = ops.reg1 },
921		.op2 = .{ .mem = Memory.sib(.qword, .{
922		.base = src_reg,
923		.disp = disp,
924		}) },
925		});
926		},
927		0b10 => {
928		const imm = emit.mir.instructions.items(.data)[inst].imm;
929		return emit.encode(.imul, .{
930		.op1 = .{ .reg = ops.reg1 },
931		.op2 = .{ .reg = ops.reg2 },
932		.op3 = .{ .imm = Immediate.u(imm) },
933		});
934		},
935		0b11 => {
936		const payload = emit.mir.instructions.items(.data)[inst].payload;
937		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
938		return emit.encode(.imul, .{
939		.op1 = .{ .reg = ops.reg1 },
940		.op2 = .{ .mem = Memory.sib(.qword, .{
941		.base = ops.reg2,
942		.disp = imm_pair.dest_off,
943		}) },
944		.op3 = .{ .imm = Immediate.u(imm_pair.operand) },
945		});
946		},
947		}
948		}
949
950		fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
951		const ops = emit.mir.instructions.items(.ops)[inst].decode();
952		const mnemonic: Instruction.Mnemonic = switch (ops.flags) {
953		0b00 => .cbw,
954		0b01 => .cwd,
955		0b10 => .cdq,
956		0b11 => .cqo,
957		};
958		return emit.encode(mnemonic, .{});
959		}
960
961		fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
962		const tag = emit.mir.instructions.items(.tag)[inst];
963		assert(tag == .lea);
964		const ops = emit.mir.instructions.items(.ops)[inst].decode();
965		switch (ops.flags) {
966		0b00 => {
967		const disp = emit.mir.instructions.items(.data)[inst].disp;
968		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
969		return emit.encode(.lea, .{
970		.op1 = .{ .reg = ops.reg1 },
971		.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
972		.base = src_reg,
973		.disp = disp,
974		}) },
975		});
976		},
977		0b01 => {
978		const start_offset = emit.code.items.len;
979		try emit.encode(.lea, .{
980		.op1 = .{ .reg = ops.reg1 },
981		.op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), 0) },
982		});
983		const end_offset = emit.code.items.len;
984		// Backpatch the displacement
985		const payload = emit.mir.instructions.items(.data)[inst].payload;
986		const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
987		const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
988		mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
989		},
990		0b10 => {
991		const payload = emit.mir.instructions.items(.data)[inst].payload;
992		const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
993		const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
994		const scale_index = Memory.ScaleIndex{
995		.scale = 1,
996		.index = index_reg_disp.index,
997		};
998		return emit.encode(.lea, .{
999		.op1 = .{ .reg = ops.reg1 },
1000		.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
1001		.base = src_reg,
1002		.scale_index = scale_index,
1003		.disp = index_reg_disp.disp,
1004		}) },
1005		});
1006		},
1007		0b11 => return emit.fail("TODO unused LEA variant 0b11", .{}),
1008		}
1009		}
1010
1011		fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1012		const tag = emit.mir.instructions.items(.tag)[inst];
1013		assert(tag == .lea_pic);
1014		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1015		const relocation = emit.mir.instructions.items(.data)[inst].relocation;
1016
1017		switch (ops.flags) {
1018		0b00, 0b01, 0b10 => {},
1019		else => return emit.fail("TODO unused LEA PIC variant 0b11", .{}),
1020		}
1021
1022		try emit.encode(.lea, .{
1023		.op1 = .{ .reg = ops.reg1 },
1024		.op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), 0) },
1025		});
1026
1027		const end_offset = emit.code.items.len;
1028
1029		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1030		const reloc_type = switch (ops.flags) {
1031		0b00 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
1032		0b01 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED),
1033		else => unreachable,
1034		};
1035		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1036		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
1037		.type = reloc_type,
1038		.target = .{ .sym_index = relocation.sym_index, .file = null },
1039		.offset = @intCast(u32, end_offset - 4),
1040		.addend = 0,
1041		.pcrel = true,
1042		.length = 2,
1043		});
1044		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
1045		const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1046		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
1047		.type = switch (ops.flags) {
1048		0b00 => .got,
1049		0b01 => .direct,
1050		0b10 => .import,
1051		else => unreachable,
1052		},
1053		.target = switch (ops.flags) {
1054		0b00, 0b01 => .{ .sym_index = relocation.sym_index, .file = null },
1055		0b10 => coff_file.getGlobalByIndex(relocation.sym_index),
1056		else => unreachable,
1057		},
1058		.offset = @intCast(u32, end_offset - 4),
1059		.addend = 0,
1060		.pcrel = true,
1061		.length = 2,
1062		});
1063		} else {
1064		return emit.fail("TODO implement lea reg, [rip + reloc] for linking backends different than MachO", .{});
1065		}
1066		}
1067
1068		// SSE/AVX instructions
1069
1070		fn mirMovFloat(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
1071		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1072		switch (ops.flags) {
1073		0b00 => {
1074		const disp = emit.mir.instructions.items(.data)[inst].disp;
1075		return emit.encode(mnemonic, .{
1076		.op1 = .{ .reg = ops.reg1 },
1077		.op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg2.bitSize()), .{
1078		.base = ops.reg2,
1079		.disp = disp,
1080		}) },
1081		});
1082		},
1083		0b01 => {
1084		const disp = emit.mir.instructions.items(.data)[inst].disp;
1085		return emit.encode(mnemonic, .{
1086		.op1 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
1087		.base = ops.reg1,
1088		.disp = disp,
1089		}) },
1090		.op2 = .{ .reg = ops.reg2 },
1091		});
1092		},
1093		0b10 => {
1094		return emit.encode(mnemonic, .{
1095		.op1 = .{ .reg = ops.reg1 },
1096		.op2 = .{ .reg = ops.reg2 },
1097		});
1098		},
1099		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, mnemonic }),
1100		}
1101		}
1102
1103		fn mirAddFloat(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
1104		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1105		switch (ops.flags) {
1106		0b00 => {
1107		return emit.encode(mnemonic, .{
1108		.op1 = .{ .reg = ops.reg1 },
1109		.op2 = .{ .reg = ops.reg2 },
1110		});
1111		},
1112		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, mnemonic }),
1113		}
1114		}
1115
1116		fn mirCmpFloat(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
1117		const ops = emit.mir.instructions.items(.ops)[inst].decode();
1118		switch (ops.flags) {
1119		0b00 => {
1120		return emit.encode(mnemonic, .{
1121		.op1 = .{ .reg = ops.reg1 },
1122		.op2 = .{ .reg = ops.reg2 },
1123		});
1124		},
1125		else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, mnemonic }),
1126		}
1127		}
1128
1129		// Pseudo-instructions
1130
1131		fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1132		const tag = emit.mir.instructions.items(.tag)[inst];
1133		assert(tag == .call_extern);
1134		const relocation = emit.mir.instructions.items(.data)[inst].relocation;
1135
1136		const offset = blk: {
1137		// callq
1138		try emit.encode(.call, .{
1139		.op1 = .{ .imm = Immediate.s(0) },
1140		});
1141		break :blk @intCast(u32, emit.code.items.len) - 4;
1142		};
1143
1144		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1145		// Add relocation to the decl.
1146		const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1147		const target = macho_file.getGlobalByIndex(relocation.sym_index);
1148		try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
1149		.type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
1150		.target = target,
1151		.offset = offset,
1152		.addend = 0,
1153		.pcrel = true,
1154		.length = 2,
1155		});
1156		} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
1157		// Add relocation to the decl.
1158		const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1159		const target = coff_file.getGlobalByIndex(relocation.sym_index);
1160		try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
1161		.type = .direct,
1162		.target = target,
1163		.offset = offset,
1164		.addend = 0,
1165		.pcrel = true,
1166		.length = 2,
1167		});
1168		} else {
1169		return emit.fail("TODO implement call_extern for linking backends different than MachO and COFF", .{});
1170		}
1171		}
	242	// fn mirJmpCall(emit: *Emit, mnemonic: Instruction.Mnemonic, inst: Mir.Inst.Index) InnerError!void {
	243	// const ops = emit.mir.instructions.items(.ops)[inst].decode();
	244	// switch (ops.flags) {
	245	// 0b00 => {
	246	// const target = emit.mir.instructions.items(.data)[inst].inst;
	247	// const source = emit.code.items.len;
	248	// try emit.encode(mnemonic, .{
	249	// .op1 = .{ .imm = Immediate.s(0) },
	250	// });
	251	// try emit.relocs.append(emit.bin_file.allocator, .{
	252	// .source = source,
	253	// .target = target,
	254	// .offset = emit.code.items.len - 4,
	255	// .length = 5,
	256	// });
	257	// },
	258	// 0b01 => {
	259	// if (ops.reg1 == .none) {
	260	// const disp = emit.mir.instructions.items(.data)[inst].disp;
	261	// return emit.encode(mnemonic, .{
	262	// .op1 = .{ .mem = Memory.sib(.qword, .{ .disp = disp }) },
	263	// });
	264	// }
	265	// return emit.encode(mnemonic, .{
	266	// .op1 = .{ .reg = ops.reg1 },
	267	// });
	268	// },
	269	// 0b10 => {
	270	// const disp = emit.mir.instructions.items(.data)[inst].disp;
	271	// return emit.encode(mnemonic, .{
	272	// .op1 = .{ .mem = Memory.sib(.qword, .{
	273	// .base = ops.reg1,
	274	// .disp = disp,
	275	// }) },
	276	// });
	277	// },
	278	// 0b11 => return emit.fail("TODO unused variant jmp/call 0b11", .{}),
	279	// }
	280	// }
	281
	282	// fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	283	// const tag = emit.mir.instructions.items(.tag)[inst];
	284	// assert(tag == .cond_jmp);
	285	// const inst_cc = emit.mir.instructions.items(.data)[inst].inst_cc;
	286	// const mnemonic: Instruction.Mnemonic = switch (inst_cc.cc) {
	287	// .a => .ja,
	288	// .ae => .jae,
	289	// .b => .jb,
	290	// .be => .jbe,
	291	// .c => .jc,
	292	// .e => .je,
	293	// .g => .jg,
	294	// .ge => .jge,
	295	// .l => .jl,
	296	// .le => .jle,
	297	// .na => .jna,
	298	// .nae => .jnae,
	299	// .nb => .jnb,
	300	// .nbe => .jnbe,
	301	// .nc => .jnc,
	302	// .ne => .jne,
	303	// .ng => .jng,
	304	// .nge => .jnge,
	305	// .nl => .jnl,
	306	// .nle => .jnle,
	307	// .no => .jno,
	308	// .np => .jnp,
	309	// .ns => .jns,
	310	// .nz => .jnz,
	311	// .o => .jo,
	312	// .p => .jp,
	313	// .pe => .jpe,
	314	// .po => .jpo,
	315	// .s => .js,
	316	// .z => .jz,
	317	// };
	318	// const source = emit.code.items.len;
	319	// try emit.encode(mnemonic, .{
	320	// .op1 = .{ .imm = Immediate.s(0) },
	321	// });
	322	// try emit.relocs.append(emit.bin_file.allocator, .{
	323	// .source = source,
	324	// .target = inst_cc.inst,
	325	// .offset = emit.code.items.len - 4,
	326	// .length = 6,
	327	// });
	328	// }
	329
	330	// fn mirCondSetByte(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	331	// const tag = emit.mir.instructions.items(.tag)[inst];
	332	// assert(tag == .cond_set_byte);
	333	// const ops = emit.mir.instructions.items(.ops)[inst].decode();
	334	// const cc = emit.mir.instructions.items(.data)[inst].cc;
	335	// const mnemonic: Instruction.Mnemonic = switch (cc) {
	336	// .a => .seta,
	337	// .ae => .setae,
	338	// .b => .setb,
	339	// .be => .setbe,
	340	// .c => .setc,
	341	// .e => .sete,
	342	// .g => .setg,
	343	// .ge => .setge,
	344	// .l => .setl,
	345	// .le => .setle,
	346	// .na => .setna,
	347	// .nae => .setnae,
	348	// .nb => .setnb,
	349	// .nbe => .setnbe,
	350	// .nc => .setnc,
	351	// .ne => .setne,
	352	// .ng => .setng,
	353	// .nge => .setnge,
	354	// .nl => .setnl,
	355	// .nle => .setnle,
	356	// .no => .setno,
	357	// .np => .setnp,
	358	// .ns => .setns,
	359	// .nz => .setnz,
	360	// .o => .seto,
	361	// .p => .setp,
	362	// .pe => .setpe,
	363	// .po => .setpo,
	364	// .s => .sets,
	365	// .z => .setz,
	366	// };
	367	// return emit.encode(mnemonic, .{ .op1 = .{ .reg = ops.reg1 } });
	368	// }
	369
	370	// fn mirCondMov(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	371	// const tag = emit.mir.instructions.items(.tag)[inst];
	372	// assert(tag == .cond_mov);
	373	// const ops = emit.mir.instructions.items(.ops)[inst].decode();
	374	// const cc = emit.mir.instructions.items(.data)[inst].cc;
	375	// const mnemonic: Instruction.Mnemonic = switch (cc) {
	376	// .a => .cmova,
	377	// .ae => .cmovae,
	378	// .b => .cmovb,
	379	// .be => .cmovbe,
	380	// .c => .cmovc,
	381	// .e => .cmove,
	382	// .g => .cmovg,
	383	// .ge => .cmovge,
	384	// .l => .cmovl,
	385	// .le => .cmovle,
	386	// .na => .cmovna,
	387	// .nae => .cmovnae,
	388	// .nb => .cmovnb,
	389	// .nbe => .cmovnbe,
	390	// .nc => .cmovnc,
	391	// .ne => .cmovne,
	392	// .ng => .cmovng,
	393	// .nge => .cmovnge,
	394	// .nl => .cmovnl,
	395	// .nle => .cmovnle,
	396	// .no => .cmovno,
	397	// .np => .cmovnp,
	398	// .ns => .cmovns,
	399	// .nz => .cmovnz,
	400	// .o => .cmovo,
	401	// .p => .cmovp,
	402	// .pe => .cmovpe,
	403	// .po => .cmovpo,
	404	// .s => .cmovs,
	405	// .z => .cmovz,
	406	// };
	407	// const op1: Instruction.Operand = .{ .reg = ops.reg1 };
	408
	409	// if (ops.flags == 0b00) {
	410	// return emit.encode(mnemonic, .{
	411	// .op1 = op1,
	412	// .op2 = .{ .reg = ops.reg2 },
	413	// });
	414	// }
	415	// const disp = emit.mir.instructions.items(.data)[inst].disp;
	416	// const ptr_size: Memory.PtrSize = switch (ops.flags) {
	417	// 0b00 => unreachable,
	418	// 0b01 => .word,
	419	// 0b10 => .dword,
	420	// 0b11 => .qword,
	421	// };
	422	// return emit.encode(mnemonic, .{
	423	// .op1 = op1,
	424	// .op2 = .{ .mem = Memory.sib(ptr_size, .{
	425	// .base = ops.reg2,
	426	// .disp = disp,
	427	// }) },
	428	// });
	429	// }
	430
	431	// fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	432	// const tag = emit.mir.instructions.items(.tag)[inst];
	433	// assert(tag == .lea);
	434	// const ops = emit.mir.instructions.items(.ops)[inst].decode();
	435	// switch (ops.flags) {
	436	// 0b00 => {
	437	// const disp = emit.mir.instructions.items(.data)[inst].disp;
	438	// const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
	439	// return emit.encode(.lea, .{
	440	// .op1 = .{ .reg = ops.reg1 },
	441	// .op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
	442	// .base = src_reg,
	443	// .disp = disp,
	444	// }) },
	445	// });
	446	// },
	447	// 0b01 => {
	448	// const start_offset = emit.code.items.len;
	449	// try emit.encode(.lea, .{
	450	// .op1 = .{ .reg = ops.reg1 },
	451	// .op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), 0) },
	452	// });
	453	// const end_offset = emit.code.items.len;
	454	// // Backpatch the displacement
	455	// const payload = emit.mir.instructions.items(.data)[inst].payload;
	456	// const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
	457	// const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
	458	// mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
	459	// },
	460	// 0b10 => {
	461	// const payload = emit.mir.instructions.items(.data)[inst].payload;
	462	// const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
	463	// const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
	464	// const scale_index = Memory.ScaleIndex{
	465	// .scale = 1,
	466	// .index = index_reg_disp.index,
	467	// };
	468	// return emit.encode(.lea, .{
	469	// .op1 = .{ .reg = ops.reg1 },
	470	// .op2 = .{ .mem = Memory.sib(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), .{
	471	// .base = src_reg,
	472	// .scale_index = scale_index,
	473	// .disp = index_reg_disp.disp,
	474	// }) },
	475	// });
	476	// },
	477	// 0b11 => return emit.fail("TODO unused LEA variant 0b11", .{}),
	478	// }
	479	// }
	480
	481	// fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	482	// const tag = emit.mir.instructions.items(.tag)[inst];
	483	// assert(tag == .lea_pic);
	484	// const ops = emit.mir.instructions.items(.ops)[inst].decode();
	485	// const relocation = emit.mir.instructions.items(.data)[inst].relocation;
	486
	487	// switch (ops.flags) {
	488	// 0b00, 0b01, 0b10 => {},
	489	// else => return emit.fail("TODO unused LEA PIC variant 0b11", .{}),
	490	// }
	491
	492	// try emit.encode(.lea, .{
	493	// .op1 = .{ .reg = ops.reg1 },
	494	// .op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(ops.reg1.bitSize()), 0) },
	495	// });
	496
	497	// const end_offset = emit.code.items.len;
	498
	499	// if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	500	// const reloc_type = switch (ops.flags) {
	501	// 0b00 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	502	// 0b01 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED),
	503	// else => unreachable,
	504	// };
	505	// const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	506	// try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	507	// .type = reloc_type,
	508	// .target = .{ .sym_index = relocation.sym_index, .file = null },
	509	// .offset = @intCast(u32, end_offset - 4),
	510	// .addend = 0,
	511	// .pcrel = true,
	512	// .length = 2,
	513	// });
	514	// } else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	515	// const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	516	// try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	517	// .type = switch (ops.flags) {
	518	// 0b00 => .got,
	519	// 0b01 => .direct,
	520	// 0b10 => .import,
	521	// else => unreachable,
	522	// },
	523	// .target = switch (ops.flags) {
	524	// 0b00, 0b01 => .{ .sym_index = relocation.sym_index, .file = null },
	525	// 0b10 => coff_file.getGlobalByIndex(relocation.sym_index),
	526	// else => unreachable,
	527	// },
	528	// .offset = @intCast(u32, end_offset - 4),
	529	// .addend = 0,
	530	// .pcrel = true,
	531	// .length = 2,
	532	// });
	533	// } else {
	534	// return emit.fail("TODO implement lea reg, [rip + reloc] for linking backends different than MachO", .{});
	535	// }
	536	// }
	537
	538	// fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	539	// const tag = emit.mir.instructions.items(.tag)[inst];
	540	// assert(tag == .call_extern);
	541	// const relocation = emit.mir.instructions.items(.data)[inst].relocation;
	542
	543	// const offset = blk: {
	544	// // callq
	545	// try emit.encode(.call, .{
	546	// .op1 = .{ .imm = Immediate.s(0) },
	547	// });
	548	// break :blk @intCast(u32, emit.code.items.len) - 4;
	549	// };
	550
	551	// if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	552	// // Add relocation to the decl.
	553	// const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	554	// const target = macho_file.getGlobalByIndex(relocation.sym_index);
	555	// try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
	556	// .type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	557	// .target = target,
	558	// .offset = offset,
	559	// .addend = 0,
	560	// .pcrel = true,
	561	// .length = 2,
	562	// });
	563	// } else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
	564	// // Add relocation to the decl.
	565	// const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
	566	// const target = coff_file.getGlobalByIndex(relocation.sym_index);
	567	// try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
	568	// .type = .direct,
	569	// .target = target,
	570	// .offset = offset,
	571	// .addend = 0,
	572	// .pcrel = true,
	573	// .length = 2,
	574	// });
	575	// } else {
	576	// return emit.fail("TODO implement call_extern for linking backends different than MachO and COFF", .{});
	577	// }
	578	// }
1172	579
1173	580	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1174		const tag = emit.mir.instructions.items(.tag)[inst];
1175		assert(tag == .dbg_line);
1176	581	const payload = emit.mir.instructions.items(.data)[inst].payload;
1177	582	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
1178	583	log.debug("mirDbgLine", .{});
...	...	@@ -1230,8 +635,7 @@ fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1230	635	}
1231	636
1232	637	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1233		const tag = emit.mir.instructions.items(.tag)[inst];
1234		assert(tag == .dbg_prologue_end);
	638	_ = inst;
1235	639	switch (emit.debug_output) {
1236	640	.dwarf => \|dw\| {
1237	641	try dw.setPrologueEnd();
...	...	@@ -1247,8 +651,7 @@ fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1247	651	}
1248	652
1249	653	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1250		const tag = emit.mir.instructions.items(.tag)[inst];
1251		assert(tag == .dbg_epilogue_begin);
	654	_ = inst;
1252	655	switch (emit.debug_output) {
1253	656	.dwarf => \|dw\| {
1254	657	try dw.setEpilogueBegin();

src/arch/x86_64/Mir.zig+215-441

...	...	@@ -12,6 +12,8 @@ const builtin = @import("builtin");
12	12	const assert = std.debug.assert;
13	13
14	14	const bits = @import("bits.zig");
	15	const encoder = @import("encoder.zig");
	16
15	17	const Air = @import("../../Air.zig");
16	18	const CodeGen = @import("CodeGen.zig");
17	19	const IntegerBitSet = std.bit_set.IntegerBitSet;
...	...	@@ -21,421 +23,239 @@ instructions: std.MultiArrayList(Inst).Slice,
21	23	/// The meaning of this data is determined by `Inst.Tag` value.
22	24	extra: []const u32,
23	25
	26	pub const Mnemonic = encoder.Instruction.Mnemonic;
	27	pub const Operand = encoder.Instruction.Operand;
	28
24	29	pub const Inst = struct {
25	30	tag: Tag,
26	31	ops: Ops,
27		/// The meaning of this depends on `tag` and `ops`.
28	32	data: Data,
29	33
30		pub const Tag = enum(u16) {
31		/// ops flags: form:
32		/// 0b00 reg1, reg2
33		/// 0b00 reg1, imm32
34		/// 0b01 reg1, [reg2 + imm32]
35		/// 0b01 reg1, [ds:imm32]
36		/// 0b10 [reg1 + imm32], reg2
37		/// 0b11 reg1, imm_s
38		/// Notes:
39		/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
40		/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
41		adc,
42
43		/// ops flags: form:
44		/// 0b00 byte ptr [reg1 + imm32], imm8
45		/// 0b01 word ptr [reg1 + imm32], imm16
46		/// 0b10 dword ptr [reg1 + imm32], imm32
47		/// 0b11 qword ptr [reg1 + imm32], imm32 (sign-extended to imm64)
48		/// Notes:
49		/// * Uses `ImmPair` as payload
50		adc_mem_imm,
51
52		/// form: reg1, [reg2 + scale*index + imm32]
53		/// ops flags scale
54		/// 0b00 1
55		/// 0b01 2
56		/// 0b10 4
57		/// 0b11 8
58		/// Notes:
59		/// * Uses `IndexRegisterDisp` as payload
60		adc_scale_src,
61
62		/// form: [reg1 + scale*index + imm32], reg2
63		/// ops flags scale
64		/// 0b00 1
65		/// 0b01 2
66		/// 0b10 4
67		/// 0b11 8
68		/// Notes:
69		/// * Uses `IndexRegisterDisp` payload.
70		adc_scale_dst,
71
72		/// form: [reg1 + scale*rax + imm32], imm32
73		/// ops flags scale
74		/// 0b00 1
75		/// 0b01 2
76		/// 0b10 4
77		/// 0b11 8
78		/// Notes:
79		/// * Uses `IndexRegisterDispImm` payload.
80		adc_scale_imm,
81
82		/// ops flags: form:
83		/// 0b00 byte ptr [reg1 + index + imm32], imm8
84		/// 0b01 word ptr [reg1 + index + imm32], imm16
85		/// 0b10 dword ptr [reg1 + index + imm32], imm32
86		/// 0b11 qword ptr [reg1 + index + imm32], imm32 (sign-extended to imm64)
87		/// Notes:
88		/// * Uses `IndexRegisterDispImm` payload.
89		adc_mem_index_imm,
90
91		// The following instructions all have the same encoding as `adc`.
	34	pub const Index = u32;
92	35
	36	pub const Tag = enum(u8) {
	37	/// Add with carry
	38	adc,
	39	/// Add
93	40	add,
94		add_mem_imm,
95		add_scale_src,
96		add_scale_dst,
97		add_scale_imm,
98		add_mem_index_imm,
99		sub,
100		sub_mem_imm,
101		sub_scale_src,
102		sub_scale_dst,
103		sub_scale_imm,
104		sub_mem_index_imm,
105		xor,
106		xor_mem_imm,
107		xor_scale_src,
108		xor_scale_dst,
109		xor_scale_imm,
110		xor_mem_index_imm,
	41	/// Logical and
111	42	@"and",
112		and_mem_imm,
113		and_scale_src,
114		and_scale_dst,
115		and_scale_imm,
116		and_mem_index_imm,
117		@"or",
118		or_mem_imm,
119		or_scale_src,
120		or_scale_dst,
121		or_scale_imm,
122		or_mem_index_imm,
123		rol,
124		rol_mem_imm,
125		rol_scale_src,
126		rol_scale_dst,
127		rol_scale_imm,
128		rol_mem_index_imm,
129		ror,
130		ror_mem_imm,
131		ror_scale_src,
132		ror_scale_dst,
133		ror_scale_imm,
134		ror_mem_index_imm,
135		rcl,
136		rcl_mem_imm,
137		rcl_scale_src,
138		rcl_scale_dst,
139		rcl_scale_imm,
140		rcl_mem_index_imm,
141		rcr,
142		rcr_mem_imm,
143		rcr_scale_src,
144		rcr_scale_dst,
145		rcr_scale_imm,
146		rcr_mem_index_imm,
147		sbb,
148		sbb_mem_imm,
149		sbb_scale_src,
150		sbb_scale_dst,
151		sbb_scale_imm,
152		sbb_mem_index_imm,
	43	/// Call
	44	call,
	45	/// Convert byte to word
	46	cbw,
	47	/// Convert word to doubleword
	48	cwde,
	49	/// Convert doubleword to quadword
	50	cdqe,
	51	/// Convert word to doubleword
	52	cwd,
	53	/// Convert doubleword to quadword
	54	cdq,
	55	/// Convert doubleword to quadword
	56	cqo,
	57	/// Logical compare
153	58	cmp,
154		cmp_mem_imm,
155		cmp_scale_src,
156		cmp_scale_dst,
157		cmp_scale_imm,
158		cmp_mem_index_imm,
159		mov,
160		mov_mem_imm,
161		mov_scale_src,
162		mov_scale_dst,
163		mov_scale_imm,
164		mov_mem_index_imm,
165
166		/// ops flags: form:
167		/// 0b00 reg1, reg2,
168		/// 0b01 reg1, byte ptr [reg2 + imm32]
169		/// 0b10 reg1, word ptr [reg2 + imm32]
170		/// 0b11 reg1, dword ptr [reg2 + imm32]
171		mov_sign_extend,
172
173		/// ops flags: form:
174		/// 0b00 reg1, reg2
175		/// 0b01 reg1, byte ptr [reg2 + imm32]
176		/// 0b10 reg1, word ptr [reg2 + imm32]
177		mov_zero_extend,
178
179		/// ops flags: form:
180		/// 0b00 reg1, [reg2 + imm32]
181		/// 0b00 reg1, [ds:imm32]
182		/// 0b01 reg1, [rip + imm32]
183		/// 0b10 reg1, [reg2 + index + imm32]
184		/// Notes:
185		/// * 0b10 uses `IndexRegisterDisp` payload
186		lea,
187
188		/// ops flags: form:
189		/// 0b00 reg1, [rip + reloc] // via GOT PIC
190		/// 0b01 reg1, [rip + reloc] // direct load PIC
191		/// 0b10 reg1, [rip + reloc] // via imports table PIC
192		/// Notes:
193		/// * `Data` contains `relocation`
194		lea_pic,
195
196		/// ops flags: form:
197		/// 0b00 reg1, 1
198		/// 0b01 reg1, .cl
199		/// 0b10 reg1, imm8
200		/// Notes:
201		/// * If flags == 0b10, uses `imm`.
202		shl,
203		shl_mem_imm,
204		shl_scale_src,
205		shl_scale_dst,
206		shl_scale_imm,
207		shl_mem_index_imm,
208		sal,
209		sal_mem_imm,
210		sal_scale_src,
211		sal_scale_dst,
212		sal_scale_imm,
213		sal_mem_index_imm,
214		shr,
215		shr_mem_imm,
216		shr_scale_src,
217		shr_scale_dst,
218		shr_scale_imm,
219		shr_mem_index_imm,
220		sar,
221		sar_mem_imm,
222		sar_scale_src,
223		sar_scale_dst,
224		sar_scale_imm,
225		sar_mem_index_imm,
226
227		/// ops flags: form:
228		/// 0b00 reg1
229		/// 0b00 byte ptr [reg2 + imm32]
230		/// 0b01 word ptr [reg2 + imm32]
231		/// 0b10 dword ptr [reg2 + imm32]
232		/// 0b11 qword ptr [reg2 + imm32]
233		imul,
234		idiv,
235		mul,
	59	/// Conditional move
	60	cmovcc,
	61	/// Unsigned division
236	62	div,
237
238		/// ops flags: form:
239		/// 0b00 AX <- AL
240		/// 0b01 DX:AX <- AX
241		/// 0b10 EDX:EAX <- EAX
242		/// 0b11 RDX:RAX <- RAX
243		cwd,
244
245		/// ops flags: form:
246		/// 0b00 reg1, reg2
247		/// 0b01 reg1, [reg2 + imm32]
248		/// 0b01 reg1, [imm32] if reg2 is none
249		/// 0b10 reg1, reg2, imm32
250		/// 0b11 reg1, [reg2 + imm32], imm32
251		imul_complex,
252
253		/// ops flags: form:
254		/// 0b00 reg1, imm64
255		/// 0b01 rax, moffs64
256		/// Notes:
257		/// * If reg1 is 64-bit, the immediate is 64-bit and stored
258		/// within extra data `Imm64`.
259		/// * For 0b01, reg1 (or reg2) need to be
260		/// a version of rax. If reg1 == .none, then reg2 == .rax,
261		/// or vice versa.
262		movabs,
263
264		/// ops flags: form:
265		/// 0b00 word ptr [reg1 + imm32]
266		/// 0b01 dword ptr [reg1 + imm32]
267		/// 0b10 qword ptr [reg1 + imm32]
268		/// Notes:
269		/// * source is always ST(0)
270		/// * only supports memory operands as destination
	63	/// Store integer with truncation
271	64	fisttp,
272
273		/// ops flags: form:
274		/// 0b01 dword ptr [reg1 + imm32]
275		/// 0b10 qword ptr [reg1 + imm32]
	65	/// Load floating-point value
276	66	fld,
277
278		/// ops flags: form:
279		/// 0b00 inst
280		/// 0b01 reg1
281		/// 0b01 [imm32] if reg1 is none
282		/// 0b10 [reg1 + imm32]
	67	/// Signed division
	68	idiv,
	69	/// Signed multiplication
	70	imul,
	71	///
	72	int3,
	73	/// Conditional jump
	74	jcc,
	75	/// Jump
283	76	jmp,
284		call,
285
286		/// ops flags:
287		/// unused
288		/// Notes:
289		/// * uses `inst_cc` in Data.
290		cond_jmp,
291
292		/// ops flags:
293		/// 0b00 reg1
294		/// Notes:
295		/// * uses condition code (CC) stored as part of data
296		cond_set_byte,
297
298		/// ops flags:
299		/// 0b00 reg1, reg2,
300		/// 0b01 reg1, word ptr [reg2 + imm]
301		/// 0b10 reg1, dword ptr [reg2 + imm]
302		/// 0b11 reg1, qword ptr [reg2 + imm]
303		/// Notes:
304		/// * uses condition code (CC) stored as part of data
305		cond_mov,
306
307		/// ops flags: form:
308		/// 0b00 reg1
309		/// 0b01 [reg1 + imm32]
310		/// 0b10 imm32
311		/// Notes:
312		/// * If 0b10 is specified and the tag is push, pushes immediate onto the stack
313		/// using the mnemonic PUSH imm32.
314		push,
	77	/// Load effective address
	78	lea,
	79	/// Move
	80	mov,
	81	/// Move with sign extension
	82	movsx,
	83	/// Move with zero extension
	84	movzx,
	85	/// Multiply
	86	mul,
	87	/// No-op
	88	nop,
	89	/// Logical or
	90	@"or",
	91	/// Pop
315	92	pop,
316
317		/// ops flags: form:
318		/// 0b00 retf imm16
319		/// 0b01 retf
320		/// 0b10 retn imm16
321		/// 0b11 retn
	93	/// Push
	94	push,
	95	/// Return
322	96	ret,
323
324		/// Fast system call
	97	/// Arithmetic shift left
	98	sal,
	99	/// Arithmetic shift right
	100	sar,
	101	/// Integer subtraction with borrow
	102	sbb,
	103	/// Set byte on condition
	104	setcc,
	105	/// Logical shift left
	106	shl,
	107	/// Logical shift right
	108	shr,
	109	/// Subtract
	110	sub,
	111	/// Syscall
325	112	syscall,
326
327		/// ops flags: form:
328		/// 0b00 reg1, imm32 if reg2 == .none
329		/// 0b00 reg1, reg2
330		/// TODO handle more cases
	113	/// Test condition
331	114	@"test",
	115	/// Undefined instruction
	116	ud2,
	117	/// Logical exclusive-or
	118	xor,
332	119
333		/// Undefined Instruction
334		ud,
335
336		/// Breakpoint form:
337		/// 0b00 int3
338		interrupt,
339
340		/// Nop
341		nop,
342
343		/// SSE/AVX instructions
344		/// ops flags: form:
345		/// 0b00 reg1, qword ptr [reg2 + imm32]
346		/// 0b01 qword ptr [reg1 + imm32], reg2
347		/// 0b10 reg1, reg2
348		mov_f64,
349		mov_f32,
350
351		/// ops flags: form:
352		/// 0b00 reg1, reg2
353		add_f64,
354		add_f32,
355
356		/// ops flags: form:
357		/// 0b00 reg1, reg2
358		cmp_f64,
359		cmp_f32,
360
361		/// Pseudo-instructions
362		/// call extern function
363		/// Notes:
364		/// * target of the call is stored as `relocation` in `Data` union.
365		call_extern,
366
367		/// end of prologue
	120	/// Add single precision floating point
	121	addss,
	122	/// Compare scalar single-precision floating-point values
	123	cmpss,
	124	/// Move scalar single-precision floating-point value
	125	movss,
	126	/// Unordered compare scalar single-precision floating-point values
	127	ucomiss,
	128	/// Add double precision floating point
	129	addsd,
	130	/// Compare scalar double-precision floating-point values
	131	cmpsd,
	132	/// Move scalar double-precision floating-point value
	133	movsd,
	134	/// Unordered compare scalar double-precision floating-point values
	135	ucomisd,
	136
	137	/// End of prologue
368	138	dbg_prologue_end,
369
370		/// start of epilogue
	139	/// Start of epilogue
371	140	dbg_epilogue_begin,
372
373		/// update debug line
	141	/// Update debug line
	142	/// Uses `payload` payload with data of type `DbgLineColumn`.
374	143	dbg_line,
375
376		/// push registers
377		/// Uses `payload` field with `SaveRegisterList` as payload.
	144	/// Push registers
	145	/// Uses `payload` payload with data of type `SaveRegisterList`.
378	146	push_regs,
379
380		/// pop registers
381		/// Uses `payload` field with `SaveRegisterList` as payload.
	147	/// Pop registers
	148	/// Uses `payload` payload with data of type `SaveRegisterList`.
382	149	pop_regs,
383	150	};
384		/// The position of an MIR instruction within the `Mir` instructions array.
385		pub const Index = u32;
386
387		pub const Ops = packed struct {
388		reg1: u7,
389		reg2: u7,
390		flags: u2,
391	151
392		pub fn encode(vals: struct {
393		reg1: Register = .none,
394		reg2: Register = .none,
395		flags: u2 = 0b00,
396		}) Ops {
397		return .{
398		.reg1 = @enumToInt(vals.reg1),
399		.reg2 = @enumToInt(vals.reg2),
400		.flags = vals.flags,
401		};
402		}
403
404		pub fn decode(ops: Ops) struct {
405		reg1: Register,
406		reg2: Register,
407		flags: u2,
408		} {
409		return .{
410		.reg1 = @intToEnum(Register, ops.reg1),
411		.reg2 = @intToEnum(Register, ops.reg2),
412		.flags = ops.flags,
413		};
414		}
	152	pub const Ops = enum(u8) {
	153	/// No data associated with this instruction (only mnemonic is used).
	154	none,
	155	/// Single register operand.
	156	/// Uses `r` payload.
	157	r,
	158	/// Register, register operands.
	159	/// Uses `rr` payload.
	160	rr,
	161	/// Register, register, register operands.
	162	/// Uses `rrr` payload.
	163	rrr,
	164	/// Register, immediate (sign-extended) operands.
	165	/// Uses `ri_s` payload.
	166	ri_s,
	167	/// Register, immediate (unsigned) operands.
	168	/// Uses `ri_u` payload.
	169	ri_u,
	170	/// Register, 64-bit unsigned immediate operands.
	171	/// Uses `rx` payload with payload type `Imm64`.
	172	ri64,
	173	/// Immediate (sign-extended) operand.
	174	/// Uses `imm_s` payload.
	175	imm_s,
	176	/// Immediate (unsigned) operand.
	177	/// Uses `imm_u` payload.
	178	imm_u,
	179	/// Relative displacement operand.
	180	/// Uses `rel` payload.
	181	rel,
	182	/// Register, memory operands.
	183	/// Uses `rx` payload.
	184	rm,
	185	/// Register, memory, immediate (unsigned) operands
	186	/// Uses `rx` payload.
	187	rmi_u,
	188	/// Register, memory, immediate (sign-extended) operands
	189	/// Uses `rx` payload.
	190	rmi_s,
	191	/// Memory, immediate (unsigned) operands.
	192	/// Uses `payload` payload.
	193	mi_u,
	194	/// Memory, immediate (sign-extend) operands.
	195	/// Uses `payload` payload.
	196	mi_s,
	197	/// Memory, register operands.
	198	/// Uses `payload` payload.
	199	mr,
	200	/// Lea into register with linker relocation.
	201	/// Uses `payload` payload with data of type `LeaRegisterReloc`.
	202	lea_r_reloc,
	203	/// References another Mir instruction directly.
	204	/// Uses `inst` payload.
	205	inst,
	206	/// References another Mir instruction directly with condition code (CC).
	207	/// Uses `inst_cc` payload.
	208	inst_cc,
	209	/// Uses `payload` payload with data of type `MemoryConditionCode`.
	210	m_cc,
	211	/// Uses `rx` payload with extra data of type `MemoryConditionCode`.
	212	rm_cc,
	213	/// Uses `reloc` payload.
	214	reloc,
415	215	};
416	216
417		/// All instructions have a 4-byte payload, which is contained within
418		/// this union. `Tag` determines which union field is active, as well as
419		/// how to interpret the data within.
420	217	pub const Data = union {
421		/// Another instruction.
	218	/// References another Mir instruction.
422	219	inst: Index,
423		/// A 32-bit immediate value.
424		imm: u32,
425		/// A 32-bit signed immediate value.
426		imm_s: i32,
427		/// A 32-bit signed displacement value.
428		disp: i32,
429		/// A condition code for use with EFLAGS register.
430		cc: bits.Condition,
431		/// Another instruction with condition code.
432		/// Used by `cond_jmp`.
	220	/// Another instruction with condition code (CC).
	221	/// Used by `jcc`.
433	222	inst_cc: struct {
434	223	/// Another instruction.
435	224	inst: Index,
436	225	/// A condition code for use with EFLAGS register.
437	226	cc: bits.Condition,
438	227	},
	228	/// A 32-bit signed immediate value.
	229	imm_s: i32,
	230	/// A 32-bit unsigned immediate value.
	231	imm_u: u32,
	232	/// A 32-bit signed relative offset value.
	233	rel: i32,
	234	r: Register,
	235	rr: struct {
	236	r1: Register,
	237	r2: Register,
	238	},
	239	rrr: struct {
	240	r1: Register,
	241	r2: Register,
	242	r3: Register,
	243	},
	244	/// Register, signed immediate.
	245	ri_s: struct {
	246	r1: Register,
	247	imm: i32,
	248	},
	249	/// Register, unsigned immediate.
	250	ri_u: struct {
	251	r1: Register,
	252	imm: u32,
	253	},
	254	/// Register, followed by custom payload found in extra.
	255	rx: struct {
	256	r1: Register,
	257	payload: u32,
	258	},
439	259	/// Relocation for the linker where:
440	260	/// * `atom_index` is the index of the source
441	261	/// * `sym_index` is the index of the target
...	...	@@ -458,62 +278,19 @@ pub const Inst = struct {
458	278	}
459	279	};
460	280
461		pub const IndexRegisterDisp = struct {
462		/// Index register to use with SIB-based encoding
463		index: u32,
464
465		/// Displacement value
466		disp: i32,
467
468		pub fn encode(index: Register, disp: i32) IndexRegisterDisp {
469		return .{
470		.index = @enumToInt(index),
471		.disp = disp,
472		};
473		}
474
475		pub fn decode(this: IndexRegisterDisp) struct {
476		index: Register,
477		disp: i32,
478		} {
479		return .{
480		.index = @intToEnum(Register, this.index),
481		.disp = this.disp,
482		};
483		}
484		};
485
486		/// TODO: would it be worth making `IndexRegisterDisp` and `IndexRegisterDispImm` a variable length list
487		/// instead of having two structs, one a superset of the other one?
488		pub const IndexRegisterDispImm = struct {
489		/// Index register to use with SIB-based encoding
490		index: u32,
491
492		/// Displacement value
493		disp: i32,
494
495		/// Immediate
496		imm: u32,
497
498		pub fn encode(index: Register, disp: i32, imm: u32) IndexRegisterDispImm {
499		return .{
500		.index = @enumToInt(index),
501		.disp = disp,
502		.imm = imm,
503		};
504		}
505
506		pub fn decode(this: IndexRegisterDispImm) struct {
507		index: Register,
508		disp: i32,
509		imm: u32,
510		} {
511		return .{
512		.index = @intToEnum(Register, this.index),
513		.disp = this.disp,
514		.imm = this.imm,
515		};
516		}
	281	pub const LeaRegisterReloc = struct {
	282	/// Destination register.
	283	reg: Register,
	284	/// Type of the load.
	285	load_type: enum(u2) {
	286	got,
	287	direct,
	288	import,
	289	},
	290	/// Index of the containing atom.
	291	atom_index: u32,
	292	/// Index into the linker's symbol table.
	293	sym_index: u32,
517	294	};
518	295
519	296	/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers
...	...	@@ -557,16 +334,13 @@ pub const RegisterList = struct {
557	334	};
558	335
559	336	pub const SaveRegisterList = struct {
	337	/// Base register
	338	base_reg: u32,
560	339	/// Use `RegisterList` to populate.
561	340	register_list: u32,
562	341	stack_end: u32,
563	342	};
564	343
565		pub const ImmPair = struct {
566		dest_off: i32,
567		operand: u32,
568		};
569
570	344	pub const Imm64 = struct {
571	345	msb: u32,
572	346	lsb: u32,

src/arch/x86_64/encoder.zig+5-6

...	...	@@ -4,10 +4,6 @@ const math = std.math;
4	4
5	5	const bits = @import("bits.zig");
6	6	const Encoding = @import("Encoding.zig");
7		const Immediate = bits.Immediate;
8		const Memory = bits.Memory;
9		const Moffs = bits.Moffs;
10		const PtrSize = bits.PtrSize;
11	7	const Register = bits.Register;
12	8
13	9	pub const Instruction = struct {
...	...	@@ -25,6 +21,9 @@ pub const Instruction = struct {
25	21	mem: Memory,
26	22	imm: Immediate,
27	23
	24	pub const Memory = bits.Memory;
	25	pub const Immediate = bits.Immediate;
	26
28	27	/// Returns the bitsize of the operand.
29	28	pub fn bitSize(op: Operand) u64 {
30	29	return switch (op) {
...	...	@@ -296,7 +295,7 @@ pub const Instruction = struct {
296	295	try encoder.opcode_1byte(prefix);
297	296	}
298	297
299		fn encodeMemory(encoding: Encoding, mem: Memory, operand: Operand, encoder: anytype) !void {
	298	fn encodeMemory(encoding: Encoding, mem: Operand.Memory, operand: Operand, encoder: anytype) !void {
300	299	const operand_enc = switch (operand) {
301	300	.reg => \|reg\| reg.lowEnc(),
302	301	.none => encoding.modRmExt(),
...	...	@@ -379,7 +378,7 @@ pub const Instruction = struct {
379	378	}
380	379	}
381	380
382		fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {
	381	fn encodeImm(imm: Operand.Immediate, kind: Encoding.Op, encoder: anytype) !void {
383	382	const raw = imm.asUnsigned(kind.bitSize());
384	383	switch (kind.bitSize()) {
385	384	8 => try encoder.imm8(@intCast(u8, raw)),