Commit 3c7310f8cc164dbf6e6393392e8812c9f5b6a270 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2021-10-31 23:08:53+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-11-08 13:27:49-05:00
log	3c7310f8cc164dbf6e6393392e8812c9f5b6a270
tree	64d58dfe8cde58f591651c52fc13e786a0b7a9bd
parent	4acd8920d856f475bb86e3ae7d6284f91e7119d9

stage2 x86_64: add MIR->Isel lowering step for x86_64

* incorporate Andrew's MIR draft as Mir.zig
* add skeleton for Emit.zig module - Emit will lower MIR into
  machine code or textual ASM.
* implement push
* implement ret
* implement mov r/m, r
* implement sub r/m imm and sub r/m, r
* put encoding common ops together - some ops share impl such as
  MOV and cmp so put them together and vary the actual opcode
  with modRM ext only.
* implement pop
* implement movabs - movabs being a special-case of mov not
  handled by general mov MIR instruction due to requirement to
  handle 64bit immediates.
* store imm64 as a struct `Imm64{ msb: u32, lsb: u32 }` in extra data
  for use with for instance movabs inst
* implement more mov variations
* implement adc
* implement add
* implement sub
* implement xor
* implement and
* implement or
* implement sbb
* implement cmp
* implement lea - lea doesn't follow the scheme as other inst above. Similarly, I
  think bit shifts and rotates should be put in a separate basket too.
* implement adc_scale_src
* implement add_scale_src
* implement sub_scale_src
* implement xor_scale_src
* implement and_scale_src
* implement or_scale_src
* implement sbb_scale_src
* implement cmp_scale_src
* implement adc_scale_dst
* implement add_scale_dst
* implement sub_scale_dst
* implement xor_scale_dst
* implement and_scale_dst
* implement or_scale_dst
* implement sbb_scale_dst
* implement cmp_scale_dst
* implement mov_scale_src
* implement mov_scale_dst
* implement adc_scale_imm
* implement add_scale_imm
* implement sub_scale_imm
* implement xor_scale_imm
* implement and_scale_imm
* implement or_scale_imm
* implement sbb_scale_imm
* implement cmp_scale_imm
* port bin math to MIR
* backpatch stack size into prev MIR inst
* implement Function.gen() (minus dbg info)
* implement jmp/call [imm] - we can now call functions using indirect absolute
  addressing, or via registers.
* port airRet to use MIR
* port airLoop to use MIR
* patch up performReloc to use inst indices
* implement conditional jumps (without relocs)
* implement set byte on condition
* implement basic lea r64, [rip + imm]
* implement calling externs
* implement callq in PIE
* implement lea RIP in PIE context
* remove all refs to Encoder from CodeGen
* implement basic imul ops
* pass all Linux tests!
* enable most of dbg info gen
* generate arg dbg info in Emit

5 files changed, 2118 insertions(+), 820 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    bits.zig
- codegen.zig

src/arch/x86_64/CodeGen.zig+566-812

...	...	@@ -14,11 +14,12 @@ const Allocator = mem.Allocator;
14	14	const Compilation = @import("../../Compilation.zig");
15	15	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
16	16	const DW = std.dwarf;
17		const Encoder = @import("bits.zig").Encoder;
	17	const Emit = @import("Emit.zig");
18	18	const ErrorMsg = Module.ErrorMsg;
19	19	const FnResult = @import("../../codegen.zig").FnResult;
20	20	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
21	21	const Liveness = @import("../../Liveness.zig");
	22	const Mir = @import("Mir.zig");
22	23	const Module = @import("../../Module.zig");
23	24	const RegisterManager = @import("../../register_manager.zig").RegisterManager;
24	25	const Target = std.Target;
...	...	@@ -32,15 +33,12 @@ const InnerError = error{
32	33	CodegenFail,
33	34	};
34	35
35		arch: std.Target.Cpu.Arch,
36	36	gpa: *Allocator,
37	37	air: Air,
38	38	liveness: Liveness,
39	39	bin_file: *link.File,
40	40	target: *const std.Target,
41	41	mod_fn: *const Module.Fn,
42		code: *std.ArrayList(u8),
43		debug_output: DebugInfoOutput,
44	42	err_msg: ?*ErrorMsg,
45	43	args: []MCValue,
46	44	ret_mcv: MCValue,
...	...	@@ -49,18 +47,19 @@ arg_index: usize,
49	47	src_loc: Module.SrcLoc,
50	48	stack_align: u32,
51	49
52		prev_di_line: u32,
53		prev_di_column: u32,
	50	/// MIR Instructions
	51	mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
	52	/// MIR extra data
	53	mir_extra: std.ArrayListUnmanaged(u32) = .{},
	54
54	55	/// Byte offset within the source file of the ending curly.
55	56	end_di_line: u32,
56	57	end_di_column: u32,
57		/// Relative to the beginning of `code`.
58		prev_di_pc: usize,
59	58
60	59	/// The value is an offset into the `Function` `code` from the beginning.
61	60	/// To perform the reloc, write 32-bit signed little-endian integer
62	61	/// which is a relative jump, based on the address following the reloc.
63		exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{},
	62	exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
64	63
65	64	/// Whenever there is a runtime branch, we push a Branch onto this stack,
66	65	/// and pop it off when the runtime branch joins. This provides an "overlay"
...	...	@@ -89,7 +88,7 @@ air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
89	88
90	89	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
91	90
92		const MCValue = union(enum) {
	91	pub const MCValue = union(enum) {
93	92	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
94	93	/// TODO Look into deleting this tag and using `dead` instead, since every use
95	94	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
...	...	@@ -178,7 +177,7 @@ const StackAllocation = struct {
178	177	};
179	178
180	179	const BlockData = struct {
181		relocs: std.ArrayListUnmanaged(Reloc),
	180	relocs: std.ArrayListUnmanaged(Mir.Inst.Index),
182	181	/// The first break instruction encounters `null` here and chooses a
183	182	/// machine code value for the block result, populating this field.
184	183	/// Following break instructions encounter that value and use it for
...	...	@@ -186,18 +185,6 @@ const BlockData = struct {
186	185	mcv: MCValue,
187	186	};
188	187
189		const Reloc = union(enum) {
190		/// The value is an offset into the `Function` `code` from the beginning.
191		/// To perform the reloc, write 32-bit signed little-endian integer
192		/// which is a relative jump, based on the address following the reloc.
193		rel32: usize,
194		/// A branch in the ARM instruction set
195		arm_branch: struct {
196		pos: usize,
197		cond: @import("../../arch/arm/bits.zig").Condition,
198		},
199		};
200
201	188	const BigTomb = struct {
202	189	function: *Self,
203	190	inst: Air.Inst.Index,
...	...	@@ -238,7 +225,6 @@ const BigTomb = struct {
238	225	const Self = @This();
239	226
240	227	pub fn generate(
241		arch: std.Target.Cpu.Arch,
242	228	bin_file: *link.File,
243	229	src_loc: Module.SrcLoc,
244	230	module_fn: *Module.Fn,
...	...	@@ -247,7 +233,7 @@ pub fn generate(
247	233	code: *std.ArrayList(u8),
248	234	debug_output: DebugInfoOutput,
249	235	) GenerateSymbolError!FnResult {
250		if (build_options.skip_non_native and builtin.cpu.arch != arch) {
	236	if (build_options.skip_non_native and builtin.cpu.arch != bin_file.options.target.cpu.arch) {
251	237	@panic("Attempted to compile for architecture that was disabled by build configuration");
252	238	}
253	239
...	...	@@ -263,15 +249,12 @@ pub fn generate(
263	249	try branch_stack.append(.{});
264	250
265	251	var function = Self{
266		.arch = arch,
267	252	.gpa = bin_file.allocator,
268	253	.air = air,
269	254	.liveness = liveness,
270	255	.target = &bin_file.options.target,
271	256	.bin_file = bin_file,
272	257	.mod_fn = module_fn,
273		.code = code,
274		.debug_output = debug_output,
275	258	.err_msg = null,
276	259	.args = undefined, // populated after `resolveCallingConventionValues`
277	260	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
...	...	@@ -280,15 +263,14 @@ pub fn generate(
280	263	.branch_stack = &branch_stack,
281	264	.src_loc = src_loc,
282	265	.stack_align = undefined,
283		.prev_di_pc = 0,
284		.prev_di_line = module_fn.lbrace_line,
285		.prev_di_column = module_fn.lbrace_column,
286	266	.end_di_line = module_fn.rbrace_line,
287	267	.end_di_column = module_fn.rbrace_column,
288	268	};
289	269	defer function.stack.deinit(bin_file.allocator);
290	270	defer function.blocks.deinit(bin_file.allocator);
291	271	defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
	272	defer function.mir_instructions.deinit(bin_file.allocator);
	273	defer function.mir_extra.deinit(bin_file.allocator);
292	274
293	275	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
294	276	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
...	...	@@ -306,6 +288,30 @@ pub fn generate(
306	288	else => \|e\| return e,
307	289	};
308	290
	291	var mir = Mir{
	292	.function = &function,
	293	.instructions = function.mir_instructions.toOwnedSlice(),
	294	.extra = function.mir_extra.toOwnedSlice(bin_file.allocator),
	295	};
	296	defer mir.deinit(bin_file.allocator);
	297
	298	var emit = Emit{
	299	.mir = mir,
	300	.bin_file = bin_file,
	301	.debug_output = debug_output,
	302	.target = &bin_file.options.target,
	303	.src_loc = src_loc,
	304	.code = code,
	305	.prev_di_pc = 0,
	306	.prev_di_line = module_fn.lbrace_line,
	307	.prev_di_column = module_fn.lbrace_column,
	308	};
	309	defer emit.deinit();
	310	emit.emitMir() catch \|err\| switch (err) {
	311	error.EmitFail => return FnResult{ .fail = emit.err_msg.? },
	312	else => \|e\| return e,
	313	};
	314
309	315	if (function.err_msg) \|em\| {
310	316	return FnResult{ .fail = em };
311	317	} else {
...	...	@@ -313,71 +319,143 @@ pub fn generate(
313	319	}
314	320	}
315	321
316		fn gen(self: *Self) !void {
317		try self.code.ensureUnusedCapacity(11);
	322	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
	323	const gpa = self.gpa;
	324	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
	325	const result_index = @intCast(Air.Inst.Index, self.mir_instructions.len);
	326	self.mir_instructions.appendAssumeCapacity(inst);
	327	return result_index;
	328	}
	329
	330	pub fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
	331	const fields = std.meta.fields(@TypeOf(extra));
	332	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
	333	return self.addExtraAssumeCapacity(extra);
	334	}
318	335
	336	pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
	337	const fields = std.meta.fields(@TypeOf(extra));
	338	const result = @intCast(u32, self.mir_extra.items.len);
	339	inline for (fields) \|field\| {
	340	self.mir_extra.appendAssumeCapacity(switch (field.field_type) {
	341	u32 => @field(extra, field.name),
	342	i32 => @bitCast(u32, @field(extra, field.name)),
	343	else => @compileError("bad field type"),
	344	});
	345	}
	346	return result;
	347	}
	348
	349	fn gen(self: *Self) InnerError!void {
319	350	const cc = self.fn_type.fnCallingConvention();
320	351	if (cc != .Naked) {
	352	_ = try self.addInst(.{
	353	.tag = .push,
	354	.ops = (Mir.Ops{
	355	.reg1 = .rbp,
	356	}).encode(),
	357	.data = undefined, // unused for push reg,
	358	});
	359	_ = try self.addInst(.{
	360	.tag = .mov,
	361	.ops = (Mir.Ops{
	362	.reg1 = .rsp,
	363	.reg2 = .rbp,
	364	}).encode(),
	365	.data = undefined,
	366	});
321	367	// We want to subtract the aligned stack frame size from rsp here, but we don't
322	368	// yet know how big it will be, so we leave room for a 4-byte stack size.
323	369	// TODO During semantic analysis, check if there are no function calls. If there
324	370	// are none, here we can omit the part where we subtract and then add rsp.
325		self.code.appendSliceAssumeCapacity(&[_]u8{
326		0x55, // push rbp
327		0x48, 0x89, 0xe5, // mov rbp, rsp
328		0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc)
	371	const backpatch_reloc = try self.addInst(.{
	372	.tag = .sub,
	373	.ops = (Mir.Ops{
	374	.reg1 = .rsp,
	375	}).encode(),
	376	.data = .{ .imm = 0 },
	377	});
	378
	379	_ = try self.addInst(.{
	380	.tag = .dbg_prologue_end,
	381	.ops = undefined,
	382	.data = undefined,
329	383	});
330		const reloc_index = self.code.items.len;
331		self.code.items.len += 4;
332	384
333		try self.dbgSetPrologueEnd();
334	385	try self.genBody(self.air.getMainBody());
335	386
336	387	const stack_end = self.max_end_stack;
337		if (stack_end > math.maxInt(i32))
	388	if (stack_end > math.maxInt(i32)) {
338	389	return self.failSymbol("too much stack used in call parameters", .{});
	390	}
339	391	const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
340		mem.writeIntLittle(u32, self.code.items[reloc_index..][0..4], @intCast(u32, aligned_stack_end));
341
342		if (self.code.items.len >= math.maxInt(i32)) {
343		return self.failSymbol("unable to perform relocation: jump too far", .{});
	392	if (aligned_stack_end > 0) {
	393	self.mir_instructions.items(.data)[backpatch_reloc].imm = @intCast(i32, aligned_stack_end);
344	394	}
	395
345	396	if (self.exitlude_jump_relocs.items.len == 1) {
346		self.code.items.len -= 5;
	397	self.mir_instructions.len -= 1;
347	398	} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
348		const amt = self.code.items.len - (jmp_reloc + 4);
349		const s32_amt = @intCast(i32, amt);
350		mem.writeIntLittle(i32, self.code.items[jmp_reloc..][0..4], s32_amt);
	399	self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);
351	400	}
352	401
353		// Important to be after the possible self.code.items.len -= 5 above.
354		try self.dbgSetEpilogueBegin();
355
356		try self.code.ensureUnusedCapacity(9);
357		// add rsp, x
358		if (aligned_stack_end > math.maxInt(i8)) {
359		// example: 48 81 c4 ff ff ff 7f add rsp,0x7fffffff
360		self.code.appendSliceAssumeCapacity(&[_]u8{ 0x48, 0x81, 0xc4 });
361		const x = @intCast(u32, aligned_stack_end);
362		mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), x);
363		} else if (aligned_stack_end != 0) {
364		// example: 48 83 c4 7f add rsp,0x7f
365		const x = @intCast(u8, aligned_stack_end);
366		self.code.appendSliceAssumeCapacity(&[_]u8{ 0x48, 0x83, 0xc4, x });
	402	_ = try self.addInst(.{
	403	.tag = .dbg_epilogue_begin,
	404	.ops = undefined,
	405	.data = undefined,
	406	});
	407
	408	if (aligned_stack_end > 0) {
	409	// add rsp, x
	410	_ = try self.addInst(.{
	411	.tag = .add,
	412	.ops = (Mir.Ops{
	413	.reg1 = .rsp,
	414	}).encode(),
	415	.data = .{ .imm = @intCast(i32, aligned_stack_end) },
	416	});
367	417	}
368	418
369		self.code.appendSliceAssumeCapacity(&[_]u8{
370		0x5d, // pop rbp
371		0xc3, // ret
	419	_ = try self.addInst(.{
	420	.tag = .pop,
	421	.ops = (Mir.Ops{
	422	.reg1 = .rbp,
	423	}).encode(),
	424	.data = undefined,
	425	});
	426	_ = try self.addInst(.{
	427	.tag = .ret,
	428	.ops = (Mir.Ops{
	429	.flags = 0b11,
	430	}).encode(),
	431	.data = undefined,
372	432	});
373	433	} else {
374		try self.dbgSetPrologueEnd();
	434	_ = try self.addInst(.{
	435	.tag = .dbg_prologue_end,
	436	.ops = undefined,
	437	.data = undefined,
	438	});
	439
375	440	try self.genBody(self.air.getMainBody());
376		try self.dbgSetEpilogueBegin();
	441
	442	_ = try self.addInst(.{
	443	.tag = .dbg_epilogue_begin,
	444	.ops = undefined,
	445	.data = undefined,
	446	});
377	447	}
378	448
379	449	// Drop them off at the rbrace.
380		try self.dbgAdvancePCAndLine(self.end_di_line, self.end_di_column);
	450	const payload = try self.addExtra(Mir.DbgLineColumn{
	451	.line = self.end_di_line,
	452	.column = self.end_di_column,
	453	});
	454	_ = try self.addInst(.{
	455	.tag = .dbg_line,
	456	.ops = undefined,
	457	.data = .{ .payload = payload },
	458	});
381	459	}
382	460
383	461	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
...	...	@@ -518,79 +596,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
518	596	}
519	597	}
520	598
521		fn dbgSetPrologueEnd(self: *Self) InnerError!void {
522		switch (self.debug_output) {
523		.dwarf => \|dbg_out\| {
524		try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
525		try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
526		},
527		.plan9 => {},
528		.none => {},
529		}
530		}
531
532		fn dbgSetEpilogueBegin(self: *Self) InnerError!void {
533		switch (self.debug_output) {
534		.dwarf => \|dbg_out\| {
535		try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
536		try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
537		},
538		.plan9 => {},
539		.none => {},
540		}
541		}
542
543		fn dbgAdvancePCAndLine(self: *Self, line: u32, column: u32) InnerError!void {
544		const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
545		const delta_pc: usize = self.code.items.len - self.prev_di_pc;
546		switch (self.debug_output) {
547		.dwarf => \|dbg_out\| {
548		// TODO Look into using the DWARF special opcodes to compress this data.
549		// It lets you emit single-byte opcodes that add different numbers to
550		// both the PC and the line number at the same time.
551		try dbg_out.dbg_line.ensureUnusedCapacity(11);
552		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
553		leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
554		if (delta_line != 0) {
555		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
556		leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
557		}
558		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
559		self.prev_di_pc = self.code.items.len;
560		self.prev_di_line = line;
561		self.prev_di_column = column;
562		self.prev_di_pc = self.code.items.len;
563		},
564		.plan9 => \|dbg_out\| {
565		if (delta_pc <= 0) return; // only do this when the pc changes
566		// we have already checked the target in the linker to make sure it is compatable
567		const quant = @import("../../link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;
568
569		// increasing the line number
570		try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
571		// increasing the pc
572		const d_pc_p9 = @intCast(i64, delta_pc) - quant;
573		if (d_pc_p9 > 0) {
574		// minus one because if its the last one, we want to leave space to change the line which is one quanta
575		try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
576		if (dbg_out.pcop_change_index.*) \|pci\|
577		dbg_out.dbg_line.items[pci] += 1;
578		dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
579		} else if (d_pc_p9 == 0) {
580		// we don't need to do anything, because adding the quant does it for us
581		} else unreachable;
582		if (dbg_out.start_line.* == null)
583		dbg_out.start_line.* = self.prev_di_line;
584		dbg_out.end_line.* = line;
585		// only do this if the pc changed
586		self.prev_di_line = line;
587		self.prev_di_column = column;
588		self.prev_di_pc = self.code.items.len;
589		},
590		.none => {},
591		}
592		}
593
594	599	/// Asserts there is already capacity to insert into top branch inst_table.
595	600	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
596	601	const air_tags = self.air.instructions.items(.tag);
...	...	@@ -654,29 +659,6 @@ fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
654	659	try table.ensureUnusedCapacity(self.gpa, additional_count);
655	660	}
656	661
657		/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
658		/// after codegen for this symbol is done.
659		fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void {
660		switch (self.debug_output) {
661		.dwarf => \|dbg_out\| {
662		assert(ty.hasCodeGenBits());
663		const index = dbg_out.dbg_info.items.len;
664		try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
665
666		const gop = try dbg_out.dbg_info_type_relocs.getOrPut(self.gpa, ty);
667		if (!gop.found_existing) {
668		gop.value_ptr.* = .{
669		.off = undefined,
670		.relocs = .{},
671		};
672		}
673		try gop.value_ptr.relocs.append(self.gpa, @intCast(u32, index));
674		},
675		.plan9 => {},
676		.none => {},
677		}
678		}
679
680	662	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
681	663	if (abi_align > self.stack_align)
682	664	self.stack_align = abi_align;
...	...	@@ -848,7 +830,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
848	830	},
849	831	else => {},
850	832	}
851		break :result try self.genX8664BinMath(inst, ty_op.operand, .bool_true);
	833	break :result try self.genBinMathOp(inst, ty_op.operand, .bool_true);
852	834	};
853	835	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
854	836	}
...	...	@@ -886,7 +868,7 @@ fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
886	868	const result: MCValue = if (self.liveness.isUnused(inst))
887	869	.dead
888	870	else
889		try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
	871	try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
890	872	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
891	873	}
892	874
...	...	@@ -913,7 +895,7 @@ fn airSub(self: *Self, inst: Air.Inst.Index) !void {
913	895	const result: MCValue = if (self.liveness.isUnused(inst))
914	896	.dead
915	897	else
916		try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
	898	try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
917	899	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
918	900	}
919	901
...	...	@@ -940,7 +922,7 @@ fn airMul(self: *Self, inst: Air.Inst.Index) !void {
940	922	const result: MCValue = if (self.liveness.isUnused(inst))
941	923	.dead
942	924	else
943		try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
	925	try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
944	926	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
945	927	}
946	928
...	...	@@ -994,7 +976,7 @@ fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
994	976	const result: MCValue = if (self.liveness.isUnused(inst))
995	977	.dead
996	978	else
997		try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
	979	try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
998	980	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
999	981	}
1000	982
...	...	@@ -1003,7 +985,7 @@ fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
1003	985	const result: MCValue = if (self.liveness.isUnused(inst))
1004	986	.dead
1005	987	else
1006		try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
	988	try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
1007	989	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1008	990	}
1009	991
...	...	@@ -1415,7 +1397,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
1415	1397	/// Perform "binary" operators, excluding comparisons.
1416	1398	/// Currently, the following ops are supported:
1417	1399	/// ADD, SUB, XOR, OR, AND
1418		fn genX8664BinMath(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
	1400	fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
1419	1401	// We'll handle these ops in two steps.
1420	1402	// 1) Prepare an output location (register or memory)
1421	1403	// This location will be the location of the operand that dies (if one exists)
...	...	@@ -1425,9 +1407,6 @@ fn genX8664BinMath(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
1425	1407	// In this case, copy that location to a register, then perform the op to that register instead.
1426	1408	//
1427	1409	// TODO: make this algorithm less bad
1428
1429		try self.code.ensureUnusedCapacity(8);
1430
1431	1410	const lhs = try self.resolveInst(op_lhs);
1432	1411	const rhs = try self.resolveInst(op_rhs);
1433	1412
...	...	@@ -1486,107 +1465,28 @@ fn genX8664BinMath(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
1486	1465	else => {},
1487	1466	}
1488	1467
1489		// Now for step 2, we perform the actual op
1490		const inst_ty = self.air.typeOfIndex(inst);
	1468	// Now for step 2, we assing an MIR instruction
	1469	const dst_ty = self.air.typeOfIndex(inst);
1491	1470	const air_tags = self.air.instructions.items(.tag);
1492	1471	switch (air_tags[inst]) {
1493		// TODO: Generate wrapping and non-wrapping versions separately
1494		.add, .addwrap => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 0, 0x00),
1495		.bool_or, .bit_or => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 1, 0x08),
1496		.bool_and, .bit_and => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 4, 0x20),
1497		.sub, .subwrap => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 5, 0x28),
1498		.xor, .not => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 6, 0x30),
1499
1500		.mul, .mulwrap => try self.genX8664Imul(inst_ty, dst_mcv, src_mcv),
	1472	.add, .addwrap => try self.genBinMathOpMir(.add, dst_ty, dst_mcv, src_mcv),
	1473	.bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, dst_mcv, src_mcv),
	1474	.bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, dst_mcv, src_mcv),
	1475	.sub, .subwrap => try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, src_mcv),
	1476	.xor, .not => try self.genBinMathOpMir(.xor, dst_ty, dst_mcv, src_mcv),
	1477	.mul, .mulwrap => try self.genIMulOpMir(dst_ty, dst_mcv, src_mcv),
1501	1478	else => unreachable,
1502	1479	}
1503	1480
1504	1481	return dst_mcv;
1505	1482	}
1506	1483
1507		/// Wrap over Instruction.encodeInto to translate errors
1508		fn encodeX8664Instruction(self: *Self, inst: Instruction) !void {
1509		inst.encodeInto(self.code) catch \|err\| {
1510		if (err == error.OutOfMemory)
1511		return error.OutOfMemory
1512		else
1513		return self.fail("Instruction.encodeInto failed because {s}", .{@errorName(err)});
1514		};
1515		}
1516
1517		/// This function encodes a binary operation for x86_64
1518		/// intended for use with the following opcode ranges
1519		/// because they share the same structure.
1520		///
1521		/// Thus not all binary operations can be used here
1522		/// -- multiplication needs to be done with imul,
1523		/// which doesn't have as convenient an interface.
1524		///
1525		/// "opx"-style instructions use the opcode extension field to indicate which instruction to execute:
1526		///
1527		/// opx = /0: add
1528		/// opx = /1: or
1529		/// opx = /2: adc
1530		/// opx = /3: sbb
1531		/// opx = /4: and
1532		/// opx = /5: sub
1533		/// opx = /6: xor
1534		/// opx = /7: cmp
1535		///
1536		/// opcode \| operand shape
1537		/// --------+----------------------
1538		/// 80 /opx \| r/m8, imm8
1539		/// 81 /opx \| r/m16/32/64, imm16/32
1540		/// 83 /opx \| r/m16/32/64, imm8
1541		///
1542		/// "mr"-style instructions use the low bits of opcode to indicate shape of instruction:
1543		///
1544		/// mr = 00: add
1545		/// mr = 08: or
1546		/// mr = 10: adc
1547		/// mr = 18: sbb
1548		/// mr = 20: and
1549		/// mr = 28: sub
1550		/// mr = 30: xor
1551		/// mr = 38: cmp
1552		///
1553		/// opcode \| operand shape
1554		/// -------+-------------------------
1555		/// mr + 0 \| r/m8, r8
1556		/// mr + 1 \| r/m16/32/64, r16/32/64
1557		/// mr + 2 \| r8, r/m8
1558		/// mr + 3 \| r16/32/64, r/m16/32/64
1559		/// mr + 4 \| AL, imm8
1560		/// mr + 5 \| rAX, imm16/32
1561		///
1562		/// TODO: rotates and shifts share the same structure, so we can potentially implement them
1563		/// at a later date with very similar code.
1564		/// They have "opx"-style instructions, but no "mr"-style instructions.
1565		///
1566		/// opx = /0: rol,
1567		/// opx = /1: ror,
1568		/// opx = /2: rcl,
1569		/// opx = /3: rcr,
1570		/// opx = /4: shl sal,
1571		/// opx = /5: shr,
1572		/// opx = /6: sal shl,
1573		/// opx = /7: sar,
1574		///
1575		/// opcode \| operand shape
1576		/// --------+------------------
1577		/// c0 /opx \| r/m8, imm8
1578		/// c1 /opx \| r/m16/32/64, imm8
1579		/// d0 /opx \| r/m8, 1
1580		/// d1 /opx \| r/m16/32/64, 1
1581		/// d2 /opx \| r/m8, CL (for context, CL is register 1)
1582		/// d3 /opx \| r/m16/32/64, CL (for context, CL is register 1)
1583		fn genX8664BinMathCode(
	1484	fn genBinMathOpMir(
1584	1485	self: *Self,
	1486	mir_tag: Mir.Inst.Tag,
1585	1487	dst_ty: Type,
1586	1488	dst_mcv: MCValue,
1587	1489	src_mcv: MCValue,
1588		opx: u3,
1589		mr: u8,
1590	1490	) !void {
1591	1491	switch (dst_mcv) {
1592	1492	.none => unreachable,
...	...	@@ -1604,84 +1504,43 @@ fn genX8664BinMathCode(
1604	1504	.ptr_stack_offset => unreachable,
1605	1505	.ptr_embedded_in_code => unreachable,
1606	1506	.register => \|src_reg\| {
1607		// for register, register use mr + 1
1608		// addressing mode: r/m16/32/64, r16/32/64
1609		const abi_size = dst_ty.abiSize(self.target.*);
1610		const encoder = try Encoder.init(self.code, 3);
1611		encoder.rex(.{
1612		.w = abi_size == 8,
1613		.r = src_reg.isExtended(),
1614		.b = dst_reg.isExtended(),
	1507	_ = try self.addInst(.{
	1508	.tag = mir_tag,
	1509	.ops = (Mir.Ops{
	1510	.reg1 = src_reg,
	1511	.reg2 = dst_reg,
	1512	.flags = 0b11,
	1513	}).encode(),
	1514	.data = undefined,
1615	1515	});
1616		encoder.opcode_1byte(mr + 1);
1617		encoder.modRm_direct(
1618		src_reg.low_id(),
1619		dst_reg.low_id(),
1620		);
1621	1516	},
1622	1517	.immediate => \|imm\| {
1623		// register, immediate use opx = 81 or 83 addressing modes:
1624		// opx = 81: r/m16/32/64, imm16/32
1625		// opx = 83: r/m16/32/64, imm8
1626		const imm32 = @intCast(i32, imm); // This case must be handled before calling genX8664BinMathCode.
1627		if (imm32 <= math.maxInt(i8)) {
1628		const abi_size = dst_ty.abiSize(self.target.*);
1629		const encoder = try Encoder.init(self.code, 4);
1630		encoder.rex(.{
1631		.w = abi_size == 8,
1632		.b = dst_reg.isExtended(),
1633		});
1634		encoder.opcode_1byte(0x83);
1635		encoder.modRm_direct(
1636		opx,
1637		dst_reg.low_id(),
1638		);
1639		encoder.imm8(@intCast(i8, imm32));
1640		} else {
1641		const abi_size = dst_ty.abiSize(self.target.*);
1642		const encoder = try Encoder.init(self.code, 7);
1643		encoder.rex(.{
1644		.w = abi_size == 8,
1645		.b = dst_reg.isExtended(),
1646		});
1647		encoder.opcode_1byte(0x81);
1648		encoder.modRm_direct(
1649		opx,
1650		dst_reg.low_id(),
1651		);
1652		encoder.imm32(@intCast(i32, imm32));
1653		}
	1518	_ = try self.addInst(.{
	1519	.tag = mir_tag,
	1520	.ops = (Mir.Ops{
	1521	.reg1 = dst_reg,
	1522	}).encode(),
	1523	.data = .{ .imm = @intCast(i32, imm) },
	1524	});
1654	1525	},
1655	1526	.embedded_in_code, .memory => {
1656	1527	return self.fail("TODO implement x86 ADD/SUB/CMP source memory", .{});
1657	1528	},
1658	1529	.stack_offset => \|off\| {
1659		// register, indirect use mr + 3
1660		// addressing mode: r16/32/64, r/m16/32/64
1661		const abi_size = dst_ty.abiSize(self.target.*);
1662		const adj_off = off + abi_size;
1663	1530	if (off > math.maxInt(i32)) {
1664	1531	return self.fail("stack offset too large", .{});
1665	1532	}
1666		const encoder = try Encoder.init(self.code, 7);
1667		encoder.rex(.{
1668		.w = abi_size == 8,
1669		.r = dst_reg.isExtended(),
	1533	const abi_size = dst_ty.abiSize(self.target.*);
	1534	const adj_off = off + abi_size;
	1535	_ = try self.addInst(.{
	1536	.tag = mir_tag,
	1537	.ops = (Mir.Ops{
	1538	.reg1 = dst_reg,
	1539	.reg2 = .ebp,
	1540	.flags = 0b01,
	1541	}).encode(),
	1542	.data = .{ .imm = -@intCast(i32, adj_off) },
1670	1543	});
1671		encoder.opcode_1byte(mr + 3);
1672		if (adj_off <= std.math.maxInt(i8)) {
1673		encoder.modRm_indirectDisp8(
1674		dst_reg.low_id(),
1675		Register.ebp.low_id(),
1676		);
1677		encoder.disp8(-@intCast(i8, adj_off));
1678		} else {
1679		encoder.modRm_indirectDisp32(
1680		dst_reg.low_id(),
1681		Register.ebp.low_id(),
1682		);
1683		encoder.disp32(-@intCast(i32, adj_off));
1684		}
1685	1544	},
1686	1545	.compare_flags_unsigned => {
1687	1546	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
...	...	@@ -1699,7 +1558,20 @@ fn genX8664BinMathCode(
1699	1558	.ptr_stack_offset => unreachable,
1700	1559	.ptr_embedded_in_code => unreachable,
1701	1560	.register => \|src_reg\| {
1702		try self.genX8664ModRMRegToStack(dst_ty, off, src_reg, mr + 0x1);
	1561	if (off > math.maxInt(i32)) {
	1562	return self.fail("stack offset too large", .{});
	1563	}
	1564	const abi_size = dst_ty.abiSize(self.target.*);
	1565	const adj_off = off + abi_size;
	1566	_ = try self.addInst(.{
	1567	.tag = mir_tag,
	1568	.ops = (Mir.Ops{
	1569	.reg1 = src_reg,
	1570	.reg2 = .ebp,
	1571	.flags = 0b10,
	1572	}).encode(),
	1573	.data = .{ .imm = -@intCast(i32, adj_off) },
	1574	});
1703	1575	},
1704	1576	.immediate => \|imm\| {
1705	1577	_ = imm;
...	...	@@ -1722,13 +1594,8 @@ fn genX8664BinMathCode(
1722	1594	}
1723	1595	}
1724	1596
1725		/// Performs integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
1726		fn genX8664Imul(
1727		self: *Self,
1728		dst_ty: Type,
1729		dst_mcv: MCValue,
1730		src_mcv: MCValue,
1731		) !void {
	1597	// Performs integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
	1598	fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
1732	1599	switch (dst_mcv) {
1733	1600	.none => unreachable,
1734	1601	.undef => unreachable,
...	...	@@ -1746,68 +1613,30 @@ fn genX8664Imul(
1746	1613	.ptr_embedded_in_code => unreachable,
1747	1614	.register => \|src_reg\| {
1748	1615	// register, register
1749		//
1750		// Use the following imul opcode
1751		// 0F AF /r: IMUL r32/64, r/m32/64
1752		const abi_size = dst_ty.abiSize(self.target.*);
1753		const encoder = try Encoder.init(self.code, 4);
1754		encoder.rex(.{
1755		.w = abi_size == 8,
1756		.r = dst_reg.isExtended(),
1757		.b = src_reg.isExtended(),
	1616	_ = try self.addInst(.{
	1617	.tag = .imul_complex,
	1618	.ops = (Mir.Ops{
	1619	.reg1 = dst_reg,
	1620	.reg2 = src_reg,
	1621	}).encode(),
	1622	.data = undefined,
1758	1623	});
1759		encoder.opcode_2byte(0x0f, 0xaf);
1760		encoder.modRm_direct(
1761		dst_reg.low_id(),
1762		src_reg.low_id(),
1763		);
1764	1624	},
1765	1625	.immediate => \|imm\| {
1766		// register, immediate:
1767		// depends on size of immediate.
1768		//
1769		// immediate fits in i8:
1770		// 6B /r ib: IMUL r32/64, r/m32/64, imm8
1771		//
1772		// immediate fits in i32:
1773		// 69 /r id: IMUL r32/64, r/m32/64, imm32
1774		//
1775		// immediate is huge:
1776		// split into 2 instructions
1777		// 1) copy the 64 bit immediate into a tmp register
1778		// 2) perform register,register mul
1779		// 0F AF /r: IMUL r32/64, r/m32/64
1780		if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
1781		const abi_size = dst_ty.abiSize(self.target.*);
1782		const encoder = try Encoder.init(self.code, 4);
1783		encoder.rex(.{
1784		.w = abi_size == 8,
1785		.r = dst_reg.isExtended(),
1786		.b = dst_reg.isExtended(),
	1626	// register, immediate
	1627	if (imm <= math.maxInt(i32)) {
	1628	_ = try self.addInst(.{
	1629	.tag = .imul_complex,
	1630	.ops = (Mir.Ops{
	1631	.reg1 = dst_reg,
	1632	.reg2 = dst_reg,
	1633	.flags = 0b10,
	1634	}).encode(),
	1635	.data = .{ .imm = @intCast(i32, imm) },
1787	1636	});
1788		encoder.opcode_1byte(0x6B);
1789		encoder.modRm_direct(
1790		dst_reg.low_id(),
1791		dst_reg.low_id(),
1792		);
1793		encoder.imm8(@intCast(i8, imm));
1794		} else if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
1795		const abi_size = dst_ty.abiSize(self.target.*);
1796		const encoder = try Encoder.init(self.code, 7);
1797		encoder.rex(.{
1798		.w = abi_size == 8,
1799		.r = dst_reg.isExtended(),
1800		.b = dst_reg.isExtended(),
1801		});
1802		encoder.opcode_1byte(0x69);
1803		encoder.modRm_direct(
1804		dst_reg.low_id(),
1805		dst_reg.low_id(),
1806		);
1807		encoder.imm32(@intCast(i32, imm));
1808	1637	} else {
1809	1638	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
1810		return self.genX8664Imul(dst_ty, dst_mcv, MCValue{ .register = src_reg });
	1639	return self.genIMulOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
1811	1640	}
1812	1641	},
1813	1642	.embedded_in_code, .memory, .stack_offset => {
...	...	@@ -1833,20 +1662,14 @@ fn genX8664Imul(
1833	1662	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
1834	1663	// multiply into dst_reg
1835	1664	// register, register
1836		// Use the following imul opcode
1837		// 0F AF /r: IMUL r32/64, r/m32/64
1838		const abi_size = dst_ty.abiSize(self.target.*);
1839		const encoder = try Encoder.init(self.code, 4);
1840		encoder.rex(.{
1841		.w = abi_size == 8,
1842		.r = dst_reg.isExtended(),
1843		.b = src_reg.isExtended(),
	1665	_ = try self.addInst(.{
	1666	.tag = .imul_complex,
	1667	.ops = (Mir.Ops{
	1668	.reg1 = dst_reg,
	1669	.reg2 = src_reg,
	1670	}).encode(),
	1671	.data = undefined,
1844	1672	});
1845		encoder.opcode_2byte(0x0f, 0xaf);
1846		encoder.modRm_direct(
1847		dst_reg.low_id(),
1848		src_reg.low_id(),
1849		);
1850	1673	// copy dst_reg back out
1851	1674	return self.genSetStack(dst_ty, off, MCValue{ .register = dst_reg });
1852	1675	},
...	...	@@ -1871,73 +1694,6 @@ fn genX8664Imul(
1871	1694	}
1872	1695	}
1873	1696
1874		fn genX8664ModRMRegToStack(self: *Self, ty: Type, off: u32, reg: Register, opcode: u8) !void {
1875		const abi_size = ty.abiSize(self.target.*);
1876		const adj_off = off + abi_size;
1877		if (off > math.maxInt(i32)) {
1878		return self.fail("stack offset too large", .{});
1879		}
1880
1881		const i_adj_off = -@intCast(i32, adj_off);
1882		const encoder = try Encoder.init(self.code, 7);
1883		encoder.rex(.{
1884		.w = abi_size == 8,
1885		.r = reg.isExtended(),
1886		});
1887		encoder.opcode_1byte(opcode);
1888		if (i_adj_off < std.math.maxInt(i8)) {
1889		// example: 48 89 55 7f mov QWORD PTR [rbp+0x7f],rdx
1890		encoder.modRm_indirectDisp8(
1891		reg.low_id(),
1892		Register.ebp.low_id(),
1893		);
1894		encoder.disp8(@intCast(i8, i_adj_off));
1895		} else {
1896		// example: 48 89 95 80 00 00 00 mov QWORD PTR [rbp+0x80],rdx
1897		encoder.modRm_indirectDisp32(
1898		reg.low_id(),
1899		Register.ebp.low_id(),
1900		);
1901		encoder.disp32(i_adj_off);
1902		}
1903		}
1904
1905		fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
1906		const ty_str = self.air.instructions.items(.data)[inst].ty_str;
1907		const zir = &self.mod_fn.owner_decl.getFileScope().zir;
1908		const name = zir.nullTerminatedString(ty_str.str);
1909		const name_with_null = name.ptr[0 .. name.len + 1];
1910		const ty = self.air.getRefType(ty_str.ty);
1911
1912		switch (mcv) {
1913		.register => \|reg\| {
1914		switch (self.debug_output) {
1915		.dwarf => \|dbg_out\| {
1916		try dbg_out.dbg_info.ensureUnusedCapacity(3);
1917		dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
1918		dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
1919		1, // ULEB128 dwarf expression length
1920		reg.dwarfLocOp(),
1921		});
1922		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
1923		try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
1924		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
1925		},
1926		.plan9 => {},
1927		.none => {},
1928		}
1929		},
1930		.stack_offset => {
1931		switch (self.debug_output) {
1932		.dwarf => {},
1933		.plan9 => {},
1934		.none => {},
1935		}
1936		},
1937		else => {},
1938		}
1939		}
1940
1941	1697	fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1942	1698	const arg_index = self.arg_index;
1943	1699	self.arg_index += 1;
...	...	@@ -1946,8 +1702,15 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1946	1702	_ = ty;
1947	1703
1948	1704	const mcv = self.args[arg_index];
1949		try self.genArgDbgInfo(inst, mcv);
1950
	1705	const payload = try self.addExtra(Mir.ArgDbgInfo{
	1706	.air_inst = inst,
	1707	.arg_index = @intCast(u32, arg_index), // TODO can arg_index: u32?
	1708	});
	1709	_ = try self.addInst(.{
	1710	.tag = .arg_dbg_info,
	1711	.ops = undefined,
	1712	.data = .{ .payload = payload },
	1713	});
1951	1714	if (self.liveness.isUnused(inst))
1952	1715	return self.finishAirBookkeeping();
1953	1716
...	...	@@ -1962,7 +1725,11 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1962	1725	}
1963	1726
1964	1727	fn airBreakpoint(self: *Self) !void {
1965		try self.code.append(0xcc); // int3
	1728	_ = try self.addInst(.{
	1729	.tag = .brk,
	1730	.ops = undefined,
	1731	.data = undefined,
	1732	});
1966	1733	return self.finishAirBookkeeping();
1967	1734	}
1968	1735
...	...	@@ -2021,7 +1788,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2021	1788	if (self.air.value(callee)) \|func_value\| {
2022	1789	if (func_value.castTag(.function)) \|func_payload\| {
2023	1790	const func = func_payload.data;
2024
2025	1791	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2026	1792	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2027	1793	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
...	...	@@ -2031,11 +1797,13 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2031	1797	@intCast(u32, coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes)
2032	1798	else
2033	1799	unreachable;
2034
2035		// ff 14 25 xx xx xx xx call [addr]
2036		try self.code.ensureUnusedCapacity(7);
2037		self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 });
2038		mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), got_addr);
	1800	_ = try self.addInst(.{
	1801	.tag = .call,
	1802	.ops = (Mir.Ops{
	1803	.flags = 0b01,
	1804	}).encode(),
	1805	.data = .{ .imm = @bitCast(i32, got_addr) },
	1806	});
2039	1807	} else if (func_value.castTag(.extern_fn)) \|_\| {
2040	1808	return self.fail("TODO implement calling extern functions", .{});
2041	1809	} else {
...	...	@@ -2089,26 +1857,21 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2089	1857	.memory = func.owner_decl.link.macho.local_sym_index,
2090	1858	});
2091	1859	// callq *%rax
2092		try self.code.ensureUnusedCapacity(2);
2093		self.code.appendSliceAssumeCapacity(&[2]u8{ 0xff, 0xd0 });
	1860	_ = try self.addInst(.{
	1861	.tag = .call,
	1862	.ops = (Mir.Ops{
	1863	.reg1 = .rax,
	1864	.flags = 0b01,
	1865	}).encode(),
	1866	.data = undefined,
	1867	});
2094	1868	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
2095	1869	const decl = func_payload.data;
2096	1870	const n_strx = try macho_file.addExternFn(mem.spanZ(decl.name));
2097		const offset = blk: {
2098		// callq
2099		try self.code.ensureUnusedCapacity(5);
2100		self.code.appendSliceAssumeCapacity(&[5]u8{ 0xe8, 0x0, 0x0, 0x0, 0x0 });
2101		break :blk @intCast(u32, self.code.items.len) - 4;
2102		};
2103		// Add relocation to the decl.
2104		try macho_file.active_decl.?.link.macho.relocs.append(self.bin_file.allocator, .{
2105		.offset = offset,
2106		.target = .{ .global = n_strx },
2107		.addend = 0,
2108		.subtractor = null,
2109		.pcrel = true,
2110		.length = 2,
2111		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	1871	_ = try self.addInst(.{
	1872	.tag = .call_extern,
	1873	.ops = undefined,
	1874	.data = .{ .extern_fn = n_strx },
2112	1875	});
2113	1876	} else {
2114	1877	return self.fail("TODO implement calling bitcasted functions", .{});
...	...	@@ -2157,11 +1920,14 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2157	1920	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2158	1921	const got_addr = p9.bases.data;
2159	1922	const got_index = func_payload.data.owner_decl.link.plan9.got_index.?;
2160		// ff 14 25 xx xx xx xx call [addr]
2161		try self.code.ensureUnusedCapacity(7);
2162		self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 });
2163	1923	const fn_got_addr = got_addr + got_index * ptr_bytes;
2164		mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), @intCast(u32, fn_got_addr));
	1924	_ = try self.addInst(.{
	1925	.tag = .call,
	1926	.ops = (Mir.Ops{
	1927	.flags = 0b01,
	1928	}).encode(),
	1929	.data = .{ .imm = @bitCast(i32, @intCast(u32, fn_got_addr)) },
	1930	});
2165	1931	} else return self.fail("TODO implement calling extern fn on plan9", .{});
2166	1932	} else {
2167	1933	return self.fail("TODO implement calling runtime known function pointer", .{});
...	...	@@ -2201,9 +1967,14 @@ fn ret(self: *Self, mcv: MCValue) !void {
2201	1967	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
2202	1968	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
2203	1969	// which is available if the jump is 127 bytes or less forward.
2204		try self.code.resize(self.code.items.len + 5);
2205		self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
2206		try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
	1970	const jmp_reloc = try self.addInst(.{
	1971	.tag = .jmp,
	1972	.ops = (Mir.Ops{
	1973	.flags = 0b00,
	1974	}).encode(),
	1975	.data = .{ .inst = undefined },
	1976	});
	1977	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
2207	1978	}
2208	1979
2209	1980	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -2233,8 +2004,6 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2233	2004	const lhs = try self.resolveInst(bin_op.lhs);
2234	2005	const rhs = try self.resolveInst(bin_op.rhs);
2235	2006	const result: MCValue = result: {
2236		try self.code.ensureUnusedCapacity(8);
2237
2238	2007	// There are 2 operands, destination and source.
2239	2008	// Either one, but not both, can be a memory operand.
2240	2009	// Source operand can be an immediate, 8 bits or 32 bits.
...	...	@@ -2245,7 +2014,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2245	2014	// This instruction supports only signed 32-bit immediates at most.
2246	2015	const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
2247	2016
2248		try self.genX8664BinMathCode(Type.initTag(.bool), dst_mcv, src_mcv, 7, 0x38);
	2017	try self.genBinMathOpMir(.cmp, Type.initTag(.bool), dst_mcv, src_mcv);
2249	2018	break :result switch (ty.isSignedInt()) {
2250	2019	true => MCValue{ .compare_flags_signed = op },
2251	2020	false => MCValue{ .compare_flags_unsigned = op },
...	...	@@ -2256,7 +2025,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2256	2025
2257	2026	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
2258	2027	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
2259		try self.dbgAdvancePCAndLine(dbg_stmt.line, dbg_stmt.column);
	2028	const payload = try self.addExtra(Mir.DbgLineColumn{
	2029	.line = dbg_stmt.line,
	2030	.column = dbg_stmt.column,
	2031	});
	2032	_ = try self.addInst(.{
	2033	.tag = .dbg_line,
	2034	.ops = undefined,
	2035	.data = .{ .payload = payload },
	2036	});
2260	2037	return self.finishAirBookkeeping();
2261	2038	}
2262	2039
...	...	@@ -2268,58 +2045,77 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2268	2045	const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
2269	2046	const liveness_condbr = self.liveness.getCondBr(inst);
2270	2047
2271		const reloc: Reloc = reloc: {
2272		try self.code.ensureUnusedCapacity(6);
2273
2274		const opcode: u8 = switch (cond) {
2275		.compare_flags_signed => \|cmp_op\| blk: {
2276		// Here we map to the opposite opcode because the jump is to the false branch.
2277		const opcode: u8 = switch (cmp_op) {
2278		.gte => 0x8c,
2279		.gt => 0x8e,
2280		.neq => 0x84,
2281		.lt => 0x8d,
2282		.lte => 0x8f,
2283		.eq => 0x85,
	2048	const reloc: Mir.Inst.Index = reloc: {
	2049	switch (cond) {
	2050	.compare_flags_signed => \|cmp_op\| {
	2051	// Here we map the opposites since the jump is to the false branch.
	2052	const flags: u2 = switch (cmp_op) {
	2053	.gte => 0b10,
	2054	.gt => 0b11,
	2055	.neq => 0b01,
	2056	.lt => 0b00,
	2057	.lte => 0b01,
	2058	.eq => 0b00,
2284	2059	};
2285		break :blk opcode;
	2060	const tag: Mir.Inst.Tag = if (cmp_op == .neq or cmp_op == .eq)
	2061	.cond_jmp_eq_ne
	2062	else
	2063	.cond_jmp_greater_less;
	2064	const reloc = try self.addInst(.{
	2065	.tag = tag,
	2066	.ops = (Mir.Ops{
	2067	.flags = flags,
	2068	}).encode(),
	2069	.data = .{ .inst = undefined },
	2070	});
	2071	break :reloc reloc;
2286	2072	},
2287		.compare_flags_unsigned => \|cmp_op\| blk: {
2288		// Here we map to the opposite opcode because the jump is to the false branch.
2289		const opcode: u8 = switch (cmp_op) {
2290		.gte => 0x82,
2291		.gt => 0x86,
2292		.neq => 0x84,
2293		.lt => 0x83,
2294		.lte => 0x87,
2295		.eq => 0x85,
	2073	.compare_flags_unsigned => \|cmp_op\| {
	2074	// Here we map the opposites since the jump is to the false branch.
	2075	const flags: u2 = switch (cmp_op) {
	2076	.gte => 0b10,
	2077	.gt => 0b11,
	2078	.neq => 0b01,
	2079	.lt => 0b00,
	2080	.lte => 0b01,
	2081	.eq => 0b00,
2296	2082	};
2297		break :blk opcode;
	2083	const tag: Mir.Inst.Tag = if (cmp_op == .neq or cmp_op == .eq)
	2084	.cond_jmp_eq_ne
	2085	else
	2086	.cond_jmp_above_below;
	2087	const reloc = try self.addInst(.{
	2088	.tag = tag,
	2089	.ops = (Mir.Ops{
	2090	.flags = flags,
	2091	}).encode(),
	2092	.data = .{ .inst = undefined },
	2093	});
	2094	break :reloc reloc;
2298	2095	},
2299		.register => \|reg\| blk: {
2300		// test reg, 1
2301		// TODO detect al, ax, eax
2302		const encoder = try Encoder.init(self.code, 4);
2303		encoder.rex(.{
2304		// TODO audit this codegen: we force w = true here to make
2305		// the value affect the big register
2306		.w = true,
2307		.b = reg.isExtended(),
	2096	.register => \|reg\| {
	2097	_ = try self.addInst(.{
	2098	.tag = .@"test",
	2099	.ops = (Mir.Ops{
	2100	.reg1 = reg,
	2101	.flags = 0b00,
	2102	}).encode(),
	2103	.data = .{ .imm = 1 },
	2104	});
	2105	const reloc = try self.addInst(.{
	2106	.tag = .cond_jmp_eq_ne,
	2107	.ops = (Mir.Ops{
	2108	.flags = 0b01,
	2109	}).encode(),
	2110	.data = .{ .inst = undefined },
2308	2111	});
2309		encoder.opcode_1byte(0xf6);
2310		encoder.modRm_direct(
2311		0,
2312		reg.low_id(),
2313		);
2314		encoder.disp8(1);
2315		break :blk 0x84;
	2112	break :reloc reloc;
2316	2113	},
2317		else => return self.fail("TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
2318		};
2319		self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode });
2320		const reloc = Reloc{ .rel32 = self.code.items.len };
2321		self.code.items.len += 4;
2322		break :reloc reloc;
	2114	else => return self.fail("TODO implement condbr {s} when condition is {s}", .{
	2115	self.target.cpu.arch,
	2116	@tagName(cond),
	2117	}),
	2118	}
2323	2119	};
2324	2120
2325	2121	// Capture the state of register and stack allocation state so that we can revert to it.
...	...	@@ -2578,25 +2374,18 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
2578	2374	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2579	2375	const loop = self.air.extraData(Air.Block, ty_pl.payload);
2580	2376	const body = self.air.extra[loop.end..][0..loop.data.body_len];
2581		const start_index = self.code.items.len;
	2377	const jmp_target = @intCast(u32, self.mir_instructions.len);
2582	2378	try self.genBody(body);
2583		try self.jump(start_index);
	2379	_ = try self.addInst(.{
	2380	.tag = .jmp,
	2381	.ops = (Mir.Ops{
	2382	.flags = 0b00,
	2383	}).encode(),
	2384	.data = .{ .inst = jmp_target },
	2385	});
2584	2386	return self.finishAirBookkeeping();
2585	2387	}
2586	2388
2587		/// Send control flow to the `index` of `self.code`.
2588		fn jump(self: *Self, index: usize) !void {
2589		try self.code.ensureUnusedCapacity(5);
2590		if (math.cast(i8, @intCast(i32, index) - (@intCast(i32, self.code.items.len + 2)))) \|delta\| {
2591		self.code.appendAssumeCapacity(0xeb); // jmp rel8
2592		self.code.appendAssumeCapacity(@bitCast(u8, delta));
2593		} else \|_\| {
2594		const delta = @intCast(i32, index) - (@intCast(i32, self.code.items.len + 5));
2595		self.code.appendAssumeCapacity(0xe9); // jmp rel32
2596		mem.writeIntLittle(i32, self.code.addManyAsArrayAssumeCapacity(4), delta);
2597		}
2598		}
2599
2600	2389	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
2601	2390	try self.blocks.putNoClobber(self.gpa, inst, .{
2602	2391	// A block is a setup to be able to jump to the end.
...	...	@@ -2630,22 +2419,9 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
2630	2419	// return self.finishAir(inst, .dead, .{ condition, .none, .none });
2631	2420	}
2632	2421
2633		fn performReloc(self: *Self, reloc: Reloc) !void {
2634		switch (reloc) {
2635		.rel32 => \|pos\| {
2636		const amt = self.code.items.len - (pos + 4);
2637		// Here it would be tempting to implement testing for amt == 0 and then elide the
2638		// jump. However, that will cause a problem because other jumps may assume that they
2639		// can jump to this code. Or maybe I didn't understand something when I was debugging.
2640		// It could be worth another look. Anyway, that's why that isn't done here. Probably the
2641		// best place to elide jumps will be in semantic analysis, by inlining blocks that only
2642		// only have 1 break instruction.
2643		const s32_amt = math.cast(i32, amt) catch
2644		return self.fail("unable to perform relocation: jump too far", .{});
2645		mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt);
2646		},
2647		.arm_branch => unreachable,
2648		}
	2422	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
	2423	const next_inst = @intCast(u32, self.mir_instructions.len);
	2424	self.mir_instructions.items(.data)[reloc].inst = next_inst;
2649	2425	}
2650	2426
2651	2427	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -2661,9 +2437,9 @@ fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
2661	2437	.dead
2662	2438	else switch (air_tags[inst]) {
2663	2439	// lhs AND rhs
2664		.bool_and => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
	2440	.bool_and => try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs),
2665	2441	// lhs OR rhs
2666		.bool_or => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
	2442	.bool_or => try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs),
2667	2443	else => unreachable, // Not a boolean operation
2668	2444	};
2669	2445	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -2688,12 +2464,15 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
2688	2464	const block_data = self.blocks.getPtr(block).?;
2689	2465	// Emit a jump with a relocation. It will be patched up after the block ends.
2690	2466	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
2691		// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
2692		// which is available if the jump is 127 bytes or less forward.
2693		try self.code.resize(self.code.items.len + 5);
2694		self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
2695	2467	// Leave the jump offset undefined
2696		block_data.relocs.appendAssumeCapacity(.{ .rel32 = self.code.items.len - 4 });
	2468	const jmp_reloc = try self.addInst(.{
	2469	.tag = .jmp,
	2470	.ops = (Mir.Ops{
	2471	.flags = 0b00,
	2472	}).encode(),
	2473	.data = .{ .inst = undefined },
	2474	});
	2475	block_data.relocs.appendAssumeCapacity(jmp_reloc);
2697	2476	}
2698	2477
2699	2478	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -2750,22 +2529,35 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
2750	2529	var iter = std.mem.tokenize(u8, asm_source, "\n\r");
2751	2530	while (iter.next()) \|ins\| {
2752	2531	if (mem.eql(u8, ins, "syscall")) {
2753		try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });
	2532	_ = try self.addInst(.{
	2533	.tag = .syscall,
	2534	.ops = undefined,
	2535	.data = undefined,
	2536	});
2754	2537	} else if (mem.indexOf(u8, ins, "push")) \|_\| {
2755	2538	const arg = ins[4..];
2756	2539	if (mem.indexOf(u8, arg, "$")) \|l\| {
2757		const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch return self.fail("TODO implement more inline asm int parsing", .{});
2758		try self.code.appendSlice(&.{ 0x6a, n });
	2540	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch {
	2541	return self.fail("TODO implement more inline asm int parsing", .{});
	2542	};
	2543	_ = try self.addInst(.{
	2544	.tag = .push,
	2545	.ops = (Mir.Ops{
	2546	.flags = 0b10,
	2547	}).encode(),
	2548	.data = .{ .imm = n },
	2549	});
2759	2550	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
2760	2551	const reg_name = ins[4 + l + 2 ..];
2761	2552	const reg = parseRegName(reg_name) orelse
2762	2553	return self.fail("unrecognized register: '{s}'", .{reg_name});
2763		const low_id: u8 = reg.low_id();
2764		if (reg.isExtended()) {
2765		try self.code.appendSlice(&.{ 0x41, 0b1010000 \| low_id });
2766		} else {
2767		try self.code.append(0b1010000 \| low_id);
2768		}
	2554	_ = try self.addInst(.{
	2555	.tag = .push,
	2556	.ops = (Mir.Ops{
	2557	.reg1 = reg,
	2558	}).encode(),
	2559	.data = undefined,
	2560	});
2769	2561	} else return self.fail("TODO more push operands", .{});
2770	2562	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
2771	2563	const arg = ins[3..];
...	...	@@ -2773,12 +2565,13 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
2773	2565	const reg_name = ins[3 + l + 2 ..];
2774	2566	const reg = parseRegName(reg_name) orelse
2775	2567	return self.fail("unrecognized register: '{s}'", .{reg_name});
2776		const low_id: u8 = reg.low_id();
2777		if (reg.isExtended()) {
2778		try self.code.appendSlice(&.{ 0x41, 0b1011000 \| low_id });
2779		} else {
2780		try self.code.append(0b1011000 \| low_id);
2781		}
	2568	_ = try self.addInst(.{
	2569	.tag = .pop,
	2570	.ops = (Mir.Ops{
	2571	.reg1 = reg,
	2572	}).encode(),
	2573	.data = undefined,
	2574	});
2782	2575	} else return self.fail("TODO more pop operands", .{});
2783	2576	} else {
2784	2577	return self.fail("TODO implement support for more x86 assembly instructions", .{});
...	...	@@ -2870,7 +2663,6 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2870	2663	if (adj_off > 128) {
2871	2664	return self.fail("TODO implement set stack variable with large stack offset", .{});
2872	2665	}
2873		try self.code.ensureUnusedCapacity(8);
2874	2666	switch (abi_size) {
2875	2667	1 => {
2876	2668	return self.fail("TODO implement set abi_size=1 stack variable with immediate", .{});
...	...	@@ -2879,34 +2671,57 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2879	2671	return self.fail("TODO implement set abi_size=2 stack variable with immediate", .{});
2880	2672	},
2881	2673	4 => {
2882		const x = @intCast(u32, x_big);
2883	2674	// We have a positive stack offset value but we want a twos complement negative
2884	2675	// offset from rbp, which is at the top of the stack frame.
2885		const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
2886		const twos_comp = @bitCast(u8, negative_offset);
2887	2676	// mov DWORD PTR [rbp+offset], immediate
2888		self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp });
2889		mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), x);
	2677	const payload = try self.addExtra(Mir.ImmPair{
	2678	.dest_off = -@intCast(i32, adj_off),
	2679	.operand = @bitCast(i32, @intCast(u32, x_big)),
	2680	});
	2681	_ = try self.addInst(.{
	2682	.tag = .mov,
	2683	.ops = (Mir.Ops{
	2684	.reg1 = .rbp,
	2685	.flags = 0b11,
	2686	}).encode(),
	2687	.data = .{ .payload = payload },
	2688	});
2890	2689	},
2891	2690	8 => {
2892	2691	// We have a positive stack offset value but we want a twos complement negative
2893	2692	// offset from rbp, which is at the top of the stack frame.
2894		const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
2895		const twos_comp = @bitCast(u8, negative_offset);
	2693	const negative_offset = -@intCast(i32, adj_off);
2896	2694
2897	2695	// 64 bit write to memory would take two mov's anyways so we
2898	2696	// insted just use two 32 bit writes to avoid register allocation
2899		try self.code.ensureUnusedCapacity(14);
2900		var buf: [8]u8 = undefined;
2901		mem.writeIntLittle(u64, &buf, x_big);
2902
2903		// mov DWORD PTR [rbp+offset+4], immediate
2904		self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp + 4 });
2905		self.code.appendSliceAssumeCapacity(buf[4..8]);
2906
2907		// mov DWORD PTR [rbp+offset], immediate
2908		self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp });
2909		self.code.appendSliceAssumeCapacity(buf[0..4]);
	2697	{
	2698	const payload = try self.addExtra(Mir.ImmPair{
	2699	.dest_off = negative_offset + 4,
	2700	.operand = @bitCast(i32, @truncate(u32, x_big >> 32)),
	2701	});
	2702	_ = try self.addInst(.{
	2703	.tag = .mov,
	2704	.ops = (Mir.Ops{
	2705	.reg1 = .rbp,
	2706	.flags = 0b11,
	2707	}).encode(),
	2708	.data = .{ .payload = payload },
	2709	});
	2710	}
	2711	{
	2712	const payload = try self.addExtra(Mir.ImmPair{
	2713	.dest_off = negative_offset,
	2714	.operand = @bitCast(i32, @truncate(u32, x_big)),
	2715	});
	2716	_ = try self.addInst(.{
	2717	.tag = .mov,
	2718	.ops = (Mir.Ops{
	2719	.reg1 = .rbp,
	2720	.flags = 0b11,
	2721	}).encode(),
	2722	.data = .{ .payload = payload },
	2723	});
	2724	}
2910	2725	},
2911	2726	else => {
2912	2727	return self.fail("TODO implement set abi_size=large stack variable with immediate", .{});
...	...	@@ -2920,7 +2735,20 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2920	2735	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2921	2736	},
2922	2737	.register => \|reg\| {
2923		try self.genX8664ModRMRegToStack(ty, stack_offset, reg, 0x89);
	2738	if (stack_offset > math.maxInt(i32)) {
	2739	return self.fail("stack offset too large", .{});
	2740	}
	2741	const abi_size = ty.abiSize(self.target.*);
	2742	const adj_off = stack_offset + abi_size;
	2743	_ = try self.addInst(.{
	2744	.tag = .mov,
	2745	.ops = (Mir.Ops{
	2746	.reg1 = reg,
	2747	.reg2 = .ebp,
	2748	.flags = 0b10,
	2749	}).encode(),
	2750	.data = .{ .imm = -@intCast(i32, adj_off) },
	2751	});
2924	2752	},
2925	2753	.memory => \|vaddr\| {
2926	2754	_ = vaddr;
...	...	@@ -2958,25 +2786,26 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2958	2786	}
2959	2787	},
2960	2788	.compare_flags_unsigned => \|op\| {
2961		const encoder = try Encoder.init(self.code, 7);
2962		// TODO audit this codegen: we force w = true here to make
2963		// the value affect the big register
2964		encoder.rex(.{
2965		.w = true,
2966		.b = reg.isExtended(),
	2789	const tag: Mir.Inst.Tag = switch (op) {
	2790	.gte, .gt, .lt, .lte => .cond_set_byte_above_below,
	2791	.eq, .neq => .cond_set_byte_eq_ne,
	2792	};
	2793	const flags: u2 = switch (op) {
	2794	.gte => 0b00,
	2795	.gt => 0b01,
	2796	.lt => 0b10,
	2797	.lte => 0b11,
	2798	.eq => 0b01,
	2799	.neq => 0b00,
	2800	};
	2801	_ = try self.addInst(.{
	2802	.tag = tag,
	2803	.ops = (Mir.Ops{
	2804	.reg1 = reg,
	2805	.flags = flags,
	2806	}).encode(),
	2807	.data = undefined,
2967	2808	});
2968		encoder.opcode_2byte(0x0f, switch (op) {
2969		.gte => 0x93,
2970		.gt => 0x97,
2971		.neq => 0x95,
2972		.lt => 0x92,
2973		.lte => 0x96,
2974		.eq => 0x94,
2975		});
2976		encoder.modRm_direct(
2977		0,
2978		reg.low_id(),
2979		);
2980	2809	},
2981	2810	.compare_flags_signed => \|op\| {
2982	2811	_ = op;
...	...	@@ -2986,44 +2815,25 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2986	2815	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
2987	2816	// register is the fastest way to zero a register.
2988	2817	if (x == 0) {
2989		// The encoding for `xor r32, r32` is `0x31 /r`.
2990		const encoder = try Encoder.init(self.code, 3);
2991
2992		// If we're accessing e.g. r8d, we need to use a REX prefix before the actual operation. Since
2993		// this is a 32-bit operation, the W flag is set to zero. X is also zero, as we're not using a SIB.
2994		// Both R and B are set, as we're extending, in effect, the register bits and the operand.
2995		encoder.rex(.{
2996		.r = reg.isExtended(),
2997		.b = reg.isExtended(),
	2818	_ = try self.addInst(.{
	2819	.tag = .xor,
	2820	.ops = (Mir.Ops{
	2821	.reg1 = reg,
	2822	.reg2 = reg,
	2823	}).encode(),
	2824	.data = undefined,
2998	2825	});
2999		encoder.opcode_1byte(0x31);
3000		// Section 3.1.1.1 of the Intel x64 Manual states that "/r indicates that the
3001		// ModR/M byte of the instruction contains a register operand and an r/m operand."
3002		encoder.modRm_direct(
3003		reg.low_id(),
3004		reg.low_id(),
3005		);
3006
3007	2826	return;
3008	2827	}
3009	2828	if (x <= math.maxInt(i32)) {
3010	2829	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
3011		//
3012		// The encoding for `mov IMM32 -> REG` is (0xB8 + R) IMM.
3013
3014		const encoder = try Encoder.init(self.code, 6);
3015		// Just as with XORing, we need a REX prefix. This time though, we only
3016		// need the B bit set, as we're extending the opcode's register field,
3017		// and there is no Mod R/M byte.
3018		encoder.rex(.{
3019		.b = reg.isExtended(),
	2830	_ = try self.addInst(.{
	2831	.tag = .mov,
	2832	.ops = (Mir.Ops{
	2833	.reg1 = reg,
	2834	}).encode(),
	2835	.data = .{ .imm = @intCast(i32, x) },
3020	2836	});
3021		encoder.opcode_withReg(0xB8, reg.low_id());
3022
3023		// no ModR/M byte
3024
3025		// IMM
3026		encoder.imm32(@intCast(i32, x));
3027	2837	return;
3028	2838	}
3029	2839	// Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls
...	...	@@ -3033,137 +2843,87 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3033	2843	// This encoding is, in fact, the same as the one used for 32-bit loads. The only
3034	2844	// difference is that we set REX.W before the instruction, which extends the load to
3035	2845	// 64-bit and uses the full bit-width of the register.
3036		{
3037		const encoder = try Encoder.init(self.code, 10);
3038		encoder.rex(.{
3039		.w = true,
3040		.b = reg.isExtended(),
3041		});
3042		encoder.opcode_withReg(0xB8, reg.low_id());
3043		encoder.imm64(x);
3044		}
	2846	const payload = try self.addExtra(Mir.Imm64.encode(x));
	2847	_ = try self.addInst(.{
	2848	.tag = .movabs,
	2849	.ops = (Mir.Ops{
	2850	.reg1 = reg,
	2851	}).encode(),
	2852	.data = .{ .payload = payload },
	2853	});
3045	2854	},
3046	2855	.embedded_in_code => \|code_offset\| {
3047	2856	// We need the offset from RIP in a signed i32 twos complement.
3048		// The instruction is 7 bytes long and RIP points to the next instruction.
3049
3050		// 64-bit LEA is encoded as REX.W 8D /r.
3051		const rip = self.code.items.len + 7;
3052		const big_offset = @intCast(i64, code_offset) - @intCast(i64, rip);
3053		const offset = @intCast(i32, big_offset);
3054		const encoder = try Encoder.init(self.code, 7);
3055
3056		// byte 1, always exists because w = true
3057		encoder.rex(.{
3058		.w = true,
3059		.r = reg.isExtended(),
	2857	const payload = try self.addExtra(Mir.Imm64.encode(code_offset));
	2858	_ = try self.addInst(.{
	2859	.tag = .lea_rip,
	2860	.ops = (Mir.Ops{
	2861	.reg1 = reg,
	2862	}).encode(),
	2863	.data = .{ .payload = payload },
3060	2864	});
3061		// byte 2
3062		encoder.opcode_1byte(0x8D);
3063		// byte 3
3064		encoder.modRm_RIPDisp32(reg.low_id());
3065		// byte 4-7
3066		encoder.disp32(offset);
3067
3068		// Double check that we haven't done any math errors
3069		assert(rip == self.code.items.len);
3070	2865	},
3071	2866	.register => \|src_reg\| {
3072	2867	// If the registers are the same, nothing to do.
3073	2868	if (src_reg.id() == reg.id())
3074	2869	return;
3075	2870
3076		// This is a variant of 8B /r.
3077		const abi_size = ty.abiSize(self.target.*);
3078		const encoder = try Encoder.init(self.code, 3);
3079		encoder.rex(.{
3080		.w = abi_size == 8,
3081		.r = reg.isExtended(),
3082		.b = src_reg.isExtended(),
	2871	_ = try self.addInst(.{
	2872	.tag = .mov,
	2873	.ops = (Mir.Ops{
	2874	.reg1 = reg,
	2875	.reg2 = src_reg,
	2876	.flags = 0b11,
	2877	}).encode(),
	2878	.data = undefined,
3083	2879	});
3084		encoder.opcode_1byte(0x8B);
3085		encoder.modRm_direct(reg.low_id(), src_reg.low_id());
3086	2880	},
3087	2881	.memory => \|x\| {
	2882	// TODO can we move this entire logic into Emit.zig like with aarch64?
3088	2883	if (self.bin_file.options.pie) {
3089		// RIP-relative displacement to the entry in the GOT table.
3090		const abi_size = ty.abiSize(self.target.*);
3091		const encoder = try Encoder.init(self.code, 10);
3092
3093		// LEA reg, [<offset>]
3094
3095		// We encode the instruction FIRST because prefixes may or may not appear.
3096		// After we encode the instruction, we will know that the displacement bytes
3097		// for [<offset>] will be at self.code.items.len - 4.
3098		encoder.rex(.{
3099		.w = true, // force 64 bit because loading an address (to the GOT)
3100		.r = reg.isExtended(),
	2884	// TODO we should flag up `x` as GOT symbol entry explicitly rather than as a hack.
	2885	_ = try self.addInst(.{
	2886	.tag = .lea_rip,
	2887	.ops = (Mir.Ops{
	2888	.reg1 = reg,
	2889	.flags = 0b01,
	2890	}).encode(),
	2891	.data = .{ .got_entry = @intCast(u32, x) },
3101	2892	});
3102		encoder.opcode_1byte(0x8D);
3103		encoder.modRm_RIPDisp32(reg.low_id());
3104		encoder.disp32(0);
3105
3106		const offset = @intCast(u32, self.code.items.len);
3107
3108		if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
3109		// TODO I think the reloc might be in the wrong place.
3110		const decl = macho_file.active_decl.?;
3111		// Load reloc for LEA instruction.
3112		try decl.link.macho.relocs.append(self.bin_file.allocator, .{
3113		.offset = offset - 4,
3114		.target = .{ .local = @intCast(u32, x) },
3115		.addend = 0,
3116		.subtractor = null,
3117		.pcrel = true,
3118		.length = 2,
3119		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
3120		});
3121		} else {
3122		return self.fail("TODO implement genSetReg for PIE GOT indirection on this platform", .{});
3123		}
3124
3125	2893	// MOV reg, [reg]
3126		encoder.rex(.{
3127		.w = abi_size == 8,
3128		.r = reg.isExtended(),
3129		.b = reg.isExtended(),
	2894	_ = try self.addInst(.{
	2895	.tag = .mov,
	2896	.ops = (Mir.Ops{
	2897	.reg1 = reg,
	2898	.reg2 = reg,
	2899	.flags = 0b01,
	2900	}).encode(),
	2901	.data = .{ .imm = 0 },
3130	2902	});
3131		encoder.opcode_1byte(0x8B);
3132		encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
3133	2903	} else if (x <= math.maxInt(i32)) {
3134		// Moving from memory to a register is a variant of `8B /r`.
3135		// Since we're using 64-bit moves, we require a REX.
3136		// This variant also requires a SIB, as it would otherwise be RIP-relative.
3137		// We want mode zero with the lower three bits set to four to indicate an SIB with no other displacement.
3138		// The SIB must be 0x25, to indicate a disp32 with no scaled index.
3139		// 0b00RRR100, where RRR is the lower three bits of the register ID.
3140		// The instruction is thus eight bytes; REX 0x8B 0b00RRR100 0x25 followed by a four-byte disp32.
3141		const abi_size = ty.abiSize(self.target.*);
3142		const encoder = try Encoder.init(self.code, 8);
3143		encoder.rex(.{
3144		.w = abi_size == 8,
3145		.r = reg.isExtended(),
	2904	// mov reg, [ds:imm32]
	2905	_ = try self.addInst(.{
	2906	.tag = .mov,
	2907	.ops = (Mir.Ops{
	2908	.reg1 = reg,
	2909	.flags = 0b01,
	2910	}).encode(),
	2911	.data = .{ .imm = @intCast(i32, x) },
3146	2912	});
3147		encoder.opcode_1byte(0x8B);
3148		// effective address = [SIB]
3149		encoder.modRm_SIBDisp0(reg.low_id());
3150		// SIB = disp32
3151		encoder.sib_disp32();
3152		encoder.disp32(@intCast(i32, x));
3153	2913	} else {
3154		// If this is RAX, we can use a direct load; otherwise, we need to load the address, then indirectly load
3155		// the value.
	2914	// If this is RAX, we can use a direct load.
	2915	// Otherwise, we need to load the address, then indirectly load the value.
3156	2916	if (reg.id() == 0) {
3157		// REX.W 0xA1 moffs64*
3158		// moffs64* is a 64-bit offset "relative to segment base", which really just means the
3159		// absolute address for all practical purposes.
3160
3161		const encoder = try Encoder.init(self.code, 10);
3162		encoder.rex(.{
3163		.w = true,
	2917	// movabs rax, ds:moffs64
	2918	const payload = try self.addExtra(Mir.Imm64.encode(x));
	2919	_ = try self.addInst(.{
	2920	.tag = .movabs,
	2921	.ops = (Mir.Ops{
	2922	.reg1 = .rax,
	2923	.flags = 0b01, // imm64 will become moffs64
	2924	}).encode(),
	2925	.data = .{ .payload = payload },
3164	2926	});
3165		encoder.opcode_1byte(0xA1);
3166		encoder.writeIntLittle(u64, x);
3167	2927	} else {
3168	2928	// This requires two instructions; a move imm as used above, followed by an indirect load using the register
3169	2929	// as the address and the register as the destination.
...	...	@@ -3181,16 +2941,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3181	2941	// Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant.
3182	2942	// TODO: determine whether to allow other sized registers, and if so, handle them properly.
3183	2943
3184		// mov reg, [reg]
3185		const abi_size = ty.abiSize(self.target.*);
3186		const encoder = try Encoder.init(self.code, 3);
3187		encoder.rex(.{
3188		.w = abi_size == 8,
3189		.r = reg.isExtended(),
3190		.b = reg.isExtended(),
	2944	// mov reg, [reg + 0x0]
	2945	_ = try self.addInst(.{
	2946	.tag = .mov,
	2947	.ops = (Mir.Ops{
	2948	.reg1 = reg,
	2949	.reg2 = reg,
	2950	.flags = 0b01,
	2951	}).encode(),
	2952	.data = .{ .imm = 0 },
3191	2953	});
3192		encoder.opcode_1byte(0x8B);
3193		encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
3194	2954	}
3195	2955	}
3196	2956	},
...	...	@@ -3201,21 +2961,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3201	2961	return self.fail("stack offset too large", .{});
3202	2962	}
3203	2963	const ioff = -@intCast(i32, off);
3204		const encoder = try Encoder.init(self.code, 3);
3205		encoder.rex(.{
3206		.w = abi_size == 8,
3207		.r = reg.isExtended(),
	2964	_ = try self.addInst(.{
	2965	.tag = .mov,
	2966	.ops = (Mir.Ops{
	2967	.reg1 = reg,
	2968	.reg2 = .ebp,
	2969	.flags = 0b01,
	2970	}).encode(),
	2971	.data = .{ .imm = ioff },
3208	2972	});
3209		encoder.opcode_1byte(0x8B);
3210		if (std.math.minInt(i8) <= ioff and ioff <= std.math.maxInt(i8)) {
3211		// Example: 48 8b 4d 7f mov rcx,QWORD PTR [rbp+0x7f]
3212		encoder.modRm_indirectDisp8(reg.low_id(), Register.ebp.low_id());
3213		encoder.disp8(@intCast(i8, ioff));
3214		} else {
3215		// Example: 48 8b 8d 80 00 00 00 mov rcx,QWORD PTR [rbp+0x80]
3216		encoder.modRm_indirectDisp32(reg.low_id(), Register.ebp.low_id());
3217		encoder.disp32(ioff);
3218		}
3219	2973	},
3220	2974	}
3221	2975	}

src/arch/x86_64/Emit.zig created+1161

...	...	@@ -0,0 +1,1161 @@
	1	//! This file contains the functionality for lowering x86_64 MIR into
	2	//! machine code
	3
	4	const Emit = @This();
	5
	6	const std = @import("std");
	7	const assert = std.debug.assert;
	8	const bits = @import("bits.zig");
	9	const leb128 = std.leb;
	10	const link = @import("../../link.zig");
	11	const log = std.log.scoped(.codegen);
	12	const math = std.math;
	13	const mem = std.mem;
	14
	15	const Air = @import("../../Air.zig");
	16	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
	17	const DW = std.dwarf;
	18	const Encoder = bits.Encoder;
	19	const ErrorMsg = Module.ErrorMsg;
	20	const MCValue = @import("CodeGen.zig").MCValue;
	21	const Mir = @import("Mir.zig");
	22	const Module = @import("../../Module.zig");
	23	const Instruction = bits.Instruction;
	24	const Register = bits.Register;
	25	const Type = @import("../../type.zig").Type;
	26
	27	mir: Mir,
	28	bin_file: *link.File,
	29	debug_output: DebugInfoOutput,
	30	target: *const std.Target,
	31	err_msg: ?*ErrorMsg = null,
	32	src_loc: Module.SrcLoc,
	33	code: *std.ArrayList(u8),
	34
	35	prev_di_line: u32,
	36	prev_di_column: u32,
	37	/// Relative to the beginning of `code`.
	38	prev_di_pc: usize,
	39
	40	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
	41	relocs: std.ArrayListUnmanaged(Reloc) = .{},
	42
	43	const InnerError = error{
	44	OutOfMemory,
	45	EmitFail,
	46	};
	47
	48	const Reloc = struct {
	49	/// Offset of the instruction.
	50	source: u64,
	51	/// Target of the relocation.
	52	target: Mir.Inst.Index,
	53	/// Offset of the relocation within the instruction.
	54	offset: u64,
	55	/// Length of the instruction.
	56	length: u5,
	57	};
	58
	59	pub fn emitMir(emit: *Emit) InnerError!void {
	60	const mir_tags = emit.mir.instructions.items(.tag);
	61
	62	for (mir_tags) \|tag, index\| {
	63	const inst = @intCast(u32, index);
	64	try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
	65	switch (tag) {
	66	.adc => try emit.mirArith(.adc, inst),
	67	.add => try emit.mirArith(.add, inst),
	68	.sub => try emit.mirArith(.sub, inst),
	69	.xor => try emit.mirArith(.xor, inst),
	70	.@"and" => try emit.mirArith(.@"and", inst),
	71	.@"or" => try emit.mirArith(.@"or", inst),
	72	.sbb => try emit.mirArith(.sbb, inst),
	73	.cmp => try emit.mirArith(.cmp, inst),
	74
	75	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
	76	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
	77	.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
	78	.xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
	79	.and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
	80	.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
	81	.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
	82	.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
	83
	84	.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
	85	.add_scale_dst => try emit.mirArithScaleDst(.add, inst),
	86	.sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
	87	.xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
	88	.and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
	89	.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
	90	.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
	91	.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
	92
	93	.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
	94	.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
	95	.sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
	96	.xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
	97	.and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
	98	.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
	99	.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
	100	.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
	101
	102	// Even though MOV is technically not an arithmetic op,
	103	// its structure can be represented using the same set of
	104	// opcode primitives.
	105	.mov => try emit.mirArith(.mov, inst),
	106	.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
	107	.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
	108	.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
	109	.movabs => try emit.mirMovabs(inst),
	110
	111	.lea => try emit.mirLea(inst),
	112	.lea_rip => try emit.mirLeaRip(inst),
	113
	114	.imul_complex => try emit.mirIMulComplex(inst),
	115
	116	.push => try emit.mirPushPop(.push, inst),
	117	.pop => try emit.mirPushPop(.pop, inst),
	118
	119	.jmp => try emit.mirJmpCall(.jmp, inst),
	120	.call => try emit.mirJmpCall(.call, inst),
	121
	122	.cond_jmp_greater_less => try emit.mirCondJmp(.cond_jmp_greater_less, inst),
	123	.cond_jmp_above_below => try emit.mirCondJmp(.cond_jmp_above_below, inst),
	124	.cond_jmp_eq_ne => try emit.mirCondJmp(.cond_jmp_eq_ne, inst),
	125
	126	.cond_set_byte_greater_less => try emit.mirCondSetByte(.cond_set_byte_greater_less, inst),
	127	.cond_set_byte_above_below => try emit.mirCondSetByte(.cond_set_byte_above_below, inst),
	128	.cond_set_byte_eq_ne => try emit.mirCondSetByte(.cond_set_byte_eq_ne, inst),
	129
	130	.ret => try emit.mirRet(inst),
	131
	132	.syscall => try emit.mirSyscall(),
	133
	134	.@"test" => try emit.mirTest(inst),
	135
	136	.brk => try emit.mirBrk(),
	137
	138	.call_extern => try emit.mirCallExtern(inst),
	139
	140	.dbg_line => try emit.mirDbgLine(inst),
	141	.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
	142	.dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
	143	.arg_dbg_info => try emit.mirArgDbgInfo(inst),
	144
	145	else => {
	146	return emit.fail("Implement MIR->Isel lowering for x86_64 for pseudo-inst: {s}", .{tag});
	147	},
	148	}
	149	}
	150
	151	try emit.fixupRelocs();
	152	}
	153
	154	pub fn deinit(emit: *Emit) void {
	155	emit.relocs.deinit(emit.bin_file.allocator);
	156	emit.code_offset_mapping.deinit(emit.bin_file.allocator);
	157	emit.* = undefined;
	158	}
	159
	160	fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
	161	@setCold(true);
	162	assert(emit.err_msg == null);
	163	emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args);
	164	return error.EmitFail;
	165	}
	166
	167	fn fixupRelocs(emit: *Emit) InnerError!void {
	168	// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
	169	// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
	170	// possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
	171	// until the entire decl is correctly emitted with all JMP/CALL instructions within range.
	172	for (emit.relocs.items) \|reloc\| {
	173	const target = emit.code_offset_mapping.get(reloc.target) orelse
	174	return emit.fail("JMP/CALL relocation target not found!", .{});
	175	const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
	176	mem.writeIntLittle(i32, emit.code.items[reloc.offset..][0..4], disp);
	177	}
	178	}
	179
	180	fn mirBrk(emit: *Emit) InnerError!void {
	181	const encoder = try Encoder.init(emit.code, 1);
	182	encoder.opcode_1byte(0xcc);
	183	}
	184
	185	fn mirSyscall(emit: *Emit) InnerError!void {
	186	const encoder = try Encoder.init(emit.code, 2);
	187	encoder.opcode_2byte(0x0f, 0x05);
	188	}
	189
	190	fn mirPushPop(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	191	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	192	switch (ops.flags) {
	193	0b00 => {
	194	// PUSH/POP reg
	195	const opc: u8 = switch (tag) {
	196	.push => 0x50,
	197	.pop => 0x58,
	198	else => unreachable,
	199	};
	200	const encoder = try Encoder.init(emit.code, 1);
	201	encoder.opcode_withReg(opc, ops.reg1.lowId());
	202	},
	203	0b01 => {
	204	// PUSH/POP r/m64
	205	const imm = emit.mir.instructions.items(.data)[inst].imm;
	206	const opc: u8 = switch (tag) {
	207	.push => 0xff,
	208	.pop => 0x8f,
	209	else => unreachable,
	210	};
	211	const modrm_ext: u3 = switch (tag) {
	212	.push => 0x6,
	213	.pop => 0x0,
	214	else => unreachable,
	215	};
	216	const encoder = try Encoder.init(emit.code, 6);
	217	encoder.opcode_1byte(opc);
	218	if (math.cast(i8, imm)) \|imm_i8\| {
	219	encoder.modRm_indirectDisp8(modrm_ext, ops.reg1.lowId());
	220	encoder.imm8(@intCast(i8, imm_i8));
	221	} else \|_\| {
	222	encoder.modRm_indirectDisp32(modrm_ext, ops.reg1.lowId());
	223	encoder.imm32(imm);
	224	}
	225	},
	226	0b10 => {
	227	// PUSH imm32
	228	assert(tag == .push);
	229	const imm = emit.mir.instructions.items(.data)[inst].imm;
	230	const opc: u8 = if (imm <= math.maxInt(i8)) 0x6a else 0x6b;
	231	const encoder = try Encoder.init(emit.code, 2);
	232	encoder.opcode_1byte(opc);
	233	if (imm <= math.maxInt(i8)) {
	234	encoder.imm8(@intCast(i8, imm));
	235	} else if (imm <= math.maxInt(i16)) {
	236	encoder.imm16(@intCast(i16, imm));
	237	} else {
	238	encoder.imm32(imm);
	239	}
	240	},
	241	0b11 => unreachable,
	242	}
	243	}
	244
	245	fn mirJmpCall(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	246	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	247	const flag = @truncate(u1, ops.flags);
	248	if (flag == 0) {
	249	const target = emit.mir.instructions.items(.data)[inst].inst;
	250	const opc: u8 = switch (tag) {
	251	.jmp => 0xe9,
	252	.call => 0xe8,
	253	else => unreachable,
	254	};
	255	const source = emit.code.items.len;
	256	const encoder = try Encoder.init(emit.code, 5);
	257	encoder.opcode_1byte(opc);
	258	try emit.relocs.append(emit.bin_file.allocator, .{
	259	.source = source,
	260	.target = target,
	261	.offset = emit.code.items.len,
	262	.length = 5,
	263	});
	264	encoder.imm32(0x0);
	265	return;
	266	}
	267	const modrm_ext: u3 = switch (tag) {
	268	.jmp => 0x4,
	269	.call => 0x2,
	270	else => unreachable,
	271	};
	272	if (ops.reg1 == .none) {
	273	// JMP/CALL [imm]
	274	const imm = emit.mir.instructions.items(.data)[inst].imm;
	275	const encoder = try Encoder.init(emit.code, 7);
	276	encoder.opcode_1byte(0xff);
	277	encoder.modRm_SIBDisp0(modrm_ext);
	278	encoder.sib_disp32();
	279	encoder.imm32(imm);
	280	return;
	281	}
	282	// JMP/CALL reg
	283	const encoder = try Encoder.init(emit.code, 2);
	284	encoder.opcode_1byte(0xff);
	285	encoder.modRm_direct(modrm_ext, ops.reg1.lowId());
	286	}
	287
	288	const CondType = enum {
	289	/// greater than or equal
	290	gte,
	291
	292	/// greater than
	293	gt,
	294
	295	/// less than
	296	lt,
	297
	298	/// less than or equal
	299	lte,
	300
	301	/// above or equal
	302	ae,
	303
	304	/// above
	305	a,
	306
	307	/// below
	308	b,
	309
	310	/// below or equal
	311	be,
	312
	313	/// not equal
	314	ne,
	315
	316	/// equal
	317	eq,
	318
	319	fn fromTagAndFlags(tag: Mir.Inst.Tag, flags: u2) CondType {
	320	return switch (tag) {
	321	.cond_jmp_greater_less,
	322	.cond_set_byte_greater_less,
	323	=> switch (flags) {
	324	0b00 => CondType.gte,
	325	0b01 => CondType.gt,
	326	0b10 => CondType.lt,
	327	0b11 => CondType.lte,
	328	},
	329	.cond_jmp_above_below,
	330	.cond_set_byte_above_below,
	331	=> switch (flags) {
	332	0b00 => CondType.ae,
	333	0b01 => CondType.a,
	334	0b10 => CondType.b,
	335	0b11 => CondType.be,
	336	},
	337	.cond_jmp_eq_ne,
	338	.cond_set_byte_eq_ne,
	339	=> switch (@truncate(u1, flags)) {
	340	0b0 => CondType.ne,
	341	0b1 => CondType.eq,
	342	},
	343	else => unreachable,
	344	};
	345	}
	346	};
	347
	348	inline fn getCondOpCode(tag: Mir.Inst.Tag, cond: CondType) u8 {
	349	switch (cond) {
	350	.gte => return switch (tag) {
	351	.cond_jmp_greater_less => 0x8d,
	352	.cond_set_byte_greater_less => 0x9d,
	353	else => unreachable,
	354	},
	355	.gt => return switch (tag) {
	356	.cond_jmp_greater_less => 0x8f,
	357	.cond_set_byte_greater_less => 0x9f,
	358	else => unreachable,
	359	},
	360	.lt => return switch (tag) {
	361	.cond_jmp_greater_less => 0x8c,
	362	.cond_set_byte_greater_less => 0x9c,
	363	else => unreachable,
	364	},
	365	.lte => return switch (tag) {
	366	.cond_jmp_greater_less => 0x8e,
	367	.cond_set_byte_greater_less => 0x9e,
	368	else => unreachable,
	369	},
	370	.ae => return switch (tag) {
	371	.cond_jmp_above_below => 0x83,
	372	.cond_set_byte_above_below => 0x93,
	373	else => unreachable,
	374	},
	375	.a => return switch (tag) {
	376	.cond_jmp_above_below => 0x87,
	377	.cond_set_byte_greater_less => 0x97,
	378	else => unreachable,
	379	},
	380	.b => return switch (tag) {
	381	.cond_jmp_above_below => 0x82,
	382	.cond_set_byte_greater_less => 0x92,
	383	else => unreachable,
	384	},
	385	.be => return switch (tag) {
	386	.cond_jmp_above_below => 0x86,
	387	.cond_set_byte_greater_less => 0x96,
	388	else => unreachable,
	389	},
	390	.eq => return switch (tag) {
	391	.cond_jmp_eq_ne => 0x84,
	392	.cond_set_byte_eq_ne => 0x94,
	393	else => unreachable,
	394	},
	395	.ne => return switch (tag) {
	396	.cond_jmp_eq_ne => 0x85,
	397	.cond_set_byte_eq_ne => 0x95,
	398	else => unreachable,
	399	},
	400	}
	401	}
	402
	403	fn mirCondJmp(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	404	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	405	const target = emit.mir.instructions.items(.data)[inst].inst;
	406	const cond = CondType.fromTagAndFlags(tag, ops.flags);
	407	const opc = getCondOpCode(tag, cond);
	408	const source = emit.code.items.len;
	409	const encoder = try Encoder.init(emit.code, 6);
	410	encoder.opcode_2byte(0x0f, opc);
	411	try emit.relocs.append(emit.bin_file.allocator, .{
	412	.source = source,
	413	.target = target,
	414	.offset = emit.code.items.len,
	415	.length = 6,
	416	});
	417	encoder.imm32(0);
	418	}
	419
	420	fn mirCondSetByte(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	421	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	422	const cond = CondType.fromTagAndFlags(tag, ops.flags);
	423	const opc = getCondOpCode(tag, cond);
	424	const encoder = try Encoder.init(emit.code, 4);
	425	encoder.rex(.{
	426	.w = true,
	427	.b = ops.reg1.isExtended(),
	428	});
	429	encoder.opcode_2byte(0x0f, opc);
	430	encoder.modRm_direct(0x0, ops.reg1.lowId());
	431	}
	432
	433	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	434	const tag = emit.mir.instructions.items(.tag)[inst];
	435	assert(tag == .@"test");
	436	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	437	switch (ops.flags) {
	438	0b00 => blk: {
	439	if (ops.reg2 == .none) {
	440	// TEST r/m64, imm32
	441	const imm = emit.mir.instructions.items(.data)[inst].imm;
	442	if (ops.reg1.to64() == .rax) {
	443	// TODO reduce the size of the instruction if the immediate
	444	// is smaller than 32 bits
	445	const encoder = try Encoder.init(emit.code, 6);
	446	encoder.rex(.{
	447	.w = true,
	448	});
	449	encoder.opcode_1byte(0xa9);
	450	encoder.imm32(imm);
	451	break :blk;
	452	}
	453	const opc: u8 = if (ops.reg1.size() == 8) 0xf6 else 0xf7;
	454	const encoder = try Encoder.init(emit.code, 7);
	455	encoder.rex(.{
	456	.w = true,
	457	.b = ops.reg1.isExtended(),
	458	});
	459	encoder.opcode_1byte(opc);
	460	encoder.modRm_direct(0, ops.reg1.lowId());
	461	encoder.imm8(@intCast(i8, imm));
	462	break :blk;
	463	}
	464	// TEST r/m64, r64
	465	return emit.fail("TODO TEST r/m64, r64", .{});
	466	},
	467	else => return emit.fail("TODO more TEST alternatives", .{}),
	468	}
	469	}
	470
	471	fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	472	const tag = emit.mir.instructions.items(.tag)[inst];
	473	assert(tag == .ret);
	474	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	475	const encoder = try Encoder.init(emit.code, 3);
	476	switch (ops.flags) {
	477	0b00 => {
	478	// RETF imm16
	479	const imm = emit.mir.instructions.items(.data)[inst].imm;
	480	encoder.opcode_1byte(0xca);
	481	encoder.imm16(@intCast(i16, imm));
	482	},
	483	0b01 => encoder.opcode_1byte(0xcb), // RETF
	484	0b10 => {
	485	// RET imm16
	486	const imm = emit.mir.instructions.items(.data)[inst].imm;
	487	encoder.opcode_1byte(0xc2);
	488	encoder.imm16(@intCast(i16, imm));
	489	},
	490	0b11 => encoder.opcode_1byte(0xc3), // RET
	491	}
	492	}
	493
	494	const EncType = enum {
	495	/// OP r/m64, imm32
	496	mi,
	497
	498	/// OP r/m64, r64
	499	mr,
	500
	501	/// OP r64, r/m64
	502	rm,
	503	};
	504
	505	const OpCode = struct {
	506	opc: u8,
	507	/// Only used if `EncType == .mi`.
	508	modrm_ext: u3,
	509	};
	510
	511	inline fn getArithOpCode(tag: Mir.Inst.Tag, enc: EncType) OpCode {
	512	switch (enc) {
	513	.mi => return switch (tag) {
	514	.adc => .{ .opc = 0x81, .modrm_ext = 0x2 },
	515	.add => .{ .opc = 0x81, .modrm_ext = 0x0 },
	516	.sub => .{ .opc = 0x81, .modrm_ext = 0x5 },
	517	.xor => .{ .opc = 0x81, .modrm_ext = 0x6 },
	518	.@"and" => .{ .opc = 0x81, .modrm_ext = 0x4 },
	519	.@"or" => .{ .opc = 0x81, .modrm_ext = 0x1 },
	520	.sbb => .{ .opc = 0x81, .modrm_ext = 0x3 },
	521	.cmp => .{ .opc = 0x81, .modrm_ext = 0x7 },
	522	.mov => .{ .opc = 0xc7, .modrm_ext = 0x0 },
	523	else => unreachable,
	524	},
	525	.mr => {
	526	const opc: u8 = switch (tag) {
	527	.adc => 0x11,
	528	.add => 0x01,
	529	.sub => 0x29,
	530	.xor => 0x31,
	531	.@"and" => 0x21,
	532	.@"or" => 0x09,
	533	.sbb => 0x19,
	534	.cmp => 0x39,
	535	.mov => 0x89,
	536	else => unreachable,
	537	};
	538	return .{ .opc = opc, .modrm_ext = undefined };
	539	},
	540	.rm => {
	541	const opc: u8 = switch (tag) {
	542	.adc => 0x13,
	543	.add => 0x03,
	544	.sub => 0x2b,
	545	.xor => 0x33,
	546	.@"and" => 0x23,
	547	.@"or" => 0x0b,
	548	.sbb => 0x1b,
	549	.cmp => 0x3b,
	550	.mov => 0x8b,
	551	else => unreachable,
	552	};
	553	return .{ .opc = opc, .modrm_ext = undefined };
	554	},
	555	}
	556	}
	557
	558	fn mirArith(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	559	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	560	switch (ops.flags) {
	561	0b00 => blk: {
	562	if (ops.reg2 == .none) {
	563	// OP reg1, imm32
	564	// OP r/m64, imm32
	565	const imm = emit.mir.instructions.items(.data)[inst].imm;
	566	const opcode = getArithOpCode(tag, .mi);
	567	const encoder = try Encoder.init(emit.code, 7);
	568	encoder.rex(.{
	569	.w = ops.reg1.size() == 64,
	570	.b = ops.reg1.isExtended(),
	571	});
	572	if (tag != .mov and imm <= math.maxInt(i8)) {
	573	encoder.opcode_1byte(opcode.opc + 2);
	574	encoder.modRm_direct(opcode.modrm_ext, ops.reg1.lowId());
	575	encoder.imm8(@intCast(i8, imm));
	576	} else {
	577	encoder.opcode_1byte(opcode.opc);
	578	encoder.modRm_direct(opcode.modrm_ext, ops.reg1.lowId());
	579	encoder.imm32(imm);
	580	}
	581	break :blk;
	582	}
	583	// OP reg1, reg2
	584	// OP r/m64, r64
	585	const opcode = getArithOpCode(tag, .mr);
	586	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	587	const encoder = try Encoder.init(emit.code, 3);
	588	encoder.rex(.{
	589	.w = ops.reg1.size() == 64 and ops.reg2.size() == 64,
	590	.r = ops.reg1.isExtended(),
	591	.b = ops.reg2.isExtended(),
	592	});
	593	encoder.opcode_1byte(opc);
	594	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
	595	},
	596	0b01 => blk: {
	597	const imm = emit.mir.instructions.items(.data)[inst].imm;
	598	const opcode = getArithOpCode(tag, .rm);
	599	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	600	if (ops.reg2 == .none) {
	601	// OP reg1, [imm32]
	602	// OP r64, r/m64
	603	const encoder = try Encoder.init(emit.code, 8);
	604	encoder.rex(.{
	605	.w = ops.reg1.size() == 64,
	606	.b = ops.reg1.isExtended(),
	607	});
	608	encoder.opcode_1byte(opc);
	609	encoder.modRm_SIBDisp0(ops.reg1.lowId());
	610	encoder.sib_disp32();
	611	encoder.disp32(imm);
	612	break :blk;
	613	}
	614	// OP reg1, [reg2 + imm32]
	615	// OP r64, r/m64
	616	const encoder = try Encoder.init(emit.code, 7);
	617	encoder.rex(.{
	618	.w = ops.reg1.size() == 64,
	619	.r = ops.reg1.isExtended(),
	620	.b = ops.reg2.isExtended(),
	621	});
	622	encoder.opcode_1byte(opc);
	623	if (imm <= math.maxInt(i8)) {
	624	encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId());
	625	encoder.disp8(@intCast(i8, imm));
	626	} else {
	627	encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId());
	628	encoder.disp32(imm);
	629	}
	630	},
	631	0b10 => blk: {
	632	if (ops.reg2 == .none) {
	633	// OP [reg1 + 0], imm32
	634	// OP r/m64, imm32
	635	const imm = emit.mir.instructions.items(.data)[inst].imm;
	636	const opcode = getArithOpCode(tag, .mi);
	637	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	638	const encoder = try Encoder.init(emit.code, 7);
	639	encoder.rex(.{
	640	.w = ops.reg1.size() == 64,
	641	.b = ops.reg1.isExtended(),
	642	});
	643	encoder.opcode_1byte(opc);
	644	encoder.modRm_indirectDisp0(opcode.modrm_ext, ops.reg1.lowId());
	645	if (imm <= math.maxInt(i8)) {
	646	encoder.imm8(@intCast(i8, imm));
	647	} else if (imm <= math.maxInt(i16)) {
	648	encoder.imm16(@intCast(i16, imm));
	649	} else {
	650	encoder.imm32(imm);
	651	}
	652	break :blk;
	653	}
	654	// OP [reg1 + imm32], reg2
	655	// OP r/m64, r64
	656	const imm = emit.mir.instructions.items(.data)[inst].imm;
	657	const opcode = getArithOpCode(tag, .mr);
	658	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	659	const encoder = try Encoder.init(emit.code, 7);
	660	encoder.rex(.{
	661	.w = ops.reg2.size() == 64,
	662	.r = ops.reg1.isExtended(),
	663	.b = ops.reg2.isExtended(),
	664	});
	665	encoder.opcode_1byte(opc);
	666	if (imm <= math.maxInt(i8)) {
	667	encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId());
	668	encoder.disp8(@intCast(i8, imm));
	669	} else {
	670	encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId());
	671	encoder.disp32(imm);
	672	}
	673	},
	674	0b11 => blk: {
	675	if (ops.reg2 == .none) {
	676	// OP [reg1 + imm32], imm32
	677	// OP r/m64, imm32
	678	const payload = emit.mir.instructions.items(.data)[inst].payload;
	679	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	680	const opcode = getArithOpCode(tag, .mi);
	681	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	682	const encoder = try Encoder.init(emit.code, 11);
	683	encoder.rex(.{
	684	.w = false,
	685	.b = ops.reg1.isExtended(),
	686	});
	687	encoder.opcode_1byte(opc);
	688	if (imm_pair.dest_off <= math.maxInt(i8)) {
	689	encoder.modRm_indirectDisp8(opcode.modrm_ext, ops.reg1.lowId());
	690	encoder.disp8(@intCast(i8, imm_pair.dest_off));
	691	} else {
	692	encoder.modRm_indirectDisp32(opcode.modrm_ext, ops.reg1.lowId());
	693	encoder.disp32(imm_pair.dest_off);
	694	}
	695	encoder.imm32(imm_pair.operand);
	696	break :blk;
	697	}
	698	// TODO clearly mov doesn't belong here; for other, arithemtic ops,
	699	// this is the same as 0b00.
	700	const opcode = getArithOpCode(tag, if (tag == .mov) .rm else .mr);
	701	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	702	const encoder = try Encoder.init(emit.code, 3);
	703	encoder.rex(.{
	704	.w = ops.reg1.size() == 64 and ops.reg2.size() == 64,
	705	.r = ops.reg1.isExtended(),
	706	.b = ops.reg2.isExtended(),
	707	});
	708	encoder.opcode_1byte(opc);
	709	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
	710	},
	711	}
	712	}
	713
	714	fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	715	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	716	const scale = ops.flags;
	717	// OP reg1, [reg2 + scale*rcx + imm32]
	718	const opcode = getArithOpCode(tag, .rm);
	719	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	720	const imm = emit.mir.instructions.items(.data)[inst].imm;
	721	const encoder = try Encoder.init(emit.code, 8);
	722	encoder.rex(.{
	723	.w = ops.reg1.size() == 64,
	724	.r = ops.reg1.isExtended(),
	725	.b = ops.reg2.isExtended(),
	726	});
	727	encoder.opcode_1byte(opc);
	728	if (imm <= math.maxInt(i8)) {
	729	encoder.modRm_SIBDisp8(ops.reg1.lowId());
	730	encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
	731	encoder.disp8(@intCast(i8, imm));
	732	} else {
	733	encoder.modRm_SIBDisp32(ops.reg1.lowId());
	734	encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
	735	encoder.disp32(imm);
	736	}
	737	}
	738
	739	fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	740	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	741	const scale = ops.flags;
	742	const imm = emit.mir.instructions.items(.data)[inst].imm;
	743
	744	if (ops.reg2 == .none) {
	745	// OP [reg1 + scale*rax + 0], imm32
	746	const opcode = getArithOpCode(tag, .mi);
	747	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	748	const encoder = try Encoder.init(emit.code, 8);
	749	encoder.rex(.{
	750	.w = ops.reg1.size() == 64,
	751	.b = ops.reg1.isExtended(),
	752	});
	753	encoder.opcode_1byte(opc);
	754	encoder.modRm_SIBDisp0(opcode.modrm_ext);
	755	encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
	756	if (imm <= math.maxInt(i8)) {
	757	encoder.imm8(@intCast(i8, imm));
	758	} else if (imm <= math.maxInt(i16)) {
	759	encoder.imm16(@intCast(i16, imm));
	760	} else {
	761	encoder.imm32(imm);
	762	}
	763	return;
	764	}
	765
	766	// OP [reg1 + scale*rax + imm32], reg2
	767	const opcode = getArithOpCode(tag, .mr);
	768	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	769	const encoder = try Encoder.init(emit.code, 8);
	770	encoder.rex(.{
	771	.w = ops.reg1.size() == 64,
	772	.r = ops.reg2.isExtended(),
	773	.b = ops.reg1.isExtended(),
	774	});
	775	encoder.opcode_1byte(opc);
	776	if (imm <= math.maxInt(i8)) {
	777	encoder.modRm_SIBDisp8(ops.reg2.lowId());
	778	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	779	encoder.disp8(@intCast(i8, imm));
	780	} else {
	781	encoder.modRm_SIBDisp32(ops.reg2.lowId());
	782	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	783	encoder.disp32(imm);
	784	}
	785	}
	786
	787	fn mirArithScaleImm(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	788	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	789	const scale = ops.flags;
	790	const payload = emit.mir.instructions.items(.data)[inst].payload;
	791	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	792	const opcode = getArithOpCode(tag, .mi);
	793	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
	794	const encoder = try Encoder.init(emit.code, 2);
	795	encoder.rex(.{
	796	.w = ops.reg1.size() == 64,
	797	.b = ops.reg1.isExtended(),
	798	});
	799	encoder.opcode_1byte(opc);
	800	if (imm_pair.dest_off <= math.maxInt(i8)) {
	801	encoder.modRm_SIBDisp8(opcode.modrm_ext);
	802	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	803	encoder.disp8(@intCast(i8, imm_pair.dest_off));
	804	} else {
	805	encoder.modRm_SIBDisp32(opcode.modrm_ext);
	806	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	807	encoder.disp32(imm_pair.dest_off);
	808	}
	809	encoder.imm32(imm_pair.operand);
	810	}
	811
	812	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	813	const tag = emit.mir.instructions.items(.tag)[inst];
	814	assert(tag == .movabs);
	815	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	816
	817	const encoder = try Encoder.init(emit.code, 10);
	818	const is_64 = blk: {
	819	if (ops.flags == 0b00) {
	820	// movabs reg, imm64
	821	const opc: u8 = if (ops.reg1.size() == 8) 0xb0 else 0xb8;
	822	if (ops.reg1.size() == 64) {
	823	encoder.rex(.{
	824	.w = true,
	825	.b = ops.reg1.isExtended(),
	826	});
	827	encoder.opcode_withReg(opc, ops.reg1.lowId());
	828	break :blk true;
	829	}
	830	break :blk false;
	831	}
	832	if (ops.reg1 == .none) {
	833	// movabs moffs64, rax
	834	const opc: u8 = if (ops.reg2.size() == 8) 0xa2 else 0xa3;
	835	encoder.rex(.{
	836	.w = ops.reg2.size() == 64,
	837	});
	838	encoder.opcode_1byte(opc);
	839	break :blk ops.reg2.size() == 64;
	840	} else {
	841	// movabs rax, moffs64
	842	const opc: u8 = if (ops.reg2.size() == 8) 0xa0 else 0xa1;
	843	encoder.rex(.{
	844	.w = ops.reg1.size() == 64,
	845	});
	846	encoder.opcode_1byte(opc);
	847	break :blk ops.reg1.size() == 64;
	848	}
	849	};
	850
	851	if (is_64) {
	852	const payload = emit.mir.instructions.items(.data)[inst].payload;
	853	const imm64 = emit.mir.extraData(Mir.Imm64, payload).data;
	854	encoder.imm64(imm64.decode());
	855	} else {
	856	const imm = emit.mir.instructions.items(.data)[inst].imm;
	857	if (imm <= math.maxInt(i8)) {
	858	encoder.imm8(@intCast(i8, imm));
	859	} else if (imm <= math.maxInt(i16)) {
	860	encoder.imm16(@intCast(i16, imm));
	861	} else {
	862	encoder.imm32(imm);
	863	}
	864	}
	865	}
	866
	867	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	868	const tag = emit.mir.instructions.items(.tag)[inst];
	869	assert(tag == .imul_complex);
	870	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	871	switch (ops.flags) {
	872	0b00 => {
	873	const encoder = try Encoder.init(emit.code, 4);
	874	encoder.rex(.{
	875	.w = ops.reg1.size() == 64,
	876	.r = ops.reg1.isExtended(),
	877	.b = ops.reg2.isExtended(),
	878	});
	879	encoder.opcode_2byte(0x0f, 0xaf);
	880	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
	881	},
	882	0b10 => {
	883	const imm = emit.mir.instructions.items(.data)[inst].imm;
	884	const opc: u8 = if (imm <= math.maxInt(i8)) 0x6b else 0x69;
	885	const encoder = try Encoder.init(emit.code, 7);
	886	encoder.rex(.{
	887	.w = ops.reg1.size() == 64,
	888	.r = ops.reg1.isExtended(),
	889	.b = ops.reg1.isExtended(),
	890	});
	891	encoder.opcode_1byte(opc);
	892	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
	893	if (imm <= math.maxInt(i8)) {
	894	encoder.imm8(@intCast(i8, imm));
	895	} else if (imm <= math.maxInt(i16)) {
	896	encoder.imm16(@intCast(i16, imm));
	897	} else {
	898	encoder.imm32(imm);
	899	}
	900	},
	901	else => return emit.fail("TODO implement imul", .{}),
	902	}
	903	}
	904
	905	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	906	const tag = emit.mir.instructions.items(.tag)[inst];
	907	assert(tag == .lea);
	908	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	909	assert(ops.flags == 0b01);
	910	const imm = emit.mir.instructions.items(.data)[inst].imm;
	911
	912	if (imm == 0) {
	913	const encoder = try Encoder.init(emit.code, 3);
	914	encoder.rex(.{
	915	.w = ops.reg1.size() == 64,
	916	.r = ops.reg1.isExtended(),
	917	.b = ops.reg2.isExtended(),
	918	});
	919	encoder.opcode_1byte(0x8d);
	920	encoder.modRm_indirectDisp0(ops.reg1.lowId(), ops.reg2.lowId());
	921	} else if (imm <= math.maxInt(i8)) {
	922	const encoder = try Encoder.init(emit.code, 4);
	923	encoder.rex(.{
	924	.w = ops.reg1.size() == 64,
	925	.r = ops.reg1.isExtended(),
	926	.b = ops.reg2.isExtended(),
	927	});
	928	encoder.opcode_1byte(0x8d);
	929	encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId());
	930	encoder.disp8(@intCast(i8, imm));
	931	} else {
	932	const encoder = try Encoder.init(emit.code, 7);
	933	encoder.rex(.{
	934	.w = ops.reg1.size() == 64,
	935	.r = ops.reg1.isExtended(),
	936	.b = ops.reg2.isExtended(),
	937	});
	938	encoder.opcode_1byte(0x8d);
	939	encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId());
	940	encoder.disp32(imm);
	941	}
	942	}
	943
	944	fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	945	const tag = emit.mir.instructions.items(.tag)[inst];
	946	assert(tag == .lea_rip);
	947	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	948	const start_offset = emit.code.items.len;
	949	const encoder = try Encoder.init(emit.code, 7);
	950	encoder.rex(.{
	951	.w = ops.reg1.size() == 64,
	952	.r = ops.reg1.isExtended(),
	953	});
	954	encoder.opcode_1byte(0x8d);
	955	encoder.modRm_RIPDisp32(ops.reg1.lowId());
	956	const end_offset = emit.code.items.len;
	957	if (@truncate(u1, ops.flags) == 0b0) {
	958	const payload = emit.mir.instructions.items(.data)[inst].payload;
	959	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
	960	encoder.disp32(@intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset + 4)));
	961	} else {
	962	const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
	963	encoder.disp32(0);
	964	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	965	// TODO I think the reloc might be in the wrong place.
	966	const decl = macho_file.active_decl.?;
	967	try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
	968	.offset = @intCast(u32, end_offset),
	969	.target = .{ .local = got_entry },
	970	.addend = 0,
	971	.subtractor = null,
	972	.pcrel = true,
	973	.length = 2,
	974	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	975	});
	976	} else {
	977	return emit.fail("TODO implement lea_rip for linking backends different than MachO", .{});
	978	}
	979	}
	980	}
	981
	982	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	983	const tag = emit.mir.instructions.items(.tag)[inst];
	984	assert(tag == .call_extern);
	985	const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
	986	const offset = blk: {
	987	const offset = @intCast(u32, emit.code.items.len + 1);
	988	// callq
	989	const encoder = try Encoder.init(emit.code, 5);
	990	encoder.opcode_1byte(0xe8);
	991	encoder.imm32(0x0);
	992	break :blk offset;
	993	};
	994	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	995	// Add relocation to the decl.
	996	try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
	997	.offset = offset,
	998	.target = .{ .global = n_strx },
	999	.addend = 0,
	1000	.subtractor = null,
	1001	.pcrel = true,
	1002	.length = 2,
	1003	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	1004	});
	1005	} else {
	1006	return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
	1007	}
	1008	}
	1009
	1010	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	1011	const tag = emit.mir.instructions.items(.tag)[inst];
	1012	assert(tag == .dbg_line);
	1013	const payload = emit.mir.instructions.items(.data)[inst].payload;
	1014	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
	1015	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
	1016	}
	1017
	1018	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
	1019	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
	1020	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
	1021	switch (emit.debug_output) {
	1022	.dwarf => \|dbg_out\| {
	1023	// TODO Look into using the DWARF special opcodes to compress this data.
	1024	// It lets you emit single-byte opcodes that add different numbers to
	1025	// both the PC and the line number at the same time.
	1026	try dbg_out.dbg_line.ensureUnusedCapacity(11);
	1027	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
	1028	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
	1029	if (delta_line != 0) {
	1030	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
	1031	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	1032	}
	1033	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
	1034	emit.prev_di_pc = emit.code.items.len;
	1035	emit.prev_di_line = line;
	1036	emit.prev_di_column = column;
	1037	emit.prev_di_pc = emit.code.items.len;
	1038	},
	1039	.plan9 => \|dbg_out\| {
	1040	if (delta_pc <= 0) return; // only do this when the pc changes
	1041	// we have already checked the target in the linker to make sure it is compatable
	1042	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
	1043
	1044	// increasing the line number
	1045	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
	1046	// increasing the pc
	1047	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
	1048	if (d_pc_p9 > 0) {
	1049	// minus one because if its the last one, we want to leave space to change the line which is one quanta
	1050	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
	1051	if (dbg_out.pcop_change_index.*) \|pci\|
	1052	dbg_out.dbg_line.items[pci] += 1;
	1053	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
	1054	} else if (d_pc_p9 == 0) {
	1055	// we don't need to do anything, because adding the quant does it for us
	1056	} else unreachable;
	1057	if (dbg_out.start_line.* == null)
	1058	dbg_out.start_line.* = emit.prev_di_line;
	1059	dbg_out.end_line.* = line;
	1060	// only do this if the pc changed
	1061	emit.prev_di_line = line;
	1062	emit.prev_di_column = column;
	1063	emit.prev_di_pc = emit.code.items.len;
	1064	},
	1065	.none => {},
	1066	}
	1067	}
	1068
	1069	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	1070	const tag = emit.mir.instructions.items(.tag)[inst];
	1071	assert(tag == .dbg_prologue_end);
	1072	switch (emit.debug_output) {
	1073	.dwarf => \|dbg_out\| {
	1074	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
	1075	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	1076	},
	1077	.plan9 => {},
	1078	.none => {},
	1079	}
	1080	}
	1081
	1082	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	1083	const tag = emit.mir.instructions.items(.tag)[inst];
	1084	assert(tag == .dbg_epilogue_begin);
	1085	switch (emit.debug_output) {
	1086	.dwarf => \|dbg_out\| {
	1087	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
	1088	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	1089	},
	1090	.plan9 => {},
	1091	.none => {},
	1092	}
	1093	}
	1094
	1095	fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	1096	const tag = emit.mir.instructions.items(.tag)[inst];
	1097	assert(tag == .arg_dbg_info);
	1098	const payload = emit.mir.instructions.items(.data)[inst].payload;
	1099	const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
	1100	const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
	1101	try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
	1102	}
	1103
	1104	fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
	1105	const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
	1106	const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
	1107	const name = zir.nullTerminatedString(ty_str.str);
	1108	const name_with_null = name.ptr[0 .. name.len + 1];
	1109	const ty = emit.mir.function.air.getRefType(ty_str.ty);
	1110
	1111	switch (mcv) {
	1112	.register => \|reg\| {
	1113	switch (emit.debug_output) {
	1114	.dwarf => \|dbg_out\| {
	1115	try dbg_out.dbg_info.ensureUnusedCapacity(3);
	1116	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
	1117	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
	1118	1, // ULEB128 dwarf expression length
	1119	reg.dwarfLocOp(),
	1120	});
	1121	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	1122	try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	1123	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	1124	},
	1125	.plan9 => {},
	1126	.none => {},
	1127	}
	1128	},
	1129	.stack_offset => {
	1130	switch (emit.debug_output) {
	1131	.dwarf => {},
	1132	.plan9 => {},
	1133	.none => {},
	1134	}
	1135	},
	1136	else => {},
	1137	}
	1138	}
	1139
	1140	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
	1141	/// after codegen for this symbol is done.
	1142	fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
	1143	switch (emit.debug_output) {
	1144	.dwarf => \|dbg_out\| {
	1145	assert(ty.hasCodeGenBits());
	1146	const index = dbg_out.dbg_info.items.len;
	1147	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
	1148
	1149	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
	1150	if (!gop.found_existing) {
	1151	gop.value_ptr.* = .{
	1152	.off = undefined,
	1153	.relocs = .{},
	1154	};
	1155	}
	1156	try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
	1157	},
	1158	.plan9 => {},
	1159	.none => {},
	1160	}
	1161	}

src/arch/x86_64/Mir.zig created+379

...	...	@@ -0,0 +1,379 @@
	1	//! Machine Intermediate Representation.
	2	//! This data is produced by x86_64 Codegen and consumed by x86_64 Isel.
	3	//! These instructions have a 1:1 correspondence with machine code instructions
	4	//! for the target. MIR can be lowered to source-annotated textual assembly code
	5	//! instructions, or it can be lowered to machine code.
	6	//! The main purpose of MIR is to postpone the assignment of offsets until Isel,
	7	//! so that, for example, the smaller encodings of jump instructions can be used.
	8
	9	const Mir = @This();
	10	const std = @import("std");
	11	const builtin = @import("builtin");
	12	const assert = std.debug.assert;
	13
	14	const bits = @import("bits.zig");
	15	const Air = @import("../../Air.zig");
	16	const CodeGen = @import("CodeGen.zig");
	17	const Register = bits.Register;
	18
	19	function: *const CodeGen,
	20	instructions: std.MultiArrayList(Inst).Slice,
	21	/// The meaning of this data is determined by `Inst.Tag` value.
	22	extra: []const u32,
	23
	24	pub const Inst = struct {
	25	tag: Tag,
	26	/// This is 3 fields, and the meaning of each depends on `tag`.
	27	/// reg1: Register
	28	/// reg2: Register
	29	/// flags: u2
	30	ops: u16,
	31	/// The meaning of this depends on `tag` and `ops`.
	32	data: Data,
	33
	34	pub const Tag = enum(u16) {
	35	/// ops flags: form:
	36	/// 0b00 reg1, reg2
	37	/// 0b00 reg1, imm32
	38	/// 0b01 reg1, [reg2 + imm32]
	39	/// 0b01 reg1, [ds:imm32]
	40	/// 0b10 [reg1 + imm32], reg2
	41	/// 0b10 [reg1 + 0], imm32
	42	/// 0b11 [reg1 + imm32], imm32
	43	/// Notes:
	44	/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
	45	/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
	46	adc,
	47
	48	/// form: reg1, [reg2 + scale*rcx + imm32]
	49	/// ops flags scale
	50	/// 0b00 1
	51	/// 0b01 2
	52	/// 0b10 4
	53	/// 0b11 8
	54	adc_scale_src,
	55
	56	/// form: [reg1 + scale*rax + imm32], reg2
	57	/// form: [reg1 + scale*rax + 0], imm32
	58	/// ops flags scale
	59	/// 0b00 1
	60	/// 0b01 2
	61	/// 0b10 4
	62	/// 0b11 8
	63	/// Notes:
	64	/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
	65	adc_scale_dst,
	66
	67	/// form: [reg1 + scale*rax + imm32], imm32
	68	/// ops flags scale
	69	/// 0b00 1
	70	/// 0b01 2
	71	/// 0b10 4
	72	/// 0b11 8
	73	/// Notes:
	74	/// * Data field `payload` points at `ImmPair`.
	75	adc_scale_imm,
	76
	77	// The following instructions all have the same encoding as `adc`.
	78
	79	add,
	80	add_scale_src,
	81	add_scale_dst,
	82	add_scale_imm,
	83	sub,
	84	sub_scale_src,
	85	sub_scale_dst,
	86	sub_scale_imm,
	87	xor,
	88	xor_scale_src,
	89	xor_scale_dst,
	90	xor_scale_imm,
	91	@"and",
	92	and_scale_src,
	93	and_scale_dst,
	94	and_scale_imm,
	95	@"or",
	96	or_scale_src,
	97	or_scale_dst,
	98	or_scale_imm,
	99	rol,
	100	rol_scale_src,
	101	rol_scale_dst,
	102	rol_scale_imm,
	103	ror,
	104	ror_scale_src,
	105	ror_scale_dst,
	106	ror_scale_imm,
	107	rcl,
	108	rcl_scale_src,
	109	rcl_scale_dst,
	110	rcl_scale_imm,
	111	rcr,
	112	rcr_scale_src,
	113	rcr_scale_dst,
	114	rcr_scale_imm,
	115	shl,
	116	shl_scale_src,
	117	shl_scale_dst,
	118	shl_scale_imm,
	119	sal,
	120	sal_scale_src,
	121	sal_scale_dst,
	122	sal_scale_imm,
	123	shr,
	124	shr_scale_src,
	125	shr_scale_dst,
	126	shr_scale_imm,
	127	sar,
	128	sar_scale_src,
	129	sar_scale_dst,
	130	sar_scale_imm,
	131	sbb,
	132	sbb_scale_src,
	133	sbb_scale_dst,
	134	sbb_scale_imm,
	135	cmp,
	136	cmp_scale_src,
	137	cmp_scale_dst,
	138	cmp_scale_imm,
	139	mov,
	140	mov_scale_src,
	141	mov_scale_dst,
	142	mov_scale_imm,
	143	lea,
	144	lea_scale_src,
	145	lea_scale_dst,
	146	lea_scale_imm,
	147
	148	/// ops flags: form:
	149	/// 0bX0 reg1
	150	/// 0bX1 [reg1 + imm32]
	151	imul,
	152	idiv,
	153
	154	/// ops flags: form:
	155	/// 0b00 reg1, reg2
	156	/// 0b01 reg1, [reg2 + imm32]
	157	/// 0b01 reg1, [imm32] if reg2 is none
	158	/// 0b10 reg1, reg2, imm32
	159	/// 0b11 reg1, [reg2 + imm32], imm32
	160	imul_complex,
	161
	162	/// ops flags: form:
	163	/// 0bX0 reg1, [rip + imm32]
	164	/// 0bX1 reg1, [rip + reloc]
	165	/// Notes:
	166	/// * if flags are 0bX1, `Data` contains `got_entry` for linker to generate
	167	/// valid relocation for.
	168	/// TODO handle more cases
	169	lea_rip,
	170
	171	/// ops flags: form:
	172	/// 0bX0 reg1, imm64
	173	/// 0bX1 rax, moffs64
	174	/// Notes:
	175	/// * If reg1 is 64-bit, the immediate is 64-bit and stored
	176	/// within extra data `Imm64`.
	177	/// * For 0bX1, reg1 (or reg2) need to be
	178	/// a version of rax. If reg1 == .none, then reg2 == .rax,
	179	/// or vice versa.
	180	/// TODO handle scaling
	181	movabs,
	182
	183	/// ops flags: 0bX0:
	184	/// - Uses the `inst` Data tag as the jump target.
	185	/// - reg1 and reg2 are ignored.
	186	/// ops flags: 0bX1:
	187	/// - reg1 is the jump target, reg2 and data are ignored.
	188	/// - if reg1 is none, [imm]
	189	jmp,
	190	call,
	191
	192	/// ops flags:
	193	/// 0b00 gte
	194	/// 0b01 gt
	195	/// 0b10 lt
	196	/// 0b11 lte
	197	cond_jmp_greater_less,
	198	cond_set_byte_greater_less,
	199
	200	/// ops flags:
	201	/// 0b00 above or equal
	202	/// 0b01 above
	203	/// 0b10 below
	204	/// 0b11 below or equal
	205	cond_jmp_above_below,
	206	cond_set_byte_above_below,
	207
	208	/// ops flags:
	209	/// 0bX0 ne
	210	/// 0bX1 eq
	211	cond_jmp_eq_ne,
	212	cond_set_byte_eq_ne,
	213
	214	/// ops flags: form:
	215	/// 0b00 reg1
	216	/// 0b01 [reg1 + imm32]
	217	/// 0b10 imm32
	218	/// Notes:
	219	/// * If 0b10 is specified and the tag is push, pushes immediate onto the stack
	220	/// using the mnemonic PUSH imm32.
	221	push,
	222	pop,
	223
	224	/// ops flags: form:
	225	/// 0b00 retf imm16
	226	/// 0b01 retf
	227	/// 0b10 retn imm16
	228	/// 0b11 retn
	229	ret,
	230
	231	/// Fast system call
	232	syscall,
	233
	234	/// ops flags: form:
	235	/// 0b00 reg1, reg2
	236	/// 0b00 reg1, imm32
	237	/// 0b01 reg1, [reg2 + imm32]
	238	/// 0b01 reg1, [ds:imm32]
	239	/// 0b10 [reg1 + imm32], reg2
	240	/// 0b10 [reg1 + 0], imm32
	241	/// 0b11 [reg1 + imm32], imm32
	242	/// Notes:
	243	/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
	244	/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
	245	@"test",
	246
	247	/// Breakpoint
	248	brk,
	249
	250	/// Pseudo-instructions
	251	/// call extern function
	252	/// Notes:
	253	/// * target of the call is stored as `extern_fn` in `Data` union.
	254	call_extern,
	255
	256	/// end of prologue
	257	dbg_prologue_end,
	258
	259	/// start of epilogue
	260	dbg_epilogue_begin,
	261
	262	/// update debug line
	263	dbg_line,
	264
	265	/// arg debug info
	266	arg_dbg_info,
	267	};
	268
	269	/// The position of an MIR instruction within the `Mir` instructions array.
	270	pub const Index = u32;
	271
	272	/// All instructions have a 4-byte payload, which is contained within
	273	/// this union. `Tag` determines which union field is active, as well as
	274	/// how to interpret the data within.
	275	pub const Data = union {
	276	/// Another instruction.
	277	inst: Index,
	278	/// A 32-bit immediate value.
	279	imm: i32,
	280	/// An extern function.
	281	/// Index into the linker's string table.
	282	extern_fn: u32,
	283	/// Entry in the GOT table by index.
	284	got_entry: u32,
	285	/// Index into `extra`. Meaning of what can be found there is context-dependent.
	286	payload: u32,
	287	};
	288
	289	// Make sure we don't accidentally make instructions bigger than expected.
	290	// Note that in Debug builds, Zig is allowed to insert a secret field for safety checks.
	291	comptime {
	292	if (builtin.mode != .Debug) {
	293	assert(@sizeOf(Inst) == 8);
	294	}
	295	}
	296	};
	297
	298	pub const ImmPair = struct {
	299	dest_off: i32,
	300	operand: i32,
	301	};
	302
	303	pub const Imm64 = struct {
	304	msb: u32,
	305	lsb: u32,
	306
	307	pub fn encode(v: u64) Imm64 {
	308	return .{
	309	.msb = @truncate(u32, v >> 32),
	310	.lsb = @truncate(u32, v),
	311	};
	312	}
	313
	314	pub fn decode(imm: Imm64) u64 {
	315	var res: u64 = 0;
	316	res \|= (@intCast(u64, imm.msb) << 32);
	317	res \|= @intCast(u64, imm.lsb);
	318	return res;
	319	}
	320	};
	321
	322	pub const DbgLineColumn = struct {
	323	line: u32,
	324	column: u32,
	325	};
	326
	327	pub const ArgDbgInfo = struct {
	328	air_inst: Air.Inst.Index,
	329	arg_index: u32,
	330	};
	331
	332	pub fn deinit(mir: Mir, gpa: std.mem.Allocator) void {
	333	mir.instructions.deinit(gpa);
	334	gpa.free(mir.extra);
	335	mir.* = undefined;
	336	}
	337
	338	pub const Ops = struct {
	339	reg1: Register = .none,
	340	reg2: Register = .none,
	341	flags: u2 = 0b00,
	342
	343	pub fn encode(self: Ops) u16 {
	344	var ops: u16 = 0;
	345	ops \|= @intCast(u16, @enumToInt(self.reg1)) << 9;
	346	ops \|= @intCast(u16, @enumToInt(self.reg2)) << 2;
	347	ops \|= self.flags;
	348	return ops;
	349	}
	350
	351	pub fn decode(ops: u16) Ops {
	352	const reg1 = @intToEnum(Register, @truncate(u7, ops >> 9));
	353	const reg2 = @intToEnum(Register, @truncate(u7, ops >> 2));
	354	const flags = @truncate(u2, ops);
	355	return .{
	356	.reg1 = reg1,
	357	.reg2 = reg2,
	358	.flags = flags,
	359	};
	360	}
	361	};
	362
	363	pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
	364	const fields = std.meta.fields(T);
	365	var i: usize = index;
	366	var result: T = undefined;
	367	inline for (fields) \|field\| {
	368	@field(result, field.name) = switch (field.field_type) {
	369	u32 => mir.extra[i],
	370	i32 => @bitCast(i32, mir.extra[i]),
	371	else => @compileError("bad field type"),
	372	};
	373	i += 1;
	374	}
	375	return .{
	376	.data = result,
	377	.end = i,
	378	};
	379	}

src/arch/x86_64/bits.zig+11-7

...	...	@@ -22,7 +22,7 @@ const DW = std.dwarf;
22	22	///
23	23	/// The ID can be easily determined by figuring out what range the register is
24	24	/// in, and then subtracting the base.
25		pub const Register = enum(u8) {
	25	pub const Register = enum(u7) {
26	26	// 0 through 15, 64-bit registers. 8-15 are extended.
27	27	// id is just the int value.
28	28	rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi,
...	...	@@ -43,6 +43,10 @@ pub const Register = enum(u8) {
43	43	al, cl, dl, bl, ah, ch, dh, bh,
44	44	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
45	45
	46	// Pseudo, used only for MIR to signify that the
	47	// operand is not a register but an immediate, etc.
	48	none,
	49
46	50	/// Returns the bit-width of the register.
47	51	pub fn size(self: Register) u7 {
48	52	return switch (@enumToInt(self)) {
...	...	@@ -73,7 +77,7 @@ pub const Register = enum(u8) {
73	77	}
74	78
75	79	/// Like id, but only returns the lower 3 bits.
76		pub fn low_id(self: Register) u3 {
	80	pub fn lowId(self: Register) u3 {
77	81	return @truncate(u3, @enumToInt(self));
78	82	}
79	83
...	...	@@ -577,8 +581,8 @@ test "x86_64 Encoder helpers" {
577	581	});
578	582	encoder.opcode_2byte(0x0f, 0xaf);
579	583	encoder.modRm_direct(
580		Register.eax.low_id(),
581		Register.edi.low_id(),
	584	Register.eax.lowId(),
	585	Register.edi.lowId(),
582	586	);
583	587
584	588	try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
...	...	@@ -597,8 +601,8 @@ test "x86_64 Encoder helpers" {
597	601	});
598	602	encoder.opcode_1byte(0x89);
599	603	encoder.modRm_direct(
600		Register.edi.low_id(),
601		Register.eax.low_id(),
	604	Register.edi.lowId(),
	605	Register.eax.lowId(),
602	606	);
603	607
604	608	try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
...	...	@@ -624,7 +628,7 @@ test "x86_64 Encoder helpers" {
624	628	encoder.opcode_1byte(0x81);
625	629	encoder.modRm_direct(
626	630	0,
627		Register.rcx.low_id(),
	631	Register.rcx.lowId(),
628	632	);
629	633	encoder.imm32(2147483647);
630	634

src/codegen.zig+1-1

...	...	@@ -117,7 +117,7 @@ pub fn generateFunction(
117	117	//.thumb => return Function(.thumb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
118	118	//.thumbeb => return Function(.thumbeb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
119	119	//.i386 => return Function(.i386).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
120		.x86_64 => return @import("arch/x86_64/CodeGen.zig").generate(.x86_64, bin_file, src_loc, func, air, liveness, code, debug_output),
	120	.x86_64 => return @import("arch/x86_64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
121	121	//.xcore => return Function(.xcore).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
122	122	//.nvptx => return Function(.nvptx).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
123	123	//.nvptx64 => return Function(.nvptx64).generate(bin_file, src_loc, func, air, liveness, code, debug_output),