Commit 08ea1a2eab9472389d57ddbd97e307fcb096a992 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2021-12-29 13:38:49+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2021-12-29 22:06:38+01:00
log	08ea1a2eab9472389d57ddbd97e307fcb096a992
tree	6f546b4f1cc266f56fd9640713ae3de750c64504
parent	be5130ec535456559497ac241ac9fa76c4bbb8ca

stage2: add separate tag for MI encoding

To request memory-immediate encoding at the MIR side, we should now
use a new tag such as `mov_mem_imm` where the size of the memory
pointer is encoded as the flags:

```
0b00 => .byte_ptr,
0b01 => .word_ptr,
0b10 => .dword_ptr,
0b11 => .qword_ptr,
```

4 files changed, 1325 insertions(+), 1118 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    PrintMir.zig

src/arch/x86_64/CodeGen.zig+12-12

...	...	@@ -1646,7 +1646,7 @@ fn genBinMathOpMir(
1646	1646	_ = try self.addInst(.{
1647	1647	.tag = mir_tag,
1648	1648	.ops = (Mir.Ops{
1649		.reg1 = registerAlias(dst_reg, 4),
	1649	.reg1 = dst_reg.to32(),
1650	1650	}).encode(),
1651	1651	.data = .{ .imm = @intCast(i32, imm) },
1652	1652	});
...	...	@@ -2807,10 +2807,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2807	2807	.operand = @bitCast(i32, @intCast(u32, x_big)),
2808	2808	});
2809	2809	_ = try self.addInst(.{
2810		.tag = .mov,
	2810	.tag = .mov_mem_imm,
2811	2811	.ops = (Mir.Ops{
2812	2812	.reg1 = .rbp,
2813		.flags = 0b11,
	2813	.flags = 0b10,
2814	2814	}).encode(),
2815	2815	.data = .{ .payload = payload },
2816	2816	});
...	...	@@ -2828,10 +2828,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2828	2828	.operand = @bitCast(i32, @truncate(u32, x_big >> 32)),
2829	2829	});
2830	2830	_ = try self.addInst(.{
2831		.tag = .mov,
	2831	.tag = .mov_mem_imm,
2832	2832	.ops = (Mir.Ops{
2833	2833	.reg1 = .rbp,
2834		.flags = 0b11,
	2834	.flags = 0b10,
2835	2835	}).encode(),
2836	2836	.data = .{ .payload = payload },
2837	2837	});
...	...	@@ -2842,10 +2842,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2842	2842	.operand = @bitCast(i32, @truncate(u32, x_big)),
2843	2843	});
2844	2844	_ = try self.addInst(.{
2845		.tag = .mov,
	2845	.tag = .mov_mem_imm,
2846	2846	.ops = (Mir.Ops{
2847	2847	.reg1 = .rbp,
2848		.flags = 0b11,
	2848	.flags = 0b10,
2849	2849	}).encode(),
2850	2850	.data = .{ .payload = payload },
2851	2851	});
...	...	@@ -2955,11 +2955,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2955	2955	}
2956	2956	if (x <= math.maxInt(i32)) {
2957	2957	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
2958		// TODO I am not quite sure why we need to set the size of the register here...
2959	2958	_ = try self.addInst(.{
2960	2959	.tag = .mov,
2961	2960	.ops = (Mir.Ops{
2962		.reg1 = registerAlias(reg, 4),
	2961	.reg1 = reg.to32(),
2963	2962	}).encode(),
2964	2963	.data = .{ .imm = @intCast(i32, x) },
2965	2964	});
...	...	@@ -2985,9 +2984,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2985	2984	// We need the offset from RIP in a signed i32 twos complement.
2986	2985	const payload = try self.addExtra(Mir.Imm64.encode(code_offset));
2987	2986	_ = try self.addInst(.{
2988		.tag = .lea_rip,
	2987	.tag = .lea,
2989	2988	.ops = (Mir.Ops{
2990	2989	.reg1 = reg,
	2990	.flags = 0b01,
2991	2991	}).encode(),
2992	2992	.data = .{ .payload = payload },
2993	2993	});
...	...	@@ -3011,10 +3011,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3011	3011	if (self.bin_file.options.pie) {
3012	3012	// TODO we should flag up `x` as GOT symbol entry explicitly rather than as a hack.
3013	3013	_ = try self.addInst(.{
3014		.tag = .lea_rip,
	3014	.tag = .lea,
3015	3015	.ops = (Mir.Ops{
3016	3016	.reg1 = reg,
3017		.flags = 0b01,
	3017	.flags = 0b10,
3018	3018	}).encode(),
3019	3019	.data = .{ .got_entry = @intCast(u32, x) },
3020	3020	});

src/arch/x86_64/Emit.zig+1257-1071

...	...	@@ -76,6 +76,16 @@ pub fn emitMir(emit: *Emit) InnerError!void {
76	76	.cmp => try emit.mirArith(.cmp, inst),
77	77	.mov => try emit.mirArith(.mov, inst),
78	78
	79	.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
	80	.add_mem_imm => try emit.mirArithMemImm(.add, inst),
	81	.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
	82	.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
	83	.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
	84	.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
	85	.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
	86	.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
	87	.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
	88
79	89	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
80	90	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
81	91	.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
...	...	@@ -109,7 +119,6 @@ pub fn emitMir(emit: *Emit) InnerError!void {
109	119	.movabs => try emit.mirMovabs(inst),
110	120
111	121	.lea => try emit.mirLea(inst),
112		.lea_rip => try emit.mirLeaRip(inst),
113	122
114	123	.imul_complex => try emit.mirIMulComplex(inst),
115	124
...	...	@@ -170,6 +179,14 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
170	179	return error.EmitFail;
171	180	}
172	181
	182	fn failWithLoweringError(emit: *Emit, err: LoweringError) InnerError {
	183	return switch (err) {
	184	error.RaxOperandExpected => emit.fail("Register.rax expected as destination operand", .{}),
	185	error.OperandSizeMismatch => emit.fail("operand size mismatch", .{}),
	186	else => \|e\| e,
	187	};
	188	}
	189
173	190	fn fixupRelocs(emit: *Emit) InnerError!void {
174	191	// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
175	192	// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
...	...	@@ -185,15 +202,15 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
185	202	}
186	203
187	204	fn mirBrk(emit: *Emit) InnerError!void {
188		return lowerToZoEnc(.brk, emit.code);
	205	return lowerToZoEnc(.brk, emit.code) catch \|err\| emit.failWithLoweringError(err);
189	206	}
190	207
191	208	fn mirNop(emit: *Emit) InnerError!void {
192		return lowerToZoEnc(.nop, emit.code);
	209	return lowerToZoEnc(.nop, emit.code) catch \|err\| emit.failWithLoweringError(err);
193	210	}
194	211
195	212	fn mirSyscall(emit: *Emit) InnerError!void {
196		return lowerToZoEnc(.syscall, emit.code);
	213	return lowerToZoEnc(.syscall, emit.code) catch \|err\| emit.failWithLoweringError(err);
197	214	}
198	215
199	216	fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -201,18 +218,24 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
201	218	switch (ops.flags) {
202	219	0b00 => {
203	220	// PUSH/POP reg
204		return lowerToOEnc(tag, ops.reg1, emit.code);
	221	return lowerToOEnc(tag, ops.reg1, emit.code) catch \|err\| emit.failWithLoweringError(err);
205	222	},
206	223	0b01 => {
207	224	// PUSH/POP r/m64
208	225	const imm = emit.mir.instructions.items(.data)[inst].imm;
209		return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), emit.code);
	226	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	227	16 => .word_ptr,
	228	else => .qword_ptr,
	229	};
	230	return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm, ptr_size), emit.code) catch \|err\|
	231	emit.failWithLoweringError(err);
210	232	},
211	233	0b10 => {
212	234	// PUSH imm32
213	235	assert(tag == .push);
214	236	const imm = emit.mir.instructions.items(.data)[inst].imm;
215		return lowerToIEnc(.push, imm, emit.code);
	237	return lowerToIEnc(.push, imm, emit.code) catch \|err\|
	238	emit.failWithLoweringError(err);
216	239	},
217	240	0b11 => unreachable,
218	241	}
...	...	@@ -223,7 +246,8 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
223	246	if (tag == .push) {
224	247	for (callee_preserved_regs) \|reg, i\| {
225	248	if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
226		try lowerToOEnc(.push, reg, emit.code);
	249	lowerToOEnc(.push, reg, emit.code) catch \|err\|
	250	return emit.failWithLoweringError(err);
227	251	}
228	252	} else {
229	253	// pop in the reverse direction
...	...	@@ -231,7 +255,8 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
231	255	while (i > 0) : (i -= 1) {
232	256	const reg = callee_preserved_regs[i - 1];
233	257	if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
234		try lowerToOEnc(.pop, reg, emit.code);
	258	lowerToOEnc(.pop, reg, emit.code) catch \|err\|
	259	return emit.failWithLoweringError(err);
235	260	}
236	261	}
237	262	}
...	...	@@ -242,7 +267,8 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
242	267	if (flag == 0) {
243	268	const target = emit.mir.instructions.items(.data)[inst].inst;
244	269	const source = emit.code.items.len;
245		try lowerToDEnc(tag, 0, emit.code);
	270	lowerToDEnc(tag, 0, emit.code) catch \|err\|
	271	return emit.failWithLoweringError(err);
246	272	try emit.relocs.append(emit.bin_file.allocator, .{
247	273	.source = source,
248	274	.target = target,
...	...	@@ -254,10 +280,15 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
254	280	if (ops.reg1 == .none) {
255	281	// JMP/CALL [imm]
256	282	const imm = emit.mir.instructions.items(.data)[inst].imm;
257		return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm), emit.code);
	283	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	284	16 => .word_ptr,
	285	else => .qword_ptr,
	286	};
	287	return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm, ptr_size), emit.code) catch \|err\|
	288	emit.failWithLoweringError(err);
258	289	}
259	290	// JMP/CALL reg
260		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	291	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code) catch \|err\| emit.failWithLoweringError(err);
261	292	}
262	293
263	294	fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -283,7 +314,8 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr
283	314	else => unreachable,
284	315	};
285	316	const source = emit.code.items.len;
286		try lowerToDEnc(tag, 0, emit.code);
	317	lowerToDEnc(tag, 0, emit.code) catch \|err\|
	318	return emit.failWithLoweringError(err);
287	319	try emit.relocs.append(emit.bin_file.allocator, .{
288	320	.source = source,
289	321	.target = target,
...	...	@@ -313,7 +345,8 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne
313	345	},
314	346	else => unreachable,
315	347	};
316		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	348	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code) catch \|err\|
	349	emit.failWithLoweringError(err);
317	350	}
318	351
319	352	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -329,9 +362,11 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
329	362	if (ops.reg1.to64() == .rax) {
330	363	// TEST rax, imm32
331	364	// I
332		return lowerToIEnc(.@"test", imm, emit.code);
	365	return lowerToIEnc(.@"test", imm, emit.code) catch \|err\|
	366	emit.failWithLoweringError(err);
333	367	}
334		return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
	368	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
	369	emit.failWithLoweringError(err);
335	370	}
336	371	// TEST r/m64, r64
337	372	return emit.fail("TODO TEST r/m64, r64", .{});
...	...	@@ -349,207 +384,682 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
349	384	// RETF imm16
350	385	// I
351	386	const imm = emit.mir.instructions.items(.data)[inst].imm;
352		return lowerToIEnc(.ret_far, imm, emit.code);
	387	return lowerToIEnc(.ret_far, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	388	},
	389	0b01 => {
	390	return lowerToZoEnc(.ret_far, emit.code) catch \|err\| emit.failWithLoweringError(err);
353	391	},
354		0b01 => return lowerToZoEnc(.ret_far, emit.code),
355	392	0b10 => {
356	393	// RET imm16
357	394	// I
358	395	const imm = emit.mir.instructions.items(.data)[inst].imm;
359		return lowerToIEnc(.ret_near, imm, emit.code);
	396	return lowerToIEnc(.ret_near, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	397	},
	398	0b11 => {
	399	return lowerToZoEnc(.ret_near, emit.code) catch \|err\| emit.failWithLoweringError(err);
360	400	},
361		0b11 => return lowerToZoEnc(.ret_near, emit.code),
362	401	}
363	402	}
364	403
365		const Tag = enum {
366		adc,
367		add,
368		sub,
369		xor,
370		@"and",
371		@"or",
372		sbb,
373		cmp,
374		mov,
375		lea,
376		jmp_near,
377		call_near,
378		push,
379		pop,
380		@"test",
381		brk,
382		nop,
383		imul,
384		syscall,
385		ret_near,
386		ret_far,
387		jo,
388		jno,
389		jb,
390		jbe,
391		jc,
392		jnae,
393		jnc,
394		jae,
395		je,
396		jz,
397		jne,
398		jnz,
399		jna,
400		jnb,
401		jnbe,
402		ja,
403		js,
404		jns,
405		jpe,
406		jp,
407		jpo,
408		jnp,
409		jnge,
410		jl,
411		jge,
412		jnl,
413		jle,
414		jng,
415		jg,
416		jnle,
417		seto,
418		setno,
419		setb,
420		setc,
421		setnae,
422		setnb,
423		setnc,
424		setae,
425		sete,
426		setz,
427		setne,
428		setnz,
429		setbe,
430		setna,
431		seta,
432		setnbe,
433		sets,
434		setns,
435		setp,
436		setpe,
437		setnp,
438		setop,
439		setl,
440		setnge,
441		setnl,
442		setge,
443		setle,
444		setng,
445		setnle,
446		setg,
447
448		fn isSetCC(tag: Tag) bool {
449		return switch (tag) {
450		.seto,
451		.setno,
452		.setb,
453		.setc,
454		.setnae,
455		.setnb,
456		.setnc,
457		.setae,
458		.sete,
459		.setz,
460		.setne,
461		.setnz,
462		.setbe,
463		.setna,
464		.seta,
465		.setnbe,
466		.sets,
467		.setns,
468		.setp,
469		.setpe,
470		.setnp,
471		.setop,
472		.setl,
473		.setnge,
474		.setnl,
475		.setge,
476		.setle,
477		.setng,
478		.setnle,
479		.setg,
480		=> true,
481		else => false,
482		};
	404	fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	405	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	406	switch (ops.flags) {
	407	0b00 => {
	408	if (ops.reg2 == .none) {
	409	// mov reg1, imm32
	410	// MI
	411	const imm = emit.mir.instructions.items(.data)[inst].imm;
	412	return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
	413	emit.failWithLoweringError(err);
	414	}
	415	// mov reg1, reg2
	416	// RM
	417	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
	418	emit.failWithLoweringError(err);
	419	},
	420	0b01 => {
	421	// mov reg1, [reg2 + imm32]
	422	// RM
	423	const imm = emit.mir.instructions.items(.data)[inst].imm;
	424	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	425	return lowerToRmEnc(
	426	tag,
	427	ops.reg1,
	428	RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
	429	emit.code,
	430	) catch \|err\| emit.failWithLoweringError(err);
	431	},
	432	0b10 => {
	433	if (ops.reg2 == .none) {
	434	return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
	435	}
	436	// mov [reg1 + imm32], reg2
	437	// MR
	438	const imm = emit.mir.instructions.items(.data)[inst].imm;
	439	return lowerToMrEnc(
	440	tag,
	441	RegisterOrMemory.mem(ops.reg1, imm, Memory.PtrSize.fromBits(ops.reg2.size())),
	442	ops.reg2,
	443	emit.code,
	444	) catch \|err\| emit.failWithLoweringError(err);
	445	},
	446	0b11 => {
	447	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
	448	},
483	449	}
484		};
485
486		const Encoding = enum {
487		/// OP
488		zo,
489
490		/// OP rel32
491		d,
492
493		/// OP r/m64
494		m,
495
496		/// OP r64
497		o,
498
499		/// OP imm32
500		i,
501
502		/// OP r/m64, imm32
503		mi,
504
505		/// OP r/m64, r64
506		mr,
507
508		/// OP r64, r/m64
509		rm,
510
511		/// OP r64, imm64
512		oi,
513
514		/// OP al/ax/eax/rax, moffs
515		fd,
516
517		/// OP moffs, al/ax/eax/rax
518		td,
519
520		/// OP r64, r/m64, imm32
521		rmi,
522		};
	450	}
523	451
524		const OpCode = union(enum) {
525		one_byte: u8,
526		two_byte: struct { _1: u8, _2: u8 },
	452	fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	453	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	454	assert(ops.reg2 == .none);
	455	const payload = emit.mir.instructions.items(.data)[inst].payload;
	456	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	457	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	458	0b00 => .byte_ptr,
	459	0b01 => .word_ptr,
	460	0b10 => .dword_ptr,
	461	0b11 => .qword_ptr,
	462	};
	463	return lowerToMiEnc(
	464	tag,
	465	RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off, ptr_size),
	466	imm_pair.operand,
	467	emit.code,
	468	) catch \|err\| emit.failWithLoweringError(err);
	469	}
527	470
528		fn oneByte(opc: u8) OpCode {
529		return .{ .one_byte = opc };
	471	fn immOpSize(imm: i64) u8 {
	472	blk: {
	473	_ = math.cast(i8, imm) catch break :blk;
	474	return 8;
530	475	}
531
532		fn twoByte(opc1: u8, opc2: u8) OpCode {
533		return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
	476	blk: {
	477	_ = math.cast(i16, imm) catch break :blk;
	478	return 16;
534	479	}
535
536		fn encode(opc: OpCode, encoder: Encoder) void {
537		switch (opc) {
538		.one_byte => \|v\| encoder.opcode_1byte(v),
539		.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
540		}
	480	blk: {
	481	_ = math.cast(i32, imm) catch break :blk;
	482	return 32;
541	483	}
	484	return 64;
	485	}
542	486
543		fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
544		assert(opc == .one_byte);
545		encoder.opcode_withReg(opc.one_byte, reg.lowId());
	487	// TODO
	488	fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	489	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	490	const scale = ops.flags;
	491	// OP reg1, [reg2 + scale*rcx + imm32]
	492	const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
	493	const imm = emit.mir.instructions.items(.data)[inst].imm;
	494	const encoder = try Encoder.init(emit.code, 8);
	495	encoder.rex(.{
	496	.w = ops.reg1.size() == 64,
	497	.r = ops.reg1.isExtended(),
	498	.b = ops.reg2.isExtended(),
	499	});
	500	opc.encode(encoder);
	501	if (imm <= math.maxInt(i8)) {
	502	encoder.modRm_SIBDisp8(ops.reg1.lowId());
	503	encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
	504	encoder.disp8(@intCast(i8, imm));
	505	} else {
	506	encoder.modRm_SIBDisp32(ops.reg1.lowId());
	507	encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
	508	encoder.disp32(imm);
546	509	}
547		};
	510	}
548	511
549		inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
550		switch (enc) {
551		.zo => return switch (tag) {
552		.ret_near => OpCode.oneByte(0xc3),
	512	// TODO
	513	fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	514	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	515	const scale = ops.flags;
	516	const imm = emit.mir.instructions.items(.data)[inst].imm;
	517
	518	if (ops.reg2 == .none) {
	519	// OP [reg1 + scale*rax + 0], imm32
	520	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
	521	const modrm_ext = getModRmExt(tag).?;
	522	const encoder = try Encoder.init(emit.code, 8);
	523	encoder.rex(.{
	524	.w = ops.reg1.size() == 64,
	525	.b = ops.reg1.isExtended(),
	526	});
	527	opc.encode(encoder);
	528	encoder.modRm_SIBDisp0(modrm_ext);
	529	encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
	530	if (imm <= math.maxInt(i8)) {
	531	encoder.imm8(@intCast(i8, imm));
	532	} else if (imm <= math.maxInt(i16)) {
	533	encoder.imm16(@intCast(i16, imm));
	534	} else {
	535	encoder.imm32(imm);
	536	}
	537	return;
	538	}
	539
	540	// OP [reg1 + scale*rax + imm32], reg2
	541	const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
	542	const encoder = try Encoder.init(emit.code, 8);
	543	encoder.rex(.{
	544	.w = ops.reg1.size() == 64,
	545	.r = ops.reg2.isExtended(),
	546	.b = ops.reg1.isExtended(),
	547	});
	548	opc.encode(encoder);
	549	if (imm <= math.maxInt(i8)) {
	550	encoder.modRm_SIBDisp8(ops.reg2.lowId());
	551	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	552	encoder.disp8(@intCast(i8, imm));
	553	} else {
	554	encoder.modRm_SIBDisp32(ops.reg2.lowId());
	555	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	556	encoder.disp32(imm);
	557	}
	558	}
	559
	560	// TODO
	561	fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	562	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	563	const scale = ops.flags;
	564	const payload = emit.mir.instructions.items(.data)[inst].payload;
	565	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	566	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
	567	const modrm_ext = getModRmExt(tag).?;
	568	const encoder = try Encoder.init(emit.code, 2);
	569	encoder.rex(.{
	570	.w = ops.reg1.size() == 64,
	571	.b = ops.reg1.isExtended(),
	572	});
	573	opc.encode(encoder);
	574	if (imm_pair.dest_off <= math.maxInt(i8)) {
	575	encoder.modRm_SIBDisp8(modrm_ext);
	576	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	577	encoder.disp8(@intCast(i8, imm_pair.dest_off));
	578	} else {
	579	encoder.modRm_SIBDisp32(modrm_ext);
	580	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	581	encoder.disp32(imm_pair.dest_off);
	582	}
	583	encoder.imm32(imm_pair.operand);
	584	}
	585
	586	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	587	const tag = emit.mir.instructions.items(.tag)[inst];
	588	assert(tag == .movabs);
	589	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	590	const imm: i64 = if (ops.reg1.size() == 64) blk: {
	591	const payload = emit.mir.instructions.items(.data)[inst].payload;
	592	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
	593	break :blk @bitCast(i64, imm.decode());
	594	} else emit.mir.instructions.items(.data)[inst].imm;
	595	if (ops.flags == 0b00) {
	596	// movabs reg, imm64
	597	// OI
	598	return lowerToOiEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	599	}
	600	if (ops.reg1 == .none) {
	601	// movabs moffs64, rax
	602	// TD
	603	return lowerToTdEnc(.mov, imm, ops.reg2, emit.code) catch \|err\| emit.failWithLoweringError(err);
	604	}
	605	// movabs rax, moffs64
	606	// FD
	607	return lowerToFdEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	608	}
	609
	610	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	611	const tag = emit.mir.instructions.items(.tag)[inst];
	612	assert(tag == .imul_complex);
	613	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	614	switch (ops.flags) {
	615	0b00 => {
	616	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
	617	emit.failWithLoweringError(err);
	618	},
	619	0b10 => {
	620	const imm = emit.mir.instructions.items(.data)[inst].imm;
	621	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code) catch \|err\|
	622	emit.failWithLoweringError(err);
	623	},
	624	else => return emit.fail("TODO implement imul", .{}),
	625	}
	626	}
	627
	628	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	629	const tag = emit.mir.instructions.items(.tag)[inst];
	630	assert(tag == .lea);
	631	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	632	switch (ops.flags) {
	633	0b00 => {
	634	// lea reg1, [reg2 + imm32]
	635	// RM
	636	const imm = emit.mir.instructions.items(.data)[inst].imm;
	637	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	638	return lowerToRmEnc(
	639	.lea,
	640	ops.reg1,
	641	RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
	642	emit.code,
	643	) catch \|err\| emit.failWithLoweringError(err);
	644	},
	645	0b01 => {
	646	// lea reg1, [rip + imm32]
	647	// RM
	648	const start_offset = emit.code.items.len;
	649	lowerToRmEnc(
	650	.lea,
	651	ops.reg1,
	652	RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
	653	emit.code,
	654	) catch \|err\| return emit.failWithLoweringError(err);
	655	const end_offset = emit.code.items.len;
	656	// Backpatch the displacement
	657	const payload = emit.mir.instructions.items(.data)[inst].payload;
	658	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
	659	const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
	660	mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
	661	},
	662	0b10 => {
	663	// lea reg1, [rip + reloc]
	664	// RM
	665	lowerToRmEnc(
	666	.lea,
	667	ops.reg1,
	668	RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
	669	emit.code,
	670	) catch \|err\| return emit.failWithLoweringError(err);
	671	const end_offset = emit.code.items.len;
	672	const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
	673	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	674	// TODO I think the reloc might be in the wrong place.
	675	const decl = macho_file.active_decl.?;
	676	try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
	677	.offset = @intCast(u32, end_offset - 4),
	678	.target = .{ .local = got_entry },
	679	.addend = 0,
	680	.subtractor = null,
	681	.pcrel = true,
	682	.length = 2,
	683	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	684	});
	685	} else {
	686	return emit.fail(
	687	"TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
	688	.{},
	689	);
	690	}
	691	},
	692	0b11 => return emit.fail("TODO unused variant lea reg1, reg2, 0b11", .{}),
	693	}
	694	}
	695
	696	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	697	const tag = emit.mir.instructions.items(.tag)[inst];
	698	assert(tag == .call_extern);
	699	const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
	700	const offset = blk: {
	701	// callq
	702	lowerToDEnc(.call_near, 0, emit.code) catch \|err\|
	703	return emit.failWithLoweringError(err);
	704	break :blk @intCast(u32, emit.code.items.len) - 4;
	705	};
	706	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	707	// Add relocation to the decl.
	708	try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
	709	.offset = offset,
	710	.target = .{ .global = n_strx },
	711	.addend = 0,
	712	.subtractor = null,
	713	.pcrel = true,
	714	.length = 2,
	715	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	716	});
	717	} else {
	718	return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
	719	}
	720	}
	721
	722	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	723	const tag = emit.mir.instructions.items(.tag)[inst];
	724	assert(tag == .dbg_line);
	725	const payload = emit.mir.instructions.items(.data)[inst].payload;
	726	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
	727	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
	728	}
	729
	730	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
	731	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
	732	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
	733	switch (emit.debug_output) {
	734	.dwarf => \|dbg_out\| {
	735	// TODO Look into using the DWARF special opcodes to compress this data.
	736	// It lets you emit single-byte opcodes that add different numbers to
	737	// both the PC and the line number at the same time.
	738	try dbg_out.dbg_line.ensureUnusedCapacity(11);
	739	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
	740	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
	741	if (delta_line != 0) {
	742	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
	743	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	744	}
	745	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
	746	emit.prev_di_pc = emit.code.items.len;
	747	emit.prev_di_line = line;
	748	emit.prev_di_column = column;
	749	emit.prev_di_pc = emit.code.items.len;
	750	},
	751	.plan9 => \|dbg_out\| {
	752	if (delta_pc <= 0) return; // only do this when the pc changes
	753	// we have already checked the target in the linker to make sure it is compatable
	754	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
	755
	756	// increasing the line number
	757	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
	758	// increasing the pc
	759	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
	760	if (d_pc_p9 > 0) {
	761	// minus one because if its the last one, we want to leave space to change the line which is one quanta
	762	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
	763	if (dbg_out.pcop_change_index.*) \|pci\|
	764	dbg_out.dbg_line.items[pci] += 1;
	765	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
	766	} else if (d_pc_p9 == 0) {
	767	// we don't need to do anything, because adding the quant does it for us
	768	} else unreachable;
	769	if (dbg_out.start_line.* == null)
	770	dbg_out.start_line.* = emit.prev_di_line;
	771	dbg_out.end_line.* = line;
	772	// only do this if the pc changed
	773	emit.prev_di_line = line;
	774	emit.prev_di_column = column;
	775	emit.prev_di_pc = emit.code.items.len;
	776	},
	777	.none => {},
	778	}
	779	}
	780
	781	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	782	const tag = emit.mir.instructions.items(.tag)[inst];
	783	assert(tag == .dbg_prologue_end);
	784	switch (emit.debug_output) {
	785	.dwarf => \|dbg_out\| {
	786	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
	787	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	788	},
	789	.plan9 => {},
	790	.none => {},
	791	}
	792	}
	793
	794	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	795	const tag = emit.mir.instructions.items(.tag)[inst];
	796	assert(tag == .dbg_epilogue_begin);
	797	switch (emit.debug_output) {
	798	.dwarf => \|dbg_out\| {
	799	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
	800	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	801	},
	802	.plan9 => {},
	803	.none => {},
	804	}
	805	}
	806
	807	fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	808	const tag = emit.mir.instructions.items(.tag)[inst];
	809	assert(tag == .arg_dbg_info);
	810	const payload = emit.mir.instructions.items(.data)[inst].payload;
	811	const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
	812	const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
	813	try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
	814	}
	815
	816	fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
	817	const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
	818	const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
	819	const name = zir.nullTerminatedString(ty_str.str);
	820	const name_with_null = name.ptr[0 .. name.len + 1];
	821	const ty = emit.mir.function.air.getRefType(ty_str.ty);
	822
	823	switch (mcv) {
	824	.register => \|reg\| {
	825	switch (emit.debug_output) {
	826	.dwarf => \|dbg_out\| {
	827	try dbg_out.dbg_info.ensureUnusedCapacity(3);
	828	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
	829	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
	830	1, // ULEB128 dwarf expression length
	831	reg.dwarfLocOp(),
	832	});
	833	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	834	try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	835	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	836	},
	837	.plan9 => {},
	838	.none => {},
	839	}
	840	},
	841	.stack_offset => {
	842	switch (emit.debug_output) {
	843	.dwarf => {},
	844	.plan9 => {},
	845	.none => {},
	846	}
	847	},
	848	else => {},
	849	}
	850	}
	851
	852	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
	853	/// after codegen for this symbol is done.
	854	fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
	855	switch (emit.debug_output) {
	856	.dwarf => \|dbg_out\| {
	857	assert(ty.hasCodeGenBits());
	858	const index = dbg_out.dbg_info.items.len;
	859	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
	860
	861	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
	862	if (!gop.found_existing) {
	863	gop.value_ptr.* = .{
	864	.off = undefined,
	865	.relocs = .{},
	866	};
	867	}
	868	try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
	869	},
	870	.plan9 => {},
	871	.none => {},
	872	}
	873	}
	874
	875	const Tag = enum {
	876	adc,
	877	add,
	878	sub,
	879	xor,
	880	@"and",
	881	@"or",
	882	sbb,
	883	cmp,
	884	mov,
	885	lea,
	886	jmp_near,
	887	call_near,
	888	push,
	889	pop,
	890	@"test",
	891	brk,
	892	nop,
	893	imul,
	894	syscall,
	895	ret_near,
	896	ret_far,
	897	jo,
	898	jno,
	899	jb,
	900	jbe,
	901	jc,
	902	jnae,
	903	jnc,
	904	jae,
	905	je,
	906	jz,
	907	jne,
	908	jnz,
	909	jna,
	910	jnb,
	911	jnbe,
	912	ja,
	913	js,
	914	jns,
	915	jpe,
	916	jp,
	917	jpo,
	918	jnp,
	919	jnge,
	920	jl,
	921	jge,
	922	jnl,
	923	jle,
	924	jng,
	925	jg,
	926	jnle,
	927	seto,
	928	setno,
	929	setb,
	930	setc,
	931	setnae,
	932	setnb,
	933	setnc,
	934	setae,
	935	sete,
	936	setz,
	937	setne,
	938	setnz,
	939	setbe,
	940	setna,
	941	seta,
	942	setnbe,
	943	sets,
	944	setns,
	945	setp,
	946	setpe,
	947	setnp,
	948	setop,
	949	setl,
	950	setnge,
	951	setnl,
	952	setge,
	953	setle,
	954	setng,
	955	setnle,
	956	setg,
	957
	958	fn isSetCC(tag: Tag) bool {
	959	return switch (tag) {
	960	.seto,
	961	.setno,
	962	.setb,
	963	.setc,
	964	.setnae,
	965	.setnb,
	966	.setnc,
	967	.setae,
	968	.sete,
	969	.setz,
	970	.setne,
	971	.setnz,
	972	.setbe,
	973	.setna,
	974	.seta,
	975	.setnbe,
	976	.sets,
	977	.setns,
	978	.setp,
	979	.setpe,
	980	.setnp,
	981	.setop,
	982	.setl,
	983	.setnge,
	984	.setnl,
	985	.setge,
	986	.setle,
	987	.setng,
	988	.setnle,
	989	.setg,
	990	=> true,
	991	else => false,
	992	};
	993	}
	994	};
	995
	996	const Encoding = enum {
	997	/// OP
	998	zo,
	999
	1000	/// OP rel32
	1001	d,
	1002
	1003	/// OP r/m64
	1004	m,
	1005
	1006	/// OP r64
	1007	o,
	1008
	1009	/// OP imm32
	1010	i,
	1011
	1012	/// OP r/m64, imm32
	1013	mi,
	1014
	1015	/// OP r/m64, r64
	1016	mr,
	1017
	1018	/// OP r64, r/m64
	1019	rm,
	1020
	1021	/// OP r64, imm64
	1022	oi,
	1023
	1024	/// OP al/ax/eax/rax, moffs
	1025	fd,
	1026
	1027	/// OP moffs, al/ax/eax/rax
	1028	td,
	1029
	1030	/// OP r64, r/m64, imm32
	1031	rmi,
	1032	};
	1033
	1034	const OpCode = union(enum) {
	1035	one_byte: u8,
	1036	two_byte: struct { _1: u8, _2: u8 },
	1037
	1038	fn oneByte(opc: u8) OpCode {
	1039	return .{ .one_byte = opc };
	1040	}
	1041
	1042	fn twoByte(opc1: u8, opc2: u8) OpCode {
	1043	return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
	1044	}
	1045
	1046	fn encode(opc: OpCode, encoder: Encoder) void {
	1047	switch (opc) {
	1048	.one_byte => \|v\| encoder.opcode_1byte(v),
	1049	.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
	1050	}
	1051	}
	1052
	1053	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
	1054	assert(opc == .one_byte);
	1055	encoder.opcode_withReg(opc.one_byte, reg.lowId());
	1056	}
	1057	};
	1058
	1059	inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
	1060	switch (enc) {
	1061	.zo => return switch (tag) {
	1062	.ret_near => OpCode.oneByte(0xc3),
553	1063	.ret_far => OpCode.oneByte(0xcb),
554	1064	.brk => OpCode.oneByte(0xcc),
555	1065	.nop => OpCode.oneByte(0x90),
...	...	@@ -722,7 +1232,35 @@ const Memory = struct {
722	1232	reg: ?Register,
723	1233	rip: bool = false,
724	1234	disp: i32,
	1235	ptr_size: PtrSize,
725	1236	sib: ?ScaleIndexBase = null,
	1237
	1238	const PtrSize = enum {
	1239	byte_ptr,
	1240	word_ptr,
	1241	dword_ptr,
	1242	qword_ptr,
	1243
	1244	fn fromBits(in_bits: u64) PtrSize {
	1245	return switch (in_bits) {
	1246	8 => .byte_ptr,
	1247	16 => .word_ptr,
	1248	32 => .dword_ptr,
	1249	64 => .qword_ptr,
	1250	else => unreachable,
	1251	};
	1252	}
	1253
	1254	/// Returns size in bits.
	1255	fn size(ptr_size: PtrSize) u64 {
	1256	return switch (ptr_size) {
	1257	.byte_ptr => 8,
	1258	.word_ptr => 16,
	1259	.dword_ptr => 32,
	1260	.qword_ptr => 64,
	1261	};
	1262	}
	1263	};
726	1264	};
727	1265
728	1266	const RegisterOrMemory = union(enum) {
...	...	@@ -733,33 +1271,42 @@ const RegisterOrMemory = union(enum) {
733	1271	return .{ .register = register };
734	1272	}
735	1273
736		fn mem(register: ?Register, disp: i32) RegisterOrMemory {
	1274	fn mem(register: ?Register, disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
737	1275	return .{
738	1276	.memory = .{
739	1277	.reg = register,
740	1278	.disp = disp,
	1279	.ptr_size = ptr_size,
741	1280	},
742	1281	};
743	1282	}
744	1283
745		fn rip(disp: i32) RegisterOrMemory {
	1284	fn rip(disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
746	1285	return .{
747	1286	.memory = .{
748	1287	.reg = null,
749	1288	.rip = true,
750	1289	.disp = disp,
	1290	.ptr_size = ptr_size,
751	1291	},
752	1292	};
753	1293	}
754	1294	};
755	1295
756		fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
	1296	const LoweringError = error{
	1297	OutOfMemory,
	1298	Overflow,
	1299	OperandSizeMismatch,
	1300	RaxOperandExpected,
	1301	};
	1302
	1303	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
757	1304	const opc = getOpCode(tag, .zo, false).?;
758	1305	const encoder = try Encoder.init(code, 1);
759	1306	opc.encode(encoder);
760	1307	}
761	1308
762		fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
	1309	fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
763	1310	if (tag == .ret_far or tag == .ret_near) {
764	1311	const encoder = try Encoder.init(code, 3);
765	1312	const opc = getOpCode(tag, .i, false).?;
...	...	@@ -782,928 +1329,543 @@ fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
782	1329	}
783	1330	}
784	1331
785		fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
786		if (reg.size() != 16 and reg.size() != 64) return error.EmitFail; // TODO correct for push/pop, but is it universal?
787		const opc = getOpCode(tag, .o, false).?;
788		const encoder = try Encoder.init(code, 3);
789		if (reg.size() == 16) {
790		encoder.opcode_1byte(0x66);
791		}
792		encoder.rex(.{
793		.w = false,
794		.b = reg.isExtended(),
795		});
796		opc.encodeWithReg(encoder, reg);
797		}
798
799		fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
800		const opc = getOpCode(tag, .d, false).?;
801		const encoder = try Encoder.init(code, 6);
802		opc.encode(encoder);
803		encoder.imm32(imm);
804		}
805
806		fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
807		const opc = getOpCode(tag, .m, false).?;
808		const modrm_ext = getModRmExt(tag).?;
809		switch (reg_or_mem) {
810		.register => \|reg\| {
811		// TODO clean this up!
812		if (reg.size() != 64) {
813		if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail;
814		}
815		const encoder = try Encoder.init(code, 3);
816		encoder.rex(.{
817		.w = tag.isSetCC(),
818		.b = reg.isExtended(),
819		});
820		opc.encode(encoder);
821		encoder.modRm_direct(modrm_ext, reg.lowId());
822		},
823		.memory => \|mem_op\| {
824		const encoder = try Encoder.init(code, 8);
825		if (mem_op.reg) \|reg\| {
826		// TODO clean this up!
827		if (reg.size() != 64) {
828		if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail;
829		}
830		encoder.rex(.{
831		.w = tag.isSetCC(),
832		.b = reg.isExtended(),
833		});
834		opc.encode(encoder);
835		if (reg.lowId() == 4) {
836		if (mem_op.disp == 0) {
837		encoder.modRm_SIBDisp0(modrm_ext);
838		encoder.sib_base(reg.lowId());
839		} else if (immOpSize(mem_op.disp) == 8) {
840		encoder.modRm_SIBDisp8(modrm_ext);
841		encoder.sib_baseDisp8(reg.lowId());
842		encoder.disp8(@intCast(i8, mem_op.disp));
843		} else {
844		encoder.modRm_SIBDisp32(modrm_ext);
845		encoder.sib_baseDisp32(reg.lowId());
846		encoder.disp32(mem_op.disp);
847		}
848		} else {
849		if (mem_op.disp == 0) {
850		encoder.modRm_indirectDisp0(modrm_ext, reg.lowId());
851		} else if (immOpSize(mem_op.disp) == 8) {
852		encoder.modRm_indirectDisp8(modrm_ext, reg.lowId());
853		encoder.disp8(@intCast(i8, mem_op.disp));
854		} else {
855		encoder.modRm_indirectDisp32(modrm_ext, reg.lowId());
856		encoder.disp32(mem_op.disp);
857		}
858		}
859		} else {
860		opc.encode(encoder);
861		if (mem_op.rip) {
862		encoder.modRm_RIPDisp32(modrm_ext);
863		} else {
864		encoder.modRm_SIBDisp0(modrm_ext);
865		encoder.sib_disp32();
866		}
867		encoder.disp32(mem_op.disp);
868		}
869		},
870		}
871		}
872
873		fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
874		return lowerToTdFdEnc(tag, reg, moffs, code, true);
875		}
876
877		fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) InnerError!void {
878		return lowerToTdFdEnc(tag, reg, moffs, code, false);
879		}
880
881		fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) InnerError!void {
882		if (reg.lowId() != Register.rax.lowId()) return error.EmitFail;
883		if (reg.size() != immOpSize(moffs)) return error.EmitFail;
884		const opc = if (td)
885		getOpCode(tag, .td, reg.size() == 8).?
886		else
887		getOpCode(tag, .fd, reg.size() == 8).?;
888		const encoder = try Encoder.init(code, 10);
889		if (reg.size() == 16) {
890		encoder.opcode_1byte(0x66);
891		}
892		encoder.rex(.{
893		.w = reg.size() == 64,
894		});
895		opc.encode(encoder);
896		switch (reg.size()) {
897		8 => {
898		const moffs8 = try math.cast(i8, moffs);
899		encoder.imm8(moffs8);
900		},
901		16 => {
902		const moffs16 = try math.cast(i16, moffs);
903		encoder.imm16(moffs16);
904		},
905		32 => {
906		const moffs32 = try math.cast(i32, moffs);
907		encoder.imm32(moffs32);
908		},
909		64 => {
910		encoder.imm64(@bitCast(u64, moffs));
911		},
912		else => unreachable,
	1332	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1333	if (reg.size() != 16 and reg.size() != 64) {
	1334	return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
913	1335	}
914		}
915
916		fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) InnerError!void {
917		if (reg.size() != immOpSize(imm)) return error.EmitFail;
918		const opc = getOpCode(tag, .oi, reg.size() == 8).?;
919		const encoder = try Encoder.init(code, 10);
	1336	const opc = getOpCode(tag, .o, false).?;
	1337	const encoder = try Encoder.init(code, 3);
920	1338	if (reg.size() == 16) {
921	1339	encoder.opcode_1byte(0x66);
922	1340	}
923	1341	encoder.rex(.{
924		.w = reg.size() == 64,
	1342	.w = false,
925	1343	.b = reg.isExtended(),
926	1344	});
927	1345	opc.encodeWithReg(encoder, reg);
928		switch (reg.size()) {
929		8 => {
930		const imm8 = try math.cast(i8, imm);
931		encoder.imm8(imm8);
932		},
933		16 => {
934		const imm16 = try math.cast(i16, imm);
935		encoder.imm16(imm16);
936		},
937		32 => {
938		const imm32 = try math.cast(i32, imm);
939		encoder.imm32(imm32);
940		},
941		64 => {
942		encoder.imm64(@bitCast(u64, imm));
943		},
944		else => unreachable,
945		}
946	1346	}
947	1347
948		fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
949		const modrm_ext = getModRmExt(tag).?;
950		switch (reg_or_mem) {
951		.register => \|dst_reg\| {
952		const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
953		const encoder = try Encoder.init(code, 7);
954		if (dst_reg.size() == 16) {
955		// 0x66 prefix switches to the non-default size; here we assume a switch from
956		// the default 32bits to 16bits operand-size.
957		// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
958		encoder.opcode_1byte(0x66);
959		}
960		encoder.rex(.{
961		.w = dst_reg.size() == 64,
962		.b = dst_reg.isExtended(),
963		});
964		opc.encode(encoder);
965		encoder.modRm_direct(modrm_ext, dst_reg.lowId());
966		switch (dst_reg.size()) {
967		8 => {
968		const imm8 = try math.cast(i8, imm);
969		encoder.imm8(imm8);
970		},
971		16 => {
972		const imm16 = try math.cast(i16, imm);
973		encoder.imm16(imm16);
974		},
975		32, 64 => encoder.imm32(imm),
976		else => unreachable,
977		}
978		},
979		.memory => \|dst_mem\| {
980		const opc = getOpCode(tag, .mi, false).?;
981		const encoder = try Encoder.init(code, 12);
982		if (dst_mem.reg) \|dst_reg\| {
983		// Register dst_reg can either be 64bit or 32bit in size.
984		// TODO for memory operand, immediate operand pair, we currently
985		// have no way of flagging whether the immediate can be 8-, 16- or
986		// 32-bit and whether the corresponding memory operand is respectively
987		// a byte, word or dword ptr.
988		// TODO we currently don't have a way to flag imm32 64bit sign extended
989		if (dst_reg.size() != 64) return error.EmitFail;
990		encoder.rex(.{
991		.w = false,
992		.b = dst_reg.isExtended(),
993		});
994		opc.encode(encoder);
995		if (dst_reg.lowId() == 4) {
996		if (dst_mem.disp == 0) {
997		encoder.modRm_SIBDisp0(modrm_ext);
998		encoder.sib_base(dst_reg.lowId());
999		} else if (immOpSize(dst_mem.disp) == 8) {
1000		encoder.modRm_SIBDisp8(modrm_ext);
1001		encoder.sib_baseDisp8(dst_reg.lowId());
1002		encoder.disp8(@intCast(i8, dst_mem.disp));
1003		} else {
1004		encoder.modRm_SIBDisp32(modrm_ext);
1005		encoder.sib_baseDisp32(dst_reg.lowId());
1006		encoder.disp32(dst_mem.disp);
1007		}
1008		} else {
1009		if (dst_mem.disp == 0) {
1010		encoder.modRm_indirectDisp0(modrm_ext, dst_reg.lowId());
1011		} else if (immOpSize(dst_mem.disp) == 8) {
1012		encoder.modRm_indirectDisp8(modrm_ext, dst_reg.lowId());
1013		encoder.disp8(@intCast(i8, dst_mem.disp));
1014		} else {
1015		encoder.modRm_indirectDisp32(modrm_ext, dst_reg.lowId());
1016		encoder.disp32(dst_mem.disp);
1017		}
1018		}
1019		} else {
1020		opc.encode(encoder);
1021		if (dst_mem.rip) {
1022		encoder.modRm_RIPDisp32(modrm_ext);
1023		} else {
1024		encoder.modRm_SIBDisp0(modrm_ext);
1025		encoder.sib_disp32();
1026		}
1027		encoder.disp32(dst_mem.disp);
1028		}
1029		encoder.imm32(imm);
1030		},
1031		}
	1348	fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
	1349	const opc = getOpCode(tag, .d, false).?;
	1350	const encoder = try Encoder.init(code, 6);
	1351	opc.encode(encoder);
	1352	encoder.imm32(imm);
1032	1353	}
1033	1354
1034		fn lowerToRmEnc(
1035		tag: Tag,
1036		reg: Register,
1037		reg_or_mem: RegisterOrMemory,
1038		code: *std.ArrayList(u8),
1039		) InnerError!void {
1040		const opc = getOpCode(tag, .rm, reg.size() == 8).?;
	1355	fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
	1356	const opc = getOpCode(tag, .m, false).?;
	1357	const modrm_ext = getModRmExt(tag).?;
1041	1358	switch (reg_or_mem) {
1042		.register => \|src_reg\| {
1043		if (reg.size() != src_reg.size()) return error.EmitFail;
1044		const encoder = try Encoder.init(code, 3);
1045		encoder.rex(.{
1046		.w = reg.size() == 64,
1047		.r = reg.isExtended(),
1048		.b = src_reg.isExtended(),
1049		});
1050		opc.encode(encoder);
1051		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1052		},
1053		.memory => \|src_mem\| {
1054		const encoder = try Encoder.init(code, 9);
1055		if (reg.size() == 16) {
1056		encoder.opcode_1byte(0x66);
1057		}
1058		if (src_mem.reg) \|src_reg\| {
1059		// TODO handle 32-bit base register - requires prefix 0x67
1060		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1061		if (src_reg.size() != 64) return error.EmitFail;
1062		encoder.rex(.{
1063		.w = reg.size() == 64,
1064		.r = reg.isExtended(),
1065		.b = src_reg.isExtended(),
1066		});
1067		opc.encode(encoder);
1068		if (src_reg.lowId() == 4) {
1069		if (src_mem.disp == 0) {
1070		encoder.modRm_SIBDisp0(reg.lowId());
1071		encoder.sib_base(src_reg.lowId());
1072		} else if (immOpSize(src_mem.disp) == 8) {
1073		encoder.modRm_SIBDisp8(reg.lowId());
1074		encoder.sib_baseDisp8(src_reg.lowId());
1075		encoder.disp8(@intCast(i8, src_mem.disp));
1076		} else {
1077		encoder.modRm_SIBDisp32(reg.lowId());
1078		encoder.sib_baseDisp32(src_reg.lowId());
1079		encoder.disp32(src_mem.disp);
1080		}
1081		} else {
1082		if (src_mem.disp == 0) {
1083		encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
1084		} else if (immOpSize(src_mem.disp) == 8) {
1085		encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
1086		encoder.disp8(@intCast(i8, src_mem.disp));
1087		} else {
1088		encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
1089		encoder.disp32(src_mem.disp);
1090		}
1091		}
1092		} else {
1093		encoder.rex(.{
1094		.w = reg.size() == 64,
1095		.r = reg.isExtended(),
1096		});
1097		opc.encode(encoder);
1098		if (src_mem.rip) {
1099		encoder.modRm_RIPDisp32(reg.lowId());
1100		} else {
1101		encoder.modRm_SIBDisp0(reg.lowId());
1102		encoder.sib_disp32();
1103		}
1104		encoder.disp32(src_mem.disp);
	1359	.register => \|reg\| {
	1360	const op_size_mismatch = blk: {
	1361	if (tag.isSetCC() and reg.size() == 8)
	1362	break :blk false;
	1363	break :blk reg.size() != 64 and reg.size() != 16;
	1364	};
	1365	if (op_size_mismatch) {
	1366	return error.OperandSizeMismatch;
1105	1367	}
1106		},
1107		}
1108		}
1109
1110		fn lowerToMrEnc(
1111		tag: Tag,
1112		reg_or_mem: RegisterOrMemory,
1113		reg: Register,
1114		code: *std.ArrayList(u8),
1115		) InnerError!void {
1116		// We use size of source register reg to work out which
1117		// variant of memory ptr to pick:
1118		// * reg is 64bit - qword ptr
1119		// * reg is 32bit - dword ptr
1120		// * reg is 16bit - word ptr
1121		// * reg is 8bit - byte ptr
1122		const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1123		switch (reg_or_mem) {
1124		.register => \|dst_reg\| {
1125		if (dst_reg.size() != reg.size()) return error.EmitFail;
1126		const encoder = try Encoder.init(code, 3);
1127		encoder.rex(.{
1128		.w = dst_reg.size() == 64,
1129		.r = reg.isExtended(),
1130		.b = dst_reg.isExtended(),
1131		});
1132		opc.encode(encoder);
1133		encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1134		},
1135		.memory => \|dst_mem\| {
1136		const encoder = try Encoder.init(code, 9);
	1368	const encoder = try Encoder.init(code, 4);
1137	1369	if (reg.size() == 16) {
1138	1370	encoder.opcode_1byte(0x66);
1139	1371	}
1140		if (dst_mem.reg) \|dst_reg\| {
1141		if (dst_reg.size() != 64) return error.EmitFail;
1142		encoder.rex(.{
1143		.w = reg.size() == 64,
1144		.r = reg.isExtended(),
1145		.b = dst_reg.isExtended(),
1146		});
1147		opc.encode(encoder);
1148		if (dst_reg.lowId() == 4) {
1149		if (dst_mem.disp == 0) {
1150		encoder.modRm_SIBDisp0(reg.lowId());
1151		encoder.sib_base(dst_reg.lowId());
1152		} else if (immOpSize(dst_mem.disp) == 8) {
1153		encoder.modRm_SIBDisp8(reg.lowId());
1154		encoder.sib_baseDisp8(dst_reg.lowId());
1155		encoder.disp8(@intCast(i8, dst_mem.disp));
1156		} else {
1157		encoder.modRm_SIBDisp32(reg.lowId());
1158		encoder.sib_baseDisp32(dst_reg.lowId());
1159		encoder.disp32(dst_mem.disp);
1160		}
1161		} else {
1162		if (dst_mem.disp == 0) {
1163		encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
1164		} else if (immOpSize(dst_mem.disp) == 8) {
1165		encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
1166		encoder.disp8(@intCast(i8, dst_mem.disp));
1167		} else {
1168		encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
1169		encoder.disp32(dst_mem.disp);
1170		}
1171		}
1172		} else {
1173		encoder.rex(.{
1174		.w = reg.size() == 64,
1175		.r = reg.isExtended(),
1176		});
1177		opc.encode(encoder);
1178		if (dst_mem.rip) {
1179		encoder.modRm_RIPDisp32(reg.lowId());
1180		} else {
1181		encoder.modRm_SIBDisp0(reg.lowId());
1182		encoder.sib_disp32();
1183		}
1184		encoder.disp32(dst_mem.disp);
1185		}
1186		},
1187		}
1188		}
1189
1190		fn lowerToRmiEnc(
1191		tag: Tag,
1192		reg: Register,
1193		reg_or_mem: RegisterOrMemory,
1194		imm: i32,
1195		code: *std.ArrayList(u8),
1196		) InnerError!void {
1197		const opc = getOpCode(tag, .rmi, reg.size() == 8).?;
1198		switch (reg_or_mem) {
1199		.register => \|src_reg\| {
1200		if (reg.size() != src_reg.size()) return error.EmitFail;
1201		const encoder = try Encoder.init(code, 7);
1202	1372	encoder.rex(.{
1203		.w = reg.size() == 64,
1204		.r = reg.isExtended(),
1205		.b = src_reg.isExtended(),
	1373	.w = tag.isSetCC(),
	1374	.b = reg.isExtended(),
1206	1375	});
1207	1376	opc.encode(encoder);
1208		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1209		switch (reg.size()) {
1210		8 => {
1211		const imm8 = try math.cast(i8, imm);
1212		encoder.imm8(imm8);
1213		},
1214		16 => {
1215		const imm16 = try math.cast(i16, imm);
1216		encoder.imm16(imm16);
1217		},
1218		32, 64 => encoder.imm32(imm),
1219		else => unreachable,
1220		}
	1377	encoder.modRm_direct(modrm_ext, reg.lowId());
1221	1378	},
1222		.memory => \|src_mem\| {
1223		const encoder = try Encoder.init(code, 13);
1224		if (reg.size() == 16) {
	1379	.memory => \|mem_op\| {
	1380	if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
	1381	return error.OperandSizeMismatch;
	1382	}
	1383	const encoder = try Encoder.init(code, 8);
	1384	if (mem_op.ptr_size == .word_ptr) {
1225	1385	encoder.opcode_1byte(0x66);
1226	1386	}
1227		if (src_mem.reg) \|src_reg\| {
1228		// TODO handle 32-bit base register - requires prefix 0x67
1229		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1230		if (src_reg.size() != 64) return error.EmitFail;
	1387	if (mem_op.reg) \|reg\| {
	1388	if (reg.size() != 64) {
	1389	return error.OperandSizeMismatch;
	1390	}
1231	1391	encoder.rex(.{
1232		.w = reg.size() == 64,
1233		.r = reg.isExtended(),
1234		.b = src_reg.isExtended(),
	1392	.w = tag.isSetCC(),
	1393	.b = reg.isExtended(),
1235	1394	});
1236	1395	opc.encode(encoder);
1237		if (src_reg.lowId() == 4) {
1238		if (src_mem.disp == 0) {
1239		encoder.modRm_SIBDisp0(reg.lowId());
1240		encoder.sib_base(src_reg.lowId());
1241		} else if (immOpSize(src_mem.disp) == 8) {
1242		encoder.modRm_SIBDisp8(reg.lowId());
1243		encoder.sib_baseDisp8(src_reg.lowId());
1244		encoder.disp8(@intCast(i8, src_mem.disp));
	1396	if (reg.lowId() == 4) {
	1397	if (mem_op.disp == 0) {
	1398	encoder.modRm_SIBDisp0(modrm_ext);
	1399	encoder.sib_base(reg.lowId());
	1400	} else if (immOpSize(mem_op.disp) == 8) {
	1401	encoder.modRm_SIBDisp8(modrm_ext);
	1402	encoder.sib_baseDisp8(reg.lowId());
	1403	encoder.disp8(@intCast(i8, mem_op.disp));
1245	1404	} else {
1246		encoder.modRm_SIBDisp32(reg.lowId());
1247		encoder.sib_baseDisp32(src_reg.lowId());
1248		encoder.disp32(src_mem.disp);
	1405	encoder.modRm_SIBDisp32(modrm_ext);
	1406	encoder.sib_baseDisp32(reg.lowId());
	1407	encoder.disp32(mem_op.disp);
1249	1408	}
1250	1409	} else {
1251		if (src_mem.disp == 0) {
1252		encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
1253		} else if (immOpSize(src_mem.disp) == 8) {
1254		encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
1255		encoder.disp8(@intCast(i8, src_mem.disp));
	1410	if (mem_op.disp == 0) {
	1411	encoder.modRm_indirectDisp0(modrm_ext, reg.lowId());
	1412	} else if (immOpSize(mem_op.disp) == 8) {
	1413	encoder.modRm_indirectDisp8(modrm_ext, reg.lowId());
	1414	encoder.disp8(@intCast(i8, mem_op.disp));
1256	1415	} else {
1257		encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
1258		encoder.disp32(src_mem.disp);
	1416	encoder.modRm_indirectDisp32(modrm_ext, reg.lowId());
	1417	encoder.disp32(mem_op.disp);
1259	1418	}
1260	1419	}
1261	1420	} else {
1262		encoder.rex(.{
1263		.w = reg.size() == 64,
1264		.r = reg.isExtended(),
1265		});
1266	1421	opc.encode(encoder);
1267		if (src_mem.rip) {
1268		encoder.modRm_RIPDisp32(reg.lowId());
	1422	if (mem_op.rip) {
	1423	encoder.modRm_RIPDisp32(modrm_ext);
1269	1424	} else {
1270		encoder.modRm_SIBDisp0(reg.lowId());
	1425	encoder.modRm_SIBDisp0(modrm_ext);
1271	1426	encoder.sib_disp32();
1272	1427	}
1273		encoder.disp32(src_mem.disp);
	1428	encoder.disp32(mem_op.disp);
1274	1429	}
1275		encoder.imm32(imm);
1276	1430	},
1277	1431	}
1278	1432	}
1279	1433
1280		fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1281		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1282		switch (ops.flags) {
1283		0b00 => {
1284		if (ops.reg2 == .none) {
1285		// mov reg1, imm32
1286		// MI
1287		const imm = emit.mir.instructions.items(.data)[inst].imm;
1288		return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
1289		}
1290		// mov reg1, reg2
1291		// RM
1292		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1293		},
1294		0b01 => {
1295		const imm = emit.mir.instructions.items(.data)[inst].imm;
1296		if (ops.reg2 == .none) {
1297		// mov reg1, [imm32]
1298		// RM
1299		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(null, imm), emit.code);
1300		}
1301		// mov reg1, [reg2 + imm32]
1302		// RM
1303		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code);
1304		},
1305		0b10 => {
1306		if (ops.reg2 == .none) {
1307		// mov dword ptr [reg1 + 0], imm32
1308		// MI
1309		const imm = emit.mir.instructions.items(.data)[inst].imm;
1310		return lowerToMiEnc(tag, RegisterOrMemory.mem(ops.reg1, 0), imm, emit.code);
1311		}
1312		// mov [reg1 + imm32], reg2
1313		// MR
1314		const imm = emit.mir.instructions.items(.data)[inst].imm;
1315		return lowerToMrEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), ops.reg2, emit.code);
1316		},
1317		0b11 => {
1318		if (ops.reg2 == .none) {
1319		// mov dword ptr [reg1 + imm32], imm32
1320		// MI
1321		const payload = emit.mir.instructions.items(.data)[inst].payload;
1322		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
1323		return lowerToMiEnc(
1324		tag,
1325		RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off),
1326		imm_pair.operand,
1327		emit.code,
1328		);
1329		}
1330		return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
1331		},
1332		}
	1434	fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1435	return lowerToTdFdEnc(tag, reg, moffs, code, true);
1333	1436	}
1334	1437
1335		fn immOpSize(imm: i64) u8 {
1336		blk: {
1337		_ = math.cast(i8, imm) catch break :blk;
1338		return 8;
1339		}
1340		blk: {
1341		_ = math.cast(i16, imm) catch break :blk;
1342		return 16;
1343		}
1344		blk: {
1345		_ = math.cast(i32, imm) catch break :blk;
1346		return 32;
1347		}
1348		return 64;
	1438	fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) LoweringError!void {
	1439	return lowerToTdFdEnc(tag, reg, moffs, code, false);
1349	1440	}
1350	1441
1351		// TODO
1352		fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1353		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1354		const scale = ops.flags;
1355		// OP reg1, [reg2 + scale*rcx + imm32]
1356		const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
1357		const imm = emit.mir.instructions.items(.data)[inst].imm;
1358		const encoder = try Encoder.init(emit.code, 8);
1359		encoder.rex(.{
1360		.w = ops.reg1.size() == 64,
1361		.r = ops.reg1.isExtended(),
1362		.b = ops.reg2.isExtended(),
1363		});
1364		opc.encode(encoder);
1365		if (imm <= math.maxInt(i8)) {
1366		encoder.modRm_SIBDisp8(ops.reg1.lowId());
1367		encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
1368		encoder.disp8(@intCast(i8, imm));
1369		} else {
1370		encoder.modRm_SIBDisp32(ops.reg1.lowId());
1371		encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
1372		encoder.disp32(imm);
	1442	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
	1443	if (reg.lowId() != Register.rax.lowId()) {
	1444	return error.RaxOperandExpected;
1373	1445	}
1374		}
1375
1376		// TODO
1377		fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1378		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1379		const scale = ops.flags;
1380		const imm = emit.mir.instructions.items(.data)[inst].imm;
1381
1382		if (ops.reg2 == .none) {
1383		// OP [reg1 + scale*rax + 0], imm32
1384		const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
1385		const modrm_ext = getModRmExt(tag).?;
1386		const encoder = try Encoder.init(emit.code, 8);
1387		encoder.rex(.{
1388		.w = ops.reg1.size() == 64,
1389		.b = ops.reg1.isExtended(),
1390		});
1391		opc.encode(encoder);
1392		encoder.modRm_SIBDisp0(modrm_ext);
1393		encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
1394		if (imm <= math.maxInt(i8)) {
1395		encoder.imm8(@intCast(i8, imm));
1396		} else if (imm <= math.maxInt(i16)) {
1397		encoder.imm16(@intCast(i16, imm));
1398		} else {
1399		encoder.imm32(imm);
1400		}
1401		return;
	1446	if (reg.size() != immOpSize(moffs)) {
	1447	return error.OperandSizeMismatch;
1402	1448	}
1403
1404		// OP [reg1 + scale*rax + imm32], reg2
1405		const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
1406		const encoder = try Encoder.init(emit.code, 8);
1407		encoder.rex(.{
1408		.w = ops.reg1.size() == 64,
1409		.r = ops.reg2.isExtended(),
1410		.b = ops.reg1.isExtended(),
1411		});
1412		opc.encode(encoder);
1413		if (imm <= math.maxInt(i8)) {
1414		encoder.modRm_SIBDisp8(ops.reg2.lowId());
1415		encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
1416		encoder.disp8(@intCast(i8, imm));
1417		} else {
1418		encoder.modRm_SIBDisp32(ops.reg2.lowId());
1419		encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
1420		encoder.disp32(imm);
	1449	const opc = if (td)
	1450	getOpCode(tag, .td, reg.size() == 8).?
	1451	else
	1452	getOpCode(tag, .fd, reg.size() == 8).?;
	1453	const encoder = try Encoder.init(code, 10);
	1454	if (reg.size() == 16) {
	1455	encoder.opcode_1byte(0x66);
1421	1456	}
1422		}
1423
1424		// TODO
1425		fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1426		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1427		const scale = ops.flags;
1428		const payload = emit.mir.instructions.items(.data)[inst].payload;
1429		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
1430		const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
1431		const modrm_ext = getModRmExt(tag).?;
1432		const encoder = try Encoder.init(emit.code, 2);
1433	1457	encoder.rex(.{
1434		.w = ops.reg1.size() == 64,
1435		.b = ops.reg1.isExtended(),
	1458	.w = reg.size() == 64,
1436	1459	});
1437	1460	opc.encode(encoder);
1438		if (imm_pair.dest_off <= math.maxInt(i8)) {
1439		encoder.modRm_SIBDisp8(modrm_ext);
1440		encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
1441		encoder.disp8(@intCast(i8, imm_pair.dest_off));
1442		} else {
1443		encoder.modRm_SIBDisp32(modrm_ext);
1444		encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
1445		encoder.disp32(imm_pair.dest_off);
1446		}
1447		encoder.imm32(imm_pair.operand);
1448		}
1449
1450		fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1451		const tag = emit.mir.instructions.items(.tag)[inst];
1452		assert(tag == .movabs);
1453		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1454		const imm: i64 = if (ops.reg1.size() == 64) blk: {
1455		const payload = emit.mir.instructions.items(.data)[inst].payload;
1456		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
1457		break :blk @bitCast(i64, imm.decode());
1458		} else emit.mir.instructions.items(.data)[inst].imm;
1459		if (ops.flags == 0b00) {
1460		// movabs reg, imm64
1461		// OI
1462		return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
1463		}
1464		if (ops.reg1 == .none) {
1465		// movabs moffs64, rax
1466		// TD
1467		return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
1468		} else {
1469		// movabs rax, moffs64
1470		// FD
1471		return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
	1461	switch (reg.size()) {
	1462	8 => {
	1463	const moffs8 = try math.cast(i8, moffs);
	1464	encoder.imm8(moffs8);
	1465	},
	1466	16 => {
	1467	const moffs16 = try math.cast(i16, moffs);
	1468	encoder.imm16(moffs16);
	1469	},
	1470	32 => {
	1471	const moffs32 = try math.cast(i32, moffs);
	1472	encoder.imm32(moffs32);
	1473	},
	1474	64 => {
	1475	encoder.imm64(@bitCast(u64, moffs));
	1476	},
	1477	else => unreachable,
1472	1478	}
1473	1479	}
1474	1480
1475		fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1476		const tag = emit.mir.instructions.items(.tag)[inst];
1477		assert(tag == .imul_complex);
1478		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1479		switch (ops.flags) {
1480		0b00 => return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code),
1481		0b10 => {
1482		const imm = emit.mir.instructions.items(.data)[inst].imm;
1483		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);
	1481	fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) LoweringError!void {
	1482	if (reg.size() != immOpSize(imm)) {
	1483	return error.OperandSizeMismatch;
	1484	}
	1485	const opc = getOpCode(tag, .oi, reg.size() == 8).?;
	1486	const encoder = try Encoder.init(code, 10);
	1487	if (reg.size() == 16) {
	1488	encoder.opcode_1byte(0x66);
	1489	}
	1490	encoder.rex(.{
	1491	.w = reg.size() == 64,
	1492	.b = reg.isExtended(),
	1493	});
	1494	opc.encodeWithReg(encoder, reg);
	1495	switch (reg.size()) {
	1496	8 => {
	1497	const imm8 = try math.cast(i8, imm);
	1498	encoder.imm8(imm8);
1484	1499	},
1485		else => return emit.fail("TODO implement imul", .{}),
	1500	16 => {
	1501	const imm16 = try math.cast(i16, imm);
	1502	encoder.imm16(imm16);
	1503	},
	1504	32 => {
	1505	const imm32 = try math.cast(i32, imm);
	1506	encoder.imm32(imm32);
	1507	},
	1508	64 => {
	1509	encoder.imm64(@bitCast(u64, imm));
	1510	},
	1511	else => unreachable,
1486	1512	}
1487	1513	}
1488	1514
1489		fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1490		const tag = emit.mir.instructions.items(.tag)[inst];
1491		assert(tag == .lea);
1492		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1493		assert(ops.flags == 0b01);
1494		const imm = emit.mir.instructions.items(.data)[inst].imm;
1495		return lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code);
1496		}
1497
1498		fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1499		const tag = emit.mir.instructions.items(.tag)[inst];
1500		assert(tag == .lea_rip);
1501		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1502		const start_offset = emit.code.items.len;
1503		try lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.rip(0), emit.code);
1504		const end_offset = emit.code.items.len;
1505		if (@truncate(u1, ops.flags) == 0b0) {
1506		// Backpatch the displacement
1507		// TODO figure out if this can be simplified
1508		const payload = emit.mir.instructions.items(.data)[inst].payload;
1509		const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
1510		const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
1511		mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
1512		} else {
1513		const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
1514		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1515		// TODO I think the reloc might be in the wrong place.
1516		const decl = macho_file.active_decl.?;
1517		try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
1518		.offset = @intCast(u32, end_offset - 4),
1519		.target = .{ .local = got_entry },
1520		.addend = 0,
1521		.subtractor = null,
1522		.pcrel = true,
1523		.length = 2,
1524		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	1515	fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
	1516	const modrm_ext = getModRmExt(tag).?;
	1517	switch (reg_or_mem) {
	1518	.register => \|dst_reg\| {
	1519	const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
	1520	const encoder = try Encoder.init(code, 7);
	1521	if (dst_reg.size() == 16) {
	1522	// 0x66 prefix switches to the non-default size; here we assume a switch from
	1523	// the default 32bits to 16bits operand-size.
	1524	// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
	1525	encoder.opcode_1byte(0x66);
	1526	}
	1527	encoder.rex(.{
	1528	.w = dst_reg.size() == 64,
	1529	.b = dst_reg.isExtended(),
1525	1530	});
1526		} else {
1527		return emit.fail("TODO implement lea_rip for linking backends different than MachO", .{});
1528		}
1529		}
1530		}
1531
1532		fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1533		const tag = emit.mir.instructions.items(.tag)[inst];
1534		assert(tag == .call_extern);
1535		const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
1536		const offset = blk: {
1537		// callq
1538		try lowerToDEnc(.call_near, 0, emit.code);
1539		break :blk @intCast(u32, emit.code.items.len) - 4;
1540		};
1541		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1542		// Add relocation to the decl.
1543		try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
1544		.offset = offset,
1545		.target = .{ .global = n_strx },
1546		.addend = 0,
1547		.subtractor = null,
1548		.pcrel = true,
1549		.length = 2,
1550		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
1551		});
1552		} else {
1553		return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
	1531	opc.encode(encoder);
	1532	encoder.modRm_direct(modrm_ext, dst_reg.lowId());
	1533	switch (dst_reg.size()) {
	1534	8 => {
	1535	const imm8 = try math.cast(i8, imm);
	1536	encoder.imm8(imm8);
	1537	},
	1538	16 => {
	1539	const imm16 = try math.cast(i16, imm);
	1540	encoder.imm16(imm16);
	1541	},
	1542	32, 64 => encoder.imm32(imm),
	1543	else => unreachable,
	1544	}
	1545	},
	1546	.memory => \|dst_mem\| {
	1547	const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
	1548	const encoder = try Encoder.init(code, 12);
	1549	if (dst_mem.ptr_size == .word_ptr) {
	1550	encoder.opcode_1byte(0x66);
	1551	}
	1552	if (dst_mem.reg) \|dst_reg\| {
	1553	if (dst_reg.size() != 64) {
	1554	return error.OperandSizeMismatch;
	1555	}
	1556	encoder.rex(.{
	1557	.w = dst_mem.ptr_size == .qword_ptr,
	1558	.b = dst_reg.isExtended(),
	1559	});
	1560	opc.encode(encoder);
	1561	if (dst_reg.lowId() == 4) {
	1562	if (dst_mem.disp == 0) {
	1563	encoder.modRm_SIBDisp0(modrm_ext);
	1564	encoder.sib_base(dst_reg.lowId());
	1565	} else if (immOpSize(dst_mem.disp) == 8) {
	1566	encoder.modRm_SIBDisp8(modrm_ext);
	1567	encoder.sib_baseDisp8(dst_reg.lowId());
	1568	encoder.disp8(@intCast(i8, dst_mem.disp));
	1569	} else {
	1570	encoder.modRm_SIBDisp32(modrm_ext);
	1571	encoder.sib_baseDisp32(dst_reg.lowId());
	1572	encoder.disp32(dst_mem.disp);
	1573	}
	1574	} else {
	1575	if (dst_mem.disp == 0) {
	1576	encoder.modRm_indirectDisp0(modrm_ext, dst_reg.lowId());
	1577	} else if (immOpSize(dst_mem.disp) == 8) {
	1578	encoder.modRm_indirectDisp8(modrm_ext, dst_reg.lowId());
	1579	encoder.disp8(@intCast(i8, dst_mem.disp));
	1580	} else {
	1581	encoder.modRm_indirectDisp32(modrm_ext, dst_reg.lowId());
	1582	encoder.disp32(dst_mem.disp);
	1583	}
	1584	}
	1585	} else {
	1586	opc.encode(encoder);
	1587	if (dst_mem.rip) {
	1588	encoder.modRm_RIPDisp32(modrm_ext);
	1589	} else {
	1590	encoder.modRm_SIBDisp0(modrm_ext);
	1591	encoder.sib_disp32();
	1592	}
	1593	encoder.disp32(dst_mem.disp);
	1594	}
	1595	switch (dst_mem.ptr_size) {
	1596	.byte_ptr => {
	1597	const imm8 = try math.cast(i8, imm);
	1598	encoder.imm8(imm8);
	1599	},
	1600	.word_ptr => {
	1601	const imm16 = try math.cast(i16, imm);
	1602	encoder.imm16(imm16);
	1603	},
	1604	.dword_ptr, .qword_ptr => {
	1605	encoder.imm32(imm);
	1606	},
	1607	}
	1608	},
1554	1609	}
1555	1610	}
1556	1611
1557		fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1558		const tag = emit.mir.instructions.items(.tag)[inst];
1559		assert(tag == .dbg_line);
1560		const payload = emit.mir.instructions.items(.data)[inst].payload;
1561		const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
1562		try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
1563		}
1564
1565		fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1566		const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
1567		const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
1568		switch (emit.debug_output) {
1569		.dwarf => \|dbg_out\| {
1570		// TODO Look into using the DWARF special opcodes to compress this data.
1571		// It lets you emit single-byte opcodes that add different numbers to
1572		// both the PC and the line number at the same time.
1573		try dbg_out.dbg_line.ensureUnusedCapacity(11);
1574		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
1575		leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
1576		if (delta_line != 0) {
1577		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
1578		leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	1612	fn lowerToRmEnc(
	1613	tag: Tag,
	1614	reg: Register,
	1615	reg_or_mem: RegisterOrMemory,
	1616	code: *std.ArrayList(u8),
	1617	) LoweringError!void {
	1618	const opc = getOpCode(tag, .rm, reg.size() == 8).?;
	1619	switch (reg_or_mem) {
	1620	.register => \|src_reg\| {
	1621	if (reg.size() != src_reg.size()) {
	1622	return error.OperandSizeMismatch;
1579	1623	}
1580		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
1581		emit.prev_di_pc = emit.code.items.len;
1582		emit.prev_di_line = line;
1583		emit.prev_di_column = column;
1584		emit.prev_di_pc = emit.code.items.len;
	1624	const encoder = try Encoder.init(code, 3);
	1625	encoder.rex(.{
	1626	.w = reg.size() == 64,
	1627	.r = reg.isExtended(),
	1628	.b = src_reg.isExtended(),
	1629	});
	1630	opc.encode(encoder);
	1631	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1585	1632	},
1586		.plan9 => \|dbg_out\| {
1587		if (delta_pc <= 0) return; // only do this when the pc changes
1588		// we have already checked the target in the linker to make sure it is compatable
1589		const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
1590
1591		// increasing the line number
1592		try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
1593		// increasing the pc
1594		const d_pc_p9 = @intCast(i64, delta_pc) - quant;
1595		if (d_pc_p9 > 0) {
1596		// minus one because if its the last one, we want to leave space to change the line which is one quanta
1597		try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
1598		if (dbg_out.pcop_change_index.*) \|pci\|
1599		dbg_out.dbg_line.items[pci] += 1;
1600		dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
1601		} else if (d_pc_p9 == 0) {
1602		// we don't need to do anything, because adding the quant does it for us
1603		} else unreachable;
1604		if (dbg_out.start_line.* == null)
1605		dbg_out.start_line.* = emit.prev_di_line;
1606		dbg_out.end_line.* = line;
1607		// only do this if the pc changed
1608		emit.prev_di_line = line;
1609		emit.prev_di_column = column;
1610		emit.prev_di_pc = emit.code.items.len;
	1633	.memory => \|src_mem\| {
	1634	if (reg.size() != src_mem.ptr_size.size()) {
	1635	return error.OperandSizeMismatch;
	1636	}
	1637	const encoder = try Encoder.init(code, 9);
	1638	if (reg.size() == 16) {
	1639	encoder.opcode_1byte(0x66);
	1640	}
	1641	if (src_mem.reg) \|src_reg\| {
	1642	// TODO handle 32-bit base register - requires prefix 0x67
	1643	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1644	if (src_reg.size() != 64) {
	1645	return error.OperandSizeMismatch;
	1646	}
	1647	encoder.rex(.{
	1648	.w = reg.size() == 64,
	1649	.r = reg.isExtended(),
	1650	.b = src_reg.isExtended(),
	1651	});
	1652	opc.encode(encoder);
	1653	if (src_reg.lowId() == 4) {
	1654	if (src_mem.disp == 0) {
	1655	encoder.modRm_SIBDisp0(reg.lowId());
	1656	encoder.sib_base(src_reg.lowId());
	1657	} else if (immOpSize(src_mem.disp) == 8) {
	1658	encoder.modRm_SIBDisp8(reg.lowId());
	1659	encoder.sib_baseDisp8(src_reg.lowId());
	1660	encoder.disp8(@intCast(i8, src_mem.disp));
	1661	} else {
	1662	encoder.modRm_SIBDisp32(reg.lowId());
	1663	encoder.sib_baseDisp32(src_reg.lowId());
	1664	encoder.disp32(src_mem.disp);
	1665	}
	1666	} else {
	1667	if (src_mem.disp == 0) {
	1668	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
	1669	} else if (immOpSize(src_mem.disp) == 8) {
	1670	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
	1671	encoder.disp8(@intCast(i8, src_mem.disp));
	1672	} else {
	1673	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
	1674	encoder.disp32(src_mem.disp);
	1675	}
	1676	}
	1677	} else {
	1678	encoder.rex(.{
	1679	.w = reg.size() == 64,
	1680	.r = reg.isExtended(),
	1681	});
	1682	opc.encode(encoder);
	1683	if (src_mem.rip) {
	1684	encoder.modRm_RIPDisp32(reg.lowId());
	1685	} else {
	1686	encoder.modRm_SIBDisp0(reg.lowId());
	1687	encoder.sib_disp32();
	1688	}
	1689	encoder.disp32(src_mem.disp);
	1690	}
1611	1691	},
1612		.none => {},
1613	1692	}
1614	1693	}
1615	1694
1616		fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1617		const tag = emit.mir.instructions.items(.tag)[inst];
1618		assert(tag == .dbg_prologue_end);
1619		switch (emit.debug_output) {
1620		.dwarf => \|dbg_out\| {
1621		try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
1622		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	1695	fn lowerToMrEnc(
	1696	tag: Tag,
	1697	reg_or_mem: RegisterOrMemory,
	1698	reg: Register,
	1699	code: *std.ArrayList(u8),
	1700	) LoweringError!void {
	1701	const opc = getOpCode(tag, .mr, reg.size() == 8).?;
	1702	switch (reg_or_mem) {
	1703	.register => \|dst_reg\| {
	1704	if (dst_reg.size() != reg.size()) {
	1705	return error.OperandSizeMismatch;
	1706	}
	1707	const encoder = try Encoder.init(code, 3);
	1708	encoder.rex(.{
	1709	.w = dst_reg.size() == 64,
	1710	.r = reg.isExtended(),
	1711	.b = dst_reg.isExtended(),
	1712	});
	1713	opc.encode(encoder);
	1714	encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1623	1715	},
1624		.plan9 => {},
1625		.none => {},
1626		}
1627		}
1628
1629		fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1630		const tag = emit.mir.instructions.items(.tag)[inst];
1631		assert(tag == .dbg_epilogue_begin);
1632		switch (emit.debug_output) {
1633		.dwarf => \|dbg_out\| {
1634		try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
1635		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	1716	.memory => \|dst_mem\| {
	1717	if (dst_mem.ptr_size.size() != reg.size()) {
	1718	return error.OperandSizeMismatch;
	1719	}
	1720	const encoder = try Encoder.init(code, 9);
	1721	if (reg.size() == 16) {
	1722	encoder.opcode_1byte(0x66);
	1723	}
	1724	if (dst_mem.reg) \|dst_reg\| {
	1725	if (dst_reg.size() != 64) {
	1726	return error.OperandSizeMismatch;
	1727	}
	1728	encoder.rex(.{
	1729	.w = dst_mem.ptr_size == .qword_ptr,
	1730	.r = reg.isExtended(),
	1731	.b = dst_reg.isExtended(),
	1732	});
	1733	opc.encode(encoder);
	1734	if (dst_reg.lowId() == 4) {
	1735	if (dst_mem.disp == 0) {
	1736	encoder.modRm_SIBDisp0(reg.lowId());
	1737	encoder.sib_base(dst_reg.lowId());
	1738	} else if (immOpSize(dst_mem.disp) == 8) {
	1739	encoder.modRm_SIBDisp8(reg.lowId());
	1740	encoder.sib_baseDisp8(dst_reg.lowId());
	1741	encoder.disp8(@intCast(i8, dst_mem.disp));
	1742	} else {
	1743	encoder.modRm_SIBDisp32(reg.lowId());
	1744	encoder.sib_baseDisp32(dst_reg.lowId());
	1745	encoder.disp32(dst_mem.disp);
	1746	}
	1747	} else {
	1748	if (dst_mem.disp == 0) {
	1749	encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
	1750	} else if (immOpSize(dst_mem.disp) == 8) {
	1751	encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
	1752	encoder.disp8(@intCast(i8, dst_mem.disp));
	1753	} else {
	1754	encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
	1755	encoder.disp32(dst_mem.disp);
	1756	}
	1757	}
	1758	} else {
	1759	encoder.rex(.{
	1760	.w = dst_mem.ptr_size == .qword_ptr,
	1761	.r = reg.isExtended(),
	1762	});
	1763	opc.encode(encoder);
	1764	if (dst_mem.rip) {
	1765	encoder.modRm_RIPDisp32(reg.lowId());
	1766	} else {
	1767	encoder.modRm_SIBDisp0(reg.lowId());
	1768	encoder.sib_disp32();
	1769	}
	1770	encoder.disp32(dst_mem.disp);
	1771	}
1636	1772	},
1637		.plan9 => {},
1638		.none => {},
1639	1773	}
1640	1774	}
1641	1775
1642		fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1643		const tag = emit.mir.instructions.items(.tag)[inst];
1644		assert(tag == .arg_dbg_info);
1645		const payload = emit.mir.instructions.items(.data)[inst].payload;
1646		const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
1647		const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
1648		try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
1649		}
1650
1651		fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
1652		const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
1653		const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
1654		const name = zir.nullTerminatedString(ty_str.str);
1655		const name_with_null = name.ptr[0 .. name.len + 1];
1656		const ty = emit.mir.function.air.getRefType(ty_str.ty);
1657
1658		switch (mcv) {
1659		.register => \|reg\| {
1660		switch (emit.debug_output) {
1661		.dwarf => \|dbg_out\| {
1662		try dbg_out.dbg_info.ensureUnusedCapacity(3);
1663		dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
1664		dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
1665		1, // ULEB128 dwarf expression length
1666		reg.dwarfLocOp(),
1667		});
1668		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
1669		try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
1670		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
1671		},
1672		.plan9 => {},
1673		.none => {},
	1776	fn lowerToRmiEnc(
	1777	tag: Tag,
	1778	reg: Register,
	1779	reg_or_mem: RegisterOrMemory,
	1780	imm: i32,
	1781	code: *std.ArrayList(u8),
	1782	) LoweringError!void {
	1783	if (reg.size() == 8) {
	1784	return error.OperandSizeMismatch;
	1785	}
	1786	const opc = getOpCode(tag, .rmi, false).?;
	1787	const encoder = try Encoder.init(code, 13);
	1788	if (reg.size() == 16) {
	1789	encoder.opcode_1byte(0x66);
	1790	}
	1791	switch (reg_or_mem) {
	1792	.register => \|src_reg\| {
	1793	if (reg.size() != src_reg.size()) {
	1794	return error.OperandSizeMismatch;
1674	1795	}
	1796	encoder.rex(.{
	1797	.w = reg.size() == 64,
	1798	.r = reg.isExtended(),
	1799	.b = src_reg.isExtended(),
	1800	});
	1801	opc.encode(encoder);
	1802	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1675	1803	},
1676		.stack_offset => {
1677		switch (emit.debug_output) {
1678		.dwarf => {},
1679		.plan9 => {},
1680		.none => {},
	1804	.memory => \|src_mem\| {
	1805	if (src_mem.reg) \|src_reg\| {
	1806	// TODO handle 32-bit base register - requires prefix 0x67
	1807	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1808	if (src_reg.size() != 64) {
	1809	return error.OperandSizeMismatch;
	1810	}
	1811	if (src_mem.ptr_size == .byte_ptr) {
	1812	return error.OperandSizeMismatch;
	1813	}
	1814	encoder.rex(.{
	1815	.w = reg.size() == 64,
	1816	.r = reg.isExtended(),
	1817	.b = src_reg.isExtended(),
	1818	});
	1819	opc.encode(encoder);
	1820	if (src_reg.lowId() == 4) {
	1821	if (src_mem.disp == 0) {
	1822	encoder.modRm_SIBDisp0(reg.lowId());
	1823	encoder.sib_base(src_reg.lowId());
	1824	} else if (immOpSize(src_mem.disp) == 8) {
	1825	encoder.modRm_SIBDisp8(reg.lowId());
	1826	encoder.sib_baseDisp8(src_reg.lowId());
	1827	encoder.disp8(@intCast(i8, src_mem.disp));
	1828	} else {
	1829	encoder.modRm_SIBDisp32(reg.lowId());
	1830	encoder.sib_baseDisp32(src_reg.lowId());
	1831	encoder.disp32(src_mem.disp);
	1832	}
	1833	} else {
	1834	if (src_mem.disp == 0) {
	1835	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
	1836	} else if (immOpSize(src_mem.disp) == 8) {
	1837	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
	1838	encoder.disp8(@intCast(i8, src_mem.disp));
	1839	} else {
	1840	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
	1841	encoder.disp32(src_mem.disp);
	1842	}
	1843	}
	1844	} else {
	1845	encoder.rex(.{
	1846	.w = reg.size() == 64,
	1847	.r = reg.isExtended(),
	1848	});
	1849	opc.encode(encoder);
	1850	if (src_mem.rip) {
	1851	encoder.modRm_RIPDisp32(reg.lowId());
	1852	} else {
	1853	encoder.modRm_SIBDisp0(reg.lowId());
	1854	encoder.sib_disp32();
	1855	}
	1856	encoder.disp32(src_mem.disp);
1681	1857	}
1682	1858	},
1683		else => {},
1684	1859	}
1685		}
1686
1687		/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
1688		/// after codegen for this symbol is done.
1689		fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
1690		switch (emit.debug_output) {
1691		.dwarf => \|dbg_out\| {
1692		assert(ty.hasCodeGenBits());
1693		const index = dbg_out.dbg_info.items.len;
1694		try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
1695
1696		const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
1697		if (!gop.found_existing) {
1698		gop.value_ptr.* = .{
1699		.off = undefined,
1700		.relocs = .{},
1701		};
1702		}
1703		try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
	1860	switch (reg.size()) {
	1861	// TODO 8bit immediate
	1862	8 => unreachable,
	1863	16 => {
	1864	const imm16 = try math.cast(i16, imm);
	1865	encoder.imm16(imm16);
1704	1866	},
1705		.plan9 => {},
1706		.none => {},
	1867	32, 64 => encoder.imm32(imm),
	1868	else => unreachable,
1707	1869	}
1708	1870	}
1709	1871
...	...	@@ -1757,34 +1919,44 @@ test "lower MI encoding" {
1757	1919	defer code.deinit();
1758	1920	try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, code.buffer());
1759	1921	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", code.emitted(), "mov rax, 0x10");
1760		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0), 0x10, code.buffer());
	1922	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0, .dword_ptr), 0x10, code.buffer());
1761	1923	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", code.emitted(), "mov dword ptr [r11 + 0], 0x10");
1762		try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8), 0x10, code.buffer());
	1924	try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8, .dword_ptr), 0x10, code.buffer());
1763	1925	try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", code.emitted(), "add dword ptr [rdx - 8], 0x10");
1764		try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), 0x10, code.buffer());
	1926	try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .dword_ptr), 0x10, code.buffer());
1765	1927	try expectEqualHexStrings(
1766	1928	"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1767	1929	code.emitted(),
1768	1930	"sub dword ptr [r11 + 0x10000000], 0x10",
1769	1931	);
1770		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000), 0x10, code.buffer());
	1932	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), 0x10, code.buffer());
1771	1933	try expectEqualHexStrings(
1772	1934	"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1773	1935	code.emitted(),
1774	1936	"and dword ptr [ds:0x10000000], 0x10",
1775	1937	);
1776		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000), 0x10, code.buffer());
	1938	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000, .dword_ptr), 0x10, code.buffer());
1777	1939	try expectEqualHexStrings(
1778	1940	"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1779	1941	code.emitted(),
1780	1942	"and dword ptr [r12 + 0x10000000], 0x10",
1781	1943	);
1782		try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10), 0x10, code.buffer());
	1944	try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), 0x10, code.buffer());
1783	1945	try expectEqualHexStrings(
1784	1946	"\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1785	1947	code.emitted(),
1786		"mov [rip + 0x10], 0x10",
	1948	"mov qword ptr [rip + 0x10], 0x10",
	1949	);
	1950	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, code.buffer());
	1951	try expectEqualHexStrings(
	1952	"\x48\xc7\x45\xf8\x10\x00\x00\x00",
	1953	code.emitted(),
	1954	"mov qword ptr [rbp - 8], 0x10",
1787	1955	);
	1956	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, code.buffer());
	1957	try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", code.emitted(), "mov word ptr [rbp - 2], 0x10");
	1958	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -1, .byte_ptr), 0x10, code.buffer());
	1959	try expectEqualHexStrings("\xC6\x45\xFF\x10", code.emitted(), "mov byte ptr [rbp - 1], 0x10");
1788	1960	}
1789	1961
1790	1962	test "lower RM encoding" {
...	...	@@ -1792,35 +1964,35 @@ test "lower RM encoding" {
1792	1964	defer code.deinit();
1793	1965	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), code.buffer());
1794	1966	try expectEqualHexStrings("\x48\x8b\xc3", code.emitted(), "mov rax, rbx");
1795		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0), code.buffer());
	1967	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0, .qword_ptr), code.buffer());
1796	1968	try expectEqualHexStrings("\x49\x8b\x03", code.emitted(), "mov rax, qword ptr [r11 + 0]");
1797		try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000), code.buffer());
	1969	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000, .qword_ptr), code.buffer());
1798	1970	try expectEqualHexStrings(
1799	1971	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
1800	1972	code.emitted(),
1801	1973	"add r11, qword ptr [ds:0x10000000]",
1802	1974	);
1803		try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000), code.buffer());
	1975	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), code.buffer());
1804	1976	try expectEqualHexStrings(
1805	1977	"\x44\x02\x24\x25\x00\x00\x00\x10",
1806	1978	code.emitted(),
1807	1979	"add r11b, byte ptr [ds:0x10000000]",
1808	1980	);
1809		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000), code.buffer());
	1981	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000, .qword_ptr), code.buffer());
1810	1982	try expectEqualHexStrings(
1811	1983	"\x4D\x2B\x9D\x00\x00\x00\x10",
1812	1984	code.emitted(),
1813	1985	"sub r11, qword ptr [r13 + 0x10000000]",
1814	1986	);
1815		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000), code.buffer());
	1987	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000, .qword_ptr), code.buffer());
1816	1988	try expectEqualHexStrings(
1817	1989	"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1818	1990	code.emitted(),
1819	1991	"sub r11, qword ptr [r12 + 0x10000000]",
1820	1992	);
1821		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4), code.buffer());
	1993	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), code.buffer());
1822	1994	try expectEqualHexStrings("\x48\x8B\x45\xFC", code.emitted(), "mov rax, qword ptr [rbp - 4]");
1823		try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10), code.buffer());
	1995	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10, .qword_ptr), code.buffer());
1824	1996	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", code.emitted(), "lea rax, [rip + 0x10]");
1825	1997	}
1826	1998
...	...	@@ -1829,27 +2001,27 @@ test "lower MR encoding" {
1829	2001	defer code.deinit();
1830	2002	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, code.buffer());
1831	2003	try expectEqualHexStrings("\x48\x89\xd8", code.emitted(), "mov rax, rbx");
1832		try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4), .r11, code.buffer());
	2004	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), .r11, code.buffer());
1833	2005	try expectEqualHexStrings("\x4c\x89\x5d\xfc", code.emitted(), "mov qword ptr [rbp - 4], r11");
1834		try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12b, code.buffer());
	2006	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), .r12b, code.buffer());
1835	2007	try expectEqualHexStrings(
1836	2008	"\x44\x00\x24\x25\x00\x00\x00\x10",
1837	2009	code.emitted(),
1838	2010	"add byte ptr [ds:0x10000000], r12b",
1839	2011	);
1840		try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12d, code.buffer());
	2012	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), .r12d, code.buffer());
1841	2013	try expectEqualHexStrings(
1842	2014	"\x44\x01\x24\x25\x00\x00\x00\x10",
1843	2015	code.emitted(),
1844	2016	"add dword ptr [ds:0x10000000], r12d",
1845	2017	);
1846		try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), .r12, code.buffer());
	2018	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .qword_ptr), .r12, code.buffer());
1847	2019	try expectEqualHexStrings(
1848	2020	"\x4D\x29\xA3\x00\x00\x00\x10",
1849	2021	code.emitted(),
1850	2022	"sub qword ptr [r11 + 0x10000000], r12",
1851	2023	);
1852		try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10), .r12, code.buffer());
	2024	try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), .r12, code.buffer());
1853	2025	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", code.emitted(), "mov qword ptr [rip + 0x10], r12");
1854	2026	}
1855	2027
...	...	@@ -1898,19 +2070,23 @@ test "lower M encoding" {
1898	2070	defer code.deinit();
1899	2071	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), code.buffer());
1900	2072	try expectEqualHexStrings("\x41\xFF\xE4", code.emitted(), "jmp r12");
1901		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0), code.buffer());
	2073	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), code.buffer());
	2074	try expectEqualHexStrings("\x66\x41\xFF\xE4", code.emitted(), "jmp r12w");
	2075	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .qword_ptr), code.buffer());
1902	2076	try expectEqualHexStrings("\x41\xFF\x24\x24", code.emitted(), "jmp qword ptr [r12]");
1903		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10), code.buffer());
	2077	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .word_ptr), code.buffer());
	2078	try expectEqualHexStrings("\x66\x41\xFF\x24\x24", code.emitted(), "jmp word ptr [r12]");
	2079	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10, .qword_ptr), code.buffer());
1904	2080	try expectEqualHexStrings("\x41\xFF\x64\x24\x10", code.emitted(), "jmp qword ptr [r12 + 0x10]");
1905		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000), code.buffer());
	2081	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000, .qword_ptr), code.buffer());
1906	2082	try expectEqualHexStrings(
1907	2083	"\x41\xFF\xA4\x24\x00\x10\x00\x00",
1908	2084	code.emitted(),
1909	2085	"jmp qword ptr [r12 + 0x1000]",
1910	2086	);
1911		try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10), code.buffer());
	2087	try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10, .qword_ptr), code.buffer());
1912	2088	try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [rip + 0x10]");
1913		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10), code.buffer());
	2089	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10, .qword_ptr), code.buffer());
1914	2090	try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [ds:0x10]");
1915	2091	try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), code.buffer());
1916	2092	try expectEqualHexStrings("\x49\x0F\x97\xC3", code.emitted(), "seta r11b");
...	...	@@ -1928,8 +2104,18 @@ test "lower O encoding" {
1928	2104	test "lower RMI encoding" {
1929	2105	var code = TestEmitCode.init();
1930	2106	defer code.deinit();
1931		try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8), 0x10, code.buffer());
1932		try expectEqualHexStrings("\x48\x69\x45\xF8\x10\x00\x00\x00", code.emitted(), "imul rax, [rbp - 8], 0x10");
	2107	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, code.buffer());
	2108	try expectEqualHexStrings(
	2109	"\x48\x69\x45\xF8\x10\x00\x00\x00",
	2110	code.emitted(),
	2111	"imul rax, qword ptr [rbp - 8], 0x10",
	2112	);
	2113	try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.rbp, -4, .dword_ptr), 0x10, code.buffer());
	2114	try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", code.emitted(), "imul ax, [rbp - 2], 0x10");
	2115	try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, code.buffer());
	2116	try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", code.emitted(), "imul eax, [rbp - 4], 0x10");
1933	2117	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, code.buffer());
1934	2118	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", code.emitted(), "imul r12, r12, 0x10");
	2119	try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, code.buffer());
	2120	try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", code.emitted(), "imul r12w, r12w, 0x10");
1935	2121	}

src/arch/x86_64/Mir.zig+32-23

...	...	@@ -38,13 +38,18 @@ pub const Inst = struct {
38	38	/// 0b01 reg1, [reg2 + imm32]
39	39	/// 0b01 reg1, [ds:imm32]
40	40	/// 0b10 [reg1 + imm32], reg2
41		/// 0b10 [reg1 + 0], imm32
42		/// 0b11 [reg1 + imm32], imm32
43	41	/// Notes:
44	42	/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
45	43	/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
46	44	adc,
47	45
	46	/// ops flags: form:
	47	/// 0b00 byte ptr [reg1 + imm32], imm8
	48	/// 0b01 word ptr [reg1 + imm32], imm16
	49	/// 0b10 dword ptr [reg1 + imm32], imm32
	50	/// 0b11 qword ptr [reg1 + imm32], imm32 (sign-extended to imm64)
	51	adc_mem_imm,
	52
48	53	/// form: reg1, [reg2 + scale*rcx + imm32]
49	54	/// ops flags scale
50	55	/// 0b00 1
...	...	@@ -77,74 +82,95 @@ pub const Inst = struct {
77	82	// The following instructions all have the same encoding as `adc`.
78	83
79	84	add,
	85	add_mem_imm,
80	86	add_scale_src,
81	87	add_scale_dst,
82	88	add_scale_imm,
83	89	sub,
	90	sub_mem_imm,
84	91	sub_scale_src,
85	92	sub_scale_dst,
86	93	sub_scale_imm,
87	94	xor,
	95	xor_mem_imm,
88	96	xor_scale_src,
89	97	xor_scale_dst,
90	98	xor_scale_imm,
91	99	@"and",
	100	and_mem_imm,
92	101	and_scale_src,
93	102	and_scale_dst,
94	103	and_scale_imm,
95	104	@"or",
	105	or_mem_imm,
96	106	or_scale_src,
97	107	or_scale_dst,
98	108	or_scale_imm,
99	109	rol,
	110	rol_mem_imm,
100	111	rol_scale_src,
101	112	rol_scale_dst,
102	113	rol_scale_imm,
103	114	ror,
	115	ror_mem_imm,
104	116	ror_scale_src,
105	117	ror_scale_dst,
106	118	ror_scale_imm,
107	119	rcl,
	120	rcl_mem_imm,
108	121	rcl_scale_src,
109	122	rcl_scale_dst,
110	123	rcl_scale_imm,
111	124	rcr,
	125	rcr_mem_imm,
112	126	rcr_scale_src,
113	127	rcr_scale_dst,
114	128	rcr_scale_imm,
115	129	shl,
	130	shl_mem_imm,
116	131	shl_scale_src,
117	132	shl_scale_dst,
118	133	shl_scale_imm,
119	134	sal,
	135	sal_mem_imm,
120	136	sal_scale_src,
121	137	sal_scale_dst,
122	138	sal_scale_imm,
123	139	shr,
	140	shr_mem_imm,
124	141	shr_scale_src,
125	142	shr_scale_dst,
126	143	shr_scale_imm,
127	144	sar,
	145	sar_mem_imm,
128	146	sar_scale_src,
129	147	sar_scale_dst,
130	148	sar_scale_imm,
131	149	sbb,
	150	sbb_mem_imm,
132	151	sbb_scale_src,
133	152	sbb_scale_dst,
134	153	sbb_scale_imm,
135	154	cmp,
	155	cmp_mem_imm,
136	156	cmp_scale_src,
137	157	cmp_scale_dst,
138	158	cmp_scale_imm,
139	159	mov,
	160	mov_mem_imm,
140	161	mov_scale_src,
141	162	mov_scale_dst,
142	163	mov_scale_imm,
143	164
	165	/// ops flags: form:
	166	/// 0b00 reg1, [reg2 + imm32]
	167	/// 0b00 reg1, [ds:imm32]
	168	/// 0b01 reg1, [rip + imm32]
	169	/// 0b10 reg1, [rip + reloc]
	170	/// Notes:
	171	/// * if flags are 0b10, `Data` contains `got_entry` for the linker to generate
	172	/// a valid relocation for.
144	173	lea,
145		lea_scale_src,
146		lea_scale_dst,
147		lea_scale_imm,
148	174
149	175	/// ops flags: form:
150	176	/// 0bX0 reg1
...	...	@@ -160,15 +186,6 @@ pub const Inst = struct {
160	186	/// 0b11 reg1, [reg2 + imm32], imm32
161	187	imul_complex,
162	188
163		/// ops flags: form:
164		/// 0bX0 reg1, [rip + imm32]
165		/// 0bX1 reg1, [rip + reloc]
166		/// Notes:
167		/// * if flags are 0bX1, `Data` contains `got_entry` for linker to generate
168		/// valid relocation for.
169		/// TODO handle more cases
170		lea_rip,
171
172	189	/// ops flags: form:
173	190	/// 0bX0 reg1, imm64
174	191	/// 0bX1 rax, moffs64
...	...	@@ -233,16 +250,8 @@ pub const Inst = struct {
233	250	syscall,
234	251
235	252	/// ops flags: form:
236		/// 0b00 reg1, reg2
237	253	/// 0b00 reg1, imm32
238		/// 0b01 reg1, [reg2 + imm32]
239		/// 0b01 reg1, [ds:imm32]
240		/// 0b10 [reg1 + imm32], reg2
241		/// 0b10 [reg1 + 0], imm32
242		/// 0b11 [reg1 + imm32], imm32
243		/// Notes:
244		/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
245		/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
	254	/// TODO handle more cases
246	255	@"test",
247	256
248	257	/// Breakpoint

src/arch/x86_64/PrintMir.zig+24-12

...	...	@@ -64,6 +64,15 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
64	64	.sbb => try print.mirArith(.sbb, inst, w),
65	65	.cmp => try print.mirArith(.cmp, inst, w),
66	66
	67	.adc_mem_imm => try print.mirArithMemImm(.adc, inst, w),
	68	.add_mem_imm => try print.mirArithMemImm(.add, inst, w),
	69	.sub_mem_imm => try print.mirArithMemImm(.sub, inst, w),
	70	.xor_mem_imm => try print.mirArithMemImm(.xor, inst, w),
	71	.and_mem_imm => try print.mirArithMemImm(.@"and", inst, w),
	72	.or_mem_imm => try print.mirArithMemImm(.@"or", inst, w),
	73	.sbb_mem_imm => try print.mirArithMemImm(.sbb, inst, w),
	74	.cmp_mem_imm => try print.mirArithMemImm(.cmp, inst, w),
	75
67	76	.adc_scale_src => try print.mirArithScaleSrc(.adc, inst, w),
68	77	.add_scale_src => try print.mirArithScaleSrc(.add, inst, w),
69	78	.sub_scale_src => try print.mirArithScaleSrc(.sub, inst, w),
...	...	@@ -98,7 +107,6 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
98	107	.movabs => try print.mirMovabs(inst, w),
99	108
100	109	.lea => try print.mirLea(inst, w),
101		.lea_rip => try print.mirLeaRip(inst, w),
102	110
103	111	.imul_complex => try print.mirIMulComplex(inst, w),
104	112
...	...	@@ -490,6 +498,13 @@ fn mirArith(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: any
490	498	try w.writeByte('\n');
491	499	}
492	500
	501	fn mirArithMemImm(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
	502	_ = print;
	503	_ = tag;
	504	_ = inst;
	505	return w.writeAll("TODO mirArithMemImm\n");
	506	}
	507
493	508	fn mirArithScaleSrc(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
494	509	const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
495	510	const scale = ops.flags;
...	...	@@ -560,19 +575,16 @@ fn mirIMulComplex(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
560	575	}
561	576
562	577	fn mirLea(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
563		const tag = print.mir.instructions.items(.tag)[inst];
564		assert(tag == .lea);
565		const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
566		assert(ops.flags == 0b01);
567		const imm = print.mir.instructions.items(.data)[inst].imm;
568
569		try w.print("lea {s} [{s} + {d}]\n", .{ @tagName(ops.reg1), @tagName(ops.reg2), imm });
570		}
571
572		fn mirLeaRip(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
573	578	_ = print;
574	579	_ = inst;
575		return w.writeAll("TODO lea_rip\n");
	580	return w.writeAll("TODO lea\n");
	581	// const tag = print.mir.instructions.items(.tag)[inst];
	582	// assert(tag == .lea);
	583	// const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
	584	// assert(ops.flags == 0b01);
	585	// const imm = print.mir.instructions.items(.data)[inst].imm;
	586
	587	// try w.print("lea {s} [{s} + {d}]\n", .{ @tagName(ops.reg1), @tagName(ops.reg2), imm });
576	588	}
577	589
578	590	fn mirCallExtern(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {