Commit 6740c1f0849dd2615859e4d65df355087165e073 | zig

author	David Rubin <david@vortan.dev> 2024-04-03 00:15:56-07:00
committer	David Rubin <david@vortan.dev> 2024-05-11 02:17:11-07:00
log	6740c1f0849dd2615859e4d65df355087165e073
tree	71e28e23a53dea1dd6a882a2c737ed1ad3a3c14c
parent	9d0bb6371df31dd25e86b7ef4161852740f39f07

riscv: big rewrite to use latest liveness

this one is even harder to document then the last large overhaul.

TLDR;
- split apart Emit.zig into an Emit.zig and a Lower.zig
- created seperate files for the encoding, and now adding a new instruction
is as simple as just adding it to a couple of switch statements and providing the encoding.
- relocs are handled in a more sane maner, and we have a clear defining boundary between
lea_symbol and load_symbol now.
- a lot of different abstractions for things like the stack, memory, registers, and others.
- we're using x86_64's FrameIndex now, which simplifies a lot of the tougher design process.
- a lot more that I don't have the energy to document. at this point, just read the commit itself :p

16 files changed, 3010 insertions(+), 2092 deletions(-)

lib
- compiler
  - test_runner.zig
- std
  - builtin.zig
  - start.zig
src
- arch
  - riscv64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    abi.zig
    bits.zig
    encoder.zig
- link
  - riscv.zig
- register_manager.zig
- target.zig
test
- behavior
  - align.zig
  - array.zig

lib/compiler/test_runner.zig+5-1

 @@ -252,12 +252,16 @@ pub fn mainSimple() anyerror!void {
 pub fn mainExtraSimple() !void {
     var pass_count: u8 = 0;
+    var skip_count: u8 = 0;
+    var fail_count: u8 = 0;
     for (builtin.test_functions) |test_fn| {
         test_fn.func() catch |err| {
             if (err != error.SkipZigTest) {
-                @panic(test_fn.name);
+                fail_count += 1;
+                continue;
             }
+            skip_count += 1;
             continue;
         };
         pass_count += 1;

lib/std/builtin.zig+1-9

...	@@ -775,15 +775,7 @@ pub fn default_panic(msg: []const u8, error_return_trace: ?*StackTrace, ret_addr	...	@@ -775,15 +775,7 @@ pub fn default_panic(msg: []const u8, error_return_trace: ?*StackTrace, ret_addr
775	}	775	}
776		776
777	if (builtin.zig_backend == .stage2_riscv64) {	777	if (builtin.zig_backend == .stage2_riscv64) {
778	asm volatile ("ecall"	778	unreachable;
779	:
780	: [number] "{a7}" (64),
781	[arg1] "{a0}" (1),
782	[arg2] "{a1}" (@intFromPtr(msg.ptr)),
783	[arg3] "{a2}" (msg.len),
784	: "rcx", "r11", "memory"
785	);
786	std.posix.exit(127);
787	}	779	}
788		780
789	switch (builtin.os.tag) {	781	switch (builtin.os.tag) {

lib/std/start.zig+1-2

...	@@ -208,8 +208,7 @@ fn wasi_start() callconv(.C) void {	...	@@ -208,8 +208,7 @@ fn wasi_start() callconv(.C) void {
208	}	208	}
209		209
210	fn riscv_start() callconv(.C) noreturn {	210	fn riscv_start() callconv(.C) noreturn {
211	const code = @call(.always_inline, callMain, .{});	211	std.process.exit(@call(.always_inline, callMain, .{}));
212	std.process.exit(code);
213	}	212	}
214		213
215	fn EfiMain(handle: uefi.Handle, system_table: *uefi.tables.SystemTable) callconv(.C) usize {	214	fn EfiMain(handle: uefi.Handle, system_table: *uefi.tables.SystemTable) callconv(.C) usize {

src/arch/riscv64/CodeGen.zig+1768-1000

...	@@ -19,7 +19,8 @@ const Allocator = mem.Allocator;	...	@@ -19,7 +19,8 @@ const Allocator = mem.Allocator;
19	const trace = @import("../../tracy.zig").trace;	19	const trace = @import("../../tracy.zig").trace;
20	const DW = std.dwarf;	20	const DW = std.dwarf;
21	const leb128 = std.leb;	21	const leb128 = std.leb;
22	const log = std.log.scoped(.codegen);	22	const log = std.log.scoped(.riscv_codegen);
		23	const tracking_log = std.log.scoped(.tracking);
23	const build_options = @import("build_options");	24	const build_options = @import("build_options");
24	const codegen = @import("../../codegen.zig");	25	const codegen = @import("../../codegen.zig");
25	const Alignment = InternPool.Alignment;	26	const Alignment = InternPool.Alignment;
...	@@ -31,6 +32,9 @@ const DebugInfoOutput = codegen.DebugInfoOutput;	...	@@ -31,6 +32,9 @@ const DebugInfoOutput = codegen.DebugInfoOutput;
31	const bits = @import("bits.zig");	32	const bits = @import("bits.zig");
32	const abi = @import("abi.zig");	33	const abi = @import("abi.zig");
33	const Register = bits.Register;	34	const Register = bits.Register;
		35	const Immediate = bits.Immediate;
		36	const Memory = bits.Memory;
		37	const FrameIndex = bits.FrameIndex;
34	const RegisterManager = abi.RegisterManager;	38	const RegisterManager = abi.RegisterManager;
35	const RegisterLock = RegisterManager.RegisterLock;	39	const RegisterLock = RegisterManager.RegisterLock;
36	const callee_preserved_regs = abi.callee_preserved_regs;	40	const callee_preserved_regs = abi.callee_preserved_regs;
...	@@ -58,11 +62,10 @@ code: *std.ArrayList(u8),	...	@@ -58,11 +62,10 @@ code: *std.ArrayList(u8),
58	debug_output: DebugInfoOutput,	62	debug_output: DebugInfoOutput,
59	err_msg: ?*ErrorMsg,	63	err_msg: ?*ErrorMsg,
60	args: []MCValue,	64	args: []MCValue,
61	ret_mcv: MCValue,	65	ret_mcv: InstTracking,
62	fn_type: Type,	66	fn_type: Type,
63	arg_index: usize,	67	arg_index: usize,
64	src_loc: Module.SrcLoc,	68	src_loc: Module.SrcLoc,
65	stack_align: Alignment,
66		69
67	/// MIR Instructions	70	/// MIR Instructions
68	mir_instructions: std.MultiArrayList(Mir.Inst) = .{},	71	mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
...	@@ -73,6 +76,8 @@ mir_extra: std.ArrayListUnmanaged(u32) = .{},	...	@@ -73,6 +76,8 @@ mir_extra: std.ArrayListUnmanaged(u32) = .{},
73	end_di_line: u32,	76	end_di_line: u32,
74	end_di_column: u32,	77	end_di_column: u32,
75		78
		79	scope_generation: u32,
		80
76	/// The value is an offset into the `Function` `code` from the beginning.	81	/// The value is an offset into the `Function` `code` from the beginning.
77	/// To perform the reloc, write 32-bit signed little-endian integer	82	/// To perform the reloc, write 32-bit signed little-endian integer
78	/// which is a relative jump, based on the address following the reloc.	83	/// which is a relative jump, based on the address following the reloc.
...	@@ -91,14 +96,12 @@ branch_stack: *std.ArrayList(Branch),	...	@@ -91,14 +96,12 @@ branch_stack: *std.ArrayList(Branch),
91	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},	96	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
92	register_manager: RegisterManager = .{},	97	register_manager: RegisterManager = .{},
93		98
94	/// Maps offset to what is stored there.	99	const_tracking: ConstTrackingMap = .{},
95	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},	100	inst_tracking: InstTrackingMap = .{},
96		101
97	/// Offset from the stack base, representing the end of the stack frame.	102	frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
98	max_end_stack: u32 = 0,	103	free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .{},
99	/// Represents the current end stack offset. If there is no existing slot	104	frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
100	/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
101	next_stack_offset: u32 = 0,
102		105
103	/// Debug field, used to find bugs in the compiler.	106	/// Debug field, used to find bugs in the compiler.
104	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,	107	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
...	@@ -107,6 +110,7 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}	...	@@ -107,6 +110,7 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}
107		110
108	const SymbolOffset = struct { sym: u32, off: i32 = 0 };	111	const SymbolOffset = struct { sym: u32, off: i32 = 0 };
109	const RegisterOffset = struct { reg: Register, off: i32 = 0 };	112	const RegisterOffset = struct { reg: Register, off: i32 = 0 };
		113	pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
110		114
111	const MCValue = union(enum) {	115	const MCValue = union(enum) {
112	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.	116	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
...	@@ -116,7 +120,8 @@ const MCValue = union(enum) {	...	@@ -116,7 +120,8 @@ const MCValue = union(enum) {
116	/// Control flow will not allow this value to be observed.	120	/// Control flow will not allow this value to be observed.
117	unreach,	121	unreach,
118	/// No more references to this value remain.	122	/// No more references to this value remain.
119	dead,	123	/// The payload is the value of scope_generation at the point where the death occurred
		124	dead: u32,
120	/// The value is undefined.	125	/// The value is undefined.
121	undef,	126	undef,
122	/// A pointer-sized integer that fits in a register.	127	/// A pointer-sized integer that fits in a register.
...	@@ -125,7 +130,7 @@ const MCValue = union(enum) {	...	@@ -125,7 +130,7 @@ const MCValue = union(enum) {
125	/// The value doesn't exist in memory yet.	130	/// The value doesn't exist in memory yet.
126	load_symbol: SymbolOffset,	131	load_symbol: SymbolOffset,
127	/// The address of the memory location not-yet-allocated by the linker.	132	/// The address of the memory location not-yet-allocated by the linker.
128	addr_symbol: SymbolOffset,	133	lea_symbol: SymbolOffset,
129	/// The value is in a target-specific register.	134	/// The value is in a target-specific register.
130	register: Register,	135	register: Register,
131	/// The value is split across two registers	136	/// The value is split across two registers
...	@@ -133,16 +138,21 @@ const MCValue = union(enum) {	...	@@ -133,16 +138,21 @@ const MCValue = union(enum) {
133	/// The value is in memory at a hard-coded address.	138	/// The value is in memory at a hard-coded address.
134	/// If the type is a pointer, it means the pointer address is at this memory location.	139	/// If the type is a pointer, it means the pointer address is at this memory location.
135	memory: u64,	140	memory: u64,
136	/// The value is one of the stack variables.	141	/// The value stored at an offset from a frame index
137	/// If the type is a pointer, it means the pointer address is in the stack at this offset.	142	/// Payload is a frame address.
138	stack_offset: u32,	143	load_frame: FrameAddr,
139	/// The value is a pointer to one of the stack variables (payload is stack offset).	144	/// The address of an offset from a frame index
140	ptr_stack_offset: u32,	145	/// Payload is a frame address.
		146	lea_frame: FrameAddr,
141	air_ref: Air.Inst.Ref,	147	air_ref: Air.Inst.Ref,
142	/// The value is in memory at a constant offset from the address in a register.	148	/// The value is in memory at a constant offset from the address in a register.
143	indirect: RegisterOffset,	149	indirect: RegisterOffset,
144	/// The value is a constant offset from the value in a register.	150	/// The value is a constant offset from the value in a register.
145	register_offset: RegisterOffset,	151	register_offset: RegisterOffset,
		152	/// This indicates that we have already allocated a frame index for this instruction,
		153	/// but it has not been spilled there yet in the current control flow.
		154	/// Payload is a frame index.
		155	reserved_frame: FrameIndex,
146		156
147	fn isMemory(mcv: MCValue) bool {	157	fn isMemory(mcv: MCValue) bool {
148	return switch (mcv) {	158	return switch (mcv) {
...	@@ -166,16 +176,17 @@ const MCValue = union(enum) {	...	@@ -166,16 +176,17 @@ const MCValue = union(enum) {
166		176
167	.immediate,	177	.immediate,
168	.memory,	178	.memory,
169	.ptr_stack_offset,	179	.lea_frame,
170	.undef,	180	.undef,
171	.addr_symbol,	181	.lea_symbol,
172	.air_ref,	182	.air_ref,
		183	.reserved_frame,
173	=> false,	184	=> false,
174		185
175	.register,	186	.register,
176	.register_pair,	187	.register_pair,
177	.register_offset,	188	.register_offset,
178	.stack_offset,	189	.load_frame,
179	.load_symbol,	190	.load_symbol,
180	.indirect,	191	.indirect,
181	=> true,	192	=> true,
...	@@ -188,18 +199,19 @@ const MCValue = union(enum) {	...	@@ -188,18 +199,19 @@ const MCValue = union(enum) {
188	.unreach,	199	.unreach,
189	.dead,	200	.dead,
190	.immediate,	201	.immediate,
191	.ptr_stack_offset,	202	.lea_frame,
192	.register_offset,	203	.register_offset,
193	.register_pair,	204	.register_pair,
194	.register,	205	.register,
195	.undef,	206	.undef,
196	.air_ref,	207	.air_ref,
197	.addr_symbol,	208	.lea_symbol,
		209	.reserved_frame,
198	=> unreachable, // not in memory	210	=> unreachable, // not in memory
199		211
200	.load_symbol => \|sym_off\| .{ .addr_symbol = sym_off },	212	.load_symbol => \|sym_off\| .{ .lea_symbol = sym_off },
201	.memory => \|addr\| .{ .immediate = addr },	213	.memory => \|addr\| .{ .immediate = addr },
202	.stack_offset => \|off\| .{ .ptr_stack_offset = off },	214	.load_frame => \|off\| .{ .lea_frame = off },
203	.indirect => \|reg_off\| switch (reg_off.off) {	215	.indirect => \|reg_off\| switch (reg_off.off) {
204	0 => .{ .register = reg_off.reg },	216	0 => .{ .register = reg_off.reg },
205	else => .{ .register_offset = reg_off },	217	else => .{ .register_offset = reg_off },
...	@@ -216,16 +228,17 @@ const MCValue = union(enum) {	...	@@ -216,16 +228,17 @@ const MCValue = union(enum) {
216	.indirect,	228	.indirect,
217	.undef,	229	.undef,
218	.air_ref,	230	.air_ref,
219	.stack_offset,	231	.load_frame,
220	.register_pair,	232	.register_pair,
221	.load_symbol,	233	.load_symbol,
		234	.reserved_frame,
222	=> unreachable, // not a pointer	235	=> unreachable, // not a pointer
223		236
224	.immediate => \|addr\| .{ .memory = addr },	237	.immediate => \|addr\| .{ .memory = addr },
225	.ptr_stack_offset => \|off\| .{ .stack_offset = off },	238	.lea_frame => \|off\| .{ .load_frame = off },
226	.register => \|reg\| .{ .indirect = .{ .reg = reg } },	239	.register => \|reg\| .{ .indirect = .{ .reg = reg } },
227	.register_offset => \|reg_off\| .{ .indirect = reg_off },	240	.register_offset => \|reg_off\| .{ .indirect = reg_off },
228	.addr_symbol => \|sym_off\| .{ .load_symbol = sym_off },	241	.lea_symbol => \|sym_off\| .{ .load_symbol = sym_off },
229	};	242	};
230	}	243	}
231		244
...	@@ -236,13 +249,14 @@ const MCValue = union(enum) {	...	@@ -236,13 +249,14 @@ const MCValue = union(enum) {
236	.dead,	249	.dead,
237	.undef,	250	.undef,
238	.air_ref,	251	.air_ref,
		252	.reserved_frame,
239	=> unreachable, // not valid	253	=> unreachable, // not valid
240	.register_pair,	254	.register_pair,
241	.memory,	255	.memory,
242	.indirect,	256	.indirect,
243	.stack_offset,	257	.load_frame,
244	.load_symbol,	258	.load_symbol,
245	.addr_symbol,	259	.lea_symbol,
246	=> switch (off) {	260	=> switch (off) {
247	0 => mcv,	261	0 => mcv,
248	else => unreachable, // not offsettable	262	else => unreachable, // not offsettable
...	@@ -250,7 +264,26 @@ const MCValue = union(enum) {	...	@@ -250,7 +264,26 @@ const MCValue = union(enum) {
250	.immediate => \|imm\| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) },	264	.immediate => \|imm\| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) },
251	.register => \|reg\| .{ .register_offset = .{ .reg = reg, .off = off } },	265	.register => \|reg\| .{ .register_offset = .{ .reg = reg, .off = off } },
252	.register_offset => \|reg_off\| .{ .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off } },	266	.register_offset => \|reg_off\| .{ .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off } },
253	.ptr_stack_offset => \|stack_off\| .{ .ptr_stack_offset = @intCast(@as(i64, @intCast(stack_off)) +% off) },	267	.lea_frame => \|frame_addr\| .{
		268	.lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
		269	},
		270	};
		271	}
		272
		273	fn getReg(mcv: MCValue) ?Register {
		274	return switch (mcv) {
		275	.register => \|reg\| reg,
		276	.register_offset, .indirect => \|ro\| ro.reg,
		277	else => null,
		278	};
		279	}
		280
		281	fn getRegs(mcv: *const MCValue) []const Register {
		282	return switch (mcv.*) {
		283	.register => \|reg\| @as(const [1]Register, reg),
		284	.register_pair => \|*regs\| regs,
		285	.register_offset, .indirect => \|ro\| @as(const [1]Register, &ro.reg),
		286	else => &.{},
254	};	287	};
255	}	288	}
256	};	289	};
...	@@ -264,6 +297,265 @@ const Branch = struct {	...	@@ -264,6 +297,265 @@ const Branch = struct {
264	}	297	}
265	};	298	};
266		299
		300	const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
		301	const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking);
		302	const InstTracking = struct {
		303	long: MCValue,
		304	short: MCValue,
		305
		306	fn init(result: MCValue) InstTracking {
		307	return .{ .long = switch (result) {
		308	.none,
		309	.unreach,
		310	.undef,
		311	.immediate,
		312	.memory,
		313	.load_frame,
		314	.lea_frame,
		315	.load_symbol,
		316	.lea_symbol,
		317	=> result,
		318	.dead,
		319	.reserved_frame,
		320	.air_ref,
		321	=> unreachable,
		322	.register,
		323	.register_pair,
		324	.register_offset,
		325	.indirect,
		326	=> .none,
		327	}, .short = result };
		328	}
		329
		330	fn getReg(self: InstTracking) ?Register {
		331	return self.short.getReg();
		332	}
		333
		334	fn getRegs(self: *const InstTracking) []const Register {
		335	return self.short.getRegs();
		336	}
		337
		338	fn spill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
		339	if (std.meta.eql(self.long, self.short)) return; // Already spilled
		340	// Allocate or reuse frame index
		341	switch (self.long) {
		342	.none => self.long = try function.allocRegOrMem(inst, false),
		343	.load_frame => {},
		344	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
		345	else => unreachable,
		346	}
		347	tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
		348	try function.genCopy(function.typeOfIndex(inst), self.long, self.short);
		349	}
		350
		351	fn reuseFrame(self: *InstTracking) void {
		352	switch (self.long) {
		353	.reserved_frame => \|index\| self.long = .{ .load_frame = .{ .index = index } },
		354	else => {},
		355	}
		356	self.short = switch (self.long) {
		357	.none,
		358	.unreach,
		359	.undef,
		360	.immediate,
		361	.memory,
		362	.load_frame,
		363	.lea_frame,
		364	.load_symbol,
		365	.lea_symbol,
		366	=> self.long,
		367	.dead,
		368	.register,
		369	.register_pair,
		370	.register_offset,
		371	.indirect,
		372	.reserved_frame,
		373	.air_ref,
		374	=> unreachable,
		375	};
		376	}
		377
		378	fn trackSpill(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
		379	try function.freeValue(self.short);
		380	self.reuseFrame();
		381	tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
		382	}
		383
		384	fn verifyMaterialize(self: InstTracking, target: InstTracking) void {
		385	switch (self.long) {
		386	.none,
		387	.unreach,
		388	.undef,
		389	.immediate,
		390	.memory,
		391	.lea_frame,
		392	.load_symbol,
		393	.lea_symbol,
		394	=> assert(std.meta.eql(self.long, target.long)),
		395	.load_frame,
		396	.reserved_frame,
		397	=> switch (target.long) {
		398	.none,
		399	.load_frame,
		400	.reserved_frame,
		401	=> {},
		402	else => unreachable,
		403	},
		404	.dead,
		405	.register,
		406	.register_pair,
		407	.register_offset,
		408	.indirect,
		409	.air_ref,
		410	=> unreachable,
		411	}
		412	}
		413
		414	fn materialize(
		415	self: *InstTracking,
		416	function: *Self,
		417	inst: Air.Inst.Index,
		418	target: InstTracking,
		419	) !void {
		420	self.verifyMaterialize(target);
		421	try self.materializeUnsafe(function, inst, target);
		422	}
		423
		424	fn materializeUnsafe(
		425	self: InstTracking,
		426	function: *Self,
		427	inst: Air.Inst.Index,
		428	target: InstTracking,
		429	) !void {
		430	const ty = function.typeOfIndex(inst);
		431	if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame)
		432	try function.genCopy(ty, target.long, self.short);
		433	try function.genCopy(ty, target.short, self.short);
		434	}
		435
		436	fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
		437	self.verifyMaterialize(target);
		438	// Don't clobber reserved frame indices
		439	self.long = if (target.long == .none) switch (self.long) {
		440	.load_frame => \|addr\| .{ .reserved_frame = addr.index },
		441	.reserved_frame => self.long,
		442	else => target.long,
		443	} else target.long;
		444	self.short = target.short;
		445	tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
		446	}
		447
		448	fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
		449	switch (self.short) {
		450	.dead => \|die_generation\| if (die_generation >= scope_generation) {
		451	self.reuseFrame();
		452	tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
		453	},
		454	else => {},
		455	}
		456	}
		457
		458	fn die(self: InstTracking, function: Self, inst: Air.Inst.Index) !void {
		459	if (self.short == .dead) return;
		460	try function.freeValue(self.short);
		461	self.short = .{ .dead = function.scope_generation };
		462	tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
		463	}
		464
		465	fn reuse(
		466	self: *InstTracking,
		467	function: *Self,
		468	new_inst: ?Air.Inst.Index,
		469	old_inst: Air.Inst.Index,
		470	) void {
		471	self.short = .{ .dead = function.scope_generation };
		472	if (new_inst) \|inst\|
		473	tracking_log.debug("%{d} => {} (reuse %{d})", .{ inst, self.*, old_inst })
		474	else
		475	tracking_log.debug("tmp => {} (reuse %{d})", .{ self.*, old_inst });
		476	}
		477
		478	fn liveOut(self: InstTracking, function: Self, inst: Air.Inst.Index) void {
		479	for (self.getRegs()) \|reg\| {
		480	if (function.register_manager.isRegFree(reg)) {
		481	tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* });
		482	continue;
		483	}
		484
		485	const index = RegisterManager.indexOfRegIntoTracked(reg).?;
		486	const tracked_inst = function.register_manager.registers[index];
		487	const tracking = function.getResolvedInstValue(tracked_inst);
		488
		489	// Disable death.
		490	var found_reg = false;
		491	var remaining_reg: Register = .zero;
		492	for (tracking.getRegs()) \|tracked_reg\| if (tracked_reg.id() == reg.id()) {
		493	assert(!found_reg);
		494	found_reg = true;
		495	} else {
		496	assert(remaining_reg == .zero);
		497	remaining_reg = tracked_reg;
		498	};
		499	assert(found_reg);
		500	tracking.short = switch (remaining_reg) {
		501	.zero => .{ .dead = function.scope_generation },
		502	else => .{ .register = remaining_reg },
		503	};
		504
		505	// Perform side-effects of freeValue manually.
		506	function.register_manager.freeReg(reg);
		507
		508	tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst });
		509	}
		510	}
		511
		512	pub fn format(
		513	self: InstTracking,
		514	comptime _: []const u8,
		515	_: std.fmt.FormatOptions,
		516	writer: anytype,
		517	) @TypeOf(writer).Error!void {
		518	if (!std.meta.eql(self.long, self.short)) try writer.print("\|{}\| ", .{self.long});
		519	try writer.print("{}", .{self.short});
		520	}
		521	};
		522
		523	const FrameAlloc = struct {
		524	abi_size: u31,
		525	spill_pad: u3,
		526	abi_align: Alignment,
		527	ref_count: u16,
		528
		529	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
		530	return .{
		531	.abi_size = @intCast(alloc_abi.size),
		532	.spill_pad = alloc_abi.pad,
		533	.abi_align = alloc_abi.alignment,
		534	.ref_count = 0,
		535	};
		536	}
		537	fn initType(ty: Type, zcu: *Module) FrameAlloc {
		538	return init(.{
		539	.size = ty.abiSize(zcu),
		540	.alignment = ty.abiAlignment(zcu),
		541	});
		542	}
		543	fn initSpill(ty: Type, zcu: *Module) FrameAlloc {
		544	const abi_size = ty.abiSize(zcu);
		545	const spill_size = if (abi_size < 8)
		546	math.ceilPowerOfTwoAssert(u64, abi_size)
		547	else
		548	std.mem.alignForward(u64, abi_size, 8);
		549	return init(.{
		550	.size = spill_size,
		551	.pad = @intCast(spill_size - abi_size),
		552	.alignment = ty.abiAlignment(zcu).maxStrict(
		553	Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
		554	),
		555	});
		556	}
		557	};
		558
267	const StackAllocation = struct {	559	const StackAllocation = struct {
268	inst: ?Air.Inst.Index,	560	inst: ?Air.Inst.Index,
269	/// TODO: make the size inferred from the bits of the inst	561	/// TODO: make the size inferred from the bits of the inst
...	@@ -271,36 +563,127 @@ const StackAllocation = struct {	...	@@ -271,36 +563,127 @@ const StackAllocation = struct {
271	};	563	};
272		564
273	const BlockData = struct {	565	const BlockData = struct {
274	relocs: std.ArrayListUnmanaged(Mir.Inst.Index),	566	relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
275	/// The first break instruction encounters `null` here and chooses a	567	state: State,
276	/// machine code value for the block result, populating this field.	568
277	/// Following break instructions encounter that value and use it for	569	fn deinit(self: *BlockData, gpa: Allocator) void {
278	/// the location to store their block results.	570	self.relocs.deinit(gpa);
279	mcv: MCValue,	571	self.* = undefined;
		572	}
280	};	573	};
281		574
282	const BigTomb = struct {	575	const State = struct {
283	function: *Self,	576	registers: RegisterManager.TrackedRegisters,
284	inst: Air.Inst.Index,	577	reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
285	lbt: Liveness.BigTomb,	578	free_registers: RegisterManager.RegisterBitSet,
		579	inst_tracking_len: u32,
		580	scope_generation: u32,
		581	};
		582
		583	fn initRetroactiveState(self: *Self) State {
		584	var state: State = undefined;
		585	state.inst_tracking_len = @intCast(self.inst_tracking.count());
		586	state.scope_generation = self.scope_generation;
		587	return state;
		588	}
286		589
287	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {	590	fn saveRetroactiveState(self: Self, state: State) !void {
288	const dies = bt.lbt.feed();	591	const free_registers = self.register_manager.free_registers;
289	const op_index = op_ref.toIndex() orelse return;	592	var it = free_registers.iterator(.{ .kind = .unset });
290	if (!dies) return;	593	while (it.next()) \|index\| {
291	bt.function.processDeath(op_index);	594	const tracked_inst = self.register_manager.registers[index];
		595	state.registers[index] = tracked_inst;
		596	state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?;
292	}	597	}
		598	state.free_registers = free_registers;
		599	}
		600
		601	fn saveState(self: *Self) !State {
		602	var state = self.initRetroactiveState();
		603	try self.saveRetroactiveState(&state);
		604	return state;
		605	}
		606
		607	fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
		608	emit_instructions: bool,
		609	update_tracking: bool,
		610	resurrect: bool,
		611	close_scope: bool,
		612	}) !void {
		613	if (opts.close_scope) {
		614	for (
		615	self.inst_tracking.keys()[state.inst_tracking_len..],
		616	self.inst_tracking.values()[state.inst_tracking_len..],
		617	) \|inst, *tracking\| try tracking.die(self, inst);
		618	self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
		619	}
		620
		621	if (opts.resurrect) for (
		622	self.inst_tracking.keys()[0..state.inst_tracking_len],
		623	self.inst_tracking.values()[0..state.inst_tracking_len],
		624	) \|inst, *tracking\| tracking.resurrect(inst, state.scope_generation);
		625	for (deaths) \|death\| try self.processDeath(death);
		626
		627	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
		628	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
		629	if (opts.update_tracking)
		630	{} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
		631
		632	var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
		633	stack.get(),
		634	@typeInfo(ExpectedContents).Array.len,
		635	);
		636	defer if (!opts.update_tracking) {
		637	for (reg_locks.items) \|lock\| self.register_manager.unlockReg(lock);
		638	reg_locks.deinit();
		639	};
293		640
294	fn finishAir(bt: *BigTomb, result: MCValue) void {	641	for (0..state.registers.len) \|index\| {
295	const is_used = !bt.function.liveness.isUnused(bt.inst);	642	const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
296	if (is_used) {	643	null
297	log.debug("%{d} => {}", .{ bt.inst, result });	644	else
298	const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];	645	self.register_manager.registers[index];
299	branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);	646	const target_maybe_inst = if (state.free_registers.isSet(index))
		647	null
		648	else
		649	state.registers[index];
		650	if (std.debug.runtime_safety) if (target_maybe_inst) \|target_inst\|
		651	assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
		652	if (opts.emit_instructions) {
		653	if (current_maybe_inst) \|current_inst\| {
		654	try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
		655	}
		656	if (target_maybe_inst) \|target_inst\| {
		657	const target_tracking = self.inst_tracking.getPtr(target_inst).?;
		658	try target_tracking.materialize(self, target_inst, state.reg_tracking[index]);
		659	}
300	}	660	}
301	bt.function.finishAirBookkeeping();	661	if (opts.update_tracking) {
		662	if (current_maybe_inst) \|current_inst\| {
		663	try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
		664	}
		665	{
		666	const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
		667	self.register_manager.freeReg(reg);
		668	self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
		669	}
		670	if (target_maybe_inst) \|target_inst\| {
		671	self.inst_tracking.getPtr(target_inst).?.trackMaterialize(
		672	target_inst,
		673	state.reg_tracking[index],
		674	);
		675	}
		676	} else if (target_maybe_inst) \|_\|
		677	try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
302	}	678	}
303	};	679
		680	if (opts.update_tracking and std.debug.runtime_safety) {
		681	assert(self.register_manager.free_registers.eql(state.free_registers));
		682	var used_reg_it = state.free_registers.iterator(.{ .kind = .unset });
		683	while (used_reg_it.next()) \|index\|
		684	assert(self.register_manager.registers[index] == state.registers[index]);
		685	}
		686	}
304		687
305	const Self = @This();	688	const Self = @This();
306		689
...	@@ -310,7 +693,7 @@ const CallView = enum(u1) {	...	@@ -310,7 +693,7 @@ const CallView = enum(u1) {
310	};	693	};
311		694
312	pub fn generate(	695	pub fn generate(
313	lf: *link.File,	696	bin_file: *link.File,
314	src_loc: Module.SrcLoc,	697	src_loc: Module.SrcLoc,
315	func_index: InternPool.Index,	698	func_index: InternPool.Index,
316	air: Air,	699	air: Air,
...	@@ -318,14 +701,17 @@ pub fn generate(	...	@@ -318,14 +701,17 @@ pub fn generate(
318	code: *std.ArrayList(u8),	701	code: *std.ArrayList(u8),
319	debug_output: DebugInfoOutput,	702	debug_output: DebugInfoOutput,
320	) CodeGenError!Result {	703	) CodeGenError!Result {
321	const gpa = lf.comp.gpa;	704	const comp = bin_file.comp;
322	const zcu = lf.comp.module.?;	705	const gpa = comp.gpa;
		706	const zcu = comp.module.?;
		707	const ip = &zcu.intern_pool;
323	const func = zcu.funcInfo(func_index);	708	const func = zcu.funcInfo(func_index);
324	const fn_owner_decl = zcu.declPtr(func.owner_decl);	709	const fn_owner_decl = zcu.declPtr(func.owner_decl);
325	assert(fn_owner_decl.has_tv);	710	assert(fn_owner_decl.has_tv);
326	const fn_type = fn_owner_decl.typeOf(zcu);	711	const fn_type = fn_owner_decl.typeOf(zcu);
327	const namespace = zcu.namespacePtr(fn_owner_decl.src_namespace);	712	const namespace = zcu.namespacePtr(fn_owner_decl.src_namespace);
328	const target = &namespace.file_scope.mod.resolved_target.result;	713	const target = &namespace.file_scope.mod.resolved_target.result;
		714	const mod = namespace.file_scope.mod;
329		715
330	var branch_stack = std.ArrayList(Branch).init(gpa);	716	var branch_stack = std.ArrayList(Branch).init(gpa);
331	defer {	717	defer {
...	@@ -340,7 +726,7 @@ pub fn generate(	...	@@ -340,7 +726,7 @@ pub fn generate(
340	.air = air,	726	.air = air,
341	.liveness = liveness,	727	.liveness = liveness,
342	.target = target,	728	.target = target,
343	.bin_file = lf,	729	.bin_file = bin_file,
344	.func_index = func_index,	730	.func_index = func_index,
345	.code = code,	731	.code = code,
346	.debug_output = debug_output,	732	.debug_output = debug_output,
...	@@ -351,15 +737,39 @@ pub fn generate(	...	@@ -351,15 +737,39 @@ pub fn generate(
351	.arg_index = 0,	737	.arg_index = 0,
352	.branch_stack = &branch_stack,	738	.branch_stack = &branch_stack,
353	.src_loc = src_loc,	739	.src_loc = src_loc,
354	.stack_align = undefined,
355	.end_di_line = func.rbrace_line,	740	.end_di_line = func.rbrace_line,
356	.end_di_column = func.rbrace_column,	741	.end_di_column = func.rbrace_column,
		742	.scope_generation = 0,
357	};	743	};
358	defer function.stack.deinit(gpa);	744	defer {
359	defer function.blocks.deinit(gpa);	745	function.frame_allocs.deinit(gpa);
360	defer function.exitlude_jump_relocs.deinit(gpa);	746	function.free_frame_indices.deinit(gpa);
		747	function.frame_locs.deinit(gpa);
		748	var block_it = function.blocks.valueIterator();
		749	while (block_it.next()) \|block\| block.deinit(gpa);
		750	function.blocks.deinit(gpa);
		751	function.inst_tracking.deinit(gpa);
		752	function.const_tracking.deinit(gpa);
		753	function.exitlude_jump_relocs.deinit(gpa);
		754	function.mir_instructions.deinit(gpa);
		755	function.mir_extra.deinit(gpa);
		756	}
		757
		758	try function.frame_allocs.resize(gpa, FrameIndex.named_count);
		759	function.frame_allocs.set(
		760	@intFromEnum(FrameIndex.stack_frame),
		761	FrameAlloc.init(.{
		762	.size = 0,
		763	.alignment = func.analysis(ip).stack_alignment.max(.@"1"),
		764	}),
		765	);
		766	function.frame_allocs.set(
		767	@intFromEnum(FrameIndex.call_frame),
		768	FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
		769	);
361		770
362	var call_info = function.resolveCallingConventionValues(fn_type, .callee) catch \|err\| switch (err) {	771	const fn_info = zcu.typeToFunc(fn_type).?;
		772	var call_info = function.resolveCallingConventionValues(fn_info) catch \|err\| switch (err) {
363	error.CodegenFail => return Result{ .fail = function.err_msg.? },	773	error.CodegenFail => return Result{ .fail = function.err_msg.? },
364	error.OutOfRegisters => return Result{	774	error.OutOfRegisters => return Result{
365	.fail = try ErrorMsg.create(gpa, src_loc, "CodeGen ran out of registers. This is a bug in the Zig compiler.", .{}),	775	.fail = try ErrorMsg.create(gpa, src_loc, "CodeGen ran out of registers. This is a bug in the Zig compiler.", .{}),
...	@@ -371,8 +781,25 @@ pub fn generate(	...	@@ -371,8 +781,25 @@ pub fn generate(
371		781
372	function.args = call_info.args;	782	function.args = call_info.args;
373	function.ret_mcv = call_info.return_value;	783	function.ret_mcv = call_info.return_value;
374	function.stack_align = call_info.stack_align;	784	function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{
375	function.max_end_stack = call_info.stack_byte_count;	785	.size = Type.usize.abiSize(zcu),
		786	.alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align),
		787	}));
		788	function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{
		789	.size = Type.usize.abiSize(zcu),
		790	.alignment = Alignment.min(
		791	call_info.stack_align,
		792	Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
		793	),
		794	}));
		795	function.frame_allocs.set(@intFromEnum(FrameIndex.args_frame), FrameAlloc.init(.{
		796	.size = call_info.stack_byte_count,
		797	.alignment = call_info.stack_align,
		798	}));
		799	function.frame_allocs.set(@intFromEnum(FrameIndex.spill_frame), FrameAlloc.init(.{
		800	.size = 0,
		801	.alignment = Type.usize.abiAlignment(zcu),
		802	}));
376		803
377	function.gen() catch \|err\| switch (err) {	804	function.gen() catch \|err\| switch (err) {
378	error.CodegenFail => return Result{ .fail = function.err_msg.? },	805	error.CodegenFail => return Result{ .fail = function.err_msg.? },
...	@@ -382,41 +809,47 @@ pub fn generate(	...	@@ -382,41 +809,47 @@ pub fn generate(
382	else => \|e\| return e,	809	else => \|e\| return e,
383	};	810	};
384		811
385	// Create list of registers to save in the prologue.
386	var save_reg_list = Mir.RegisterList{};
387	for (callee_preserved_regs) \|reg\| {
388	if (function.register_manager.isRegAllocated(reg)) {
389	save_reg_list.push(&callee_preserved_regs, reg);
390	}
391	}
392
393	var mir = Mir{	812	var mir = Mir{
394	.instructions = function.mir_instructions.toOwnedSlice(),	813	.instructions = function.mir_instructions.toOwnedSlice(),
395	.extra = try function.mir_extra.toOwnedSlice(gpa),	814	.extra = try function.mir_extra.toOwnedSlice(gpa),
		815	.frame_locs = function.frame_locs.toOwnedSlice(),
396	};	816	};
397	defer mir.deinit(gpa);	817	defer mir.deinit(gpa);
398		818
399	var emit = Emit{	819	var emit = Emit{
400	.mir = mir,	820	.lower = .{
401	.bin_file = lf,	821	.bin_file = bin_file,
		822	.allocator = gpa,
		823	.mir = mir,
		824	.cc = fn_info.cc,
		825	.src_loc = src_loc,
		826	.output_mode = comp.config.output_mode,
		827	.link_mode = comp.config.link_mode,
		828	.pic = mod.pic,
		829	},
402	.debug_output = debug_output,	830	.debug_output = debug_output,
403	.target = target,
404	.src_loc = src_loc,
405	.code = code,	831	.code = code,
406	.prev_di_pc = 0,	832	.prev_di_pc = 0,
407	.prev_di_line = func.lbrace_line,	833	.prev_di_line = func.lbrace_line,
408	.prev_di_column = func.lbrace_column,	834	.prev_di_column = func.lbrace_column,
409	.code_offset_mapping = .{},
410	// need to at least decrease the sp by -8
411	.stack_size = @max(8, mem.alignForward(u32, function.max_end_stack, 16)),
412	.save_reg_list = save_reg_list,
413	.output_mode = lf.comp.config.output_mode,
414	.link_mode = lf.comp.config.link_mode,
415	};	835	};
416	defer emit.deinit();	836	defer emit.deinit();
417		837
418	emit.emitMir() catch \|err\| switch (err) {	838	emit.emitMir() catch \|err\| switch (err) {
419	error.EmitFail => return Result{ .fail = emit.err_msg.? },	839	error.LowerFail, error.EmitFail => return Result{ .fail = emit.lower.err_msg.? },
		840	error.InvalidInstruction => \|e\| {
		841	const msg = switch (e) {
		842	error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
		843	};
		844	return Result{
		845	.fail = try ErrorMsg.create(
		846	gpa,
		847	src_loc,
		848	"{s} This is a bug in the Zig compiler.",
		849	.{msg},
		850	),
		851	};
		852	},
420	else => \|e\| return e,	853	else => \|e\| return e,
421	};	854	};
422		855
...	@@ -438,9 +871,26 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {	...	@@ -438,9 +871,26 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
438	}	871	}
439		872
440	fn addNop(self: *Self) error{OutOfMemory}!Mir.Inst.Index {	873	fn addNop(self: *Self) error{OutOfMemory}!Mir.Inst.Index {
441	return try self.addInst(.{	874	return self.addInst(.{
442	.tag = .nop,	875	.tag = .nop,
443	.data = .{ .nop = {} },	876	.ops = .none,
		877	.data = undefined,
		878	});
		879	}
		880
		881	fn addPseudoNone(self: *Self, ops: Mir.Inst.Ops) !void {
		882	_ = try self.addInst(.{
		883	.tag = .pseudo,
		884	.ops = ops,
		885	.data = undefined,
		886	});
		887	}
		888
		889	fn addPseudo(self: *Self, ops: Mir.Inst.Ops) !Mir.Inst.Index {
		890	return self.addInst(.{
		891	.tag = .pseudo,
		892	.ops = ops,
		893	.data = undefined,
444	});	894	});
445	}	895	}
446		896
...	@@ -464,22 +914,132 @@ pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {	...	@@ -464,22 +914,132 @@ pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
464	}	914	}
465		915
466	fn gen(self: *Self) !void {	916	fn gen(self: *Self) !void {
467	_ = try self.addInst(.{	917	const mod = self.bin_file.comp.module.?;
468	.tag = .psuedo_prologue,	918	const fn_info = mod.typeToFunc(self.fn_type).?;
469	.data = .{ .nop = {} }, // Backpatched later.
470	});
471		919
472	_ = try self.addInst(.{	920	if (fn_info.cc != .Naked) {
473	.tag = .dbg_prologue_end,	921	try self.addPseudoNone(.pseudo_dbg_prologue_end);
474	.data = .{ .nop = {} },	922
475	});	923	const backpatch_stack_alloc = try self.addPseudo(.pseudo_dead);
		924	const backpatch_ra_spill = try self.addPseudo(.pseudo_dead);
		925	const backpatch_fp_spill = try self.addPseudo(.pseudo_dead);
		926	const backpatch_fp_add = try self.addPseudo(.pseudo_dead);
		927	const backpatch_spill_callee_preserved_regs = try self.addPseudo(.pseudo_dead);
		928
		929	try self.genBody(self.air.getMainBody());
		930
		931	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
		932	self.mir_instructions.items(.data)[jmp_reloc].inst =
		933	@intCast(self.mir_instructions.len);
		934	}
		935
		936	try self.addPseudoNone(.pseudo_dbg_epilogue_begin);
		937
		938	const backpatch_restore_callee_preserved_regs = try self.addPseudo(.pseudo_dead);
		939	const backpatch_ra_restore = try self.addPseudo(.pseudo_dead);
		940	const backpatch_fp_restore = try self.addPseudo(.pseudo_dead);
		941	const backpatch_stack_alloc_restore = try self.addPseudo(.pseudo_dead);
		942	try self.addPseudoNone(.pseudo_ret);
476		943
477	try self.genBody(self.air.getMainBody());	944	const frame_layout = try self.computeFrameLayout();
		945	const need_save_reg = frame_layout.save_reg_list.count() > 0;
		946
		947	self.mir_instructions.set(backpatch_stack_alloc, .{
		948	.tag = .addi,
		949	.ops = .rri,
		950	.data = .{ .i_type = .{
		951	.rd = .sp,
		952	.rs1 = .sp,
		953	.imm12 = Immediate.s(-@as(i32, @intCast(frame_layout.stack_adjust))),
		954	} },
		955	});
		956	self.mir_instructions.set(backpatch_ra_spill, .{
		957	.tag = .pseudo,
		958	.ops = .pseudo_store_rm,
		959	.data = .{ .rm = .{
		960	.r = .ra,
		961	.m = .{
		962	.base = .{ .frame = .ret_addr },
		963	.mod = .{ .rm = .{ .size = .dword } },
		964	},
		965	} },
		966	});
		967	self.mir_instructions.set(backpatch_ra_restore, .{
		968	.tag = .pseudo,
		969	.ops = .pseudo_load_rm,
		970	.data = .{ .rm = .{
		971	.r = .ra,
		972	.m = .{
		973	.base = .{ .frame = .ret_addr },
		974	.mod = .{ .rm = .{ .size = .dword } },
		975	},
		976	} },
		977	});
		978	self.mir_instructions.set(backpatch_fp_spill, .{
		979	.tag = .pseudo,
		980	.ops = .pseudo_store_rm,
		981	.data = .{ .rm = .{
		982	.r = .s0,
		983	.m = .{
		984	.base = .{ .frame = .base_ptr },
		985	.mod = .{ .rm = .{ .size = .dword } },
		986	},
		987	} },
		988	});
		989	self.mir_instructions.set(backpatch_fp_restore, .{
		990	.tag = .pseudo,
		991	.ops = .pseudo_load_rm,
		992	.data = .{ .rm = .{
		993	.r = .s0,
		994	.m = .{
		995	.base = .{ .frame = .base_ptr },
		996	.mod = .{ .rm = .{ .size = .dword } },
		997	},
		998	} },
		999	});
		1000	self.mir_instructions.set(backpatch_fp_add, .{
		1001	.tag = .addi,
		1002	.ops = .rri,
		1003	.data = .{ .i_type = .{
		1004	.rd = .s0,
		1005	.rs1 = .sp,
		1006	.imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)),
		1007	} },
		1008	});
		1009	self.mir_instructions.set(backpatch_stack_alloc_restore, .{
		1010	.tag = .addi,
		1011	.ops = .rri,
		1012	.data = .{ .i_type = .{
		1013	.rd = .sp,
		1014	.rs1 = .sp,
		1015	.imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)),
		1016	} },
		1017	});
		1018
		1019	if (need_save_reg) {
		1020	self.mir_instructions.set(backpatch_spill_callee_preserved_regs, .{
		1021	.tag = .pseudo,
		1022	.ops = .pseudo_spill_regs,
		1023	.data = .{ .reg_list = frame_layout.save_reg_list },
		1024	});
		1025
		1026	self.mir_instructions.set(backpatch_restore_callee_preserved_regs, .{
		1027	.tag = .pseudo,
		1028	.ops = .pseudo_restore_regs,
		1029	.data = .{ .reg_list = frame_layout.save_reg_list },
		1030	});
		1031	}
		1032	} else {
		1033	try self.addPseudoNone(.pseudo_dbg_prologue_end);
		1034	try self.genBody(self.air.getMainBody());
		1035	try self.addPseudoNone(.pseudo_dbg_epilogue_begin);
		1036	}
478		1037
479	// Drop them off at the rbrace.	1038	// Drop them off at the rbrace.
480	_ = try self.addInst(.{	1039	_ = try self.addInst(.{
481	.tag = .dbg_line,	1040	.tag = .pseudo,
482	.data = .{ .dbg_line_column = .{	1041	.ops = .pseudo_dbg_line_column,
		1042	.data = .{ .pseudo_dbg_line_column = .{
483	.line = self.end_di_line,	1043	.line = self.end_di_line,
484	.column = self.end_di_column,	1044	.column = self.end_di_column,
485	} },	1045	} },
...	@@ -487,18 +1047,15 @@ fn gen(self: *Self) !void {	...	@@ -487,18 +1047,15 @@ fn gen(self: *Self) !void {
487	}	1047	}
488		1048
489	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	1049	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
490	const mod = self.bin_file.comp.module.?;	1050	const zcu = self.bin_file.comp.module.?;
491	const ip = &mod.intern_pool;	1051	const ip = &zcu.intern_pool;
492	const air_tags = self.air.instructions.items(.tag);	1052	const air_tags = self.air.instructions.items(.tag);
493		1053
494	for (body) \|inst\| {	1054	for (body) \|inst\| {
495	// TODO: remove now-redundant isUnused calls from AIR handler functions	1055	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
496	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip))
497	continue;
498		1056
499	const old_air_bookkeeping = self.air_bookkeeping;	1057	const old_air_bookkeeping = self.air_bookkeeping;
500	try self.ensureProcessDeathCapacity(Liveness.bpi);	1058	try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
501
502	switch (air_tags[@intFromEnum(inst)]) {	1059	switch (air_tags[@intFromEnum(inst)]) {
503	// zig fmt: off	1060	// zig fmt: off
504	.ptr_add => try self.airPtrArithmetic(inst, .ptr_add),	1061	.ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
...	@@ -731,30 +1288,58 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -731,30 +1288,58 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
731	.work_group_id => unreachable,	1288	.work_group_id => unreachable,
732	// zig fmt: on	1289	// zig fmt: on
733	}	1290	}
		1291
		1292	assert(!self.register_manager.lockedRegsExist());
		1293
734	if (std.debug.runtime_safety) {	1294	if (std.debug.runtime_safety) {
735	if (self.air_bookkeeping < old_air_bookkeeping + 1) {	1295	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
736	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] });	1296	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] });
737	}	1297	}
		1298
		1299	{ // check consistency of tracked registers
		1300	var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
		1301	while (it.next()) \|index\| {
		1302	const tracked_inst = self.register_manager.registers[index];
		1303	const tracking = self.getResolvedInstValue(tracked_inst);
		1304	for (tracking.getRegs()) \|reg\| {
		1305	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
		1306	} else return self.fail(
		1307	\\%{} takes up these regs: {any}, however those regs don't use it
		1308	, .{ index, tracking.getRegs() });
		1309	}
		1310	}
738	}	1311	}
739	}	1312	}
740	}	1313	}
741		1314
		1315	fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
		1316	for (value.getRegs()) \|reg\| try self.register_manager.getReg(reg, inst);
		1317	}
		1318
		1319	fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
		1320	for (value.getRegs()) \|reg\| if (self.register_manager.isRegFree(reg))
		1321	self.register_manager.getRegAssumeFree(reg, inst);
		1322	}
		1323
		1324	fn freeValue(self: *Self, value: MCValue) !void {
		1325	switch (value) {
		1326	.register => \|reg\| self.register_manager.freeReg(reg),
		1327	.register_pair => \|regs\| for (regs) \|reg\| self.register_manager.freeReg(reg),
		1328	.register_offset => \|reg_off\| self.register_manager.freeReg(reg_off.reg),
		1329	else => {}, // TODO process stack allocation death
		1330	}
		1331	}
		1332
742	fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) !void {	1333	fn feed(self: Self, bt: Liveness.BigTomb, operand: Air.Inst.Ref) !void {
743	if (bt.feed()) if (operand.toIndex()) \|inst\| self.processDeath(inst);	1334	if (bt.feed()) if (operand.toIndex()) \|inst\| {
		1335	log.debug("feed inst: %{}", .{inst});
		1336	try self.processDeath(inst);
		1337	};
744	}	1338	}
745		1339
746	/// Asserts there is already capacity to insert into top branch inst_table.	1340	/// Asserts there is already capacity to insert into top branch inst_table.
747	fn processDeath(self: *Self, inst: Air.Inst.Index) void {	1341	fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
748	// When editing this function, note that the logic must synchronize with `reuseOperand`.	1342	try self.inst_tracking.getPtr(inst).?.die(self, inst);
749	const prev_value = self.getResolvedInstValue(inst);
750	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
751	branch.inst_table.putAssumeCapacity(inst, .dead);
752	switch (prev_value) {
753	.register => \|reg\| {
754	self.register_manager.freeReg(reg);
755	},
756	else => {}, // TODO process stack allocation death by freeing it to be reused later
757	}
758	}	1343	}
759		1344
760	/// Called when there are no operands, and the instruction is always unreferenced.	1345	/// Called when there are no operands, and the instruction is always unreferenced.
...	@@ -769,23 +1354,12 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {	...	@@ -769,23 +1354,12 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
769	.none, .dead, .unreach => {},	1354	.none, .dead, .unreach => {},
770	else => unreachable, // Why didn't the result die?	1355	else => unreachable, // Why didn't the result die?
771	} else {	1356	} else {
772	log.debug("%{d} => {}", .{ inst, result });	1357	tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
773	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	1358	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
774	branch.inst_table.putAssumeCapacityNoClobber(inst, result);	1359	// In some cases, an operand may be reused as the result.
775		1360	// If that operand died and was a register, it was freed by
776	switch (result) {	1361	// processDeath, so we have to "re-allocate" the register.
777	.register => \|reg\| {	1362	self.getValueIfFree(result, inst);
778	// In some cases (such as bitcast), an operand
779	// may be the same MCValue as the result. If
780	// that operand died and was a register, it
781	// was freed by processDeath. We have to
782	// "re-allocate" the register.
783	if (self.register_manager.isRegFree(reg)) {
784	self.register_manager.getRegAssumeFree(reg, inst);
785	}
786	},
787	else => {},
788	}
789	}	1363	}
790	self.finishAirBookkeeping();	1364	self.finishAirBookkeeping();
791	}	1365	}
...	@@ -801,43 +1375,153 @@ fn finishAir(	...	@@ -801,43 +1375,153 @@ fn finishAir(
801	const dies = @as(u1, @truncate(tomb_bits)) != 0;	1375	const dies = @as(u1, @truncate(tomb_bits)) != 0;
802	tomb_bits >>= 1;	1376	tomb_bits >>= 1;
803	if (!dies) continue;	1377	if (!dies) continue;
804	self.processDeath(op.toIndexAllowNone() orelse continue);	1378	try self.processDeath(op.toIndexAllowNone() orelse continue);
805	}	1379	}
806	self.finishAirResult(inst, result);	1380	self.finishAirResult(inst, result);
807	}	1381	}
808		1382
		1383	const FrameLayout = struct {
		1384	stack_adjust: u32,
		1385	save_reg_list: Mir.RegisterList,
		1386	};
		1387
		1388	fn setFrameLoc(
		1389	self: *Self,
		1390	frame_index: FrameIndex,
		1391	base: Register,
		1392	offset: *i32,
		1393	comptime aligned: bool,
		1394	) void {
		1395	const frame_i = @intFromEnum(frame_index);
		1396	if (aligned) {
		1397	const alignment: InternPool.Alignment = self.frame_allocs.items(.abi_align)[frame_i];
		1398	offset.* = if (math.sign(offset.*) < 0)
		1399	-1 * @as(i32, @intCast(alignment.backward(@intCast(@abs(offset.*)))))
		1400	else
		1401	@intCast(alignment.forward(@intCast(@abs(offset.*))));
		1402	}
		1403	self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
		1404	offset.* += self.frame_allocs.items(.abi_size)[frame_i];
		1405	}
		1406
		1407	fn computeFrameLayout(self: *Self) !FrameLayout {
		1408	const frame_allocs_len = self.frame_allocs.len;
		1409	try self.frame_locs.resize(self.gpa, frame_allocs_len);
		1410	const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
		1411	defer self.gpa.free(stack_frame_order);
		1412
		1413	const frame_size = self.frame_allocs.items(.abi_size);
		1414	const frame_align = self.frame_allocs.items(.abi_align);
		1415
		1416	for (stack_frame_order, FrameIndex.named_count..) \|*frame_order, frame_index\|
		1417	frame_order.* = @enumFromInt(frame_index);
		1418
		1419	{
		1420	const SortContext = struct {
		1421	frame_align: @TypeOf(frame_align),
		1422	pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
		1423	return context.frame_align[@intFromEnum(lhs)].compare(.gt, context.frame_align[@intFromEnum(rhs)]);
		1424	}
		1425	};
		1426	const sort_context = SortContext{ .frame_align = frame_align };
		1427	mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
		1428	}
		1429
		1430	var save_reg_list = Mir.RegisterList{};
		1431	for (callee_preserved_regs) \|reg\| {
		1432	if (self.register_manager.isRegAllocated(reg)) {
		1433	save_reg_list.push(&callee_preserved_regs, reg);
		1434	}
		1435	}
		1436
		1437	const total_alloc_size: i32 = blk: {
		1438	var i: i32 = 0;
		1439	for (stack_frame_order) \|frame_index\| {
		1440	i += frame_size[@intFromEnum(frame_index)];
		1441	}
		1442	break :blk i;
		1443	};
		1444	const saved_reg_size = save_reg_list.size();
		1445
		1446	frame_size[@intFromEnum(FrameIndex.spill_frame)] = @intCast(saved_reg_size);
		1447
		1448	// The total frame size is calculated by the amount of s registers you need to save * 8, as each
		1449	// register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0
		1450	// register. Finally we align the frame size to the align of the base pointer.
		1451	const acc_frame_size: i32 = std.mem.alignForward(
		1452	i32,
		1453	total_alloc_size + 16 + frame_size[@intFromEnum(FrameIndex.args_frame)] + frame_size[@intFromEnum(FrameIndex.spill_frame)],
		1454	@intCast(frame_align[@intFromEnum(FrameIndex.base_ptr)].toByteUnits().?),
		1455	);
		1456	log.debug("frame size: {}", .{acc_frame_size});
		1457
		1458	// store the ra at total_size - 8, so it's the very first thing in the stack
		1459	// relative to the fp
		1460	self.frame_locs.set(
		1461	@intFromEnum(FrameIndex.ret_addr),
		1462	.{ .base = .sp, .disp = acc_frame_size - 8 },
		1463	);
		1464	self.frame_locs.set(
		1465	@intFromEnum(FrameIndex.base_ptr),
		1466	.{ .base = .sp, .disp = acc_frame_size - 16 },
		1467	);
		1468
		1469	// now we grow the stack frame from the bottom of total frame in order to
		1470	// not need to know the size of the first allocation. Stack offsets point at the "bottom"
		1471	// of variables.
		1472	var s0_offset: i32 = -acc_frame_size;
		1473	self.setFrameLoc(.stack_frame, .s0, &s0_offset, true);
		1474	for (stack_frame_order) \|frame_index\| self.setFrameLoc(frame_index, .s0, &s0_offset, true);
		1475	self.setFrameLoc(.args_frame, .s0, &s0_offset, true);
		1476	self.setFrameLoc(.call_frame, .s0, &s0_offset, true);
		1477	self.setFrameLoc(.spill_frame, .s0, &s0_offset, true);
		1478
		1479	return .{
		1480	.stack_adjust = @intCast(acc_frame_size),
		1481	.save_reg_list = save_reg_list,
		1482	};
		1483	}
		1484
809	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {	1485	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
810	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;	1486	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
811	try table.ensureUnusedCapacity(self.gpa, additional_count);	1487	try table.ensureUnusedCapacity(self.gpa, additional_count);
812	}	1488	}
813		1489
814	fn splitType(self: *Self, ty: Type) ![2]Type {	1490	fn memSize(self: *Self, ty: Type) Memory.Size {
815	const mod = self.bin_file.comp.module.?;	1491	const mod = self.bin_file.comp.module.?;
816	const classes = mem.sliceTo(&abi.classifySystemV(ty, mod), .none);	1492	return switch (ty.zigTypeTag(mod)) {
		1493	.Float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)),
		1494	else => Memory.Size.fromSize(@intCast(ty.abiSize(mod))),
		1495	};
		1496	}
		1497
		1498	fn splitType(self: *Self, ty: Type) ![2]Type {
		1499	const zcu = self.bin_file.comp.module.?;
		1500	const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none);
817	var parts: [2]Type = undefined;	1501	var parts: [2]Type = undefined;
818	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {	1502	if (classes.len == 2) for (&parts, classes, 0..) \|*part, class, part_i\| {
819	part.* = switch (class) {	1503	part.* = switch (class) {
820	.integer => switch (part_i) {	1504	.integer => switch (part_i) {
821	0 => Type.u64,	1505	0 => Type.u64,
822	1 => part: {	1506	1 => part: {
823	const elem_size = ty.abiAlignment(mod).minStrict(.@"8").toByteUnitsOptional().?;	1507	const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
824	const elem_ty = try mod.intType(.unsigned, @intCast(elem_size * 8));	1508	const elem_ty = try zcu.intType(.unsigned, @intCast(elem_size * 8));
825	break :part switch (@divExact(ty.abiSize(mod) - 8, elem_size)) {	1509	break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) {
826	1 => elem_ty,	1510	1 => elem_ty,
827	else => \|len\| try mod.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),	1511	else => \|len\| try zcu.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
828	};	1512	};
829	},	1513	},
830	else => unreachable,	1514	else => unreachable,
831	},	1515	},
832	else => break,	1516	else => break,
833	};	1517	};
834	} else if (parts[0].abiSize(mod) + parts[1].abiSize(mod) == ty.abiSize(mod)) return parts;	1518	} else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts;
835	return self.fail("TODO implement splitType for {}", .{ty.fmt(mod)});	1519	return self.fail("TODO implement splitType for {}", .{ty.fmt(zcu)});
836	}	1520	}
837		1521
838	fn symbolIndex(self: *Self) !u32 {	1522	fn symbolIndex(self: *Self) !u32 {
839	const mod = self.bin_file.comp.module.?;	1523	const zcu = self.bin_file.comp.module.?;
840	const decl_index = mod.funcOwnerDeclIndex(self.func_index);	1524	const decl_index = zcu.funcOwnerDeclIndex(self.func_index);
841	return switch (self.bin_file.tag) {	1525	return switch (self.bin_file.tag) {
842	.elf => blk: {	1526	.elf => blk: {
843	const elf_file = self.bin_file.cast(link.File.Elf).?;	1527	const elf_file = self.bin_file.cast(link.File.Elf).?;
...	@@ -848,41 +1532,49 @@ fn symbolIndex(self: *Self) !u32 {	...	@@ -848,41 +1532,49 @@ fn symbolIndex(self: *Self) !u32 {
848	};	1532	};
849	}	1533	}
850		1534
851	fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: Alignment) !u32 {	1535	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
852	self.stack_align = self.stack_align.max(abi_align);	1536	const frame_allocs_slice = self.frame_allocs.slice();
853	// TODO find a free slot instead of always appending	1537	const frame_size = frame_allocs_slice.items(.abi_size);
854	const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset));	1538	const frame_align = frame_allocs_slice.items(.abi_align);
855	self.next_stack_offset = offset + abi_size;	1539
856	if (self.next_stack_offset > self.max_end_stack)	1540	const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)];
857	self.max_end_stack = self.next_stack_offset;	1541	stack_frame_align.* = stack_frame_align.max(alloc.abi_align);
858	try self.stack.putNoClobber(self.gpa, offset, .{	1542
859	.inst = inst,	1543	for (self.free_frame_indices.keys(), 0..) \|frame_index, free_i\| {
860	.size = abi_size,	1544	const abi_size = frame_size[@intFromEnum(frame_index)];
861	});	1545	if (abi_size != alloc.abi_size) continue;
862	return offset;	1546	const abi_align = &frame_align[@intFromEnum(frame_index)];
		1547	abi_align.* = abi_align.max(alloc.abi_align);
		1548
		1549	_ = self.free_frame_indices.swapRemoveAt(free_i);
		1550	return frame_index;
		1551	}
		1552	const frame_index: FrameIndex = @enumFromInt(self.frame_allocs.len);
		1553	try self.frame_allocs.append(self.gpa, alloc);
		1554	log.debug("allocated frame {}", .{frame_index});
		1555	return frame_index;
863	}	1556	}
864		1557
865	/// Use a pointer instruction as the basis for allocating stack memory.	1558	/// Use a pointer instruction as the basis for allocating stack memory.
866	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {	1559	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
867	const mod = self.bin_file.comp.module.?;	1560	const zcu = self.bin_file.comp.module.?;
868	const elem_ty = self.typeOfIndex(inst).childType(mod);	1561	const ptr_ty = self.typeOfIndex(inst);
869	const abi_size = math.cast(u32, elem_ty.abiSize(mod)) orelse {	1562	const val_ty = ptr_ty.childType(zcu);
870	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});	1563	return self.allocFrameIndex(FrameAlloc.init(.{
871	};	1564	.size = math.cast(u32, val_ty.abiSize(zcu)) orelse {
872	// TODO swap this for inst.ty.ptrAlign	1565	return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(zcu)});
873	const abi_align = elem_ty.abiAlignment(mod);	1566	},
874	return self.allocMem(inst, abi_size, abi_align);	1567	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
		1568	}));
875	}	1569	}
876		1570
877	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {	1571	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
878	const mod = self.bin_file.comp.module.?;	1572	const zcu = self.bin_file.comp.module.?;
879	const elem_ty = self.typeOfIndex(inst);	1573	const elem_ty = self.typeOfIndex(inst);
880		1574
881	const abi_size = math.cast(u32, elem_ty.abiSize(mod)) orelse {	1575	const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse {
882	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});	1576	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(zcu)});
883	};	1577	};
884	const abi_align = elem_ty.abiAlignment(mod);
885	self.stack_align = self.stack_align.max(abi_align);
886		1578
887	if (reg_ok) {	1579	if (reg_ok) {
888	// Make sure the type can fit in a register before we try to allocate one.	1580	// Make sure the type can fit in a register before we try to allocate one.
...	@@ -894,8 +1586,9 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {	...	@@ -894,8 +1586,9 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
894	}	1586	}
895	}	1587	}
896	}	1588	}
897	const stack_offset = try self.allocMem(inst, abi_size, abi_align);	1589
898	return .{ .stack_offset = stack_offset };	1590	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(elem_ty, zcu));
		1591	return .{ .load_frame = .{ .index = frame_index } };
899	}	1592	}
900		1593
901	/// Allocates a register from the general purpose set and returns the Register and the Lock.	1594	/// Allocates a register from the general purpose set and returns the Register and the Lock.
...	@@ -938,19 +1631,12 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi	...	@@ -938,19 +1631,12 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
938	}	1631	}
939		1632
940	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {	1633	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
941	const mod = self.bin_file.comp.module.?;	1634	const tracking = self.inst_tracking.getPtr(inst) orelse return;
942	const elem_ty = self.typeOfIndex(inst);	1635	for (tracking.getRegs()) \|tracked_reg\| {
943		1636	if (tracked_reg.id() == reg.id()) break;
944	// there isn't anything to spill	1637	} else unreachable; // spilled reg not tracked with spilled instruciton
945	if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) return;	1638	try tracking.spill(self, inst);
946		1639	try tracking.trackSpill(self, inst);
947	const stack_mcv = try self.allocRegOrMem(inst, false);
948	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
949	const reg_mcv = self.getResolvedInstValue(inst);
950	assert(reg == reg_mcv.register);
951	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
952	try branch.inst_table.put(self.gpa, inst, stack_mcv);
953	try self.genSetStack(self.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
954	}	1640	}
955		1641
956	/// Copies a value to a register without tracking the register. The register is not considered	1642	/// Copies a value to a register without tracking the register. The register is not considered
...	@@ -972,39 +1658,48 @@ fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCVa	...	@@ -972,39 +1658,48 @@ fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCVa
972	}	1658	}
973		1659
974	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {	1660	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
975	const stack_offset = try self.allocMemPtr(inst);	1661	const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
976	log.debug("airAlloc offset: {}", .{stack_offset});	1662	return self.finishAir(inst, result, .{ .none, .none, .none });
977	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
978	}	1663	}
979		1664
980	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {	1665	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
981	const stack_offset = try self.allocMemPtr(inst);	1666	const result: MCValue = switch (self.ret_mcv.long) {
982	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });	1667	else => unreachable,
		1668	.none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
		1669	.load_frame => .{ .register_offset = .{
		1670	.reg = (try self.copyToNewRegister(
		1671	inst,
		1672	self.ret_mcv.long,
		1673	)).register,
		1674	.off = self.ret_mcv.short.indirect.off,
		1675	} },
		1676	};
		1677	return self.finishAir(inst, result, .{ .none, .none, .none });
983	}	1678	}
984		1679
985	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	1680	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
986	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1681	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
987	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});	1682	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
988	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	1683	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
989	}	1684	}
990		1685
991	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	1686	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
992	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1687	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
993	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});	1688	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
994	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	1689	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
995	}	1690	}
996		1691
997	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	1692	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
998	const mod = self.bin_file.comp.module.?;	1693	const zcu = self.bin_file.comp.module.?;
999	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1694	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1000	const src_ty = self.typeOf(ty_op.operand);	1695	const src_ty = self.typeOf(ty_op.operand);
1001	const dst_ty = self.typeOfIndex(inst);	1696	const dst_ty = self.typeOfIndex(inst);
1002		1697
1003	const result: MCValue = result: {	1698	const result: MCValue = result: {
1004	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));	1699	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
1005		1700
1006	const src_int_info = src_ty.intInfo(mod);	1701	const src_int_info = src_ty.intInfo(zcu);
1007	const dst_int_info = dst_ty.intInfo(mod);	1702	const dst_int_info = dst_ty.intInfo(zcu);
1008	const extend = switch (src_int_info.signedness) {	1703	const extend = switch (src_int_info.signedness) {
1009	.signed => dst_int_info,	1704	.signed => dst_int_info,
1010	.unsigned => src_int_info,	1705	.unsigned => src_int_info,
...	@@ -1019,7 +1714,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1019,7 +1714,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1019		1714
1020	const src_storage_bits: u16 = switch (src_mcv) {	1715	const src_storage_bits: u16 = switch (src_mcv) {
1021	.register => 64,	1716	.register => 64,
1022	.stack_offset => src_int_info.bits,	1717	.load_frame => src_int_info.bits,
1023	else => return self.fail("airIntCast from {s}", .{@tagName(src_mcv)}),	1718	else => return self.fail("airIntCast from {s}", .{@tagName(src_mcv)}),
1024	};	1719	};
1025		1720
...	@@ -1042,7 +1737,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1042,7 +1737,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1042		1737
1043	break :result dst_mcv;	1738	break :result dst_mcv;
1044	} orelse return self.fail("TODO implement airIntCast from {} to {}", .{	1739	} orelse return self.fail("TODO implement airIntCast from {} to {}", .{
1045	src_ty.fmt(mod), dst_ty.fmt(mod),	1740	src_ty.fmt(zcu), dst_ty.fmt(zcu),
1046	});	1741	});
1047		1742
1048	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	1743	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -1051,7 +1746,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1051,7 +1746,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1051	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	1746	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1052	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1747	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1053	if (self.liveness.isUnused(inst))	1748	if (self.liveness.isUnused(inst))
1054	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });	1749	return self.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
1055		1750
1056	const operand = try self.resolveInst(ty_op.operand);	1751	const operand = try self.resolveInst(ty_op.operand);
1057	_ = operand;	1752	_ = operand;
...	@@ -1062,19 +1757,19 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1062,19 +1757,19 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1062	fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {	1757	fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
1063	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	1758	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1064	const operand = try self.resolveInst(un_op);	1759	const operand = try self.resolveInst(un_op);
1065	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;	1760	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else operand;
1066	return self.finishAir(inst, result, .{ un_op, .none, .none });	1761	return self.finishAir(inst, result, .{ un_op, .none, .none });
1067	}	1762	}
1068		1763
1069	fn airNot(self: *Self, inst: Air.Inst.Index) !void {	1764	fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1070	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1765	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1071	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	1766	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1072	const mod = self.bin_file.comp.module.?;	1767	const zcu = self.bin_file.comp.module.?;
1073		1768
1074	const operand = try self.resolveInst(ty_op.operand);	1769	const operand = try self.resolveInst(ty_op.operand);
1075	const ty = self.typeOf(ty_op.operand);	1770	const ty = self.typeOf(ty_op.operand);
1076		1771
1077	switch (ty.zigTypeTag(mod)) {	1772	switch (ty.zigTypeTag(zcu)) {
1078	.Bool => {	1773	.Bool => {
1079	const operand_reg = blk: {	1774	const operand_reg = blk: {
1080	if (operand == .register) break :blk operand.register;	1775	if (operand == .register) break :blk operand.register;
...	@@ -1089,6 +1784,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1089,6 +1784,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1089		1784
1090	_ = try self.addInst(.{	1785	_ = try self.addInst(.{
1091	.tag = .not,	1786	.tag = .not,
		1787	.ops = .rr,
1092	.data = .{	1788	.data = .{
1093	.rr = .{	1789	.rr = .{
1094	.rs = operand_reg,	1790	.rs = operand_reg,
...	@@ -1108,20 +1804,20 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1108,20 +1804,20 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1108		1804
1109	fn airMin(self: *Self, inst: Air.Inst.Index) !void {	1805	fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1110	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	1806	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1111	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement min for {}", .{self.target.cpu.arch});	1807	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement min for {}", .{self.target.cpu.arch});
1112	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1808	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1113	}	1809	}
1114		1810
1115	fn airMax(self: *Self, inst: Air.Inst.Index) !void {	1811	fn airMax(self: *Self, inst: Air.Inst.Index) !void {
1116	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	1812	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1117	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement max for {}", .{self.target.cpu.arch});	1813	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement max for {}", .{self.target.cpu.arch});
1118	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1814	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1119	}	1815	}
1120		1816
1121	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {	1817	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1122	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	1818	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1123	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;	1819	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1124	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice for {}", .{self.target.cpu.arch});	1820	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement slice for {}", .{self.target.cpu.arch});
1125	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1821	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1126	}	1822	}
1127		1823
...	@@ -1132,7 +1828,7 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	...	@@ -1132,7 +1828,7 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1132	const lhs_ty = self.typeOf(bin_op.lhs);	1828	const lhs_ty = self.typeOf(bin_op.lhs);
1133	const rhs_ty = self.typeOf(bin_op.rhs);	1829	const rhs_ty = self.typeOf(bin_op.rhs);
1134		1830
1135	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);	1831	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
1136	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1832	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1137	}	1833	}
1138		1834
...	@@ -1175,7 +1871,8 @@ fn binOp(	...	@@ -1175,7 +1871,8 @@ fn binOp(
1175	lhs_ty: Type,	1871	lhs_ty: Type,
1176	rhs_ty: Type,	1872	rhs_ty: Type,
1177	) InnerError!MCValue {	1873	) InnerError!MCValue {
1178	const mod = self.bin_file.comp.module.?;	1874	const zcu = self.bin_file.comp.module.?;
		1875
1179	switch (tag) {	1876	switch (tag) {
1180	// Arithmetic operations on integers and floats	1877	// Arithmetic operations on integers and floats
1181	.add,	1878	.add,
...	@@ -1188,12 +1885,12 @@ fn binOp(	...	@@ -1188,12 +1885,12 @@ fn binOp(
1188	.cmp_lt,	1885	.cmp_lt,
1189	.cmp_lte,	1886	.cmp_lte,
1190	=> {	1887	=> {
1191	switch (lhs_ty.zigTypeTag(mod)) {	1888	switch (lhs_ty.zigTypeTag(zcu)) {
1192	.Float => return self.fail("TODO binary operations on floats", .{}),	1889	.Float => return self.fail("TODO binary operations on floats", .{}),
1193	.Vector => return self.fail("TODO binary operations on vectors", .{}),	1890	.Vector => return self.fail("TODO binary operations on vectors", .{}),
1194	.Int => {	1891	.Int => {
1195	assert(lhs_ty.eql(rhs_ty, mod));	1892	assert(lhs_ty.eql(rhs_ty, zcu));
1196	const int_info = lhs_ty.intInfo(mod);	1893	const int_info = lhs_ty.intInfo(zcu);
1197	if (int_info.bits <= 64) {	1894	if (int_info.bits <= 64) {
1198	if (rhs == .immediate and supportImmediate(tag)) {	1895	if (rhs == .immediate and supportImmediate(tag)) {
1199	return self.binOpImm(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);	1896	return self.binOpImm(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
...	@@ -1210,14 +1907,14 @@ fn binOp(	...	@@ -1210,14 +1907,14 @@ fn binOp(
1210	.ptr_add,	1907	.ptr_add,
1211	.ptr_sub,	1908	.ptr_sub,
1212	=> {	1909	=> {
1213	switch (lhs_ty.zigTypeTag(mod)) {	1910	switch (lhs_ty.zigTypeTag(zcu)) {
1214	.Pointer => {	1911	.Pointer => {
1215	const ptr_ty = lhs_ty;	1912	const ptr_ty = lhs_ty;
1216	const elem_ty = switch (ptr_ty.ptrSize(mod)) {	1913	const elem_ty = switch (ptr_ty.ptrSize(zcu)) {
1217	.One => ptr_ty.childType(mod).childType(mod), // ptr to array, so get array element type	1914	.One => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
1218	else => ptr_ty.childType(mod),	1915	else => ptr_ty.childType(zcu),
1219	};	1916	};
1220	const elem_size = elem_ty.abiSize(mod);	1917	const elem_size = elem_ty.abiSize(zcu);
1221		1918
1222	if (elem_size == 1) {	1919	if (elem_size == 1) {
1223	const base_tag: Air.Inst.Tag = switch (tag) {	1920	const base_tag: Air.Inst.Tag = switch (tag) {
...	@@ -1256,11 +1953,11 @@ fn binOp(	...	@@ -1256,11 +1953,11 @@ fn binOp(
1256	.shr,	1953	.shr,
1257	.shl,	1954	.shl,
1258	=> {	1955	=> {
1259	switch (lhs_ty.zigTypeTag(mod)) {	1956	switch (lhs_ty.zigTypeTag(zcu)) {
1260	.Float => return self.fail("TODO binary operations on floats", .{}),	1957	.Float => return self.fail("TODO binary operations on floats", .{}),
1261	.Vector => return self.fail("TODO binary operations on vectors", .{}),	1958	.Vector => return self.fail("TODO binary operations on vectors", .{}),
1262	.Int => {	1959	.Int => {
1263	const int_info = lhs_ty.intInfo(mod);	1960	const int_info = lhs_ty.intInfo(zcu);
1264	if (int_info.bits <= 64) {	1961	if (int_info.bits <= 64) {
1265	if (rhs == .immediate) {	1962	if (rhs == .immediate) {
1266	return self.binOpImm(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);	1963	return self.binOpImm(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
...	@@ -1332,6 +2029,7 @@ fn binOpRegister(	...	@@ -1332,6 +2029,7 @@ fn binOpRegister(
1332		2029
1333	_ = try self.addInst(.{	2030	_ = try self.addInst(.{
1334	.tag = mir_tag,	2031	.tag = mir_tag,
		2032	.ops = .rrr,
1335	.data = .{	2033	.data = .{
1336	.r_type = .{	2034	.r_type = .{
1337	.rd = dest_reg,	2035	.rd = dest_reg,
...	@@ -1402,24 +2100,26 @@ fn binOpImm(	...	@@ -1402,24 +2100,26 @@ fn binOpImm(
1402	=> {	2100	=> {
1403	_ = try self.addInst(.{	2101	_ = try self.addInst(.{
1404	.tag = mir_tag,	2102	.tag = mir_tag,
		2103	.ops = .rri,
1405	.data = .{ .i_type = .{	2104	.data = .{ .i_type = .{
1406	.rd = dest_reg,	2105	.rd = dest_reg,
1407	.rs1 = lhs_reg,	2106	.rs1 = lhs_reg,
1408	.imm12 = math.cast(i12, rhs.immediate) orelse {	2107	.imm12 = Immediate.s(math.cast(i12, rhs.immediate) orelse {
1409	return self.fail("TODO: binOpImm larger than i12 i_type payload", .{});	2108	return self.fail("TODO: binOpImm larger than i12 i_type payload", .{});
1410	},	2109	}),
1411	} },	2110	} },
1412	});	2111	});
1413	},	2112	},
1414	.addiw => {	2113	.addiw => {
1415	_ = try self.addInst(.{	2114	_ = try self.addInst(.{
1416	.tag = mir_tag,	2115	.tag = mir_tag,
		2116	.ops = .rri,
1417	.data = .{ .i_type = .{	2117	.data = .{ .i_type = .{
1418	.rd = dest_reg,	2118	.rd = dest_reg,
1419	.rs1 = lhs_reg,	2119	.rs1 = lhs_reg,
1420	.imm12 = -(math.cast(i12, rhs.immediate) orelse {	2120	.imm12 = Immediate.s(-(math.cast(i12, rhs.immediate) orelse {
1421	return self.fail("TODO: binOpImm larger than i12 i_type payload", .{});	2121	return self.fail("TODO: binOpImm larger than i12 i_type payload", .{});
1422	}),	2122	})),
1423	} },	2123	} },
1424	});	2124	});
1425	},	2125	},
...	@@ -1428,6 +2128,7 @@ fn binOpImm(	...	@@ -1428,6 +2128,7 @@ fn binOpImm(
1428		2128
1429	_ = try self.addInst(.{	2129	_ = try self.addInst(.{
1430	.tag = mir_tag,	2130	.tag = mir_tag,
		2131	.ops = .rrr,
1431	.data = .{ .r_type = .{	2132	.data = .{ .r_type = .{
1432	.rd = dest_reg,	2133	.rd = dest_reg,
1433	.rs1 = imm_reg,	2134	.rs1 = imm_reg,
...	@@ -1449,8 +2150,8 @@ fn binOpMir(	...	@@ -1449,8 +2150,8 @@ fn binOpMir(
1449	dst_mcv: MCValue,	2150	dst_mcv: MCValue,
1450	src_mcv: MCValue,	2151	src_mcv: MCValue,
1451	) !void {	2152	) !void {
1452	const mod = self.bin_file.comp.module.?;	2153	const zcu = self.bin_file.comp.module.?;
1453	const abi_size: u32 = @intCast(ty.abiSize(mod));	2154	const abi_size: u32 = @intCast(ty.abiSize(zcu));
1454		2155
1455	_ = abi_size;	2156	_ = abi_size;
1456	_ = maybe_inst;	2157	_ = maybe_inst;
...	@@ -1461,6 +2162,7 @@ fn binOpMir(	...	@@ -1461,6 +2162,7 @@ fn binOpMir(
1461		2162
1462	_ = try self.addInst(.{	2163	_ = try self.addInst(.{
1463	.tag = mir_tag,	2164	.tag = mir_tag,
		2165	.ops = .rrr,
1464	.data = .{	2166	.data = .{
1465	.r_type = .{	2167	.r_type = .{
1466	.rd = dst_reg,	2168	.rd = dst_reg,
...	@@ -1483,25 +2185,25 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void	...	@@ -1483,25 +2185,25 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void
1483	const lhs_ty = self.typeOf(bin_op.lhs);	2185	const lhs_ty = self.typeOf(bin_op.lhs);
1484	const rhs_ty = self.typeOf(bin_op.rhs);	2186	const rhs_ty = self.typeOf(bin_op.rhs);
1485		2187
1486	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);	2188	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
1487	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2189	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1488	}	2190	}
1489		2191
1490	fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {	2192	fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {
1491	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2193	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1492	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch});	2194	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch});
1493	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2195	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1494	}	2196	}
1495		2197
1496	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	2198	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1497	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2199	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1498	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});	2200	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
1499	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2201	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1500	}	2202	}
1501		2203
1502	fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {	2204	fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
1503	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2205	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1504	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2206	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1505	// RISCV arthemtic instructions already wrap, so this is simply a sub binOp with	2207	// RISCV arthemtic instructions already wrap, so this is simply a sub binOp with
1506	// no overflow checks.	2208	// no overflow checks.
1507	const lhs = try self.resolveInst(bin_op.lhs);	2209	const lhs = try self.resolveInst(bin_op.lhs);
...	@@ -1516,34 +2218,34 @@ fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1516,34 +2218,34 @@ fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
1516		2218
1517	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	2219	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1518	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2220	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1519	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});	2221	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
1520	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2222	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1521	}	2223	}
1522		2224
1523	fn airMul(self: *Self, inst: Air.Inst.Index) !void {	2225	fn airMul(self: *Self, inst: Air.Inst.Index) !void {
1524	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2226	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1525	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul for {}", .{self.target.cpu.arch});	2227	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement mul for {}", .{self.target.cpu.arch});
1526	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2228	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1527	}	2229	}
1528		2230
1529	fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {	2231	fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {
1530	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2232	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1531	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch});	2233	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch});
1532	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2234	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1533	}	2235	}
1534		2236
1535	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	2237	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1536	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2238	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1537	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});	2239	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
1538	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2240	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1539	}	2241	}
1540		2242
1541	fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	2243	fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1542	const mod = self.bin_file.comp.module.?;	2244	const zcu = self.bin_file.comp.module.?;
1543	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	2245	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1544	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;	2246	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1545		2247
1546	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2248	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1547	const lhs = try self.resolveInst(extra.lhs);	2249	const lhs = try self.resolveInst(extra.lhs);
1548	const rhs = try self.resolveInst(extra.rhs);	2250	const rhs = try self.resolveInst(extra.rhs);
1549	const lhs_ty = self.typeOf(extra.lhs);	2251	const lhs_ty = self.typeOf(extra.lhs);
...	@@ -1554,16 +2256,21 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1554,16 +2256,21 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1554	defer self.register_manager.unlockReg(add_result_lock);	2256	defer self.register_manager.unlockReg(add_result_lock);
1555		2257
1556	const tuple_ty = self.typeOfIndex(inst);	2258	const tuple_ty = self.typeOfIndex(inst);
1557	const int_info = lhs_ty.intInfo(mod);	2259	const int_info = lhs_ty.intInfo(zcu);
1558		2260
1559	// TODO: optimization, set this to true. needs the other struct access stuff to support	2261	// TODO: optimization, set this to true. needs the other struct access stuff to support
1560	// accessing registers.	2262	// accessing registers.
1561	const result_mcv = try self.allocRegOrMem(inst, false);	2263	const result_mcv = try self.allocRegOrMem(inst, false);
1562	const offset = result_mcv.stack_offset;	2264	const offset = result_mcv.load_frame;
1563
1564	const result_offset = tuple_ty.structFieldOffset(0, mod) + offset;
1565		2265
1566	try self.genSetStack(lhs_ty, @intCast(result_offset), add_result_mcv);	2266	try self.genSetStack(
		2267	lhs_ty,
		2268	.{
		2269	.index = offset.index,
		2270	.off = offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
		2271	},
		2272	add_result_mcv,
		2273	);
1567		2274
1568	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {	2275	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
1569	if (int_info.signedness == .unsigned) {	2276	if (int_info.signedness == .unsigned) {
...	@@ -1585,10 +2292,11 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1585,10 +2292,11 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1585		2292
1586	_ = try self.addInst(.{	2293	_ = try self.addInst(.{
1587	.tag = .andi,	2294	.tag = .andi,
		2295	.ops = .rri,
1588	.data = .{ .i_type = .{	2296	.data = .{ .i_type = .{
1589	.rd = overflow_reg,	2297	.rd = overflow_reg,
1590	.rs1 = add_reg,	2298	.rs1 = add_reg,
1591	.imm12 = @intCast(max_val),	2299	.imm12 = Immediate.s(max_val),
1592	} },	2300	} },
1593	});	2301	});
1594		2302
...	@@ -1601,8 +2309,14 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1601,8 +2309,14 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1601	lhs_ty,	2309	lhs_ty,
1602	);	2310	);
1603		2311
1604	const overflow_offset = tuple_ty.structFieldOffset(1, mod) + offset;	2312	try self.genSetStack(
1605	try self.genSetStack(Type.u1, @intCast(overflow_offset), overflow_mcv);	2313	Type.u1,
		2314	.{
		2315	.index = offset.index,
		2316	.off = offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
		2317	},
		2318	overflow_mcv,
		2319	);
1606		2320
1607	break :result result_mcv;	2321	break :result result_mcv;
1608	},	2322	},
...	@@ -1629,18 +2343,18 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1629,18 +2343,18 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1629	//const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];	2343	//const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
1630	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	2344	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1631	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;	2345	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1632	const mod = self.bin_file.comp.module.?;	2346	const zcu = self.bin_file.comp.module.?;
1633	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2347	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1634	const lhs = try self.resolveInst(extra.lhs);	2348	const lhs = try self.resolveInst(extra.lhs);
1635	const rhs = try self.resolveInst(extra.rhs);	2349	const rhs = try self.resolveInst(extra.rhs);
1636	const lhs_ty = self.typeOf(extra.lhs);	2350	const lhs_ty = self.typeOf(extra.lhs);
1637	const rhs_ty = self.typeOf(extra.rhs);	2351	const rhs_ty = self.typeOf(extra.rhs);
1638		2352
1639	switch (lhs_ty.zigTypeTag(mod)) {	2353	switch (lhs_ty.zigTypeTag(zcu)) {
1640	else => \|x\| return self.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}),	2354	else => \|x\| return self.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}),
1641	.Int => {	2355	.Int => {
1642	assert(lhs_ty.eql(rhs_ty, mod));	2356	assert(lhs_ty.eql(rhs_ty, zcu));
1643	const int_info = lhs_ty.intInfo(mod);	2357	const int_info = lhs_ty.intInfo(zcu);
1644	switch (int_info.bits) {	2358	switch (int_info.bits) {
1645	1...32 => {	2359	1...32 => {
1646	if (self.hasFeature(.m)) {	2360	if (self.hasFeature(.m)) {
...	@@ -1654,11 +2368,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1654,11 +2368,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1654	// TODO: optimization, set this to true. needs the other struct access stuff to support	2368	// TODO: optimization, set this to true. needs the other struct access stuff to support
1655	// accessing registers.	2369	// accessing registers.
1656	const result_mcv = try self.allocRegOrMem(inst, false);	2370	const result_mcv = try self.allocRegOrMem(inst, false);
1657	const offset = result_mcv.stack_offset;
1658		2371
1659	const result_offset = tuple_ty.structFieldOffset(0, mod) + offset;	2372	const result_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
		2373	const overflow_off: i32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
1660		2374
1661	try self.genSetStack(lhs_ty, @intCast(result_offset), dest);	2375	try self.genSetStack(lhs_ty, result_mcv.offset(result_off).load_frame, dest);
1662		2376
1663	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {	2377	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
1664	if (int_info.signedness == .unsigned) {	2378	if (int_info.signedness == .unsigned) {
...	@@ -1680,10 +2394,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1680,10 +2394,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1680		2394
1681	_ = try self.addInst(.{	2395	_ = try self.addInst(.{
1682	.tag = .andi,	2396	.tag = .andi,
		2397	.ops = .rri,
1683	.data = .{ .i_type = .{	2398	.data = .{ .i_type = .{
1684	.rd = overflow_reg,	2399	.rd = overflow_reg,
1685	.rs1 = add_reg,	2400	.rs1 = add_reg,
1686	.imm12 = @intCast(max_val),	2401	.imm12 = Immediate.s(max_val),
1687	} },	2402	} },
1688	});	2403	});
1689		2404
...	@@ -1696,8 +2411,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1696,8 +2411,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1696	lhs_ty,	2411	lhs_ty,
1697	);	2412	);
1698		2413
1699	const overflow_offset = tuple_ty.structFieldOffset(1, mod) + offset;	2414	try self.genSetStack(
1700	try self.genSetStack(Type.u1, @intCast(overflow_offset), overflow_mcv);	2415	lhs_ty,
		2416	result_mcv.offset(overflow_off).load_frame,
		2417	overflow_mcv,
		2418	);
1701		2419
1702	break :result result_mcv;	2420	break :result result_mcv;
1703	},	2421	},
...	@@ -1730,43 +2448,43 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1730,43 +2448,43 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1730		2448
1731	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {	2449	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1732	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2450	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1733	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});	2451	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
1734	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2452	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1735	}	2453	}
1736		2454
1737	fn airRem(self: *Self, inst: Air.Inst.Index) !void {	2455	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
1738	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2456	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1739	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});	2457	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});
1740	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2458	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1741	}	2459	}
1742		2460
1743	fn airMod(self: *Self, inst: Air.Inst.Index) !void {	2461	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1744	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2462	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1745	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mod for {}", .{self.target.cpu.arch});	2463	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement zcu for {}", .{self.target.cpu.arch});
1746	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2464	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1747	}	2465	}
1748		2466
1749	fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {	2467	fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
1750	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2468	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1751	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement bitwise and for {}", .{self.target.cpu.arch});	2469	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement bitwise and for {}", .{self.target.cpu.arch});
1752	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2470	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1753	}	2471	}
1754		2472
1755	fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {	2473	fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
1756	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2474	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1757	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement bitwise or for {}", .{self.target.cpu.arch});	2475	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement bitwise or for {}", .{self.target.cpu.arch});
1758	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2476	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1759	}	2477	}
1760		2478
1761	fn airXor(self: *Self, inst: Air.Inst.Index) !void {	2479	fn airXor(self: *Self, inst: Air.Inst.Index) !void {
1762	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2480	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1763	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement xor for {}", .{self.target.cpu.arch});	2481	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement xor for {}", .{self.target.cpu.arch});
1764	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2482	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1765	}	2483	}
1766		2484
1767	fn airShl(self: *Self, inst: Air.Inst.Index) !void {	2485	fn airShl(self: *Self, inst: Air.Inst.Index) !void {
1768	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2486	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1769	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2487	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1770	const lhs = try self.resolveInst(bin_op.lhs);	2488	const lhs = try self.resolveInst(bin_op.lhs);
1771	const rhs = try self.resolveInst(bin_op.rhs);	2489	const rhs = try self.resolveInst(bin_op.rhs);
1772	const lhs_ty = self.typeOf(bin_op.lhs);	2490	const lhs_ty = self.typeOf(bin_op.lhs);
...	@@ -1779,52 +2497,52 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1779,52 +2497,52 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {
1779		2497
1780	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {	2498	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
1781	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2499	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1782	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});	2500	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
1783	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2501	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1784	}	2502	}
1785		2503
1786	fn airShr(self: *Self, inst: Air.Inst.Index) !void {	2504	fn airShr(self: *Self, inst: Air.Inst.Index) !void {
1787	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2505	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1788	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shr for {}", .{self.target.cpu.arch});	2506	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement shr for {}", .{self.target.cpu.arch});
1789	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2507	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1790	}	2508	}
1791		2509
1792	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {	2510	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1793	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2511	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1794	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});	2512	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});
1795	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2513	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1796	}	2514	}
1797		2515
1798	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	2516	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1799	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2517	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1800	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});	2518	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
1801	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2519	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1802	}	2520	}
1803		2521
1804	fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	2522	fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
1805	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2523	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1806	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});	2524	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});
1807	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2525	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1808	}	2526	}
1809		2527
1810	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {	2528	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
1811	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2529	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1812	const mod = self.bin_file.comp.module.?;	2530	const zcu = self.bin_file.comp.module.?;
1813	const err_union_ty = self.typeOf(ty_op.operand);	2531	const err_union_ty = self.typeOf(ty_op.operand);
1814	const err_ty = err_union_ty.errorUnionSet(mod);	2532	const err_ty = err_union_ty.errorUnionSet(zcu);
1815	const payload_ty = err_union_ty.errorUnionPayload(mod);	2533	const payload_ty = err_union_ty.errorUnionPayload(zcu);
1816	const operand = try self.resolveInst(ty_op.operand);	2534	const operand = try self.resolveInst(ty_op.operand);
1817		2535
1818	const result: MCValue = result: {	2536	const result: MCValue = result: {
1819	if (err_ty.errorSetIsEmpty(mod)) {	2537	if (err_ty.errorSetIsEmpty(zcu)) {
1820	break :result .{ .immediate = 0 };	2538	break :result .{ .immediate = 0 };
1821	}	2539	}
1822		2540
1823	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {	2541	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
1824	break :result operand;	2542	break :result operand;
1825	}	2543	}
1826		2544
1827	const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, mod));	2545	const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
1828		2546
1829	switch (operand) {	2547	switch (operand) {
1830	.register => \|reg\| {	2548	.register => \|reg\| {
...	@@ -1865,16 +2583,18 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -1865,16 +2583,18 @@ fn genUnwrapErrUnionPayloadMir(
1865	err_union_ty: Type,	2583	err_union_ty: Type,
1866	err_union: MCValue,	2584	err_union: MCValue,
1867	) !MCValue {	2585	) !MCValue {
1868	const mod = self.bin_file.comp.module.?;	2586	const zcu = self.bin_file.comp.module.?;
1869		2587	const payload_ty = err_union_ty.errorUnionPayload(zcu);
1870	const payload_ty = err_union_ty.errorUnionPayload(mod);
1871		2588
1872	const result: MCValue = result: {	2589	const result: MCValue = result: {
1873	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;	2590	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
1874		2591
1875	const payload_off: u32 = @intCast(errUnionPayloadOffset(payload_ty, mod));	2592	const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
1876	switch (err_union) {	2593	switch (err_union) {
1877	.stack_offset => \|off\| break :result .{ .stack_offset = off + payload_off },	2594	.load_frame => \|frame_addr\| break :result .{ .load_frame = .{
		2595	.index = frame_addr.index,
		2596	.off = frame_addr.off + payload_off,
		2597	} },
1878	.register => \|reg\| {	2598	.register => \|reg\| {
1879	const eu_lock = self.register_manager.lockReg(reg);	2599	const eu_lock = self.register_manager.lockReg(reg);
1880	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);	2600	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
...	@@ -1904,26 +2624,26 @@ fn genUnwrapErrUnionPayloadMir(	...	@@ -1904,26 +2624,26 @@ fn genUnwrapErrUnionPayloadMir(
1904	// *(E!T) -> E	2624	// *(E!T) -> E
1905	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {	2625	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1906	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2626	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1907	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});	2627	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
1908	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2628	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1909	}	2629	}
1910		2630
1911	// (E!T) -> T	2631	// (E!T) -> T
1912	fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	2632	fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1913	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2633	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1914	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});	2634	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
1915	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2635	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1916	}	2636	}
1917		2637
1918	fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	2638	fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
1919	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2639	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1920	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});	2640	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});
1921	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2641	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1922	}	2642	}
1923		2643
1924	fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {	2644	fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
1925	const result: MCValue = if (self.liveness.isUnused(inst))	2645	const result: MCValue = if (self.liveness.isUnused(inst))
1926	.dead	2646	.unreach
1927	else	2647	else
1928	return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});	2648	return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
1929	return self.finishAir(inst, result, .{ .none, .none, .none });	2649	return self.finishAir(inst, result, .{ .none, .none, .none });
...	@@ -1941,12 +2661,12 @@ fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1941,12 +2661,12 @@ fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
1941		2661
1942	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	2662	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
1943	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2663	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1944	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2664	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
1945	const mod = self.bin_file.comp.module.?;	2665	const zcu = self.bin_file.comp.module.?;
1946	const optional_ty = self.typeOfIndex(inst);	2666	const optional_ty = self.typeOfIndex(inst);
1947		2667
1948	// Optional with a zero-bit payload type is just a boolean true	2668	// Optional with a zero-bit payload type is just a boolean true
1949	if (optional_ty.abiSize(mod) == 1)	2669	if (optional_ty.abiSize(zcu) == 1)
1950	break :result MCValue{ .immediate = 1 };	2670	break :result MCValue{ .immediate = 1 };
1951		2671
1952	return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});	2672	return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});
...	@@ -1957,29 +2677,29 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1957,29 +2677,29 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
1957	/// T to E!T	2677	/// T to E!T
1958	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	2678	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
1959	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2679	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1960	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});	2680	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});
1961	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2681	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1962	}	2682	}
1963		2683
1964	/// E to E!T	2684	/// E to E!T
1965	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	2685	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
1966	const mod = self.bin_file.comp.module.?;	2686	const zcu = self.bin_file.comp.module.?;
1967	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2687	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1968		2688
1969	const eu_ty = ty_op.ty.toType();	2689	const eu_ty = ty_op.ty.toType();
1970	const pl_ty = eu_ty.errorUnionPayload(mod);	2690	const pl_ty = eu_ty.errorUnionPayload(zcu);
1971	const err_ty = eu_ty.errorUnionSet(mod);	2691	const err_ty = eu_ty.errorUnionSet(zcu);
1972		2692
1973	const result: MCValue = result: {	2693	const result: MCValue = result: {
1974	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);	2694	if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
1975		2695
1976	const stack_off = try self.allocMem(null, @intCast(eu_ty.abiSize(mod)), eu_ty.abiAlignment(mod));	2696	const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
1977	const pl_off: u32 = @intCast(errUnionPayloadOffset(pl_ty, mod));	2697	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
1978	const err_off: u32 = @intCast(errUnionErrorOffset(pl_ty, mod));	2698	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
1979	try self.genSetStack(pl_ty, stack_off + pl_off, .undef);	2699	try self.genSetStack(pl_ty, .{ .index = frame_index, .off = pl_off }, .undef);
1980	const operand = try self.resolveInst(ty_op.operand);	2700	const operand = try self.resolveInst(ty_op.operand);
1981	try self.genSetStack(err_ty, stack_off + err_off, operand);	2701	try self.genSetStack(err_ty, .{ .index = frame_index, .off = err_off }, operand);
1982	break :result .{ .stack_offset = stack_off };	2702	break :result .{ .load_frame = .{ .index = frame_index } };
1983	};	2703	};
1984	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2704	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1985	}	2705	}
...	@@ -2001,67 +2721,41 @@ fn genTry(	...	@@ -2001,67 +2721,41 @@ fn genTry(
2001	operand_ty: Type,	2721	operand_ty: Type,
2002	operand_is_ptr: bool,	2722	operand_is_ptr: bool,
2003	) !MCValue {	2723	) !MCValue {
2004	const liveness_condbr = self.liveness.getCondBr(inst);
2005
2006	_ = operand_is_ptr;	2724	_ = operand_is_ptr;
2007		2725
		2726	const liveness_cond_br = self.liveness.getCondBr(inst);
		2727
2008	const operand_mcv = try self.resolveInst(operand);	2728	const operand_mcv = try self.resolveInst(operand);
2009	const is_err_mcv = try self.isErr(null, operand_ty, operand_mcv);	2729	const is_err_mcv = try self.isErr(null, operand_ty, operand_mcv);
2010		2730
2011	const cond_reg = try self.register_manager.allocReg(inst, gp);
2012	const cond_reg_lock = self.register_manager.lockRegAssumeUnused(cond_reg);
2013	defer self.register_manager.unlockReg(cond_reg_lock);
2014
2015	// A branch to the false section. Uses beq. 1 is the default "true" state.	2731	// A branch to the false section. Uses beq. 1 is the default "true" state.
2016	const reloc = try self.condBr(Type.anyerror, is_err_mcv, cond_reg);	2732	const reloc = try self.condBr(Type.anyerror, is_err_mcv);
2017		2733
2018	if (self.liveness.operandDies(inst, 0)) {	2734	if (self.liveness.operandDies(inst, 0)) {
2019	if (operand.toIndex()) \|op_inst\| self.processDeath(op_inst);	2735	if (operand.toIndex()) \|operand_inst\| try self.processDeath(operand_inst);
2020	}
2021
2022	// Save state
2023	const parent_next_stack_offset = self.next_stack_offset;
2024	const parent_free_registers = self.register_manager.free_registers;
2025	var parent_stack = try self.stack.clone(self.gpa);
2026	defer parent_stack.deinit(self.gpa);
2027	const parent_registers = self.register_manager.registers;
2028
2029	try self.branch_stack.append(.{});
2030	errdefer {
2031	_ = self.branch_stack.pop();
2032	}	2736	}
2033		2737
2034	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);	2738	self.scope_generation += 1;
2035	for (liveness_condbr.else_deaths) \|op\| {	2739	const state = try self.saveState();
2036	self.processDeath(op);
2037	}
2038		2740
		2741	for (liveness_cond_br.else_deaths) \|death\| try self.processDeath(death);
2039	try self.genBody(body);	2742	try self.genBody(body);
		2743	try self.restoreState(state, &.{}, .{
		2744	.emit_instructions = false,
		2745	.update_tracking = true,
		2746	.resurrect = true,
		2747	.close_scope = true,
		2748	});
2040		2749
2041	// Restore state	2750	self.performReloc(reloc);
2042	var saved_then_branch = self.branch_stack.pop();
2043	defer saved_then_branch.deinit(self.gpa);
2044
2045	self.register_manager.registers = parent_registers;
2046
2047	self.stack.deinit(self.gpa);
2048	self.stack = parent_stack;
2049	parent_stack = .{};
2050
2051	self.next_stack_offset = parent_next_stack_offset;
2052	self.register_manager.free_registers = parent_free_registers;
2053		2751
2054	try self.performReloc(reloc, @intCast(self.mir_instructions.len));	2752	for (liveness_cond_br.then_deaths) \|death\| try self.processDeath(death);
2055
2056	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
2057	for (liveness_condbr.then_deaths) \|op\| {
2058	self.processDeath(op);
2059	}
2060		2753
2061	const result = if (self.liveness.isUnused(inst))	2754	const result = if (self.liveness.isUnused(inst))
2062	.unreach	2755	.unreach
2063	else	2756	else
2064	try self.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv);	2757	try self.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv);
		2758
2065	return result;	2759	return result;
2066	}	2760	}
2067		2761
...	@@ -2081,11 +2775,14 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2081,11 +2775,14 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2081		2775
2082	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {	2776	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2083	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2777	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2084	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2778	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2085	const src_mcv = try self.resolveInst(ty_op.operand);	2779	const src_mcv = try self.resolveInst(ty_op.operand);
2086	switch (src_mcv) {	2780	switch (src_mcv) {
2087	.stack_offset => \|off\| {	2781	.load_frame => \|frame_addr\| {
2088	const len_mcv: MCValue = .{ .stack_offset = off + 8 };	2782	const len_mcv: MCValue = .{ .load_frame = .{
		2783	.index = frame_addr.index,
		2784	.off = frame_addr.off + 8,
		2785	} };
2089	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;	2786	if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
2090		2787
2091	const dst_mcv = try self.allocRegOrMem(inst, true);	2788	const dst_mcv = try self.allocRegOrMem(inst, true);
...	@@ -2109,29 +2806,33 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2109,29 +2806,33 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2109		2806
2110	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {	2807	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2111	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2808	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2112	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});	2809	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});
2113	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2810	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2114	}	2811	}
2115		2812
2116	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {	2813	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
2117	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2814	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2118	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});	2815	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});
2119	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2816	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2120	}	2817	}
2121		2818
2122	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	2819	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2123	const mod = self.bin_file.comp.module.?;	2820	const zcu = self.bin_file.comp.module.?;
2124	const is_volatile = false; // TODO	2821	const is_volatile = false; // TODO
2125	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2822	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2126		2823
2127	if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });	2824	if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(
		2825	inst,
		2826	.unreach,
		2827	.{ bin_op.lhs, bin_op.rhs, .none },
		2828	);
2128	const result: MCValue = result: {	2829	const result: MCValue = result: {
2129	const slice_mcv = try self.resolveInst(bin_op.lhs);	2830	const slice_mcv = try self.resolveInst(bin_op.lhs);
2130	const index_mcv = try self.resolveInst(bin_op.rhs);	2831	const index_mcv = try self.resolveInst(bin_op.rhs);
2131		2832
2132	const slice_ty = self.typeOf(bin_op.lhs);	2833	const slice_ty = self.typeOf(bin_op.lhs);
2133		2834
2134	const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);	2835	const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu);
2135		2836
2136	const index_lock: ?RegisterLock = if (index_mcv == .register)	2837	const index_lock: ?RegisterLock = if (index_mcv == .register)
2137	self.register_manager.lockRegAssumeUnused(index_mcv.register)	2838	self.register_manager.lockRegAssumeUnused(index_mcv.register)
...	@@ -2140,7 +2841,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2140,7 +2841,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2140	defer if (index_lock) \|reg\| self.register_manager.unlockReg(reg);	2841	defer if (index_lock) \|reg\| self.register_manager.unlockReg(reg);
2141		2842
2142	const base_mcv: MCValue = switch (slice_mcv) {	2843	const base_mcv: MCValue = switch (slice_mcv) {
2143	.stack_offset => \|off\| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },	2844	.load_frame,
		2845	.load_symbol,
		2846	=> .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, slice_mcv) },
2144	else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),	2847	else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
2145	};	2848	};
2146		2849
...	@@ -2156,38 +2859,34 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2156,38 +2859,34 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2156	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {	2859	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2157	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	2860	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2158	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;	2861	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2159	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});	2862	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});
2160	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });	2863	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2161	}	2864	}
2162		2865
2163	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	2866	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2164	const mod = self.bin_file.comp.module.?;	2867	const zcu = self.bin_file.comp.module.?;
2165	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2868	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2166	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2869	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2167	const array_ty = self.typeOf(bin_op.lhs);	2870	const array_ty = self.typeOf(bin_op.lhs);
2168	const array_mcv = try self.resolveInst(bin_op.lhs);	2871	const array_mcv = try self.resolveInst(bin_op.lhs);
2169		2872
2170	const index_mcv = try self.resolveInst(bin_op.rhs);	2873	const index_mcv = try self.resolveInst(bin_op.rhs);
2171	const index_ty = self.typeOf(bin_op.rhs);	2874	const index_ty = self.typeOf(bin_op.rhs);
2172		2875
2173	const elem_ty = array_ty.childType(mod);	2876	const elem_ty = array_ty.childType(zcu);
2174	const elem_abi_size = elem_ty.abiSize(mod);	2877	const elem_abi_size = elem_ty.abiSize(zcu);
2175		2878
2176	const addr_reg, const addr_reg_lock = try self.allocReg();	2879	const addr_reg, const addr_reg_lock = try self.allocReg();
2177	defer self.register_manager.unlockReg(addr_reg_lock);	2880	defer self.register_manager.unlockReg(addr_reg_lock);
2178		2881
2179	switch (array_mcv) {	2882	switch (array_mcv) {
2180	.register => {	2883	.register => {
2181	const stack_offset = try self.allocMem(	2884	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
2182	null,	2885	try self.genSetStack(array_ty, .{ .index = frame_index }, array_mcv);
2183	@intCast(array_ty.abiSize(mod)),	2886	try self.genSetReg(Type.usize, addr_reg, .{ .lea_frame = .{ .index = frame_index } });
2184	array_ty.abiAlignment(mod),
2185	);
2186	try self.genSetStack(array_ty, stack_offset, array_mcv);
2187	try self.genSetReg(Type.usize, addr_reg, .{ .ptr_stack_offset = stack_offset });
2188	},	2887	},
2189	.stack_offset => \|off\| {	2888	.load_frame => \|frame_addr\| {
2190	try self.genSetReg(Type.usize, addr_reg, .{ .ptr_stack_offset = off });	2889	try self.genSetReg(Type.usize, addr_reg, .{ .lea_frame = frame_addr });
2191	},	2890	},
2192	else => try self.genSetReg(Type.usize, addr_reg, array_mcv.address()),	2891	else => try self.genSetReg(Type.usize, addr_reg, array_mcv.address()),
2193	}	2892	}
...	@@ -2213,14 +2912,14 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2213,14 +2912,14 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2213	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	2912	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2214	const is_volatile = false; // TODO	2913	const is_volatile = false; // TODO
2215	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2914	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2216	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch});	2915	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch});
2217	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2916	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2218	}	2917	}
2219		2918
2220	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {	2919	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2221	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	2920	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2222	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;	2921	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2223	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});	2922	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});
2224	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });	2923	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2225	}	2924	}
2226		2925
...	@@ -2233,19 +2932,19 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2233,19 +2932,19 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2233		2932
2234	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	2933	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2235	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2934	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2236	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});	2935	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
2237	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2936	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2238	}	2937	}
2239		2938
2240	fn airClz(self: *Self, inst: Air.Inst.Index) !void {	2939	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
2241	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2940	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2242	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});	2941	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
2243	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2942	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2244	}	2943	}
2245		2944
2246	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	2945	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
2247	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2946	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2248	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2947	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2249	const operand = try self.resolveInst(ty_op.operand);	2948	const operand = try self.resolveInst(ty_op.operand);
2250	const operand_ty = self.typeOf(ty_op.operand);	2949	const operand_ty = self.typeOf(ty_op.operand);
2251		2950
...	@@ -2270,8 +2969,8 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2270,8 +2969,8 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
2270	}	2969	}
2271		2970
2272	fn ctz(self: *Self, src: Register, dst: Register, ty: Type) !void {	2971	fn ctz(self: *Self, src: Register, dst: Register, ty: Type) !void {
2273	const mod = self.bin_file.comp.module.?;	2972	const zcu = self.bin_file.comp.module.?;
2274	const length = (ty.abiSize(mod) * 8) - 1;	2973	const length = (ty.abiSize(zcu) * 8) - 1;
2275		2974
2276	const count_reg, const count_lock = try self.allocReg();	2975	const count_reg, const count_lock = try self.allocReg();
2277	defer self.register_manager.unlockReg(count_lock);	2976	defer self.register_manager.unlockReg(count_lock);
...	@@ -2282,17 +2981,6 @@ fn ctz(self: *Self, src: Register, dst: Register, ty: Type) !void {	...	@@ -2282,17 +2981,6 @@ fn ctz(self: *Self, src: Register, dst: Register, ty: Type) !void {
2282	try self.genSetReg(Type.usize, count_reg, .{ .immediate = 0 });	2981	try self.genSetReg(Type.usize, count_reg, .{ .immediate = 0 });
2283	try self.genSetReg(Type.usize, len_reg, .{ .immediate = length });	2982	try self.genSetReg(Type.usize, len_reg, .{ .immediate = length });
2284		2983
2285	_ = try self.addInst(.{
2286	.tag = .beq,
2287	.data = .{
2288	.b_type = .{
2289	.rs1 = count_reg,
2290	.rs2 = len_reg,
2291	.inst = @intCast(self.mir_instructions.len + 0),
2292	},
2293	},
2294	});
2295
2296	_ = src;	2984	_ = src;
2297	_ = dst;	2985	_ = dst;
2298		2986
...	@@ -2301,23 +2989,23 @@ fn ctz(self: *Self, src: Register, dst: Register, ty: Type) !void {	...	@@ -2301,23 +2989,23 @@ fn ctz(self: *Self, src: Register, dst: Register, ty: Type) !void {
2301		2989
2302	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {	2990	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
2303	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2991	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2304	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});	2992	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
2305	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2993	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2306	}	2994	}
2307		2995
2308	fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	2996	fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
2309	const mod = self.bin_file.comp.module.?;	2997	const zcu = self.bin_file.comp.module.?;
2310	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2998	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2311	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2999	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2312	const ty = self.typeOf(ty_op.operand);	3000	const ty = self.typeOf(ty_op.operand);
2313	const scalar_ty = ty.scalarType(mod);	3001	const scalar_ty = ty.scalarType(zcu);
2314	const operand = try self.resolveInst(ty_op.operand);	3002	const operand = try self.resolveInst(ty_op.operand);
2315		3003
2316	switch (scalar_ty.zigTypeTag(mod)) {	3004	switch (scalar_ty.zigTypeTag(zcu)) {
2317	.Int => if (ty.zigTypeTag(mod) == .Vector) {	3005	.Int => if (ty.zigTypeTag(zcu) == .Vector) {
2318	return self.fail("TODO implement airAbs for {}", .{ty.fmt(mod)});	3006	return self.fail("TODO implement airAbs for {}", .{ty.fmt(zcu)});
2319	} else {	3007	} else {
2320	const int_bits = ty.intInfo(mod).bits;	3008	const int_bits = ty.intInfo(zcu).bits;
2321		3009
2322	if (int_bits > 32) {	3010	if (int_bits > 32) {
2323	return self.fail("TODO: airAbs for larger than 32 bits", .{});	3011	return self.fail("TODO: airAbs for larger than 32 bits", .{});
...	@@ -2330,18 +3018,19 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2330,18 +3018,19 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
2330		3018
2331	_ = try self.addInst(.{	3019	_ = try self.addInst(.{
2332	.tag = .abs,	3020	.tag = .abs,
		3021	.ops = .rri,
2333	.data = .{	3022	.data = .{
2334	.i_type = .{	3023	.i_type = .{
2335	.rs1 = src_mcv.register,	3024	.rs1 = src_mcv.register,
2336	.rd = temp_reg,	3025	.rd = temp_reg,
2337	.imm12 = @intCast(int_bits - 1),	3026	.imm12 = Immediate.s(int_bits - 1),
2338	},	3027	},
2339	},	3028	},
2340	});	3029	});
2341		3030
2342	break :result src_mcv;	3031	break :result src_mcv;
2343	},	3032	},
2344	else => return self.fail("TODO: implement airAbs {}", .{scalar_ty.fmt(mod)}),	3033	else => return self.fail("TODO: implement airAbs {}", .{scalar_ty.fmt(zcu)}),
2345	}	3034	}
2346	};	3035	};
2347	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	3036	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -2349,12 +3038,12 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2349,12 +3038,12 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
2349		3038
2350	fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {	3039	fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
2351	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	3040	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2352	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3041	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2353	const mod = self.bin_file.comp.module.?;	3042	const zcu = self.bin_file.comp.module.?;
2354	const ty = self.typeOf(ty_op.operand);	3043	const ty = self.typeOf(ty_op.operand);
2355	const operand = try self.resolveInst(ty_op.operand);	3044	const operand = try self.resolveInst(ty_op.operand);
2356		3045
2357	const int_bits = ty.intInfo(mod).bits;	3046	const int_bits = ty.intInfo(zcu).bits;
2358		3047
2359	// bytes are no-op	3048	// bytes are no-op
2360	if (int_bits == 8 and self.reuseOperand(inst, ty_op.operand, 0, operand)) {	3049	if (int_bits == 8 and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
...	@@ -2372,14 +3061,16 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2372,14 +3061,16 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
2372	assert(temp == .register);	3061	assert(temp == .register);
2373	_ = try self.addInst(.{	3062	_ = try self.addInst(.{
2374	.tag = .slli,	3063	.tag = .slli,
		3064	.ops = .rri,
2375	.data = .{ .i_type = .{	3065	.data = .{ .i_type = .{
2376	.imm12 = 8,	3066	.imm12 = Immediate.s(8),
2377	.rd = dest_reg,	3067	.rd = dest_reg,
2378	.rs1 = dest_reg,	3068	.rs1 = dest_reg,
2379	} },	3069	} },
2380	});	3070	});
2381	_ = try self.addInst(.{	3071	_ = try self.addInst(.{
2382	.tag = .@"or",	3072	.tag = .@"or",
		3073	.ops = .rri,
2383	.data = .{ .r_type = .{	3074	.data = .{ .r_type = .{
2384	.rd = dest_reg,	3075	.rd = dest_reg,
2385	.rs1 = dest_reg,	3076	.rs1 = dest_reg,
...	@@ -2397,62 +3088,78 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2397,62 +3088,78 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
2397		3088
2398	fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	3089	fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
2399	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	3090	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2400	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});	3091	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
2401	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	3092	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2402	}	3093	}
2403		3094
2404	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {	3095	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
2405	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3096	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2406	const result: MCValue = if (self.liveness.isUnused(inst))	3097	const result: MCValue = if (self.liveness.isUnused(inst))
2407	.dead	3098	.unreach
2408	else	3099	else
2409	return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});	3100	return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});
2410	return self.finishAir(inst, result, .{ un_op, .none, .none });	3101	return self.finishAir(inst, result, .{ un_op, .none, .none });
2411	}	3102	}
2412		3103
2413	fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {	3104	fn reuseOperand(
		3105	self: *Self,
		3106	inst: Air.Inst.Index,
		3107	operand: Air.Inst.Ref,
		3108	op_index: Liveness.OperandInt,
		3109	mcv: MCValue,
		3110	) bool {
		3111	return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
		3112	}
		3113
		3114	fn reuseOperandAdvanced(
		3115	self: *Self,
		3116	inst: Air.Inst.Index,
		3117	operand: Air.Inst.Ref,
		3118	op_index: Liveness.OperandInt,
		3119	mcv: MCValue,
		3120	maybe_tracked_inst: ?Air.Inst.Index,
		3121	) bool {
2414	if (!self.liveness.operandDies(inst, op_index))	3122	if (!self.liveness.operandDies(inst, op_index))
2415	return false;	3123	return false;
2416		3124
2417	switch (mcv) {	3125	switch (mcv) {
2418	.register => \|reg\| {	3126	.register,
2419	// If it's in the registers table, need to associate the register with the	3127	.register_pair,
		3128	=> for (mcv.getRegs()) \|reg\| {
		3129	// If it's in the registers table, need to associate the register(s) with the
2420	// new instruction.	3130	// new instruction.
2421	if (RegisterManager.indexOfRegIntoTracked(reg)) \|index\| {	3131	if (maybe_tracked_inst) \|tracked_inst\| {
2422	if (!self.register_manager.isRegFree(reg)) {	3132	if (!self.register_manager.isRegFree(reg)) {
2423	self.register_manager.registers[index] = inst;	3133	if (RegisterManager.indexOfRegIntoTracked(reg)) \|index\| {
		3134	self.register_manager.registers[index] = tracked_inst;
		3135	}
2424	}	3136	}
2425	}	3137	} else self.register_manager.freeReg(reg);
2426	log.debug("%{d} => {} (reused)", .{ inst, reg });
2427	},
2428	.stack_offset => \|off\| {
2429	log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
2430	},	3138	},
		3139	.load_frame => \|frame_addr\| if (frame_addr.index.isNamed()) return false,
2431	else => return false,	3140	else => return false,
2432	}	3141	}
2433		3142
2434	// Prevent the operand deaths processing code from deallocating it.	3143	// Prevent the operand deaths processing code from deallocating it.
2435	self.liveness.clearOperandDeath(inst, op_index);	3144	self.liveness.clearOperandDeath(inst, op_index);
2436		3145	const op_inst = operand.toIndex().?;
2437	// That makes us responsible for doing the rest of the stuff that processDeath would have done.	3146	self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst);
2438	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
2439	branch.inst_table.putAssumeCapacity(operand.toIndex().?, .dead);
2440		3147
2441	return true;	3148	return true;
2442	}	3149	}
2443		3150
2444	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	3151	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
2445	const mod = self.bin_file.comp.module.?;	3152	const zcu = self.bin_file.comp.module.?;
2446	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	3153	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2447	const elem_ty = self.typeOfIndex(inst);	3154	const elem_ty = self.typeOfIndex(inst);
2448	const result: MCValue = result: {	3155	const result: MCValue = result: {
2449	if (!elem_ty.hasRuntimeBits(mod))	3156	if (!elem_ty.hasRuntimeBits(zcu))
2450	break :result .none;	3157	break :result .none;
2451		3158
2452	const ptr = try self.resolveInst(ty_op.operand);	3159	const ptr = try self.resolveInst(ty_op.operand);
2453	const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(mod);	3160	const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(zcu);
2454	if (self.liveness.isUnused(inst) and !is_volatile)	3161	if (self.liveness.isUnused(inst) and !is_volatile)
2455	break :result .dead;	3162	break :result .unreach;
2456		3163
2457	const dst_mcv: MCValue = blk: {	3164	const dst_mcv: MCValue = blk: {
2458	if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {	3165	if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
...	@@ -2462,6 +3169,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2462,6 +3169,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
2462	break :blk try self.allocRegOrMem(inst, true);	3169	break :blk try self.allocRegOrMem(inst, true);
2463	}	3170	}
2464	};	3171	};
		3172
2465	try self.load(dst_mcv, ptr, self.typeOf(ty_op.operand));	3173	try self.load(dst_mcv, ptr, self.typeOf(ty_op.operand));
2466	break :result dst_mcv;	3174	break :result dst_mcv;
2467	};	3175	};
...	@@ -2469,10 +3177,10 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2469,10 +3177,10 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
2469	}	3177	}
2470		3178
2471	fn load(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {	3179	fn load(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {
2472	const mod = self.bin_file.comp.module.?;	3180	const zcu = self.bin_file.comp.module.?;
2473	const dst_ty = ptr_ty.childType(mod);	3181	const dst_ty = ptr_ty.childType(zcu);
2474		3182
2475	log.debug("loading {}:{} into {}", .{ ptr_mcv, ptr_ty.fmt(mod), dst_mcv });	3183	log.debug("loading {}:{} into {}", .{ ptr_mcv, ptr_ty.fmt(zcu), dst_mcv });
2476		3184
2477	switch (ptr_mcv) {	3185	switch (ptr_mcv) {
2478	.none,	3186	.none,
...	@@ -2480,19 +3188,20 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerErro	...	@@ -2480,19 +3188,20 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerErro
2480	.unreach,	3188	.unreach,
2481	.dead,	3189	.dead,
2482	.register_pair,	3190	.register_pair,
		3191	.reserved_frame,
2483	=> unreachable, // not a valid pointer	3192	=> unreachable, // not a valid pointer
2484		3193
2485	.immediate,	3194	.immediate,
2486	.register,	3195	.register,
2487	.register_offset,	3196	.register_offset,
2488	.ptr_stack_offset,	3197	.lea_frame,
2489	.addr_symbol,	3198	.lea_symbol,
2490	=> try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),	3199	=> try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
2491		3200
2492	.memory,	3201	.memory,
2493	.indirect,	3202	.indirect,
2494	.load_symbol,	3203	.load_symbol,
2495	.stack_offset,	3204	.load_frame,
2496	=> {	3205	=> {
2497	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);	3206	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
2498	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	3207	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
...	@@ -2518,14 +3227,14 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -2518,14 +3227,14 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
2518		3227
2519	try self.store(ptr, value, ptr_ty, value_ty);	3228	try self.store(ptr, value, ptr_ty, value_ty);
2520		3229
2521	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });	3230	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
2522	}	3231	}
2523		3232
2524	/// Loads `value` into the "payload" of `pointer`.	3233	/// Loads `value` into the "payload" of `pointer`.
2525	fn store(self: *Self, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty: Type) !void {	3234	fn store(self: *Self, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty: Type) !void {
2526	const mod = self.bin_file.comp.module.?;	3235	const zcu = self.bin_file.comp.module.?;
2527		3236
2528	log.debug("storing {}:{} in {}:{}", .{ src_mcv, src_ty.fmt(mod), ptr_mcv, ptr_ty.fmt(mod) });	3237	log.debug("storing {}:{} in {}:{}", .{ src_mcv, src_ty.fmt(zcu), ptr_mcv, ptr_ty.fmt(zcu) });
2529		3238
2530	switch (ptr_mcv) {	3239	switch (ptr_mcv) {
2531	.none => unreachable,	3240	.none => unreachable,
...	@@ -2533,18 +3242,19 @@ fn store(self: *Self, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty:	...	@@ -2533,18 +3242,19 @@ fn store(self: *Self, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty:
2533	.unreach => unreachable,	3242	.unreach => unreachable,
2534	.dead => unreachable,	3243	.dead => unreachable,
2535	.register_pair => unreachable,	3244	.register_pair => unreachable,
		3245	.reserved_frame => unreachable,
2536		3246
2537	.immediate,	3247	.immediate,
2538	.register,	3248	.register,
2539	.register_offset,	3249	.register_offset,
2540	.addr_symbol,	3250	.lea_symbol,
2541	.ptr_stack_offset,	3251	.lea_frame,
2542	=> try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),	3252	=> try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
2543		3253
2544	.memory,	3254	.memory,
2545	.indirect,	3255	.indirect,
2546	.load_symbol,	3256	.load_symbol,
2547	.stack_offset,	3257	.load_frame,
2548	=> {	3258	=> {
2549	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);	3259	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
2550	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);	3260	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
...	@@ -2570,24 +3280,24 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {	...	@@ -2570,24 +3280,24 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
2570	}	3280	}
2571		3281
2572	fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {	3282	fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
2573	const mod = self.bin_file.comp.module.?;	3283	const zcu = self.bin_file.comp.module.?;
2574	const ptr_field_ty = self.typeOfIndex(inst);	3284	const ptr_field_ty = self.typeOfIndex(inst);
2575	const ptr_container_ty = self.typeOf(operand);	3285	const ptr_container_ty = self.typeOf(operand);
2576	const ptr_container_ty_info = ptr_container_ty.ptrInfo(mod);	3286	const ptr_container_ty_info = ptr_container_ty.ptrInfo(zcu);
2577	const container_ty = ptr_container_ty.childType(mod);	3287	const container_ty = ptr_container_ty.childType(zcu);
2578		3288
2579	const field_offset: i32 = if (mod.typeToPackedStruct(container_ty)) \|struct_obj\|	3289	const field_offset: i32 = if (zcu.typeToPackedStruct(container_ty)) \|struct_obj\|
2580	if (ptr_field_ty.ptrInfo(mod).packed_offset.host_size == 0)	3290	if (ptr_field_ty.ptrInfo(zcu).packed_offset.host_size == 0)
2581	@divExact(mod.structPackedFieldBitOffset(struct_obj, index) +	3291	@divExact(zcu.structPackedFieldBitOffset(struct_obj, index) +
2582	ptr_container_ty_info.packed_offset.bit_offset, 8)	3292	ptr_container_ty_info.packed_offset.bit_offset, 8)
2583	else	3293	else
2584	0	3294	0
2585	else	3295	else
2586	@intCast(container_ty.structFieldOffset(index, mod));	3296	@intCast(container_ty.structFieldOffset(index, zcu));
2587		3297
2588	const src_mcv = try self.resolveInst(operand);	3298	const src_mcv = try self.resolveInst(operand);
2589	const dst_mcv = if (switch (src_mcv) {	3299	const dst_mcv = if (switch (src_mcv) {
2590	.immediate, .ptr_stack_offset => true,	3300	.immediate, .lea_frame => true,
2591	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),	3301	.register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv),
2592	else => false,	3302	else => false,
2593	}) src_mcv else try self.copyToNewRegister(inst, src_mcv);	3303	}) src_mcv else try self.copyToNewRegister(inst, src_mcv);
...	@@ -2595,21 +3305,24 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde	...	@@ -2595,21 +3305,24 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
2595	}	3305	}
2596		3306
2597	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	3307	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
		3308	const mod = self.bin_file.comp.module.?;
		3309
2598	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	3310	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2599	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;	3311	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2600	const operand = extra.struct_operand;	3312	const operand = extra.struct_operand;
2601	const index = extra.field_index;	3313	const index = extra.field_index;
2602	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3314
2603	const mod = self.bin_file.comp.module.?;	3315	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
		3316	const zcu = self.bin_file.comp.module.?;
2604	const src_mcv = try self.resolveInst(operand);	3317	const src_mcv = try self.resolveInst(operand);
2605	const struct_ty = self.typeOf(operand);	3318	const struct_ty = self.typeOf(operand);
2606	const field_ty = struct_ty.structFieldType(index, mod);	3319	const field_ty = struct_ty.structFieldType(index, zcu);
2607	if (!field_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;	3320	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
2608		3321
2609	const field_off: u32 = switch (struct_ty.containerLayout(mod)) {	3322	const field_off: u32 = switch (struct_ty.containerLayout(zcu)) {
2610	.auto, .@"extern" => @intCast(struct_ty.structFieldOffset(index, mod) * 8),	3323	.auto, .@"extern" => @intCast(struct_ty.structFieldOffset(index, zcu) * 8),
2611	.@"packed" => if (mod.typeToStruct(struct_ty)) \|struct_type\|	3324	.@"packed" => if (zcu.typeToStruct(struct_ty)) \|struct_type\|
2612	mod.structPackedFieldBitOffset(struct_type, index)	3325	zcu.structPackedFieldBitOffset(struct_type, index)
2613	else	3326	else
2614	0,	3327	0,
2615	};	3328	};
...	@@ -2632,13 +3345,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2632,13 +3345,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2632	if (field_off > 0) {	3345	if (field_off > 0) {
2633	_ = try self.addInst(.{	3346	_ = try self.addInst(.{
2634	.tag = .srli,	3347	.tag = .srli,
2635	.data = .{	3348	.ops = .rri,
2636	.i_type = .{	3349	.data = .{ .i_type = .{
2637	.imm12 = @intCast(field_off),	3350	.imm12 = Immediate.s(@intCast(field_off)),
2638	.rd = dst_reg,	3351	.rd = dst_reg,
2639	.rs1 = dst_reg,	3352	.rs1 = dst_reg,
2640	},	3353	} },
2641	},
2642	});	3354	});
2643		3355
2644	return self.fail("TODO: airStructFieldVal register with field_off > 0", .{});	3356	return self.fail("TODO: airStructFieldVal register with field_off > 0", .{});
...	@@ -2646,10 +3358,49 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2646,10 +3358,49 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2646		3358
2647	break :result if (field_off == 0) dst_mcv else try self.copyToNewRegister(inst, dst_mcv);	3359	break :result if (field_off == 0) dst_mcv else try self.copyToNewRegister(inst, dst_mcv);
2648	},	3360	},
2649	.stack_offset => \|off\| {	3361	.load_frame => {
2650	log.debug("airStructFieldVal off: {}", .{field_off});	3362	const field_abi_size: u32 = @intCast(field_ty.abiSize(mod));
2651	const field_byte_off: u32 = @divExact(field_off, 8);	3363	if (field_off % 8 == 0) {
2652	break :result MCValue{ .stack_offset = off + field_byte_off };	3364	const field_byte_off = @divExact(field_off, 8);
		3365	const off_mcv = src_mcv.address().offset(@intCast(field_byte_off)).deref();
		3366	const field_bit_size = field_ty.bitSize(mod);
		3367
		3368	if (field_abi_size <= 8) {
		3369	const int_ty = try mod.intType(
		3370	if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,
		3371	@intCast(field_bit_size),
		3372	);
		3373
		3374	const dst_reg, const dst_lock = try self.allocReg();
		3375	const dst_mcv = MCValue{ .register = dst_reg };
		3376	defer self.register_manager.unlockReg(dst_lock);
		3377
		3378	try self.genCopy(int_ty, dst_mcv, off_mcv);
		3379	break :result try self.copyToNewRegister(inst, dst_mcv);
		3380	}
		3381
		3382	const container_abi_size: u32 = @intCast(struct_ty.abiSize(mod));
		3383	const dst_mcv = if (field_byte_off + field_abi_size <= container_abi_size and
		3384	self.reuseOperand(inst, operand, 0, src_mcv))
		3385	off_mcv
		3386	else dst: {
		3387	const dst_mcv = try self.allocRegOrMem(inst, true);
		3388	try self.genCopy(field_ty, dst_mcv, off_mcv);
		3389	break :dst dst_mcv;
		3390	};
		3391	if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
		3392	const tmp_reg, const tmp_lock = try self.allocReg();
		3393	defer self.register_manager.unlockReg(tmp_lock);
		3394
		3395	const hi_mcv =
		3396	dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
		3397	try self.genSetReg(Type.usize, tmp_reg, hi_mcv);
		3398	try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
		3399	}
		3400	break :result dst_mcv;
		3401	}
		3402
		3403	return self.fail("TODO: airStructFieldVal load_frame field_off non multiple of 8", .{});
2653	},	3404	},
2654	else => return self.fail("TODO: airStructField {s}", .{@tagName(src_mcv)}),	3405	else => return self.fail("TODO: airStructField {s}", .{@tagName(src_mcv)}),
2655	}	3406	}
...	@@ -2664,18 +3415,18 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2664,18 +3415,18 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
2664	}	3415	}
2665		3416
2666	fn genArgDbgInfo(self: Self, inst: Air.Inst.Index, mcv: MCValue) !void {	3417	fn genArgDbgInfo(self: Self, inst: Air.Inst.Index, mcv: MCValue) !void {
2667	const mod = self.bin_file.comp.module.?;	3418	const zcu = self.bin_file.comp.module.?;
2668	const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;	3419	const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
2669	const ty = arg.ty.toType();	3420	const ty = arg.ty.toType();
2670	const owner_decl = mod.funcOwnerDeclIndex(self.func_index);	3421	const owner_decl = zcu.funcOwnerDeclIndex(self.func_index);
2671	const name = mod.getParamName(self.func_index, arg.src_index);	3422	const name = zcu.getParamName(self.func_index, arg.src_index);
2672		3423
2673	switch (self.debug_output) {	3424	switch (self.debug_output) {
2674	.dwarf => \|dw\| switch (mcv) {	3425	.dwarf => \|dw\| switch (mcv) {
2675	.register => \|reg\| try dw.genArgDbgInfo(name, ty, owner_decl, .{	3426	.register => \|reg\| try dw.genArgDbgInfo(name, ty, owner_decl, .{
2676	.register = reg.dwarfLocOp(),	3427	.register = reg.dwarfLocOp(),
2677	}),	3428	}),
2678	.stack_offset => {},	3429	.load_frame => {},
2679	else => {},	3430	else => {},
2680	},	3431	},
2681	.plan9 => {},	3432	.plan9 => {},
...	@@ -2694,12 +3445,8 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2694,12 +3445,8 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
2694	const src_mcv = self.args[arg_index];	3445	const src_mcv = self.args[arg_index];
2695		3446
2696	const dst_mcv = switch (src_mcv) {	3447	const dst_mcv = switch (src_mcv) {
2697	.register => \|src_reg\| dst: {	3448	.register, .register_pair, .load_frame => dst: {
2698	self.register_manager.getRegAssumeFree(src_reg, null);	3449	for (src_mcv.getRegs()) \|reg\| self.register_manager.getRegAssumeFree(reg, inst);
2699	break :dst src_mcv;
2700	},
2701	.register_pair => \|pair\| dst: {
2702	for (pair) \|reg\| self.register_manager.getRegAssumeFree(reg, null);
2703	break :dst src_mcv;	3450	break :dst src_mcv;
2704	},	3451	},
2705	else => return self.fail("TODO: airArg {s}", .{@tagName(src_mcv)}),	3452	else => return self.fail("TODO: airArg {s}", .{@tagName(src_mcv)}),
...	@@ -2715,7 +3462,8 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2715,7 +3462,8 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
2715	fn airTrap(self: *Self) !void {	3462	fn airTrap(self: *Self) !void {
2716	_ = try self.addInst(.{	3463	_ = try self.addInst(.{
2717	.tag = .unimp,	3464	.tag = .unimp,
2718	.data = .{ .nop = {} },	3465	.ops = .none,
		3466	.data = undefined,
2719	});	3467	});
2720	return self.finishAirBookkeeping();	3468	return self.finishAirBookkeeping();
2721	}	3469	}
...	@@ -2723,19 +3471,22 @@ fn airTrap(self: *Self) !void {	...	@@ -2723,19 +3471,22 @@ fn airTrap(self: *Self) !void {
2723	fn airBreakpoint(self: *Self) !void {	3471	fn airBreakpoint(self: *Self) !void {
2724	_ = try self.addInst(.{	3472	_ = try self.addInst(.{
2725	.tag = .ebreak,	3473	.tag = .ebreak,
2726	.data = .{ .nop = {} },	3474	.ops = .none,
		3475	.data = undefined,
2727	});	3476	});
2728	return self.finishAirBookkeeping();	3477	return self.finishAirBookkeeping();
2729	}	3478	}
2730		3479
2731	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {	3480	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
2732	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airRetAddr for riscv64", .{});	3481	const dst_mcv = try self.allocRegOrMem(inst, true);
2733	return self.finishAir(inst, result, .{ .none, .none, .none });	3482	try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
		3483	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
2734	}	3484	}
2735		3485
2736	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {	3486	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
2737	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFrameAddress for riscv64", .{});	3487	const dst_mcv = try self.allocRegOrMem(inst, true);
2738	return self.finishAir(inst, result, .{ .none, .none, .none });	3488	try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
		3489	return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
2739	}	3490	}
2740		3491
2741	fn airFence(self: *Self) !void {	3492	fn airFence(self: *Self) !void {
...	@@ -2790,39 +3541,55 @@ fn genCall(	...	@@ -2790,39 +3541,55 @@ fn genCall(
2790	arg_tys: []const Type,	3541	arg_tys: []const Type,
2791	args: []const MCValue,	3542	args: []const MCValue,
2792	) !MCValue {	3543	) !MCValue {
2793	const mod = self.bin_file.comp.module.?;	3544	const zcu = self.bin_file.comp.module.?;
2794		3545
2795	const fn_ty = switch (info) {	3546	const fn_ty = switch (info) {
2796	.air => \|callee\| fn_info: {	3547	.air => \|callee\| fn_info: {
2797	const callee_ty = self.typeOf(callee);	3548	const callee_ty = self.typeOf(callee);
2798	break :fn_info switch (callee_ty.zigTypeTag(mod)) {	3549	break :fn_info switch (callee_ty.zigTypeTag(zcu)) {
2799	.Fn => callee_ty,	3550	.Fn => callee_ty,
2800	.Pointer => callee_ty.childType(mod),	3551	.Pointer => callee_ty.childType(zcu),
2801	else => unreachable,	3552	else => unreachable,
2802	};	3553	};
2803	},	3554	},
2804	.lib => \|lib\| try mod.funcType(.{	3555	.lib => \|lib\| try zcu.funcType(.{
2805	.param_types = lib.param_types,	3556	.param_types = lib.param_types,
2806	.return_type = lib.return_type,	3557	.return_type = lib.return_type,
2807	.cc = .C,	3558	.cc = .C,
2808	}),	3559	}),
2809	};	3560	};
2810		3561
2811	var call_info = try self.resolveCallingConventionValues(fn_ty, .caller);	3562	const fn_info = zcu.typeToFunc(fn_ty).?;
		3563	var call_info = try self.resolveCallingConventionValues(fn_info);
2812	defer call_info.deinit(self);	3564	defer call_info.deinit(self);
2813		3565
		3566	// We need a properly aligned and sized call frame to be able to call this function.
		3567	{
		3568	const needed_call_frame = FrameAlloc.init(.{
		3569	.size = call_info.stack_byte_count,
		3570	.alignment = call_info.stack_align,
		3571	});
		3572	const frame_allocs_slice = self.frame_allocs.slice();
		3573	const stack_frame_size =
		3574	&frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)];
		3575	stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
		3576	const stack_frame_align =
		3577	&frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)];
		3578	stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align);
		3579	}
		3580
2814	for (call_info.args, 0..) \|mc_arg, arg_i\| try self.genCopy(arg_tys[arg_i], mc_arg, args[arg_i]);	3581	for (call_info.args, 0..) \|mc_arg, arg_i\| try self.genCopy(arg_tys[arg_i], mc_arg, args[arg_i]);
2815		3582
2816	// Due to incremental compilation, how function calls are generated depends	3583	// Due to incremental compilation, how function calls are generated depends
2817	// on linking.	3584	// on linking.
2818	switch (info) {	3585	switch (info) {
2819	.air => \|callee\| {	3586	.air => \|callee\| {
2820	if (try self.air.value(callee, mod)) \|func_value\| {	3587	if (try self.air.value(callee, zcu)) \|func_value\| {
2821	const func_key = mod.intern_pool.indexToKey(func_value.ip_index);	3588	const func_key = zcu.intern_pool.indexToKey(func_value.ip_index);
2822	switch (switch (func_key) {	3589	switch (switch (func_key) {
2823	else => func_key,	3590	else => func_key,
2824	.ptr => \|ptr\| switch (ptr.addr) {	3591	.ptr => \|ptr\| switch (ptr.addr) {
2825	.decl => \|decl\| mod.intern_pool.indexToKey(mod.declPtr(decl).val.toIntern()),	3592	.decl => \|decl\| zcu.intern_pool.indexToKey(zcu.declPtr(decl).val.toIntern()),
2826	else => func_key,	3593	else => func_key,
2827	},	3594	},
2828	}) {	3595	}) {
...	@@ -2835,10 +3602,11 @@ fn genCall(	...	@@ -2835,10 +3602,11 @@ fn genCall(
2835	try self.genSetReg(Type.usize, .ra, .{ .memory = got_addr });	3602	try self.genSetReg(Type.usize, .ra, .{ .memory = got_addr });
2836	_ = try self.addInst(.{	3603	_ = try self.addInst(.{
2837	.tag = .jalr,	3604	.tag = .jalr,
		3605	.ops = .rri,
2838	.data = .{ .i_type = .{	3606	.data = .{ .i_type = .{
2839	.rd = .ra,	3607	.rd = .ra,
2840	.rs1 = .ra,	3608	.rs1 = .ra,
2841	.imm12 = 0,	3609	.imm12 = Immediate.s(0),
2842	} },	3610	} },
2843	});	3611	});
2844	} else if (self.bin_file.cast(link.File.Coff)) \|_\| {	3612	} else if (self.bin_file.cast(link.File.Coff)) \|_\| {
...	@@ -2855,16 +3623,17 @@ fn genCall(	...	@@ -2855,16 +3623,17 @@ fn genCall(
2855	else => return self.fail("TODO implement calling bitcasted functions", .{}),	3623	else => return self.fail("TODO implement calling bitcasted functions", .{}),
2856	}	3624	}
2857	} else {	3625	} else {
2858	assert(self.typeOf(callee).zigTypeTag(mod) == .Pointer);	3626	assert(self.typeOf(callee).zigTypeTag(zcu) == .Pointer);
2859	const addr_reg, const addr_lock = try self.allocReg();	3627	const addr_reg, const addr_lock = try self.allocReg();
2860	defer self.register_manager.unlockReg(addr_lock);	3628	defer self.register_manager.unlockReg(addr_lock);
2861	try self.genSetReg(Type.usize, addr_reg, .{ .air_ref = callee });	3629	try self.genSetReg(Type.usize, addr_reg, .{ .air_ref = callee });
2862	_ = try self.addInst(.{	3630	_ = try self.addInst(.{
2863	.tag = .jalr,	3631	.tag = .jalr,
		3632	.ops = .rri,
2864	.data = .{ .i_type = .{	3633	.data = .{ .i_type = .{
2865	.rd = .ra,	3634	.rd = .ra,
2866	.rs1 = addr_reg,	3635	.rs1 = addr_reg,
2867	.imm12 = 0,	3636	.imm12 = Immediate.s(0),
2868	} },	3637	} },
2869	});	3638	});
2870	}	3639	}
...	@@ -2872,11 +3641,12 @@ fn genCall(	...	@@ -2872,11 +3641,12 @@ fn genCall(
2872	.lib => return self.fail("TODO: lib func calls", .{}),	3641	.lib => return self.fail("TODO: lib func calls", .{}),
2873	}	3642	}
2874		3643
2875	return call_info.return_value;	3644	return call_info.return_value.short;
2876	}	3645	}
2877		3646
2878	fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	3647	fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
2879	const mod = self.bin_file.comp.module.?;	3648	const zcu = self.bin_file.comp.module.?;
		3649	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2880		3650
2881	if (safety) {	3651	if (safety) {
2882	// safe	3652	// safe
...	@@ -2884,32 +3654,35 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -2884,32 +3654,35 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
2884	// not safe	3654	// not safe
2885	}	3655	}
2886		3656
2887	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3657	const ret_ty = self.fn_type.fnReturnType(zcu);
2888	const operand = try self.resolveInst(un_op);	3658	switch (self.ret_mcv.short) {
2889		3659	.none => {},
2890	_ = try self.addInst(.{	3660	.register,
2891	.tag = .dbg_epilogue_begin,	3661	.register_pair,
2892	.data = .{ .nop = {} },	3662	=> try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }),
2893	});	3663	.indirect => \|reg_off\| {
2894		3664	try self.register_manager.getReg(reg_off.reg, null);
2895	const ret_ty = self.fn_type.fnReturnType(mod);	3665	const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
2896	try self.genCopy(ret_ty, self.ret_mcv, operand);	3666	defer self.register_manager.unlockReg(lock);
2897		3667
2898	try self.ret();	3668	try self.genSetReg(Type.usize, reg_off.reg, self.ret_mcv.long);
2899		3669	try self.genCopy(
2900	return self.finishAir(inst, .dead, .{ un_op, .none, .none });	3670	ret_ty,
2901	}	3671	.{ .register_offset = reg_off },
		3672	.{ .air_ref = un_op },
		3673	);
		3674	},
		3675	else => unreachable,
		3676	}
2902		3677
2903	fn ret(self: *Self) !void {	3678	self.ret_mcv.liveOut(self, inst);
2904	_ = try self.addInst(.{	3679	try self.finishAir(inst, .unreach, .{ un_op, .none, .none });
2905	.tag = .psuedo_epilogue,
2906	.data = .{ .nop = {} },
2907	});
2908		3680
2909	// Just add space for an instruction, patch this later	3681	// Just add space for an instruction, reloced this later
2910	const index = try self.addInst(.{	3682	const index = try self.addInst(.{
2911	.tag = .ret,	3683	.tag = .pseudo,
2912	.data = .{ .nop = {} },	3684	.ops = .pseudo_j,
		3685	.data = .{ .inst = undefined },
2913	});	3686	});
2914		3687
2915	try self.exitlude_jump_relocs.append(self.gpa, index);	3688	try self.exitlude_jump_relocs.append(self.gpa, index);
...	@@ -2918,37 +3691,49 @@ fn ret(self: *Self) !void {	...	@@ -2918,37 +3691,49 @@ fn ret(self: *Self) !void {
2918	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	3691	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
2919	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3692	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2920	const ptr = try self.resolveInst(un_op);	3693	const ptr = try self.resolveInst(un_op);
2921	const ptr_ty = self.typeOf(un_op);
2922		3694
2923	try self.load(self.ret_mcv, ptr, ptr_ty);	3695	const ptr_ty = self.typeOf(un_op);
		3696	switch (self.ret_mcv.short) {
		3697	.none => {},
		3698	.register, .register_pair => try self.load(self.ret_mcv.short, ptr, ptr_ty),
		3699	.indirect => \|reg_off\| try self.genSetReg(ptr_ty, reg_off.reg, ptr),
		3700	else => unreachable,
		3701	}
		3702	self.ret_mcv.liveOut(self, inst);
		3703	try self.finishAir(inst, .unreach, .{ un_op, .none, .none });
2924		3704
2925	try self.ret();	3705	// Just add space for an instruction, reloced this later
		3706	const index = try self.addInst(.{
		3707	.tag = .pseudo,
		3708	.ops = .pseudo_j,
		3709	.data = .{ .inst = undefined },
		3710	});
2926		3711
2927	return self.finishAir(inst, .dead, .{ un_op, .none, .none });	3712	try self.exitlude_jump_relocs.append(self.gpa, index);
2928	}	3713	}
2929		3714
2930	fn airCmp(self: *Self, inst: Air.Inst.Index) !void {	3715	fn airCmp(self: *Self, inst: Air.Inst.Index) !void {
2931	const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];	3716	const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
2932	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	3717	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2933	const mod = self.bin_file.comp.module.?;	3718	const zcu = self.bin_file.comp.module.?;
2934		3719
2935	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3720	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
2936	const lhs = try self.resolveInst(bin_op.lhs);	3721	const lhs = try self.resolveInst(bin_op.lhs);
2937	const rhs = try self.resolveInst(bin_op.rhs);	3722	const rhs = try self.resolveInst(bin_op.rhs);
2938	const lhs_ty = self.typeOf(bin_op.lhs);	3723	const lhs_ty = self.typeOf(bin_op.lhs);
2939		3724
2940	const int_ty = switch (lhs_ty.zigTypeTag(mod)) {	3725	const int_ty = switch (lhs_ty.zigTypeTag(zcu)) {
2941	.Vector => unreachable, // Handled by cmp_vector.	3726	.Vector => unreachable, // Handled by cmp_vector.
2942	.Enum => lhs_ty.intTagType(mod),	3727	.Enum => lhs_ty.intTagType(zcu),
2943	.Int => lhs_ty,	3728	.Int => lhs_ty,
2944	.Bool => Type.u1,	3729	.Bool => Type.u1,
2945	.Pointer => Type.usize,	3730	.Pointer => Type.usize,
2946	.ErrorSet => Type.u16,	3731	.ErrorSet => Type.u16,
2947	.Optional => blk: {	3732	.Optional => blk: {
2948	const payload_ty = lhs_ty.optionalChild(mod);	3733	const payload_ty = lhs_ty.optionalChild(zcu);
2949	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {	3734	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
2950	break :blk Type.u1;	3735	break :blk Type.u1;
2951	} else if (lhs_ty.isPtrLikeOptional(mod)) {	3736	} else if (lhs_ty.isPtrLikeOptional(zcu)) {
2952	break :blk Type.usize;	3737	break :blk Type.usize;
2953	} else {	3738	} else {
2954	return self.fail("TODO riscv cmp non-pointer optionals", .{});	3739	return self.fail("TODO riscv cmp non-pointer optionals", .{});
...	@@ -2958,7 +3743,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2958,7 +3743,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index) !void {
2958	else => unreachable,	3743	else => unreachable,
2959	};	3744	};
2960		3745
2961	const int_info = int_ty.intInfo(mod);	3746	const int_info = int_ty.intInfo(zcu);
2962	if (int_info.bits <= 64) {	3747	if (int_info.bits <= 64) {
2963	break :result try self.binOp(tag, null, lhs, rhs, int_ty, int_ty);	3748	break :result try self.binOp(tag, null, lhs, rhs, int_ty, int_ty);
2964	} else {	3749	} else {
...	@@ -2978,7 +3763,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2978,7 +3763,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
2978	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3763	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2979	const operand = try self.resolveInst(un_op);	3764	const operand = try self.resolveInst(un_op);
2980	_ = operand;	3765	_ = operand;
2981	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});	3766	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});
2982	return self.finishAir(inst, result, .{ un_op, .none, .none });	3767	return self.finishAir(inst, result, .{ un_op, .none, .none });
2983	}	3768	}
2984		3769
...	@@ -2986,8 +3771,9 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2986,8 +3771,9 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
2986	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;	3771	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
2987		3772
2988	_ = try self.addInst(.{	3773	_ = try self.addInst(.{
2989	.tag = .dbg_line,	3774	.tag = .pseudo,
2990	.data = .{ .dbg_line_column = .{	3775	.ops = .pseudo_dbg_line_column,
		3776	.data = .{ .pseudo_dbg_line_column = .{
2991	.line = dbg_stmt.line,	3777	.line = dbg_stmt.line,
2992	.column = dbg_stmt.column,	3778	.column = dbg_stmt.column,
2993	} },	3779	} },
...	@@ -3023,7 +3809,7 @@ fn genVarDbgInfo(	...	@@ -3023,7 +3809,7 @@ fn genVarDbgInfo(
3023	mcv: MCValue,	3809	mcv: MCValue,
3024	name: [:0]const u8,	3810	name: [:0]const u8,
3025	) !void {	3811	) !void {
3026	const mod = self.bin_file.comp.module.?;	3812	const zcu = self.bin_file.comp.module.?;
3027	const is_ptr = switch (tag) {	3813	const is_ptr = switch (tag) {
3028	.dbg_var_ptr => true,	3814	.dbg_var_ptr => true,
3029	.dbg_var_val => false,	3815	.dbg_var_val => false,
...	@@ -3043,11 +3829,11 @@ fn genVarDbgInfo(	...	@@ -3043,11 +3829,11 @@ fn genVarDbgInfo(
3043	.undef => .undef,	3829	.undef => .undef,
3044	.none => .none,	3830	.none => .none,
3045	else => blk: {	3831	else => blk: {
3046	log.debug("TODO generate debug info for {}", .{mcv});	3832	log.warn("TODO generate debug info for {}", .{mcv});
3047	break :blk .nop;	3833	break :blk .nop;
3048	},	3834	},
3049	};	3835	};
3050	try dw.genVarDbgInfo(name, ty, mod.funcOwnerDeclIndex(self.func_index), is_ptr, loc);	3836	try dw.genVarDbgInfo(name, ty, zcu.funcOwnerDeclIndex(self.func_index), is_ptr, loc);
3051	},	3837	},
3052	.plan9 => {},	3838	.plan9 => {},
3053	.none => {},	3839	.none => {},
...	@@ -3061,146 +3847,49 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3061,146 +3847,49 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
3061	const extra = self.air.extraData(Air.CondBr, pl_op.payload);	3847	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
3062	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);	3848	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
3063	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);	3849	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
3064	const liveness_condbr = self.liveness.getCondBr(inst);	3850	const liveness_cond_br = self.liveness.getCondBr(inst);
3065
3066	const cond_reg = try self.register_manager.allocReg(inst, gp);
3067	const cond_reg_lock = self.register_manager.lockRegAssumeUnused(cond_reg);
3068	defer self.register_manager.unlockReg(cond_reg_lock);
3069
3070	// A branch to the false section. Uses beq. 1 is the default "true" state.
3071	const reloc = try self.condBr(cond_ty, cond, cond_reg);
3072		3851
3073	// If the condition dies here in this condbr instruction, process	3852	// If the condition dies here in this condbr instruction, process
3074	// that death now instead of later as this has an effect on	3853	// that death now instead of later as this has an effect on
3075	// whether it needs to be spilled in the branches	3854	// whether it needs to be spilled in the branches
3076	if (self.liveness.operandDies(inst, 0)) {	3855	if (self.liveness.operandDies(inst, 0)) {
3077	if (pl_op.operand.toIndex()) \|op_inst\| self.processDeath(op_inst);	3856	if (pl_op.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst);
3078	}	3857	}
3079		3858
3080	// Save state	3859	self.scope_generation += 1;
3081	const parent_next_stack_offset = self.next_stack_offset;	3860	const state = try self.saveState();
3082	const parent_free_registers = self.register_manager.free_registers;	3861	const reloc = try self.condBr(cond_ty, cond);
3083	var parent_stack = try self.stack.clone(self.gpa);
3084	defer parent_stack.deinit(self.gpa);
3085	const parent_registers = self.register_manager.registers;
3086		3862
3087	try self.branch_stack.append(.{});	3863	for (liveness_cond_br.then_deaths) \|death\| try self.processDeath(death);
3088	errdefer {
3089	_ = self.branch_stack.pop();
3090	}
3091
3092	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
3093	for (liveness_condbr.then_deaths) \|operand\| {
3094	self.processDeath(operand);
3095	}
3096	try self.genBody(then_body);	3864	try self.genBody(then_body);
		3865	try self.restoreState(state, &.{}, .{
		3866	.emit_instructions = false,
		3867	.update_tracking = true,
		3868	.resurrect = true,
		3869	.close_scope = true,
		3870	});
3097		3871
3098	// Restore state	3872	self.performReloc(reloc);
3099	var saved_then_branch = self.branch_stack.pop();
3100	defer saved_then_branch.deinit(self.gpa);
3101
3102	self.register_manager.registers = parent_registers;
3103
3104	self.stack.deinit(self.gpa);
3105	self.stack = parent_stack;
3106	parent_stack = .{};
3107
3108	self.next_stack_offset = parent_next_stack_offset;
3109	self.register_manager.free_registers = parent_free_registers;
3110
3111	const else_branch = self.branch_stack.addOneAssumeCapacity();
3112	else_branch.* = .{};
3113
3114	try self.performReloc(reloc, @intCast(self.mir_instructions.len));
3115		3873
3116	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);	3874	for (liveness_cond_br.else_deaths) \|death\| try self.processDeath(death);
3117	for (liveness_condbr.else_deaths) \|operand\| {
3118	self.processDeath(operand);
3119	}
3120	try self.genBody(else_body);	3875	try self.genBody(else_body);
		3876	try self.restoreState(state, &.{}, .{
		3877	.emit_instructions = false,
		3878	.update_tracking = true,
		3879	.resurrect = true,
		3880	.close_scope = true,
		3881	});
3121		3882
3122	// At this point, each branch will possibly have conflicting values for where	3883	// We already took care of pl_op.operand earlier, so there's nothing left to do.
3123	// each instruction is stored. They agree, however, on which instructions are alive/dead.	3884	self.finishAirBookkeeping();
3124	// We use the first ("then") branch as canonical, and here emit
3125	// instructions into the second ("else") branch to make it conform.
3126	// We continue respect the data structure semantic guarantees of the else_branch so
3127	// that we can use all the code emitting abstractions. This is why at the bottom we
3128	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
3129	// rather than assigning it.
3130	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
3131	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
3132	const else_slice = else_branch.inst_table.entries.slice();
3133	const else_keys = else_slice.items(.key);
3134	const else_values = else_slice.items(.value);
3135	for (else_keys, 0..) \|else_key, else_idx\| {
3136	const else_value = else_values[else_idx];
3137	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
3138	// The instruction's MCValue is overridden in both branches.
3139	log.debug("condBr put branch table (key = %{d}, value = {})", .{ else_key, then_entry.value });
3140	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
3141	if (else_value == .dead) {
3142	assert(then_entry.value == .dead);
3143	continue;
3144	}
3145	break :blk then_entry.value;
3146	} else blk: {
3147	if (else_value == .dead)
3148	continue;
3149	// The instruction is only overridden in the else branch.
3150	var i: usize = self.branch_stack.items.len - 2;
3151	while (true) {
3152	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
3153	if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
3154	assert(mcv != .dead);
3155	break :blk mcv;
3156	}
3157	}
3158	};
3159	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
3160	// TODO make sure the destination stack offset / register does not already have something
3161	// going on there.
3162	try self.genCopy(self.typeOfIndex(else_key), canon_mcv, else_value);
3163	// TODO track the new register / stack allocation
3164	}
3165	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
3166	const then_slice = saved_then_branch.inst_table.entries.slice();
3167	const then_keys = then_slice.items(.key);
3168	const then_values = then_slice.items(.value);
3169	for (then_keys, 0..) \|then_key, then_idx\| {
3170	const then_value = then_values[then_idx];
3171	// We already deleted the items from this table that matched the else_branch.
3172	// So these are all instructions that are only overridden in the then branch.
3173	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
3174	if (then_value == .dead)
3175	continue;
3176	const parent_mcv = blk: {
3177	var i: usize = self.branch_stack.items.len - 2;
3178	while (true) {
3179	i -= 1;
3180	if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
3181	assert(mcv != .dead);
3182	break :blk mcv;
3183	}
3184	}
3185	};
3186	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
3187	// TODO make sure the destination stack offset / register does not already have something
3188	// going on there.
3189	try self.genCopy(self.typeOfIndex(then_key), parent_mcv, then_value);
3190	// TODO track the new register / stack allocation
3191	}
3192
3193	{
3194	var item = self.branch_stack.pop();
3195	item.deinit(self.gpa);
3196	}
3197	}	3885	}
3198		3886
3199	fn condBr(self: *Self, cond_ty: Type, condition: MCValue, cond_reg: Register) !Mir.Inst.Index {	3887	fn condBr(self: *Self, cond_ty: Type, condition: MCValue) !Mir.Inst.Index {
3200	try self.genSetReg(cond_ty, cond_reg, condition);	3888	const cond_reg = try self.copyToTmpRegister(cond_ty, condition);
3201		3889
3202	return try self.addInst(.{	3890	return try self.addInst(.{
3203	.tag = .beq,	3891	.tag = .beq,
		3892	.ops = .rr_inst,
3204	.data = .{	3893	.data = .{
3205	.b_type = .{	3894	.b_type = .{
3206	.rs1 = cond_reg,	3895	.rs1 = cond_reg,
...	@@ -3213,7 +3902,7 @@ fn condBr(self: *Self, cond_ty: Type, condition: MCValue, cond_reg: Register) !M	...	@@ -3213,7 +3902,7 @@ fn condBr(self: *Self, cond_ty: Type, condition: MCValue, cond_reg: Register) !M
3213		3902
3214	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {	3903	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
3215	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3904	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3216	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3905	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3217	const operand = try self.resolveInst(un_op);	3906	const operand = try self.resolveInst(un_op);
3218	break :result try self.isNull(operand);	3907	break :result try self.isNull(operand);
3219	};	3908	};
...	@@ -3222,7 +3911,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3222,7 +3911,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
3222		3911
3223	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {	3912	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
3224	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3913	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3225	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3914	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3226	const operand_ptr = try self.resolveInst(un_op);	3915	const operand_ptr = try self.resolveInst(un_op);
3227	const operand: MCValue = blk: {	3916	const operand: MCValue = blk: {
3228	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	3917	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
...	@@ -3247,7 +3936,7 @@ fn isNull(self: *Self, operand: MCValue) !MCValue {	...	@@ -3247,7 +3936,7 @@ fn isNull(self: *Self, operand: MCValue) !MCValue {
3247		3936
3248	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {	3937	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
3249	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3938	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3250	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3939	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3251	const operand = try self.resolveInst(un_op);	3940	const operand = try self.resolveInst(un_op);
3252	break :result try self.isNonNull(operand);	3941	break :result try self.isNonNull(operand);
3253	};	3942	};
...	@@ -3263,7 +3952,7 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {	...	@@ -3263,7 +3952,7 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {
3263		3952
3264	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {	3953	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
3265	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3954	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3266	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3955	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3267	const operand_ptr = try self.resolveInst(un_op);	3956	const operand_ptr = try self.resolveInst(un_op);
3268	const operand: MCValue = blk: {	3957	const operand: MCValue = blk: {
3269	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	3958	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
...	@@ -3281,7 +3970,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3281,7 +3970,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
3281		3970
3282	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {	3971	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
3283	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3972	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3284	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3973	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3285	const operand = try self.resolveInst(un_op);	3974	const operand = try self.resolveInst(un_op);
3286	const operand_ty = self.typeOf(un_op);	3975	const operand_ty = self.typeOf(un_op);
3287	break :result try self.isErr(inst, operand_ty, operand);	3976	break :result try self.isErr(inst, operand_ty, operand);
...	@@ -3290,9 +3979,9 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3290,9 +3979,9 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
3290	}	3979	}
3291		3980
3292	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	3981	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3293	const mod = self.bin_file.comp.module.?;	3982	const zcu = self.bin_file.comp.module.?;
3294	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3983	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3295	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3984	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3296	const operand_ptr = try self.resolveInst(un_op);	3985	const operand_ptr = try self.resolveInst(un_op);
3297	const operand: MCValue = blk: {	3986	const operand: MCValue = blk: {
3298	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	3987	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
...	@@ -3304,7 +3993,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3304,7 +3993,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3304	};	3993	};
3305	try self.load(operand, operand_ptr, self.typeOf(un_op));	3994	try self.load(operand, operand_ptr, self.typeOf(un_op));
3306	const operand_ptr_ty = self.typeOf(un_op);	3995	const operand_ptr_ty = self.typeOf(un_op);
3307	const operand_ty = operand_ptr_ty.childType(mod);	3996	const operand_ty = operand_ptr_ty.childType(zcu);
3308		3997
3309	break :result try self.isErr(inst, operand_ty, operand);	3998	break :result try self.isErr(inst, operand_ty, operand);
3310	};	3999	};
...	@@ -3315,13 +4004,13 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3315,13 +4004,13 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3315	///	4004	///
3316	/// Result is in the return register.	4005	/// Result is in the return register.
3317	fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {	4006	fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
3318	const mod = self.bin_file.comp.module.?;	4007	const zcu = self.bin_file.comp.module.?;
3319	const err_ty = eu_ty.errorUnionSet(mod);	4008	const err_ty = eu_ty.errorUnionSet(zcu);
3320	if (err_ty.errorSetIsEmpty(mod)) return MCValue{ .immediate = 0 }; // always false	4009	if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false
3321		4010
3322	_ = maybe_inst;	4011	_ = maybe_inst;
3323		4012
3324	const err_off = errUnionErrorOffset(eu_ty.errorUnionPayload(mod), mod);	4013	const err_off = errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu);
3325		4014
3326	switch (eu_mcv) {	4015	switch (eu_mcv) {
3327	.register => \|reg\| {	4016	.register => \|reg\| {
...	@@ -3361,7 +4050,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)	...	@@ -3361,7 +4050,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
3361		4050
3362	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {	4051	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
3363	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	4052	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3364	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	4053	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3365	const operand = try self.resolveInst(un_op);	4054	const operand = try self.resolveInst(un_op);
3366	const ty = self.typeOf(un_op);	4055	const ty = self.typeOf(un_op);
3367	break :result try self.isNonErr(inst, ty, operand);	4056	break :result try self.isNonErr(inst, ty, operand);
...	@@ -3375,6 +4064,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC	...	@@ -3375,6 +4064,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC
3375	.register => \|reg\| {	4064	.register => \|reg\| {
3376	_ = try self.addInst(.{	4065	_ = try self.addInst(.{
3377	.tag = .not,	4066	.tag = .not,
		4067	.ops = .rr,
3378	.data = .{	4068	.data = .{
3379	.rr = .{	4069	.rr = .{
3380	.rd = reg,	4070	.rd = reg,
...	@@ -3394,9 +4084,9 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC	...	@@ -3394,9 +4084,9 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC
3394	}	4084	}
3395		4085
3396	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	4086	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3397	const mod = self.bin_file.comp.module.?;	4087	const zcu = self.bin_file.comp.module.?;
3398	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	4088	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3399	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	4089	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
3400	const operand_ptr = try self.resolveInst(un_op);	4090	const operand_ptr = try self.resolveInst(un_op);
3401	const operand: MCValue = blk: {	4091	const operand: MCValue = blk: {
3402	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	4092	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
...	@@ -3407,7 +4097,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3407,7 +4097,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3407	}	4097	}
3408	};	4098	};
3409	const operand_ptr_ty = self.typeOf(un_op);	4099	const operand_ptr_ty = self.typeOf(un_op);
3410	const operand_ty = operand_ptr_ty.childType(mod);	4100	const operand_ty = operand_ptr_ty.childType(zcu);
3411		4101
3412	try self.load(operand, operand_ptr, self.typeOf(un_op));	4102	try self.load(operand, operand_ptr, self.typeOf(un_op));
3413	break :result try self.isNonErr(inst, operand_ty, operand);	4103	break :result try self.isNonErr(inst, operand_ty, operand);
...	@@ -3421,18 +4111,27 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3421,18 +4111,27 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
3421	const loop = self.air.extraData(Air.Block, ty_pl.payload);	4111	const loop = self.air.extraData(Air.Block, ty_pl.payload);
3422	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);	4112	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
3423		4113
3424	const start_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);	4114	self.scope_generation += 1;
		4115	const state = try self.saveState();
3425		4116
		4117	const jmp_target: Mir.Inst.Index = @intCast(self.mir_instructions.len);
3426	try self.genBody(body);	4118	try self.genBody(body);
3427	try self.jump(start_index);	4119	try self.restoreState(state, &.{}, .{
		4120	.emit_instructions = true,
		4121	.update_tracking = false,
		4122	.resurrect = false,
		4123	.close_scope = true,
		4124	});
		4125	_ = try self.jump(jmp_target);
3428		4126
3429	return self.finishAirBookkeeping();	4127	self.finishAirBookkeeping();
3430	}	4128	}
3431		4129
3432	/// Send control flow to the `index` of `self.code`.	4130	/// Send control flow to the `index` of `self.code`.
3433	fn jump(self: *Self, index: Mir.Inst.Index) !void {	4131	fn jump(self: *Self, index: Mir.Inst.Index) !Mir.Inst.Index {
3434	_ = try self.addInst(.{	4132	return self.addInst(.{
3435	.tag = .j,	4133	.tag = .pseudo,
		4134	.ops = .pseudo_j,
3436	.data = .{	4135	.data = .{
3437	.inst = index,	4136	.inst = index,
3438	},	4137	},
...	@@ -3446,33 +4145,34 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3446,33 +4145,34 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
3446	}	4145	}
3447		4146
3448	fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {	4147	fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
3449	try self.blocks.putNoClobber(self.gpa, inst, .{	4148	// A block is a setup to be able to jump to the end.
3450	// A block is a setup to be able to jump to the end.	4149	const inst_tracking_i = self.inst_tracking.count();
3451	.relocs = .{},	4150	self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
3452	// It also acts as a receptacle for break operands.	4151
3453	// Here we use `MCValue.none` to represent a null value so that the first	4152	self.scope_generation += 1;
3454	// break instruction will choose a MCValue for the block result and overwrite	4153	try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
3455	// this field. Following break instructions will use that MCValue to put their	4154	const liveness = self.liveness.getBlock(inst);
3456	// block results.
3457	.mcv = MCValue{ .none = {} },
3458	});
3459	defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);
3460		4155
3461	// TODO emit debug info lexical block	4156	// TODO emit debug info lexical block
3462	try self.genBody(body);	4157	try self.genBody(body);
3463		4158
3464	for (self.blocks.getPtr(inst).?.relocs.items) \|reloc\| {	4159	var block_data = self.blocks.fetchRemove(inst).?;
3465	// here we are relocing to point at the instruction after the block.	4160	defer block_data.value.deinit(self.gpa);
3466	// [then case]	4161	if (block_data.value.relocs.items.len > 0) {
3467	// [jump to end] // this is reloced	4162	try self.restoreState(block_data.value.state, liveness.deaths, .{
3468	// [else case]	4163	.emit_instructions = false,
3469	// [jump to end] // this is reloced	4164	.update_tracking = true,
3470	// [this isn't generated yet] // point to here	4165	.resurrect = true,
3471	try self.performReloc(reloc, @intCast(self.mir_instructions.len));	4166	.close_scope = true,
		4167	});
		4168	for (block_data.value.relocs.items) \|reloc\| self.performReloc(reloc);
3472	}	4169	}
3473		4170
3474	const result = self.blocks.getPtr(inst).?.mcv;	4171	if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
3475	return self.finishAir(inst, result, .{ .none, .none, .none });	4172	const tracking = &self.inst_tracking.values()[inst_tracking_i];
		4173	if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
		4174	self.getValueIfFree(tracking.short, inst);
		4175	self.finishAirBookkeeping();
3476	}	4176	}
3477		4177
3478	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {	4178	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -3483,8 +4183,10 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3483,8 +4183,10 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
3483	// return self.finishAir(inst, .dead, .{ condition, .none, .none });	4183	// return self.finishAir(inst, .dead, .{ condition, .none, .none });
3484	}	4184	}
3485		4185
3486	fn performReloc(self: *Self, inst: Mir.Inst.Index, target: Mir.Inst.Index) !void {	4186	fn performReloc(self: *Self, inst: Mir.Inst.Index) void {
3487	const tag = self.mir_instructions.items(.tag)[inst];	4187	const tag = self.mir_instructions.items(.tag)[inst];
		4188	const ops = self.mir_instructions.items(.ops)[inst];
		4189	const target: Mir.Inst.Index = @intCast(self.mir_instructions.len);
3488		4190
3489	switch (tag) {	4191	switch (tag) {
3490	.bne,	4192	.bne,
...	@@ -3492,52 +4194,81 @@ fn performReloc(self: *Self, inst: Mir.Inst.Index, target: Mir.Inst.Index) !void	...	@@ -3492,52 +4194,81 @@ fn performReloc(self: *Self, inst: Mir.Inst.Index, target: Mir.Inst.Index) !void
3492	=> self.mir_instructions.items(.data)[inst].b_type.inst = target,	4194	=> self.mir_instructions.items(.data)[inst].b_type.inst = target,
3493	.jal,	4195	.jal,
3494	=> self.mir_instructions.items(.data)[inst].j_type.inst = target,	4196	=> self.mir_instructions.items(.data)[inst].j_type.inst = target,
3495	.j,	4197	.pseudo => switch (ops) {
3496	=> self.mir_instructions.items(.data)[inst].inst = target,	4198	.pseudo_j => self.mir_instructions.items(.data)[inst].inst = target,
3497	else => return self.fail("TODO: performReloc {s}", .{@tagName(tag)}),	4199	else => std.debug.panic("TODO: performReloc {s}", .{@tagName(ops)}),
		4200	},
		4201	else => std.debug.panic("TODO: performReloc {s}", .{@tagName(tag)}),
3498	}	4202	}
3499	}	4203	}
3500		4204
3501	fn airBr(self: *Self, inst: Air.Inst.Index) !void {	4205	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
3502	const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
3503	try self.br(branch.block_inst, branch.operand);
3504	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
3505	}
3506
3507	fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
3508	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3509	const air_tags = self.air.instructions.items(.tag);
3510	_ = air_tags;
3511	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement boolean operations for {}", .{self.target.cpu.arch});
3512	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3513	}
3514
3515	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
3516	const block_data = self.blocks.getPtr(block).?;
3517
3518	const mod = self.bin_file.comp.module.?;	4206	const mod = self.bin_file.comp.module.?;
3519	if (self.typeOf(operand).hasRuntimeBits(mod)) {	4207	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
3520	const operand_mcv = try self.resolveInst(operand);	4208
3521	const block_mcv = block_data.mcv;	4209	const block_ty = self.typeOfIndex(br.block_inst);
3522	if (block_mcv == .none) {	4210	const block_unused =
3523	block_data.mcv = operand_mcv;	4211	!block_ty.hasRuntimeBitsIgnoreComptime(mod) or self.liveness.isUnused(br.block_inst);
3524	} else {	4212	const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
3525	try self.genCopy(self.typeOfIndex(block), block_mcv, operand_mcv);	4213	const block_data = self.blocks.getPtr(br.block_inst).?;
		4214	const first_br = block_data.relocs.items.len == 0;
		4215	const block_result = result: {
		4216	if (block_unused) break :result .none;
		4217
		4218	if (!first_br) try self.getValue(block_tracking.short, null);
		4219	const src_mcv = try self.resolveInst(br.operand);
		4220
		4221	if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
		4222	if (first_br) break :result src_mcv;
		4223
		4224	try self.getValue(block_tracking.short, br.block_inst);
		4225	// .long = .none to avoid merging operand and block result stack frames.
		4226	const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv };
		4227	try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
		4228	for (current_tracking.getRegs()) \|src_reg\| self.register_manager.freeReg(src_reg);
		4229	break :result block_tracking.short;
3526	}	4230	}
		4231
		4232	const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: {
		4233	try self.getValue(block_tracking.short, br.block_inst);
		4234	break :dst block_tracking.short;
		4235	};
		4236	try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand));
		4237	break :result dst_mcv;
		4238	};
		4239
		4240	// Process operand death so that it is properly accounted for in the State below.
		4241	if (self.liveness.operandDies(inst, 0)) {
		4242	if (br.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst);
3527	}	4243	}
3528	return self.brVoid(block);
3529	}
3530		4244
3531	fn brVoid(self: *Self, block: Air.Inst.Index) !void {	4245	if (first_br) {
3532	const block_data = self.blocks.getPtr(block).?;	4246	block_tracking.* = InstTracking.init(block_result);
		4247	try self.saveRetroactiveState(&block_data.state);
		4248	} else try self.restoreState(block_data.state, &.{}, .{
		4249	.emit_instructions = true,
		4250	.update_tracking = false,
		4251	.resurrect = false,
		4252	.close_scope = false,
		4253	});
3533		4254
3534	// Emit a jump with a relocation. It will be patched up after the block ends.	4255	// Emit a jump with a relocation. It will be patched up after the block ends.
3535	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);	4256	// Leave the jump offset undefined
		4257	const jmp_reloc = try self.jump(undefined);
		4258	try block_data.relocs.append(self.gpa, jmp_reloc);
3536		4259
3537	block_data.relocs.appendAssumeCapacity(try self.addInst(.{	4260	// Stop tracking block result without forgetting tracking info
3538	.tag = .j,	4261	try self.freeValue(block_tracking.short);
3539	.data = .{ .inst = undefined },	4262
3540	}));	4263	self.finishAirBookkeeping();
		4264	}
		4265
		4266	fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
		4267	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
		4268	const air_tags = self.air.instructions.items(.tag);
		4269	_ = air_tags;
		4270	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement boolean operations for {}", .{self.target.cpu.arch});
		4271	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3541	}	4272	}
3542		4273
3543	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	4274	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -3546,13 +4277,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3546,13 +4277,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
3546	const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;	4277	const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;
3547	const clobbers_len: u31 = @truncate(extra.data.flags);	4278	const clobbers_len: u31 = @truncate(extra.data.flags);
3548	var extra_i: usize = extra.end;	4279	var extra_i: usize = extra.end;
3549	const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);	4280	const outputs: []const Air.Inst.Ref =
		4281	@ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
3550	extra_i += outputs.len;	4282	extra_i += outputs.len;
3551	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);	4283	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
3552	extra_i += inputs.len;	4284	extra_i += inputs.len;
3553		4285
		4286	log.debug("airAsm input: {any}", .{inputs});
		4287
3554	const dead = !is_volatile and self.liveness.isUnused(inst);	4288	const dead = !is_volatile and self.liveness.isUnused(inst);
3555	const result: MCValue = if (dead) .dead else result: {	4289	const result: MCValue = if (dead) .unreach else result: {
3556	if (outputs.len > 1) {	4290	if (outputs.len > 1) {
3557	return self.fail("TODO implement codegen for asm with more than 1 output", .{});	4291	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
3558	}	4292	}
...	@@ -3599,19 +4333,25 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3599,19 +4333,25 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
3599	// for the string, we still use the next u32 for the null terminator.	4333	// for the string, we still use the next u32 for the null terminator.
3600	extra_i += clobber.len / 4 + 1;	4334	extra_i += clobber.len / 4 + 1;
3601		4335
3602	// TODO honor these	4336	if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) {
		4337	// nothing really to do
		4338	} else {
		4339	try self.register_manager.getReg(parseRegName(clobber) orelse
		4340	return self.fail("invalid clobber: '{s}'", .{clobber}), null);
		4341	}
3603	}	4342	}
3604	}	4343	}
3605		4344
3606	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];	4345	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
3607		4346
3608	if (mem.eql(u8, asm_source, "ecall")) {	4347	if (std.meta.stringToEnum(Mir.Inst.Tag, asm_source)) \|tag\| {
3609	_ = try self.addInst(.{	4348	_ = try self.addInst(.{
3610	.tag = .ecall,	4349	.tag = tag,
3611	.data = .{ .nop = {} },	4350	.ops = .none,
		4351	.data = undefined,
3612	});	4352	});
3613	} else {	4353	} else {
3614	return self.fail("TODO implement support for more riscv64 assembly instructions", .{});	4354	return self.fail("TODO: asm_source {s}", .{asm_source});
3615	}	4355	}
3616		4356
3617	if (output_constraint) \|output\| {	4357	if (output_constraint) \|output\| {
...	@@ -3621,11 +4361,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3621,11 +4361,12 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
3621	const reg_name = output[2 .. output.len - 1];	4361	const reg_name = output[2 .. output.len - 1];
3622	const reg = parseRegName(reg_name) orelse	4362	const reg = parseRegName(reg_name) orelse
3623	return self.fail("unrecognized register: '{s}'", .{reg_name});	4363	return self.fail("unrecognized register: '{s}'", .{reg_name});
3624	break :result MCValue{ .register = reg };	4364	break :result .{ .register = reg };
3625	} else {	4365	} else {
3626	break :result MCValue{ .none = {} };	4366	break :result .{ .none = {} };
3627	}	4367	}
3628	};	4368	};
		4369
3629	simple: {	4370	simple: {
3630	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);	4371	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
3631	var buf_index: usize = 0;	4372	var buf_index: usize = 0;
...	@@ -3640,30 +4381,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3640,30 +4381,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
3640	@memcpy(buf[buf_index..][0..inputs.len], inputs);	4381	@memcpy(buf[buf_index..][0..inputs.len], inputs);
3641	return self.finishAir(inst, result, buf);	4382	return self.finishAir(inst, result, buf);
3642	}	4383	}
3643	var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len);	4384	var bt = self.liveness.iterateBigTomb(inst);
3644	for (outputs) \|output\| {	4385	for (outputs) \|output\| if (output != .none) try self.feed(&bt, output);
3645	if (output == .none) continue;	4386	for (inputs) \|input\| try self.feed(&bt, input);
3646		4387	return self.finishAirResult(inst, result);
3647	bt.feed(output);
3648	}
3649	for (inputs) \|input\| {
3650	bt.feed(input);
3651	}
3652	return bt.finishAir(result);
3653	}
3654
3655	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
3656	try self.ensureProcessDeathCapacity(operand_count + 1);
3657	return BigTomb{
3658	.function = self,
3659	.inst = inst,
3660	.lbt = self.liveness.iterateBigTomb(inst),
3661	};
3662	}	4388	}
3663		4389
3664	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.	4390	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
3665	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {	4391	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
3666	const mod = self.bin_file.comp.module.?;	4392	const zcu = self.bin_file.comp.module.?;
3667		4393
3668	// There isn't anything to store	4394	// There isn't anything to store
3669	if (dst_mcv == .none) return;	4395	if (dst_mcv == .none) return;
...	@@ -3690,11 +4416,11 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {	...	@@ -3690,11 +4416,11 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
3690	.off = -dst_reg_off.off,	4416	.off = -dst_reg_off.off,
3691	} },	4417	} },
3692	}),	4418	}),
3693	.stack_offset => \|off\| return self.genSetStack(ty, off, src_mcv),	4419	.load_frame => \|frame\| return self.genSetStack(ty, frame, src_mcv),
3694	.memory => \|addr\| return self.genSetMem(ty, addr, src_mcv),	4420	.memory => return self.fail("TODO: genCopy memory", .{}),
3695	.register_pair => \|dst_regs\| {	4421	.register_pair => \|dst_regs\| {
3696	const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {	4422	const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {
3697	.register_pair, .memory, .indirect, .stack_offset => null,	4423	.register_pair, .memory, .indirect, .load_frame => null,
3698	.load_symbol => src: {	4424	.load_symbol => src: {
3699	const src_addr_reg, const src_addr_lock = try self.allocReg();	4425	const src_addr_reg, const src_addr_lock = try self.allocReg();
3700	errdefer self.register_manager.unlockReg(src_addr_lock);	4426	errdefer self.register_manager.unlockReg(src_addr_lock);
...	@@ -3708,7 +4434,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {	...	@@ -3708,7 +4434,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
3708	try self.resolveInst(src_ref),	4434	try self.resolveInst(src_ref),
3709	),	4435	),
3710	else => return self.fail("TODO implement genCopy for {s} of {}", .{	4436	else => return self.fail("TODO implement genCopy for {s} of {}", .{
3711	@tagName(src_mcv), ty.fmt(mod),	4437	@tagName(src_mcv), ty.fmt(zcu),
3712	}),	4438	}),
3713	};	4439	};
3714	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);	4440	defer if (src_info) \|info\| self.register_manager.unlockReg(info.addr_lock);
...	@@ -3717,34 +4443,38 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {	...	@@ -3717,34 +4443,38 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
3717	for (dst_regs, try self.splitType(ty), 0..) \|dst_reg, dst_ty, part_i\| {	4443	for (dst_regs, try self.splitType(ty), 0..) \|dst_reg, dst_ty, part_i\| {
3718	try self.genSetReg(dst_ty, dst_reg, switch (src_mcv) {	4444	try self.genSetReg(dst_ty, dst_reg, switch (src_mcv) {
3719	.register_pair => \|src_regs\| .{ .register = src_regs[part_i] },	4445	.register_pair => \|src_regs\| .{ .register = src_regs[part_i] },
3720	.memory, .indirect, .stack_offset => src_mcv.address().offset(part_disp).deref(),	4446	.memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
3721	.load_symbol => .{ .indirect = .{	4447	.load_symbol => .{ .indirect = .{
3722	.reg = src_info.?.addr_reg,	4448	.reg = src_info.?.addr_reg,
3723	.off = part_disp,	4449	.off = part_disp,
3724	} },	4450	} },
3725	else => unreachable,	4451	else => unreachable,
3726	});	4452	});
3727	part_disp += @intCast(dst_ty.abiSize(mod));	4453	part_disp += @intCast(dst_ty.abiSize(zcu));
3728	}	4454	}
3729	},	4455	},
3730	else => return std.debug.panic("TODO: genCopy {s} with {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }),	4456	else => return std.debug.panic("TODO: genCopy {s} with {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }),
3731	}	4457	}
3732	}	4458	}
3733		4459
3734	/// Sets the value of `src_mcv` into stack memory at `stack_offset`.	4460	fn genSetStack(
3735	fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) InnerError!void {	4461	self: *Self,
3736	const mod = self.bin_file.comp.module.?;	4462	ty: Type,
3737	const abi_size: u32 = @intCast(ty.abiSize(mod));	4463	frame: FrameAddr,
		4464	src_mcv: MCValue,
		4465	) InnerError!void {
		4466	const zcu = self.bin_file.comp.module.?;
		4467	const abi_size: u32 = @intCast(ty.abiSize(zcu));
3738		4468
3739	switch (src_mcv) {	4469	switch (src_mcv) {
3740	.none => return,	4470	.none => return,
3741	.dead => unreachable,	4471	.dead => unreachable,
3742	.undef => {	4472	.undef => {
3743	if (!self.wantSafety()) return;	4473	if (!self.wantSafety()) return;
3744	try self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa });	4474	try self.genSetStack(ty, frame, .{ .immediate = 0xaaaaaaaaaaaaaaaa });
3745	},	4475	},
3746	.immediate,	4476	.immediate,
3747	.ptr_stack_offset,	4477	.lea_frame,
3748	=> {	4478	=> {
3749	// TODO: remove this lock in favor of a copyToTmpRegister when we load 64 bit immediates with	4479	// TODO: remove this lock in favor of a copyToTmpRegister when we load 64 bit immediates with
3750	// a register allocation.	4480	// a register allocation.
...	@@ -3753,26 +4483,24 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) Inner	...	@@ -3753,26 +4483,24 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) Inner
3753		4483
3754	try self.genSetReg(ty, reg, src_mcv);	4484	try self.genSetReg(ty, reg, src_mcv);
3755		4485
3756	return self.genSetStack(ty, stack_offset, .{ .register = reg });	4486	return self.genSetStack(ty, frame, .{ .register = reg });
3757	},	4487	},
3758	.register => \|reg\| {	4488	.register => \|reg\| {
3759	switch (abi_size) {	4489	switch (abi_size) {
3760	1, 2, 4, 8 => {	4490	1, 2, 4, 8 => {
3761	const tag: Mir.Inst.Tag = switch (abi_size) {
3762	1 => .sb,
3763	2 => .sh,
3764	4 => .sw,
3765	8 => .sd,
3766	else => unreachable,
3767	};
3768
3769	_ = try self.addInst(.{	4491	_ = try self.addInst(.{
3770	.tag = tag,	4492	.tag = .pseudo,
3771	.data = .{ .i_type = .{	4493	.ops = .pseudo_store_rm,
3772	.rd = reg,	4494	.data = .{ .rm = .{
3773	.rs1 = .sp,	4495	.r = reg,
3774	.imm12 = math.cast(i12, stack_offset) orelse {	4496	.m = .{
3775	return self.fail("TODO: genSetStack bigger stack values", .{});	4497	.base = .{ .frame = frame.index },
		4498	.mod = .{
		4499	.rm = .{
		4500	.size = self.memSize(ty),
		4501	.disp = frame.off,
		4502	},
		4503	},
3776	},	4504	},
3777	} },	4505	} },
3778	});	4506	});
...	@@ -3780,38 +4508,26 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) Inner	...	@@ -3780,38 +4508,26 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) Inner
3780	else => unreachable, // register can hold a max of 8 bytes	4508	else => unreachable, // register can hold a max of 8 bytes
3781	}	4509	}
3782	},	4510	},
3783	.stack_offset,	4511	.load_frame,
3784	.indirect,	4512	.indirect,
3785	.load_symbol,	4513	.load_symbol,
3786	=> {	4514	=> {
3787	if (src_mcv == .stack_offset and src_mcv.stack_offset == stack_offset) return;
3788
3789	if (abi_size <= 8) {	4515	if (abi_size <= 8) {
3790	const reg = try self.copyToTmpRegister(ty, src_mcv);	4516	const reg = try self.copyToTmpRegister(ty, src_mcv);
3791	return self.genSetStack(ty, stack_offset, .{ .register = reg });	4517	return self.genSetStack(ty, frame, .{ .register = reg });
3792	}	4518	}
3793		4519
3794	try self.genInlineMemcpy(	4520	try self.genInlineMemcpy(
3795	.{ .ptr_stack_offset = stack_offset },	4521	.{ .lea_frame = frame },
3796	src_mcv.address(),	4522	src_mcv.address(),
3797	.{ .immediate = abi_size },	4523	.{ .immediate = abi_size },
3798	);	4524	);
3799	},	4525	},
3800	.air_ref => \|ref\| try self.genSetStack(ty, stack_offset, try self.resolveInst(ref)),	4526	.air_ref => \|ref\| try self.genSetStack(ty, frame, try self.resolveInst(ref)),
3801	else => return self.fail("TODO: genSetStack {s}", .{@tagName(src_mcv)}),	4527	else => return self.fail("TODO: genSetStack {s}", .{@tagName(src_mcv)}),
3802	}	4528	}
3803	}	4529	}
3804		4530
3805	fn genSetMem(self: *Self, ty: Type, addr: u64, src_mcv: MCValue) InnerError!void {
3806	const mod = self.bin_file.comp.module.?;
3807	const abi_size: u32 = @intCast(ty.abiSize(mod));
3808	_ = abi_size;
3809	_ = addr;
3810	_ = src_mcv;
3811
3812	return self.fail("TODO: genSetMem", .{});
3813	}
3814
3815	fn genInlineMemcpy(	4531	fn genInlineMemcpy(
3816	self: *Self,	4532	self: *Self,
3817	dst_ptr: MCValue,	4533	dst_ptr: MCValue,
...	@@ -3834,11 +4550,12 @@ fn genInlineMemcpy(	...	@@ -3834,11 +4550,12 @@ fn genInlineMemcpy(
3834	// lb tmp, 0(src)	4550	// lb tmp, 0(src)
3835	const first_inst = try self.addInst(.{	4551	const first_inst = try self.addInst(.{
3836	.tag = .lb,	4552	.tag = .lb,
		4553	.ops = .rri,
3837	.data = .{	4554	.data = .{
3838	.i_type = .{	4555	.i_type = .{
3839	.rd = tmp,	4556	.rd = tmp,
3840	.rs1 = src,	4557	.rs1 = src,
3841	.imm12 = 0,	4558	.imm12 = Immediate.s(0),
3842	},	4559	},
3843	},	4560	},
3844	});	4561	});
...	@@ -3846,11 +4563,12 @@ fn genInlineMemcpy(	...	@@ -3846,11 +4563,12 @@ fn genInlineMemcpy(
3846	// sb tmp, 0(dst)	4563	// sb tmp, 0(dst)
3847	_ = try self.addInst(.{	4564	_ = try self.addInst(.{
3848	.tag = .sb,	4565	.tag = .sb,
		4566	.ops = .rri,
3849	.data = .{	4567	.data = .{
3850	.i_type = .{	4568	.i_type = .{
3851	.rd = tmp,	4569	.rd = tmp,
3852	.rs1 = dst,	4570	.rs1 = dst,
3853	.imm12 = 0,	4571	.imm12 = Immediate.s(0),
3854	},	4572	},
3855	},	4573	},
3856	});	4574	});
...	@@ -3858,11 +4576,12 @@ fn genInlineMemcpy(	...	@@ -3858,11 +4576,12 @@ fn genInlineMemcpy(
3858	// dec count by 1	4576	// dec count by 1
3859	_ = try self.addInst(.{	4577	_ = try self.addInst(.{
3860	.tag = .addi,	4578	.tag = .addi,
		4579	.ops = .rri,
3861	.data = .{	4580	.data = .{
3862	.i_type = .{	4581	.i_type = .{
3863	.rd = count,	4582	.rd = count,
3864	.rs1 = count,	4583	.rs1 = count,
3865	.imm12 = -1,	4584	.imm12 = Immediate.s(-1),
3866	},	4585	},
3867	},	4586	},
3868	});	4587	});
...	@@ -3870,6 +4589,7 @@ fn genInlineMemcpy(	...	@@ -3870,6 +4589,7 @@ fn genInlineMemcpy(
3870	// branch if count is 0	4589	// branch if count is 0
3871	_ = try self.addInst(.{	4590	_ = try self.addInst(.{
3872	.tag = .beq,	4591	.tag = .beq,
		4592	.ops = .rr_inst,
3873	.data = .{	4593	.data = .{
3874	.b_type = .{	4594	.b_type = .{
3875	.inst = @intCast(self.mir_instructions.len + 4), // points after the last inst	4595	.inst = @intCast(self.mir_instructions.len + 4), // points after the last inst
...	@@ -3882,29 +4602,32 @@ fn genInlineMemcpy(	...	@@ -3882,29 +4602,32 @@ fn genInlineMemcpy(
3882	// increment the pointers	4602	// increment the pointers
3883	_ = try self.addInst(.{	4603	_ = try self.addInst(.{
3884	.tag = .addi,	4604	.tag = .addi,
		4605	.ops = .rri,
3885	.data = .{	4606	.data = .{
3886	.i_type = .{	4607	.i_type = .{
3887	.rd = src,	4608	.rd = src,
3888	.rs1 = src,	4609	.rs1 = src,
3889	.imm12 = 1,	4610	.imm12 = Immediate.s(1),
3890	},	4611	},
3891	},	4612	},
3892	});	4613	});
3893		4614
3894	_ = try self.addInst(.{	4615	_ = try self.addInst(.{
3895	.tag = .addi,	4616	.tag = .addi,
		4617	.ops = .rri,
3896	.data = .{	4618	.data = .{
3897	.i_type = .{	4619	.i_type = .{
3898	.rd = dst,	4620	.rd = dst,
3899	.rs1 = dst,	4621	.rs1 = dst,
3900	.imm12 = 1,	4622	.imm12 = Immediate.s(1),
3901	},	4623	},
3902	},	4624	},
3903	});	4625	});
3904		4626
3905	// jump back to start of loop	4627	// jump back to start of loop
3906	_ = try self.addInst(.{	4628	_ = try self.addInst(.{
3907	.tag = .j,	4629	.tag = .pseudo,
		4630	.ops = .pseudo_j,
3908	.data = .{	4631	.data = .{
3909	.inst = first_inst,	4632	.inst = first_inst,
3910	},	4633	},
...	@@ -3913,31 +4636,13 @@ fn genInlineMemcpy(	...	@@ -3913,31 +4636,13 @@ fn genInlineMemcpy(
3913		4636
3914	/// Sets the value of `src_mcv` into `reg`. Assumes you have a lock on it.	4637	/// Sets the value of `src_mcv` into `reg`. Assumes you have a lock on it.
3915	fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!void {	4638	fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!void {
3916	const mod = self.bin_file.comp.module.?;	4639	const zcu = self.bin_file.comp.module.?;
3917	const abi_size: u32 = @intCast(ty.abiSize(mod));	4640	const abi_size: u32 = @intCast(ty.abiSize(zcu));
3918		4641
3919	const load_tag: Mir.Inst.Tag = switch (abi_size) {	4642	if (abi_size > 8) return self.fail("tried to set reg with size {}", .{abi_size});
3920	1 => .lb,
3921	2 => .lh,
3922	4 => .lw,
3923	8 => .ld,
3924	else => return self.fail("TODO: genSetReg for size {d}", .{abi_size}),
3925	};
3926		4643
3927	switch (src_mcv) {	4644	switch (src_mcv) {
3928	.dead => unreachable,	4645	.dead => unreachable,
3929	.ptr_stack_offset => \|off\| {
3930	_ = try self.addInst(.{
3931	.tag = .addi,
3932	.data = .{ .i_type = .{
3933	.rd = reg,
3934	.rs1 = .sp,
3935	.imm12 = math.cast(i12, off) orelse {
3936	return self.fail("TODO: bigger stack sizes", .{});
3937	},
3938	} },
3939	});
3940	},
3941	.unreach, .none => return, // Nothing to do.	4646	.unreach, .none => return, // Nothing to do.
3942	.undef => {	4647	.undef => {
3943	if (!self.wantSafety())	4648	if (!self.wantSafety())
...	@@ -3950,10 +4655,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -3950,10 +4655,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
3950	if (math.minInt(i12) <= x and x <= math.maxInt(i12)) {	4655	if (math.minInt(i12) <= x and x <= math.maxInt(i12)) {
3951	_ = try self.addInst(.{	4656	_ = try self.addInst(.{
3952	.tag = .addi,	4657	.tag = .addi,
		4658	.ops = .rri,
3953	.data = .{ .i_type = .{	4659	.data = .{ .i_type = .{
3954	.rd = reg,	4660	.rd = reg,
3955	.rs1 = .zero,	4661	.rs1 = .zero,
3956	.imm12 = @intCast(x),	4662	.imm12 = Immediate.s(@intCast(x)),
3957	} },	4663	} },
3958	});	4664	});
3959	} else if (math.minInt(i32) <= x and x <= math.maxInt(i32)) {	4665	} else if (math.minInt(i32) <= x and x <= math.maxInt(i32)) {
...	@@ -3963,17 +4669,19 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -3963,17 +4669,19 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
3963		4669
3964	_ = try self.addInst(.{	4670	_ = try self.addInst(.{
3965	.tag = .lui,	4671	.tag = .lui,
		4672	.ops = .ri,
3966	.data = .{ .u_type = .{	4673	.data = .{ .u_type = .{
3967	.rd = reg,	4674	.rd = reg,
3968	.imm20 = hi20,	4675	.imm20 = Immediate.s(hi20),
3969	} },	4676	} },
3970	});	4677	});
3971	_ = try self.addInst(.{	4678	_ = try self.addInst(.{
3972	.tag = .addi,	4679	.tag = .addi,
		4680	.ops = .rri,
3973	.data = .{ .i_type = .{	4681	.data = .{ .i_type = .{
3974	.rd = reg,	4682	.rd = reg,
3975	.rs1 = reg,	4683	.rs1 = reg,
3976	.imm12 = lo12,	4684	.imm12 = Immediate.s(lo12),
3977	} },	4685	} },
3978	});	4686	});
3979	} else {	4687	} else {
...	@@ -3992,15 +4700,17 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -3992,15 +4700,17 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
3992		4700
3993	_ = try self.addInst(.{	4701	_ = try self.addInst(.{
3994	.tag = .slli,	4702	.tag = .slli,
		4703	.ops = .rri,
3995	.data = .{ .i_type = .{	4704	.data = .{ .i_type = .{
3996	.imm12 = 32,
3997	.rd = reg,	4705	.rd = reg,
3998	.rs1 = reg,	4706	.rs1 = reg,
		4707	.imm12 = Immediate.s(32),
3999	} },	4708	} },
4000	});	4709	});
4001		4710
4002	_ = try self.addInst(.{	4711	_ = try self.addInst(.{
4003	.tag = .add,	4712	.tag = .add,
		4713	.ops = .rrr,
4004	.data = .{ .r_type = .{	4714	.data = .{ .r_type = .{
4005	.rd = reg,	4715	.rd = reg,
4006	.rs1 = reg,	4716	.rs1 = reg,
...	@@ -4016,7 +4726,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -4016,7 +4726,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
4016		4726
4017	// mov reg, src_reg	4727	// mov reg, src_reg
4018	_ = try self.addInst(.{	4728	_ = try self.addInst(.{
4019	.tag = .mv,	4729	.tag = .pseudo,
		4730	.ops = .pseudo_mv,
4020	.data = .{ .rr = .{	4731	.data = .{ .rr = .{
4021	.rd = reg,	4732	.rd = reg,
4022	.rs = src_reg,	4733	.rs = src_reg,
...	@@ -4029,21 +4740,46 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -4029,21 +4740,46 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
4029		4740
4030	_ = try self.addInst(.{	4741	_ = try self.addInst(.{
4031	.tag = .ld,	4742	.tag = .ld,
		4743	.ops = .rri,
4032	.data = .{ .i_type = .{	4744	.data = .{ .i_type = .{
4033	.rd = reg,	4745	.rd = reg,
4034	.rs1 = reg,	4746	.rs1 = reg,
4035	.imm12 = 0,	4747	.imm12 = Immediate.s(0),
4036	} },	4748	} },
4037	});	4749	});
4038	},	4750	},
4039	.stack_offset => \|off\| {	4751	.load_frame => \|frame\| {
4040	_ = try self.addInst(.{	4752	_ = try self.addInst(.{
4041	.tag = load_tag,	4753	.tag = .pseudo,
4042	.data = .{ .i_type = .{	4754	.ops = .pseudo_load_rm,
4043	.rd = reg,	4755	.data = .{ .rm = .{
4044	.rs1 = .sp,	4756	.r = reg,
4045	.imm12 = math.cast(i12, off) orelse {	4757	.m = .{
4046	return self.fail("TODO: genSetReg support larger stack sizes", .{});	4758	.base = .{ .frame = frame.index },
		4759	.mod = .{
		4760	.rm = .{
		4761	.size = self.memSize(ty),
		4762	.disp = frame.off,
		4763	},
		4764	},
		4765	},
		4766	} },
		4767	});
		4768	},
		4769	.lea_frame => \|frame\| {
		4770	_ = try self.addInst(.{
		4771	.tag = .pseudo,
		4772	.ops = .pseudo_lea_rm,
		4773	.data = .{ .rm = .{
		4774	.r = reg,
		4775	.m = .{
		4776	.base = .{ .frame = frame.index },
		4777	.mod = .{
		4778	.rm = .{
		4779	.size = self.memSize(ty),
		4780	.disp = frame.off,
		4781	},
		4782	},
4047	},	4783	},
4048	} },	4784	} },
4049	});	4785	});
...	@@ -4052,35 +4788,41 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -4052,35 +4788,41 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
4052	try self.genSetReg(ty, reg, src_mcv.address());	4788	try self.genSetReg(ty, reg, src_mcv.address());
4053	try self.genSetReg(ty, reg, .{ .indirect = .{ .reg = reg } });	4789	try self.genSetReg(ty, reg, .{ .indirect = .{ .reg = reg } });
4054	},	4790	},
4055	.air_ref => \|ref\| try self.genSetReg(ty, reg, try self.resolveInst(ref)),
4056	.indirect => \|reg_off\| {	4791	.indirect => \|reg_off\| {
		4792	const load_tag: Mir.Inst.Tag = switch (abi_size) {
		4793	1 => .lb,
		4794	2 => .lh,
		4795	4 => .lw,
		4796	8 => .ld,
		4797	else => return self.fail("TODO: genSetReg for size {d}", .{abi_size}),
		4798	};
		4799
4057	_ = try self.addInst(.{	4800	_ = try self.addInst(.{
4058	.tag = load_tag,	4801	.tag = load_tag,
4059	.data = .{	4802	.ops = .rri,
4060	.i_type = .{	4803	.data = .{ .i_type = .{
4061	.rd = reg,	4804	.rd = reg,
4062	.rs1 = reg_off.reg,	4805	.rs1 = reg_off.reg,
4063	.imm12 = @intCast(reg_off.off),	4806	.imm12 = Immediate.s(reg_off.off),
4064	},	4807	} },
4065	},
4066	});	4808	});
4067	},	4809	},
4068	.addr_symbol => \|sym_off\| {	4810	.lea_symbol => \|sym_off\| {
4069	assert(sym_off.off == 0);	4811	assert(sym_off.off == 0);
4070		4812
4071	const atom_index = try self.symbolIndex();	4813	const atom_index = try self.symbolIndex();
4072		4814
4073	_ = try self.addInst(.{	4815	_ = try self.addInst(.{
4074	.tag = .load_symbol,	4816	.tag = .pseudo,
4075	.data = .{	4817	.ops = .pseudo_load_symbol,
4076	.payload = try self.addExtra(Mir.LoadSymbolPayload{	4818	.data = .{ .payload = try self.addExtra(Mir.LoadSymbolPayload{
4077	.register = reg.id(),	4819	.register = reg.id(),
4078	.atom_index = atom_index,	4820	.atom_index = atom_index,
4079	.sym_index = sym_off.sym,	4821	.sym_index = sym_off.sym,
4080	}),	4822	}) },
4081	},
4082	});	4823	});
4083	},	4824	},
		4825	.air_ref => \|ref\| try self.genSetReg(ty, reg, try self.resolveInst(ref)),
4084	else => return self.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}),	4826	else => return self.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}),
4085	}	4827	}
4086	}	4828	}
...	@@ -4100,27 +4842,44 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4100,27 +4842,44 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
4100	}	4842	}
4101		4843
4102	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	4844	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
		4845	const zcu = self.bin_file.comp.module.?;
		4846
4103	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	4847	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4104	const result = if (self.liveness.isUnused(inst)) .dead else result: {	4848	const result = if (self.liveness.isUnused(inst)) .unreach else result: {
4105	const operand = try self.resolveInst(ty_op.operand);	4849	const src_mcv = try self.resolveInst(ty_op.operand);
4106	if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
4107		4850
4108	const operand_lock = switch (operand) {	4851	const dst_ty = self.typeOfIndex(inst);
4109	.register => \|reg\| self.register_manager.lockReg(reg),	4852	const src_ty = self.typeOf(ty_op.operand);
4110	else => null,	4853
		4854	const src_lock = if (src_mcv.getReg()) \|reg\| self.register_manager.lockReg(reg) else null;
		4855	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
		4856
		4857	const dst_mcv = if (dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and
		4858	self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
		4859	const dst_mcv = try self.allocRegOrMem(inst, true);
		4860	try self.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
		4861	.lt => dst_ty,
		4862	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
		4863	.gt => src_ty,
		4864	}, dst_mcv, src_mcv);
		4865	break :dst dst_mcv;
4111	};	4866	};
4112	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
4113		4867
4114	const dest = try self.allocRegOrMem(inst, true);	4868	if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and
4115	try self.genCopy(self.typeOfIndex(inst), dest, operand);	4869	dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
4116	break :result dest;	4870
		4871	const abi_size = dst_ty.abiSize(zcu);
		4872	const bit_size = dst_ty.bitSize(zcu);
		4873	if (abi_size * 8 <= bit_size) break :result dst_mcv;
		4874
		4875	return self.fail("TODO: airBitCast {} to {}", .{ src_ty.fmt(zcu), dst_ty.fmt(zcu) });
4117	};	4876	};
4118	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	4877	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4119	}	4878	}
4120		4879
4121	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {	4880	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
4122	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	4881	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4123	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airArrayToSlice for {}", .{	4882	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airArrayToSlice for {}", .{
4124	self.target.cpu.arch,	4883	self.target.cpu.arch,
4125	});	4884	});
4126	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	4885	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -4128,7 +4887,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4128,7 +4887,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
4128		4887
4129	fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	4888	fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
4130	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	4889	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4131	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatFromInt for {}", .{	4890	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airFloatFromInt for {}", .{
4132	self.target.cpu.arch,	4891	self.target.cpu.arch,
4133	});	4892	});
4134	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	4893	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -4136,7 +4895,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4136,7 +4895,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
4136		4895
4137	fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {	4896	fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
4138	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	4897	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4139	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntFromFloat for {}", .{	4898	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airIntFromFloat for {}", .{
4140	self.target.cpu.arch,	4899	self.target.cpu.arch,
4141	});	4900	});
4142	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	4901	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -4186,7 +4945,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4186,7 +4945,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
4186	fn airTagName(self: *Self, inst: Air.Inst.Index) !void {	4945	fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
4187	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	4946	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4188	const operand = try self.resolveInst(un_op);	4947	const operand = try self.resolveInst(un_op);
4189	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {	4948	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else {
4190	_ = operand;	4949	_ = operand;
4191	return self.fail("TODO implement airTagName for riscv64", .{});	4950	return self.fail("TODO implement airTagName for riscv64", .{});
4192	};	4951	};
...	@@ -4194,7 +4953,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4194,7 +4953,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
4194	}	4953	}
4195		4954
4196	fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	4955	fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
4197	const mod = self.bin_file.comp.module.?;	4956	const zcu = self.bin_file.comp.module.?;
4198	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	4957	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4199		4958
4200	const err_ty = self.typeOf(un_op);	4959	const err_ty = self.typeOf(un_op);
...	@@ -4207,7 +4966,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4207,7 +4966,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
4207	const addr_reg, const addr_lock = try self.allocReg();	4966	const addr_reg, const addr_lock = try self.allocReg();
4208	defer self.register_manager.unlockReg(addr_lock);	4967	defer self.register_manager.unlockReg(addr_lock);
4209		4968
4210	const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);	4969	const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, zcu);
4211	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {	4970	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
4212	const sym_index = elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, lazy_sym) catch \|err\|	4971	const sym_index = elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, lazy_sym) catch \|err\|
4213	return self.fail("{s} creating lazy symbol", .{@errorName(err)});	4972	return self.fail("{s} creating lazy symbol", .{@errorName(err)});
...	@@ -4223,69 +4982,45 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4223,69 +4982,45 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
4223	const end_reg, const end_lock = try self.allocReg();	4982	const end_reg, const end_lock = try self.allocReg();
4224	defer self.register_manager.unlockReg(end_lock);	4983	defer self.register_manager.unlockReg(end_lock);
4225		4984
4226	_ = try self.addInst(.{	4985	_ = start_reg;
4227	.tag = .slli,	4986	_ = end_reg;
4228	.data = .{
4229	.i_type = .{
4230	.rd = err_reg,
4231	.rs1 = err_reg,
4232	.imm12 = 4,
4233	},
4234	},
4235	});
4236
4237	try self.binOpMir(
4238	.add,
4239	null,
4240	Type.usize,
4241	.{ .register = err_reg },
4242	.{ .register = addr_reg },
4243	);
4244
4245	try self.genSetReg(Type.usize, start_reg, .{ .indirect = .{ .reg = err_reg } });
4246	try self.genSetReg(Type.usize, end_reg, .{ .indirect = .{ .reg = err_reg, .off = 8 } });
4247		4987
4248	const dst_mcv = try self.allocRegOrMem(inst, false);	4988	return self.fail("TODO: airErrorName", .{});
4249
4250	try self.genSetStack(Type.usize, dst_mcv.stack_offset, .{ .register = start_reg });
4251	try self.genSetStack(Type.usize, dst_mcv.stack_offset + 8, .{ .register = end_reg });
4252
4253	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4254	}	4989	}
4255		4990
4256	fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	4991	fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
4257	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	4992	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4258	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for riscv64", .{});	4993	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airSplat for riscv64", .{});
4259	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	4994	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4260	}	4995	}
4261		4996
4262	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	4997	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
4263	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	4998	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4264	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	4999	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
4265	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSelect for riscv64", .{});	5000	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airSelect for riscv64", .{});
4266	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });	5001	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
4267	}	5002	}
4268		5003
4269	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	5004	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
4270	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	5005	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4271	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airShuffle for riscv64", .{});	5006	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airShuffle for riscv64", .{});
4272	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	5007	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4273	}	5008	}
4274		5009
4275	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	5010	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
4276	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;	5011	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
4277	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airReduce for riscv64", .{});	5012	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else return self.fail("TODO implement airReduce for riscv64", .{});
4278	return self.finishAir(inst, result, .{ reduce.operand, .none, .none });	5013	return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
4279	}	5014	}
4280		5015
4281	fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	5016	fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
4282	const mod = self.bin_file.comp.module.?;	5017	const zcu = self.bin_file.comp.module.?;
4283	const vector_ty = self.typeOfIndex(inst);	5018	const vector_ty = self.typeOfIndex(inst);
4284	const len = vector_ty.vectorLen(mod);	5019	const len = vector_ty.vectorLen(zcu);
4285	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5020	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4286	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);	5021	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
4287	const result: MCValue = res: {	5022	const result: MCValue = res: {
4288	if (self.liveness.isUnused(inst)) break :res MCValue.dead;	5023	if (self.liveness.isUnused(inst)) break :res .unreach;
4289	return self.fail("TODO implement airAggregateInit for riscv64", .{});	5024	return self.fail("TODO implement airAggregateInit for riscv64", .{});
4290	};	5025	};
4291		5026
...	@@ -4294,11 +5029,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4294,11 +5029,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
4294	@memcpy(buf[0..elements.len], elements);	5029	@memcpy(buf[0..elements.len], elements);
4295	return self.finishAir(inst, result, buf);	5030	return self.finishAir(inst, result, buf);
4296	}	5031	}
4297	var bt = try self.iterateBigTomb(inst, elements.len);	5032	var bt = self.liveness.iterateBigTomb(inst);
4298	for (elements) \|elem\| {	5033	for (elements) \|elem\| try self.feed(&bt, elem);
4299	bt.feed(elem);	5034	return self.finishAirResult(inst, result);
4300	}
4301	return bt.finishAir(result);
4302	}	5035	}
4303		5036
4304	fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {	5037	fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -4313,49 +5046,55 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4313,49 +5046,55 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
4313	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;	5046	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
4314	// TODO: RISC-V does have prefetch instruction variants.	5047	// TODO: RISC-V does have prefetch instruction variants.
4315	// see here: https://raw.githubusercontent.com/riscv/riscv-CMOs/master/specifications/cmobase-v1.0.1.pdf	5048	// see here: https://raw.githubusercontent.com/riscv/riscv-CMOs/master/specifications/cmobase-v1.0.1.pdf
4316	return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });	5049	return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
4317	}	5050	}
4318		5051
4319	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {	5052	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
4320	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	5053	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4321	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	5054	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
4322	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {	5055	const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else {
4323	return self.fail("TODO implement airMulAdd for riscv64", .{});	5056	return self.fail("TODO implement airMulAdd for riscv64", .{});
4324	};	5057	};
4325	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });	5058	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
4326	}	5059	}
4327		5060
4328	fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {	5061	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
4329	const mod = self.bin_file.comp.module.?;	5062	const zcu = self.bin_file.comp.module.?;
4330		5063
4331	// If the type has no codegen bits, no need to store it.	5064	// If the type has no codegen bits, no need to store it.
4332	const inst_ty = self.typeOf(inst);	5065	const inst_ty = self.typeOf(ref);
4333	if (!inst_ty.hasRuntimeBits(mod))	5066	if (!inst_ty.hasRuntimeBits(zcu))
4334	return MCValue{ .none = {} };	5067	return .none;
4335		5068
4336	const inst_index = inst.toIndex() orelse return self.genTypedValue((try self.air.value(inst, mod)).?);	5069	const mcv = if (ref.toIndex()) \|inst\| mcv: {
4337	return self.getResolvedInstValue(inst_index);	5070	break :mcv self.inst_tracking.getPtr(inst).?.short;
4338	}	5071	} else mcv: {
4339		5072	const ip_index = ref.toInterned().?;
4340	fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {	5073	const gop = try self.const_tracking.getOrPut(self.gpa, ip_index);
4341	// Treat each stack item as a "layer" on top of the previous one.	5074	if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(
4342	var i: usize = self.branch_stack.items.len;	5075	try self.genTypedValue(Value.fromInterned(ip_index)),
4343	while (true) {	5076	);
4344	i -= 1;	5077	break :mcv gop.value_ptr.short;
4345	if (self.branch_stack.items[i].inst_table.get(inst)) \|mcv\| {	5078	};
4346	assert(mcv != .dead);	5079
4347	return mcv;	5080	return mcv;
4348	}	5081	}
4349	}	5082
		5083	fn getResolvedInstValue(self: Self, inst: Air.Inst.Index) InstTracking {
		5084	const tracking = self.inst_tracking.getPtr(inst).?;
		5085	return switch (tracking.short) {
		5086	.none, .unreach, .dead => unreachable,
		5087	else => tracking,
		5088	};
4350	}	5089	}
4351		5090
4352	fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {	5091	fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
4353	const mod = self.bin_file.comp.module.?;	5092	const zcu = self.bin_file.comp.module.?;
4354	const result = try codegen.genTypedValue(	5093	const result = try codegen.genTypedValue(
4355	self.bin_file,	5094	self.bin_file,
4356	self.src_loc,	5095	self.src_loc,
4357	val,	5096	val,
4358	mod.funcOwnerDeclIndex(self.func_index),	5097	zcu.funcOwnerDeclIndex(self.func_index),
4359	);	5098	);
4360	const mcv: MCValue = switch (result) {	5099	const mcv: MCValue = switch (result) {
4361	.mcv => \|mcv\| switch (mcv) {	5100	.mcv => \|mcv\| switch (mcv) {
...	@@ -4378,8 +5117,8 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {	...	@@ -4378,8 +5117,8 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
4378		5117
4379	const CallMCValues = struct {	5118	const CallMCValues = struct {
4380	args: []MCValue,	5119	args: []MCValue,
4381	return_value: MCValue,	5120	return_value: InstTracking,
4382	stack_byte_count: u32,	5121	stack_byte_count: u31,
4383	stack_align: Alignment,	5122	stack_align: Alignment,
4384		5123
4385	fn deinit(self: CallMCValues, func: Self) void {	5124	fn deinit(self: CallMCValues, func: Self) void {
...	@@ -4389,86 +5128,115 @@ const CallMCValues = struct {	...	@@ -4389,86 +5128,115 @@ const CallMCValues = struct {
4389	};	5128	};
4390		5129
4391	/// Caller must call `CallMCValues.deinit`.	5130	/// Caller must call `CallMCValues.deinit`.
4392	fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: CallView) !CallMCValues {	5131	fn resolveCallingConventionValues(
4393	const mod = self.bin_file.comp.module.?;	5132	self: *Self,
4394	const ip = &mod.intern_pool;	5133	fn_info: InternPool.Key.FuncType,
		5134	) !CallMCValues {
		5135	const zcu = self.bin_file.comp.module.?;
		5136	const ip = &zcu.intern_pool;
		5137
		5138	const param_types = try self.gpa.alloc(Type, fn_info.param_types.len);
		5139	defer self.gpa.free(param_types);
4395		5140
4396	_ = role;	5141	for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) \|*dest, src\| {
		5142	dest.* = Type.fromInterned(src);
		5143	}
4397		5144
4398	const fn_info = mod.typeToFunc(fn_ty).?;
4399	const cc = fn_info.cc;	5145	const cc = fn_info.cc;
4400	var result: CallMCValues = .{	5146	var result: CallMCValues = .{
4401	.args = try self.gpa.alloc(MCValue, fn_info.param_types.len),	5147	.args = try self.gpa.alloc(MCValue, param_types.len),
4402	// These undefined values must be populated before returning from this function.	5148	// These undefined values must be populated before returning from this function.
4403	.return_value = undefined,	5149	.return_value = undefined,
4404	.stack_byte_count = undefined,	5150	.stack_byte_count = 0,
4405	.stack_align = undefined,	5151	.stack_align = undefined,
4406	};	5152	};
4407	errdefer self.gpa.free(result.args);	5153	errdefer self.gpa.free(result.args);
4408		5154
4409	const ret_ty = fn_ty.fnReturnType(mod);	5155	const ret_ty = Type.fromInterned(fn_info.return_type);
4410		5156
4411	switch (cc) {	5157	switch (cc) {
4412	.Naked => {	5158	.Naked => {
4413	assert(result.args.len == 0);	5159	assert(result.args.len == 0);
4414	result.return_value = .{ .unreach = {} };	5160	result.return_value = InstTracking.init(.unreach);
4415	result.stack_byte_count = 0;	5161	result.stack_align = .@"8";
4416	result.stack_align = .@"1";
4417	return result;
4418	},	5162	},
4419	.Unspecified, .C => {	5163	.C, .Unspecified => {
4420	if (result.args.len > 8) {	5164	if (result.args.len > 8) {
4421	return self.fail("TODO: support more than 8 function args", .{});	5165	return self.fail("RISC-V calling convention does not support more than 8 arguments", .{});
4422	}	5166	}
4423		5167
4424	var fa_reg_i: u32 = 0;	5168	var ret_int_reg_i: u32 = 0;
		5169	var param_int_reg_i: u32 = 0;
4425		5170
4426	// spill the needed argument registers	5171	result.stack_align = .@"16";
4427	for (fn_info.param_types.get(ip), result.args) \|ty, *result_arg\| {
4428	const param_ty = Type.fromInterned(ty);
4429	const param_size = param_ty.abiSize(mod);
4430		5172
4431	switch (param_size) {	5173	// Return values
4432	1...8 => {	5174	if (ret_ty.zigTypeTag(zcu) == .NoReturn) {
4433	const arg_reg: Register = abi.function_arg_regs[fa_reg_i];	5175	result.return_value = InstTracking.init(.unreach);
4434	fa_reg_i += 1;	5176	} else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4435	try self.register_manager.getReg(arg_reg, null);	5177	result.return_value = InstTracking.init(.none);
4436	result_arg.* = .{ .register = arg_reg };	5178	} else {
		5179	var ret_tracking: [2]InstTracking = undefined;
		5180	var ret_tracking_i: usize = 0;
		5181
		5182	const classes = mem.sliceTo(&abi.classifySystem(ret_ty, zcu), .none);
		5183
		5184	for (classes) \|class\| switch (class) {
		5185	.integer => {
		5186	const ret_int_reg = abi.function_arg_regs[ret_int_reg_i];
		5187	ret_int_reg_i += 1;
		5188
		5189	ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });
		5190	ret_tracking_i += 1;
4437	},	5191	},
4438	9...16 => {	5192	else => return self.fail("TODO: C calling convention return class {}", .{class}),
4439	const arg_regs: [2]Register = abi.function_arg_regs[fa_reg_i..][0..2].*;	5193	};
4440	fa_reg_i += 2;	5194
4441	for (arg_regs) \|reg\| try self.register_manager.getReg(reg, null);	5195	result.return_value = switch (ret_tracking_i) {
4442	result_arg.* = .{ .register_pair = arg_regs };	5196	else => return self.fail("ty {} took {} tracking return indices", .{ ret_ty.fmt(zcu), ret_tracking_i }),
		5197	1 => ret_tracking[0],
		5198	2 => InstTracking.init(.{ .register_pair = .{
		5199	ret_tracking[0].short.register, ret_tracking[1].short.register,
		5200	} }),
		5201	};
		5202	}
		5203
		5204	for (param_types, result.args) \|ty, *arg\| {
		5205	assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
		5206
		5207	var arg_mcv: [2]MCValue = undefined;
		5208	var arg_mcv_i: usize = 0;
		5209
		5210	const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none);
		5211
		5212	for (classes) \|class\| switch (class) {
		5213	.integer => {
		5214	const param_int_regs = abi.function_arg_regs;
		5215	if (param_int_reg_i >= param_int_regs.len) break;
		5216
		5217	const param_int_reg = param_int_regs[param_int_reg_i];
		5218	param_int_reg_i += 1;
		5219
		5220	arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };
		5221	arg_mcv_i += 1;
4443	},	5222	},
4444	else => return self.fail("TODO: support args of size {}", .{param_size}),	5223	else => return self.fail("TODO: C calling convention arg class {}", .{class}),
		5224	} else {
		5225	arg.* = switch (arg_mcv_i) {
		5226	else => return self.fail("ty {} took {} tracking arg indices", .{ ty.fmt(zcu), arg_mcv_i }),
		5227	1 => arg_mcv[0],
		5228	2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } },
		5229	};
		5230	continue;
4445	}	5231	}
4446	}
4447		5232
4448	result.stack_byte_count = self.max_end_stack;	5233	return self.fail("TODO: pass args by stack", .{});
4449	result.stack_align = .@"16";	5234	}
4450	},	5235	},
4451	else => return self.fail("TODO implement function parameters for {} on riscv64", .{cc}),	5236	else => return self.fail("TODO implement function parameters for {} on riscv64", .{cc}),
4452	}	5237	}
4453		5238
4454	if (ret_ty.zigTypeTag(mod) == .NoReturn) {	5239	result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));
4455	result.return_value = .{ .unreach = {} };
4456	} else if (!ret_ty.hasRuntimeBits(mod)) {
4457	result.return_value = .{ .none = {} };
4458	} else switch (cc) {
4459	.Naked => unreachable,
4460	.Unspecified, .C => {
4461	const ret_ty_size: u32 = @intCast(ret_ty.abiSize(mod));
4462	if (ret_ty_size <= 8) {
4463	result.return_value = .{ .register = .a0 };
4464	} else if (ret_ty_size <= 16) {
4465	return self.fail("TODO support returning with a0 + a1", .{});
4466	} else {
4467	return self.fail("TODO support return by reference", .{});
4468	}
4469	},
4470	else => return self.fail("TODO implement function return values for {}", .{cc}),
4471	}
4472	return result;	5240	return result;
4473	}	5241	}
4474		5242
...	@@ -4504,36 +5272,36 @@ fn parseRegName(name: []const u8) ?Register {	...	@@ -4504,36 +5272,36 @@ fn parseRegName(name: []const u8) ?Register {
4504	}	5272	}
4505		5273
4506	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {	5274	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
4507	const mod = self.bin_file.comp.module.?;	5275	const zcu = self.bin_file.comp.module.?;
4508	return self.air.typeOf(inst, &mod.intern_pool);	5276	return self.air.typeOf(inst, &zcu.intern_pool);
4509	}	5277	}
4510		5278
4511	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {	5279	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
4512	const mod = self.bin_file.comp.module.?;	5280	const zcu = self.bin_file.comp.module.?;
4513	return self.air.typeOfIndex(inst, &mod.intern_pool);	5281	return self.air.typeOfIndex(inst, &zcu.intern_pool);
4514	}	5282	}
4515		5283
4516	fn hasFeature(self: *Self, feature: Target.riscv.Feature) bool {	5284	fn hasFeature(self: *Self, feature: Target.riscv.Feature) bool {
4517	return Target.riscv.featureSetHas(self.target.cpu.features, feature);	5285	return Target.riscv.featureSetHas(self.target.cpu.features, feature);
4518	}	5286	}
4519		5287
4520	pub fn errUnionPayloadOffset(payload_ty: Type, mod: *Module) u64 {	5288	pub fn errUnionPayloadOffset(payload_ty: Type, zcu: *Module) u64 {
4521	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) return 0;	5289	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0;
4522	const payload_align = payload_ty.abiAlignment(mod);	5290	const payload_align = payload_ty.abiAlignment(zcu);
4523	const error_align = Type.anyerror.abiAlignment(mod);	5291	const error_align = Type.anyerror.abiAlignment(zcu);
4524	if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {	5292	if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4525	return 0;	5293	return 0;
4526	} else {	5294	} else {
4527	return payload_align.forward(Type.anyerror.abiSize(mod));	5295	return payload_align.forward(Type.anyerror.abiSize(zcu));
4528	}	5296	}
4529	}	5297	}
4530		5298
4531	pub fn errUnionErrorOffset(payload_ty: Type, mod: *Module) u64 {	5299	pub fn errUnionErrorOffset(payload_ty: Type, zcu: *Module) u64 {
4532	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) return 0;	5300	if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0;
4533	const payload_align = payload_ty.abiAlignment(mod);	5301	const payload_align = payload_ty.abiAlignment(zcu);
4534	const error_align = Type.anyerror.abiAlignment(mod);	5302	const error_align = Type.anyerror.abiAlignment(zcu);
4535	if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {	5303	if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4536	return error_align.forward(payload_ty.abiSize(mod));	5304	return error_align.forward(payload_ty.abiSize(zcu));
4537	} else {	5305	} else {
4538	return 0;	5306	return 0;
4539	}	5307	}

src/arch/riscv64/Emit.zig+118-575

...	@@ -1,620 +1,163 @@	...	@@ -1,620 +1,163 @@
1	//! This file contains the functionality for lowering RISCV64 MIR into	1	//! This file contains the functionality for emitting RISC-V MIR as machine code
2	//! machine code
3		2
4	mir: Mir,	3	lower: Lower,
5	bin_file: *link.File,
6	debug_output: DebugInfoOutput,	4	debug_output: DebugInfoOutput,
7	output_mode: std.builtin.OutputMode,
8	link_mode: std.builtin.LinkMode,
9	target: *const std.Target,
10	err_msg: ?*ErrorMsg = null,
11	src_loc: Module.SrcLoc,
12	code: *std.ArrayList(u8),	5	code: *std.ArrayList(u8),
13		6
14	/// List of registers to save in the prologue.
15	save_reg_list: Mir.RegisterList,
16
17	prev_di_line: u32,	7	prev_di_line: u32,
18	prev_di_column: u32,	8	prev_di_column: u32,
19	/// Relative to the beginning of `code`.	9	/// Relative to the beginning of `code`.
20	prev_di_pc: usize,	10	prev_di_pc: usize,
21		11
22	/// Function's stack size. Used for backpatching.
23	stack_size: u32,
24
25	/// For backward branches: stores the code offset of the target
26	/// instruction
27	///
28	/// For forward branches: stores the code offset of the branch
29	/// instruction
30	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},	12	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
		13	relocs: std.ArrayListUnmanaged(Reloc) = .{},
31		14
32	const log = std.log.scoped(.emit);	15	pub const Error = Lower.Error \|\| error{
33
34	const InnerError = error{
35	OutOfMemory,
36	EmitFail,	16	EmitFail,
37	};	17	};
38		18
39	pub fn emitMir(	19	pub fn emitMir(emit: *Emit) Error!void {
40	emit: *Emit,	20	log.debug("mir instruction len: {}", .{emit.lower.mir.instructions.len});
41	) InnerError!void {	21	for (0..emit.lower.mir.instructions.len) \|mir_i\| {
42	const mir_tags = emit.mir.instructions.items(.tag);	22	const mir_index: Mir.Inst.Index = @intCast(mir_i);
43		23	try emit.code_offset_mapping.putNoClobber(
44	try emit.lowerMir();	24	emit.lower.allocator,
45		25	mir_index,
46	for (mir_tags, 0..) \|tag, index\| {	26	@intCast(emit.code.items.len),
47	const inst = @as(u32, @intCast(index));	27	);
48	log.debug("emitMir: {s}", .{@tagName(tag)});	28	const lowered = try emit.lower.lowerMir(mir_index);
49	switch (tag) {	29	var lowered_relocs = lowered.relocs;
50	.add => try emit.mirRType(inst),	30	for (lowered.insts, 0..) \|lowered_inst, lowered_index\| {
51	.sub => try emit.mirRType(inst),	31	const start_offset: u32 = @intCast(emit.code.items.len);
52	.mul => try emit.mirRType(inst),	32	try lowered_inst.encode(emit.code.writer());
53	.@"or" => try emit.mirRType(inst),	33
54		34	while (lowered_relocs.len > 0 and
55	.cmp_eq => try emit.mirRType(inst),	35	lowered_relocs[0].lowered_inst_index == lowered_index) : ({
56	.cmp_neq => try emit.mirRType(inst),	36	lowered_relocs = lowered_relocs[1..];
57	.cmp_gt => try emit.mirRType(inst),	37	}) switch (lowered_relocs[0].target) {
58	.cmp_gte => try emit.mirRType(inst),	38	.inst => \|target\| try emit.relocs.append(emit.lower.allocator, .{
59	.cmp_lt => try emit.mirRType(inst),	39	.source = start_offset,
60	.cmp_imm_gte => try emit.mirRType(inst),	40	.target = target,
61	.cmp_imm_eq => try emit.mirIType(inst),	41	.offset = 0,
62	.cmp_imm_neq => try emit.mirIType(inst),	42	.enc = std.meta.activeTag(lowered_inst.encoding.data),
63	.cmp_imm_lte => try emit.mirIType(inst),	43	}),
64	.cmp_imm_lt => try emit.mirIType(inst),	44	else => \|x\| return emit.fail("TODO: emitMir {s}", .{@tagName(x)}),
65		45	};
66	.beq => try emit.mirBType(inst),	46	}
67	.bne => try emit.mirBType(inst),	47	std.debug.assert(lowered_relocs.len == 0);
68		48
69	.addi => try emit.mirIType(inst),	49	if (lowered.insts.len == 0) {
70	.addiw => try emit.mirIType(inst),	50	const mir_inst = emit.lower.mir.instructions.get(mir_index);
71	.andi => try emit.mirIType(inst),	51	switch (mir_inst.tag) {
72	.jalr => try emit.mirIType(inst),	52	else => unreachable,
73	.abs => try emit.mirIType(inst),	53	.pseudo => switch (mir_inst.ops) {
74		54	else => unreachable,
75	.jal => try emit.mirJType(inst),	55	.pseudo_dbg_prologue_end => {
76		56	switch (emit.debug_output) {
77	.ebreak => try emit.mirSystem(inst),	57	.dwarf => \|dw\| {
78	.ecall => try emit.mirSystem(inst),	58	try dw.setPrologueEnd();
79	.unimp => try emit.mirSystem(inst),	59	log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
80		60	emit.prev_di_line, emit.prev_di_column,
81	.dbg_line => try emit.mirDbgLine(inst),	61	});
82	.dbg_prologue_end => try emit.mirDebugPrologueEnd(),	62	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
83	.dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(),	63	},
84		64	.plan9 => {},
85	.psuedo_prologue => try emit.mirPsuedo(inst),	65	.none => {},
86	.psuedo_epilogue => try emit.mirPsuedo(inst),	66	}
87		67	},
88	.j => try emit.mirPsuedo(inst),	68	.pseudo_dbg_line_column => try emit.dbgAdvancePCAndLine(
89		69	mir_inst.data.pseudo_dbg_line_column.line,
90	.mv => try emit.mirRR(inst),	70	mir_inst.data.pseudo_dbg_line_column.column,
91	.not => try emit.mirRR(inst),	71	),
92		72	.pseudo_dbg_epilogue_begin => {
93	.nop => try emit.mirNop(inst),	73	switch (emit.debug_output) {
94	.ret => try emit.mirNop(inst),	74	.dwarf => \|dw\| {
95		75	try dw.setEpilogueBegin();
96	.lui => try emit.mirUType(inst),	76	log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
97		77	emit.prev_di_line, emit.prev_di_column,
98	.ld => try emit.mirIType(inst),	78	});
99	.lw => try emit.mirIType(inst),	79	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
100	.lh => try emit.mirIType(inst),	80	},
101	.lb => try emit.mirIType(inst),	81	.plan9 => {},
102		82	.none => {},
103	.sd => try emit.mirIType(inst),	83	}
104	.sw => try emit.mirIType(inst),	84	},
105	.sh => try emit.mirIType(inst),	85	.pseudo_dead => {},
106	.sb => try emit.mirIType(inst),	86	},
107		87	}
108	.srlw => try emit.mirRType(inst),
109	.sllw => try emit.mirRType(inst),
110
111	.srli => try emit.mirIType(inst),
112	.slli => try emit.mirIType(inst),
113
114	.ldr_ptr_stack => try emit.mirIType(inst),
115
116	.load_symbol => try emit.mirLoadSymbol(inst),
117	}	88	}
118	}	89	}
		90	try emit.fixupRelocs();
119	}	91	}
120		92
121	pub fn deinit(emit: *Emit) void {	93	pub fn deinit(emit: *Emit) void {
122	const comp = emit.bin_file.comp;	94	emit.relocs.deinit(emit.lower.allocator);
123	const gpa = comp.gpa;	95	emit.code_offset_mapping.deinit(emit.lower.allocator);
124
125	emit.code_offset_mapping.deinit(gpa);
126	emit.* = undefined;	96	emit.* = undefined;
127	}	97	}
128		98
129	fn writeInstruction(emit: *Emit, instruction: Instruction) !void {	99	const Reloc = struct {
130	const endian = emit.target.cpu.arch.endian();	100	/// Offset of the instruction.
131	std.mem.writeInt(u32, try emit.code.addManyAsArray(4), instruction.toU32(), endian);	101	source: usize,
132	}	102	/// Target of the relocation.
		103	target: Mir.Inst.Index,
		104	/// Offset of the relocation within the instruction.
		105	offset: u32,
		106	/// Encoding of the instruction, used to determine how to modify it.
		107	enc: Encoding.InstEnc,
		108	};
		109
		110	fn fixupRelocs(emit: *Emit) Error!void {
		111	for (emit.relocs.items) \|reloc\| {
		112	log.debug("target inst: {}", .{emit.lower.mir.instructions.get(reloc.target)});
		113	const target = emit.code_offset_mapping.get(reloc.target) orelse
		114	return emit.fail("relocation target not found!", .{});
133		115
134	fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {	116	const disp = @as(i32, @intCast(target)) - @as(i32, @intCast(reloc.source));
135	@setCold(true);	117	const code: *[4]u8 = emit.code.items[reloc.source + reloc.offset ..][0..4];
136	assert(emit.err_msg == null);	118
137	const comp = emit.bin_file.comp;	119	log.debug("disp: {x}", .{disp});
138	const gpa = comp.gpa;	120
139	emit.err_msg = try ErrorMsg.create(gpa, emit.src_loc, format, args);	121	switch (reloc.enc) {
140	return error.EmitFail;	122	.J => riscv_util.writeInstJ(code, @bitCast(disp)),
		123	else => return emit.fail("tried to reloc encoding type {s}", .{@tagName(reloc.enc)}),
		124	}
		125	}
141	}	126	}
142		127
143	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) !void {	128	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) Error!void {
144	const delta_line = @as(i32, @intCast(line)) - @as(i32, @intCast(emit.prev_di_line));	129	const delta_line = @as(i33, line) - @as(i33, emit.prev_di_line);
145	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;	130	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
		131	log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line });
146	switch (emit.debug_output) {	132	switch (emit.debug_output) {
147	.dwarf => \|dw\| {	133	.dwarf => \|dw\| {
148	if (column != emit.prev_di_column) try dw.setColumn(column);	134	if (column != emit.prev_di_column) try dw.setColumn(column);
149	if (delta_line == 0) return; // TODO: remove this	135	if (delta_line == 0) return; // TODO: fix these edge cases.
150	try dw.advancePCAndLine(delta_line, delta_pc);	136	try dw.advancePCAndLine(delta_line, delta_pc);
151	emit.prev_di_line = line;	137	emit.prev_di_line = line;
152	emit.prev_di_column = column;	138	emit.prev_di_column = column;
153	emit.prev_di_pc = emit.code.items.len;	139	emit.prev_di_pc = emit.code.items.len;
154	},	140	},
155	.plan9 => \|dbg_out\| {
156	if (delta_pc <= 0) return; // only do this when the pc changes
157
158	// increasing the line number
159	try link.File.Plan9.changeLine(&dbg_out.dbg_line, delta_line);
160	// increasing the pc
161	const d_pc_p9 = @as(i64, @intCast(delta_pc)) - dbg_out.pc_quanta;
162	if (d_pc_p9 > 0) {
163	// minus one because if its the last one, we want to leave space to change the line which is one pc quanta
164	try dbg_out.dbg_line.append(@as(u8, @intCast(@divExact(d_pc_p9, dbg_out.pc_quanta) + 128)) - dbg_out.pc_quanta);
165	if (dbg_out.pcop_change_index) \|pci\|
166	dbg_out.dbg_line.items[pci] += 1;
167	dbg_out.pcop_change_index = @as(u32, @intCast(dbg_out.dbg_line.items.len - 1));
168	} else if (d_pc_p9 == 0) {
169	// we don't need to do anything, because adding the pc quanta does it for us
170	} else unreachable;
171	if (dbg_out.start_line == null)
172	dbg_out.start_line = emit.prev_di_line;
173	dbg_out.end_line = line;
174	// only do this if the pc changed
175	emit.prev_di_line = line;
176	emit.prev_di_column = column;
177	emit.prev_di_pc = emit.code.items.len;
178	},
179	.none => {},
180	}
181	}
182
183	fn mirRType(emit: *Emit, inst: Mir.Inst.Index) !void {
184	const tag = emit.mir.instructions.items(.tag)[inst];
185	const r_type = emit.mir.instructions.items(.data)[inst].r_type;
186
187	const rd = r_type.rd;
188	const rs1 = r_type.rs1;
189	const rs2 = r_type.rs2;
190
191	switch (tag) {
192	.add => try emit.writeInstruction(Instruction.add(rd, rs1, rs2)),
193	.sub => try emit.writeInstruction(Instruction.sub(rd, rs1, rs2)),
194	.mul => try emit.writeInstruction(Instruction.mul(rd, rs1, rs2)),
195	.cmp_gt => {
196	// rs1 > rs2
197	try emit.writeInstruction(Instruction.sltu(rd, rs2, rs1));
198	},
199	.cmp_gte => {
200	// rs1 >= rs2
201	try emit.writeInstruction(Instruction.sltu(rd, rs1, rs2));
202	try emit.writeInstruction(Instruction.xori(rd, rd, 1));
203	},
204	.cmp_eq => {
205	// rs1 == rs2
206
207	try emit.writeInstruction(Instruction.xor(rd, rs1, rs2));
208	try emit.writeInstruction(Instruction.sltiu(rd, rd, 1)); // seqz
209	},
210	.cmp_neq => {
211	// rs1 != rs2
212
213	try emit.writeInstruction(Instruction.xor(rd, rs1, rs2));
214	try emit.writeInstruction(Instruction.sltu(rd, .zero, rd)); // snez
215	},
216	.cmp_lt => {
217	// rd = 1 if rs1 < rs2
218	try emit.writeInstruction(Instruction.slt(rd, rs1, rs2));
219	},
220	.sllw => try emit.writeInstruction(Instruction.sllw(rd, rs1, rs2)),
221	.srlw => try emit.writeInstruction(Instruction.srlw(rd, rs1, rs2)),
222	.@"or" => try emit.writeInstruction(Instruction.@"or"(rd, rs1, rs2)),
223	.cmp_imm_gte => {
224	// rd = 1 if rs1 >= imm12
225	// see the docstring of cmp_imm_gte to see why we use r_type here
226
227	// (rs1 >= imm12) == !(imm12 > rs1)
228	try emit.writeInstruction(Instruction.sltu(rd, rs1, rs2));
229	},
230	else => unreachable,
231	}
232	}
233
234	fn mirBType(emit: *Emit, inst: Mir.Inst.Index) !void {
235	const tag = emit.mir.instructions.items(.tag)[inst];
236	const b_type = emit.mir.instructions.items(.data)[inst].b_type;
237
238	const offset = @as(i64, @intCast(emit.code_offset_mapping.get(b_type.inst).?)) - @as(i64, @intCast(emit.code.items.len));
239
240	switch (tag) {
241	.beq => {
242	log.debug("beq: {} offset={}", .{ inst, offset });
243	try emit.writeInstruction(Instruction.beq(b_type.rs1, b_type.rs2, @intCast(offset)));
244	},
245	.bne => {
246	log.debug("bne: {} offset={}", .{ inst, offset });
247	try emit.writeInstruction(Instruction.bne(b_type.rs1, b_type.rs2, @intCast(offset)));
248	},
249	else => unreachable,
250	}
251	}
252
253	fn mirIType(emit: *Emit, inst: Mir.Inst.Index) !void {
254	const tag = emit.mir.instructions.items(.tag)[inst];
255	const i_type = emit.mir.instructions.items(.data)[inst].i_type;
256
257	const rd = i_type.rd;
258	const rs1 = i_type.rs1;
259	const imm12 = i_type.imm12;
260
261	switch (tag) {
262	.addi => try emit.writeInstruction(Instruction.addi(rd, rs1, imm12)),
263	.addiw => try emit.writeInstruction(Instruction.addiw(rd, rs1, imm12)),
264	.jalr => try emit.writeInstruction(Instruction.jalr(rd, imm12, rs1)),
265
266	.andi => try emit.writeInstruction(Instruction.andi(rd, rs1, imm12)),
267
268	.ld => try emit.writeInstruction(Instruction.ld(rd, imm12, rs1)),
269	.lw => try emit.writeInstruction(Instruction.lw(rd, imm12, rs1)),
270	.lh => try emit.writeInstruction(Instruction.lh(rd, imm12, rs1)),
271	.lb => try emit.writeInstruction(Instruction.lb(rd, imm12, rs1)),
272
273	.sd => try emit.writeInstruction(Instruction.sd(rd, imm12, rs1)),
274	.sw => try emit.writeInstruction(Instruction.sw(rd, imm12, rs1)),
275	.sh => try emit.writeInstruction(Instruction.sh(rd, imm12, rs1)),
276	.sb => try emit.writeInstruction(Instruction.sb(rd, imm12, rs1)),
277
278	.ldr_ptr_stack => try emit.writeInstruction(Instruction.add(rd, rs1, .sp)),
279
280	.abs => {
281	try emit.writeInstruction(Instruction.sraiw(rd, rs1, @intCast(imm12)));
282	try emit.writeInstruction(Instruction.xor(rs1, rs1, rd));
283	try emit.writeInstruction(Instruction.subw(rs1, rs1, rd));
284	},
285
286	.srli => try emit.writeInstruction(Instruction.srli(rd, rs1, @intCast(imm12))),
287	.slli => try emit.writeInstruction(Instruction.slli(rd, rs1, @intCast(imm12))),
288
289	.cmp_imm_eq => {
290	try emit.writeInstruction(Instruction.xori(rd, rs1, imm12));
291	try emit.writeInstruction(Instruction.sltiu(rd, rd, 1));
292	},
293	.cmp_imm_neq => {
294	try emit.writeInstruction(Instruction.xori(rd, rs1, imm12));
295	try emit.writeInstruction(Instruction.sltu(rd, .x0, rd));
296	},
297
298	.cmp_imm_lt => {
299	try emit.writeInstruction(Instruction.slti(rd, rs1, imm12));
300	},
301
302	.cmp_imm_lte => {
303	try emit.writeInstruction(Instruction.sltiu(rd, rs1, @bitCast(imm12)));
304	},
305
306	else => unreachable,
307	}
308	}
309
310	fn mirJType(emit: *Emit, inst: Mir.Inst.Index) !void {
311	const tag = emit.mir.instructions.items(.tag)[inst];
312	const j_type = emit.mir.instructions.items(.data)[inst].j_type;
313
314	const offset = @as(i64, @intCast(emit.code_offset_mapping.get(j_type.inst).?)) - @as(i64, @intCast(emit.code.items.len));
315
316	switch (tag) {
317	.jal => {
318	log.debug("jal: {} offset={}", .{ inst, offset });
319	try emit.writeInstruction(Instruction.jal(j_type.rd, @intCast(offset)));
320	},
321	else => unreachable,
322	}
323	}
324
325	fn mirSystem(emit: *Emit, inst: Mir.Inst.Index) !void {
326	const tag = emit.mir.instructions.items(.tag)[inst];
327
328	switch (tag) {
329	.ebreak => try emit.writeInstruction(Instruction.ebreak),
330	.ecall => try emit.writeInstruction(Instruction.ecall),
331	.unimp => try emit.writeInstruction(Instruction.unimp),
332	else => unreachable,
333	}
334	}
335
336	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void {
337	const tag = emit.mir.instructions.items(.tag)[inst];
338	const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column;
339
340	switch (tag) {
341	.dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column),
342	else => unreachable,
343	}
344	}
345
346	fn mirDebugPrologueEnd(emit: *Emit) !void {
347	switch (emit.debug_output) {
348	.dwarf => \|dw\| {
349	try dw.setPrologueEnd();
350	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
351	},
352	.plan9 => {},
353	.none => {},
354	}
355	}
356
357	fn mirDebugEpilogueBegin(emit: *Emit) !void {
358	switch (emit.debug_output) {
359	.dwarf => \|dw\| {
360	try dw.setEpilogueBegin();
361	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
362	},
363	.plan9 => {},	141	.plan9 => {},
364	.none => {},	142	.none => {},
365	}	143	}
366	}	144	}
367		145
368	fn mirPsuedo(emit: *Emit, inst: Mir.Inst.Index) !void {	146	fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error {
369	const tag = emit.mir.instructions.items(.tag)[inst];	147	return switch (emit.lower.fail(format, args)) {
370	const data = emit.mir.instructions.items(.data)[inst];	148	error.LowerFail => error.EmitFail,
371		149	else => \|e\| e,
372	switch (tag) {
373	.psuedo_prologue => {
374	const stack_size: i12 = math.cast(i12, emit.stack_size) orelse {
375	return emit.fail("TODO: mirPsuedo support larger stack sizes", .{});
376	};
377
378	// Decrement sp by (num s registers * 8) + local var space
379	try emit.writeInstruction(Instruction.addi(.sp, .sp, -stack_size));
380
381	// Spill ra
382	try emit.writeInstruction(Instruction.sd(.ra, 0, .sp));
383
384	// Spill callee saved registers.
385	var s_reg_iter = emit.save_reg_list.iterator(.{});
386	var i: i12 = 8;
387	while (s_reg_iter.next()) \|reg_i\| {
388	const reg = abi.callee_preserved_regs[reg_i];
389	try emit.writeInstruction(Instruction.sd(reg, i, .sp));
390	i += 8;
391	}
392	},
393	.psuedo_epilogue => {
394	const stack_size: i12 = math.cast(i12, emit.stack_size) orelse {
395	return emit.fail("TODO: mirPsuedo support larger stack sizes", .{});
396	};
397
398	// Restore ra
399	try emit.writeInstruction(Instruction.ld(.ra, 0, .sp));
400
401	// Restore spilled callee saved registers
402	var s_reg_iter = emit.save_reg_list.iterator(.{});
403	var i: i12 = 8;
404	while (s_reg_iter.next()) \|reg_i\| {
405	const reg = abi.callee_preserved_regs[reg_i];
406	try emit.writeInstruction(Instruction.ld(reg, i, .sp));
407	i += 8;
408	}
409
410	// Increment sp back to previous value
411	try emit.writeInstruction(Instruction.addi(.sp, .sp, stack_size));
412	},
413
414	.j => {
415	const offset = @as(i64, @intCast(emit.code_offset_mapping.get(data.inst).?)) - @as(i64, @intCast(emit.code.items.len));
416	try emit.writeInstruction(Instruction.jal(.zero, @intCast(offset)));
417	},
418
419	else => unreachable,
420	}
421	}
422
423	fn mirRR(emit: *Emit, inst: Mir.Inst.Index) !void {
424	const tag = emit.mir.instructions.items(.tag)[inst];
425	const rr = emit.mir.instructions.items(.data)[inst].rr;
426
427	const rd = rr.rd;
428	const rs = rr.rs;
429
430	switch (tag) {
431	.mv => try emit.writeInstruction(Instruction.addi(rd, rs, 0)),
432	.not => try emit.writeInstruction(Instruction.xori(rd, rs, 1)),
433	else => unreachable,
434	}
435	}
436
437	fn mirUType(emit: *Emit, inst: Mir.Inst.Index) !void {
438	const tag = emit.mir.instructions.items(.tag)[inst];
439	const u_type = emit.mir.instructions.items(.data)[inst].u_type;
440
441	switch (tag) {
442	.lui => try emit.writeInstruction(Instruction.lui(u_type.rd, u_type.imm20)),
443	else => unreachable,
444	}
445	}
446
447	fn mirNop(emit: *Emit, inst: Mir.Inst.Index) !void {
448	const tag = emit.mir.instructions.items(.tag)[inst];
449
450	switch (tag) {
451	.nop => try emit.writeInstruction(Instruction.addi(.zero, .zero, 0)),
452	.ret => try emit.writeInstruction(Instruction.jalr(.zero, 0, .ra)),
453	else => unreachable,
454	}
455	}
456
457	fn mirLoadSymbol(emit: *Emit, inst: Mir.Inst.Index) !void {
458	const payload = emit.mir.instructions.items(.data)[inst].payload;
459	const data = emit.mir.extraData(Mir.LoadSymbolPayload, payload).data;
460	const reg = @as(Register, @enumFromInt(data.register));
461
462	const start_offset = @as(u32, @intCast(emit.code.items.len));
463	try emit.writeInstruction(Instruction.lui(reg, 0));
464	try emit.writeInstruction(Instruction.addi(reg, reg, 0));
465
466	switch (emit.bin_file.tag) {
467	.elf => {
468	const elf_file = emit.bin_file.cast(link.File.Elf).?;
469	const atom_ptr = elf_file.symbol(data.atom_index).atom(elf_file).?;
470	const sym_index = elf_file.zigObjectPtr().?.symbol(data.sym_index);
471	const sym = elf_file.symbol(sym_index);
472
473	var hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20);
474	var lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I);
475
476	if (sym.flags.needs_zig_got) {
477	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
478
479	hi_r_type = Elf.R_ZIG_GOT_HI20;
480	lo_r_type = Elf.R_ZIG_GOT_LO12;
481	}
482
483	try atom_ptr.addReloc(elf_file, .{
484	.r_offset = start_offset,
485	.r_info = (@as(u64, @intCast(data.sym_index)) << 32) \| hi_r_type,
486	.r_addend = 0,
487	});
488
489	try atom_ptr.addReloc(elf_file, .{
490	.r_offset = start_offset + 4,
491	.r_info = (@as(u64, @intCast(data.sym_index)) << 32) \| lo_r_type,
492	.r_addend = 0,
493	});
494	},
495	else => unreachable,
496	}
497	}
498
499	fn isStore(tag: Mir.Inst.Tag) bool {
500	return switch (tag) {
501	.sb => true,
502	.sh => true,
503	.sw => true,
504	.sd => true,
505	.addi => true, // needed for ptr_stack_offset stores
506	else => false,
507	};
508	}
509
510	fn isLoad(tag: Mir.Inst.Tag) bool {
511	return switch (tag) {
512	.lb => true,
513	.lh => true,
514	.lw => true,
515	.ld => true,
516	else => false,
517	};
518	}
519
520	pub fn isBranch(tag: Mir.Inst.Tag) bool {
521	return switch (tag) {
522	.beq => true,
523	.bne => true,
524	.jal => true,
525	.j => true,
526	else => false,
527	};	150	};
528	}	151	}
529		152
530	pub fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index {	153	const link = @import("../../link.zig");
531	const tag = emit.mir.instructions.items(.tag)[inst];	154	const log = std.log.scoped(.emit);
532	const data = emit.mir.instructions.items(.data)[inst];	155	const mem = std.mem;
533		156	const std = @import("std");
534	switch (tag) {
535	.bne,
536	.beq,
537	=> return data.b_type.inst,
538	.jal => return data.j_type.inst,
539	.j => return data.inst,
540	else => std.debug.panic("branchTarget {s}", .{@tagName(tag)}),
541	}
542	}
543
544	fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
545	const tag = emit.mir.instructions.items(.tag)[inst];
546
547	return switch (tag) {
548	.dbg_line,
549	.dbg_epilogue_begin,
550	.dbg_prologue_end,
551	=> 0,
552
553	.cmp_eq,
554	.cmp_neq,
555	.cmp_imm_eq,
556	.cmp_imm_neq,
557	.cmp_gte,
558	.load_symbol,
559	.abs,
560	=> 8,
561
562	.psuedo_epilogue, .psuedo_prologue => size: {
563	const count = emit.save_reg_list.count() * 4;
564	break :size count + 8;
565	},
566
567	else => 4,
568	};
569	}
570
571	fn lowerMir(emit: *Emit) !void {
572	const comp = emit.bin_file.comp;
573	const gpa = comp.gpa;
574	const mir_tags = emit.mir.instructions.items(.tag);
575	const mir_datas = emit.mir.instructions.items(.data);
576
577	const proglogue_size: u32 = @intCast(emit.save_reg_list.size());
578	emit.stack_size += proglogue_size;
579
580	for (mir_tags, 0..) \|tag, index\| {
581	const inst: u32 = @intCast(index);
582
583	if (isStore(tag) or isLoad(tag)) {
584	const data = mir_datas[inst].i_type;
585	if (data.rs1 == .sp) {
586	const offset = mir_datas[inst].i_type.imm12;
587	mir_datas[inst].i_type.imm12 = offset + @as(i12, @intCast(proglogue_size)) + 8;
588	}
589	}
590
591	if (isBranch(tag)) {
592	const target_inst = emit.branchTarget(inst);
593	try emit.code_offset_mapping.put(gpa, target_inst, 0);
594	}
595	}
596	var current_code_offset: usize = 0;
597
598	for (0..mir_tags.len) \|index\| {
599	const inst = @as(u32, @intCast(index));
600	if (emit.code_offset_mapping.getPtr(inst)) \|offset\| {
601	offset.* = current_code_offset;
602	}
603	current_code_offset += emit.instructionSize(inst);
604	}
605	}
606		157
		158	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
607	const Emit = @This();	159	const Emit = @This();
608	const std = @import("std");	160	const Lower = @import("Lower.zig");
609	const math = std.math;
610	const Mir = @import("Mir.zig");	161	const Mir = @import("Mir.zig");
611	const bits = @import("bits.zig");	162	const riscv_util = @import("../../link/riscv.zig");
612	const abi = @import("abi.zig");	163	const Encoding = @import("Encoding.zig");
613	const link = @import("../../link.zig");
614	const Module = @import("../../Module.zig");
615	const Elf = @import("../../link/Elf.zig");
616	const ErrorMsg = Module.ErrorMsg;
617	const assert = std.debug.assert;
618	const Instruction = bits.Instruction;
619	const Register = bits.Register;
620	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;

src/arch/riscv64/Encoding.zig created+333

...	@@ -0,0 +1,333 @@
1	mnemonic: Mnemonic,
2	data: Data,
3
4	pub const Mnemonic = enum {
5	// R Type
6	add,
7
8	// I Type
9	ld,
10	lw,
11	lwu,
12	lh,
13	lhu,
14	lb,
15	lbu,
16
17	addi,
18	jalr,
19
20	// U Type
21	lui,
22
23	// S Type
24	sd,
25	sw,
26	sh,
27	sb,
28
29	// J Type
30	jal,
31
32	// System
33	ecall,
34	ebreak,
35	unimp,
36
37	pub fn encoding(mnem: Mnemonic) Enc {
38	return switch (mnem) {
39	// zig fmt: off
40	.add => .{ .opcode = 0b0110011, .funct3 = 0b000, .funct7 = 0b0000000 },
41
42	.ld => .{ .opcode = 0b0000011, .funct3 = 0b011, .funct7 = null },
43	.lw => .{ .opcode = 0b0000011, .funct3 = 0b010, .funct7 = null },
44	.lwu => .{ .opcode = 0b0000011, .funct3 = 0b110, .funct7 = null },
45	.lh => .{ .opcode = 0b0000011, .funct3 = 0b001, .funct7 = null },
46	.lhu => .{ .opcode = 0b0000011, .funct3 = 0b101, .funct7 = null },
47	.lb => .{ .opcode = 0b0000011, .funct3 = 0b000, .funct7 = null },
48	.lbu => .{ .opcode = 0b0000011, .funct3 = 0b100, .funct7 = null },
49
50
51	.addi => .{ .opcode = 0b0010011, .funct3 = 0b000, .funct7 = null },
52	.jalr => .{ .opcode = 0b1100111, .funct3 = 0b000, .funct7 = null },
53
54	.lui => .{ .opcode = 0b0110111, .funct3 = null, .funct7 = null },
55
56	.sd => .{ .opcode = 0b0100011, .funct3 = 0b011, .funct7 = null },
57	.sw => .{ .opcode = 0b0100011, .funct3 = 0b010, .funct7 = null },
58	.sh => .{ .opcode = 0b0100011, .funct3 = 0b001, .funct7 = null },
59	.sb => .{ .opcode = 0b0100011, .funct3 = 0b000, .funct7 = null },
60
61	.jal => .{ .opcode = 0b1101111, .funct3 = null, .funct7 = null },
62
63	.ecall => .{ .opcode = 0b1110011, .funct3 = 0b000, .funct7 = null },
64	.ebreak => .{ .opcode = 0b1110011, .funct3 = 0b000, .funct7 = null },
65	.unimp => .{ .opcode = 0b0000000, .funct3 = 0b000, .funct7 = null },
66	// zig fmt: on
67	};
68	}
69	};
70
71	pub const InstEnc = enum {
72	R,
73	I,
74	S,
75	B,
76	U,
77	J,
78
79	/// extras that have unusual op counts
80	system,
81
82	pub fn fromMnemonic(mnem: Mnemonic) InstEnc {
83	return switch (mnem) {
84	.add,
85	=> .R,
86
87	.addi,
88	.ld,
89	.lw,
90	.lwu,
91	.lh,
92	.lhu,
93	.lb,
94	.lbu,
95	.jalr,
96	=> .I,
97
98	.lui,
99	=> .U,
100
101	.sd,
102	.sw,
103	.sh,
104	.sb,
105	=> .S,
106
107	.jal,
108	=> .J,
109
110	.ecall,
111	.ebreak,
112	.unimp,
113	=> .system,
114	};
115	}
116
117	pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {
118	return switch (enc) {
119	.R => .{ .reg, .reg, .reg, .none },
120	.I => .{ .reg, .reg, .imm, .none },
121	.S => .{ .reg, .reg, .imm, .none },
122	.B => .{ .imm, .reg, .reg, .imm },
123	.U => .{ .reg, .imm, .none, .none },
124	.J => .{ .reg, .imm, .none, .none },
125	.system => .{ .none, .none, .none, .none },
126	};
127	}
128	};
129
130	pub const Data = union(InstEnc) {
131	R: packed struct {
132	opcode: u7,
133	rd: u5,
134	funct3: u3,
135	rs1: u5,
136	rs2: u5,
137	funct7: u7,
138	},
139	I: packed struct {
140	opcode: u7,
141	rd: u5,
142	funct3: u3,
143	rs1: u5,
144	imm0_11: u12,
145	},
146	S: packed struct {
147	opcode: u7,
148	imm0_4: u5,
149	funct3: u3,
150	rs1: u5,
151	rs2: u5,
152	imm5_11: u7,
153	},
154	B: packed struct {
155	opcode: u7,
156	imm11: u1,
157	imm1_4: u4,
158	funct3: u3,
159	rs1: u5,
160	rs2: u5,
161	imm5_10: u6,
162	imm12: u1,
163	},
164	U: packed struct {
165	opcode: u7,
166	rd: u5,
167	imm12_31: u20,
168	},
169	J: packed struct {
170	opcode: u7,
171	rd: u5,
172	imm12_19: u8,
173	imm11: u1,
174	imm1_10: u10,
175	imm20: u1,
176	},
177	system: void,
178
179	pub fn toU32(self: Data) u32 {
180	return switch (self) {
181	.R => \|v\| @as(u32, @bitCast(v)),
182	.I => \|v\| @as(u32, @bitCast(v)),
183	.S => \|v\| @as(u32, @bitCast(v)),
184	.B => \|v\| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
185	.U => \|v\| @as(u32, @bitCast(v)),
186	.J => \|v\| @as(u32, @bitCast(v)),
187	.system => unreachable,
188	};
189	}
190
191	pub fn construct(mnem: Mnemonic, ops: []const Operand) !Data {
192	const inst_enc = InstEnc.fromMnemonic(mnem);
193
194	const enc = mnem.encoding();
195
196	// special mnemonics
197	switch (mnem) {
198	.ecall,
199	.ebreak,
200	.unimp,
201	=> {
202	assert(ops.len == 0);
203	return .{
204	.I = .{
205	.rd = Register.zero.id(),
206	.rs1 = Register.zero.id(),
207	.imm0_11 = switch (mnem) {
208	.ecall => 0x000,
209	.ebreak => 0x001,
210	.unimp => 0,
211	else => unreachable,
212	},
213
214	.opcode = enc.opcode,
215	.funct3 = enc.funct3.?,
216	},
217	};
218	},
219	else => {},
220	}
221
222	switch (inst_enc) {
223	.R => {
224	assert(ops.len == 3);
225	return .{
226	.R = .{
227	.rd = ops[0].reg.id(),
228	.rs1 = ops[1].reg.id(),
229	.rs2 = ops[2].reg.id(),
230
231	.opcode = enc.opcode,
232	.funct3 = enc.funct3.?,
233	.funct7 = enc.funct7.?,
234	},
235	};
236	},
237	.S => {
238	assert(ops.len == 3);
239	const umm = ops[2].imm.asBits(u12);
240
241	return .{
242	.S = .{
243	.imm0_4 = @truncate(umm),
244	.rs1 = ops[0].reg.id(),
245	.rs2 = ops[1].reg.id(),
246	.imm5_11 = @truncate(umm >> 5),
247
248	.opcode = enc.opcode,
249	.funct3 = enc.funct3.?,
250	},
251	};
252	},
253	.I => {
254	assert(ops.len == 3);
255	return .{
256	.I = .{
257	.rd = ops[0].reg.id(),
258	.rs1 = ops[1].reg.id(),
259	.imm0_11 = ops[2].imm.asBits(u12),
260
261	.opcode = enc.opcode,
262	.funct3 = enc.funct3.?,
263	},
264	};
265	},
266	.U => {
267	assert(ops.len == 2);
268	return .{
269	.U = .{
270	.rd = ops[0].reg.id(),
271	.imm12_31 = ops[1].imm.asBits(u20),
272
273	.opcode = enc.opcode,
274	},
275	};
276	},
277	.J => {
278	assert(ops.len == 2);
279
280	const umm = ops[1].imm.asBits(u21);
281	assert(umm % 4 == 0); // misaligned jump target
282
283	return .{
284	.J = .{
285	.rd = ops[0].reg.id(),
286	.imm1_10 = @truncate(umm >> 1),
287	.imm11 = @truncate(umm >> 11),
288	.imm12_19 = @truncate(umm >> 12),
289	.imm20 = @truncate(umm >> 20),
290
291	.opcode = enc.opcode,
292	},
293	};
294	},
295
296	else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
297	}
298	}
299	};
300
301	pub fn findByMnemonic(mnem: Mnemonic, ops: []const Operand) !?Encoding {
302	if (!verifyOps(mnem, ops)) return null;
303
304	return .{
305	.mnemonic = mnem,
306	.data = try Data.construct(mnem, ops),
307	};
308	}
309
310	const Enc = struct {
311	opcode: u7,
312	funct3: ?u3,
313	funct7: ?u7,
314	};
315
316	fn verifyOps(mnem: Mnemonic, ops: []const Operand) bool {
317	const inst_enc = InstEnc.fromMnemonic(mnem);
318	const list = std.mem.sliceTo(&inst_enc.opsList(), .none);
319	for (list, ops) \|l, o\| if (l != std.meta.activeTag(o)) return false;
320	return true;
321	}
322
323	const std = @import("std");
324	const assert = std.debug.assert;
325	const log = std.log.scoped(.encoding);
326
327	const Encoding = @This();
328	const bits = @import("bits.zig");
329	const Register = bits.Register;
330	const encoder = @import("encoder.zig");
331	const Instruction = encoder.Instruction;
332	const Operand = Instruction.Operand;
333	const OperandEnum = std.meta.FieldEnum(Operand);

src/arch/riscv64/Lower.zig created+222

...	@@ -0,0 +1,222 @@
1	//! This file contains the functionality for lowering RISC-V MIR to Instructions
2
3	bin_file: *link.File,
4	output_mode: std.builtin.OutputMode,
5	link_mode: std.builtin.LinkMode,
6	pic: bool,
7	allocator: Allocator,
8	mir: Mir,
9	cc: std.builtin.CallingConvention,
10	err_msg: ?*ErrorMsg = null,
11	src_loc: Module.SrcLoc,
12	result_insts_len: u8 = undefined,
13	result_relocs_len: u8 = undefined,
14	result_insts: [
15	@max(
16	1, // non-pseudo instruction
17	abi.callee_preserved_regs.len, // spill / restore regs,
18	)
19	]Instruction = undefined,
20	result_relocs: [1]Reloc = undefined,
21
22	pub const Error = error{
23	OutOfMemory,
24	LowerFail,
25	InvalidInstruction,
26	};
27
28	pub const Reloc = struct {
29	lowered_inst_index: u8,
30	target: Target,
31
32	const Target = union(enum) {
33	inst: Mir.Inst.Index,
34	linker_reloc: bits.Symbol,
35	};
36	};
37
38	/// The returned slice is overwritten by the next call to lowerMir.
39	pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
40	insts: []const Instruction,
41	relocs: []const Reloc,
42	} {
43	lower.result_insts = undefined;
44	lower.result_relocs = undefined;
45	errdefer lower.result_insts = undefined;
46	errdefer lower.result_relocs = undefined;
47	lower.result_insts_len = 0;
48	lower.result_relocs_len = 0;
49	defer lower.result_insts_len = undefined;
50	defer lower.result_relocs_len = undefined;
51
52	const inst = lower.mir.instructions.get(index);
53	log.debug("lowerMir {}", .{inst});
54	switch (inst.tag) {
55	else => try lower.generic(inst),
56	.pseudo => switch (inst.ops) {
57	.pseudo_dbg_line_column,
58	.pseudo_dbg_epilogue_begin,
59	.pseudo_dbg_prologue_end,
60	.pseudo_dead,
61	=> {},
62	.pseudo_load_rm, .pseudo_store_rm => {
63	const rm = inst.data.rm;
64
65	const frame_loc = rm.m.toFrameLoc(lower.mir);
66
67	switch (inst.ops) {
68	.pseudo_load_rm => {
69	const tag: Encoding.Mnemonic = switch (rm.m.mod.rm.size) {
70	.byte => .lb,
71	.hword => .lh,
72	.word => .lw,
73	.dword => .ld,
74	};
75
76	try lower.emit(tag, &.{
77	.{ .reg = rm.r },
78	.{ .reg = frame_loc.base },
79	.{ .imm = Immediate.s(frame_loc.disp) },
80	});
81	},
82	.pseudo_store_rm => {
83	const tag: Encoding.Mnemonic = switch (rm.m.mod.rm.size) {
84	.byte => .sb,
85	.hword => .sh,
86	.word => .sw,
87	.dword => .sd,
88	};
89
90	try lower.emit(tag, &.{
91	.{ .reg = frame_loc.base },
92	.{ .reg = rm.r },
93	.{ .imm = Immediate.s(frame_loc.disp) },
94	});
95	},
96	else => unreachable,
97	}
98	},
99
100	.pseudo_mv => {
101	const rr = inst.data.rr;
102
103	try lower.emit(.addi, &.{
104	.{ .reg = rr.rd },
105	.{ .reg = rr.rs },
106	.{ .imm = Immediate.s(0) },
107	});
108	},
109	.pseudo_ret => {
110	try lower.emit(.jalr, &.{
111	.{ .reg = .zero },
112	.{ .reg = .ra },
113	.{ .imm = Immediate.s(0) },
114	});
115	},
116	.pseudo_j => {
117	try lower.emit(.jal, &.{
118	.{ .reg = .zero },
119	.{ .imm = lower.reloc(.{ .inst = inst.data.inst }) },
120	});
121	},
122
123	.pseudo_spill_regs => try lower.pushPopRegList(true, inst.data.reg_list),
124	.pseudo_restore_regs => try lower.pushPopRegList(false, inst.data.reg_list),
125
126	else => return lower.fail("TODO: psuedo {s}", .{@tagName(inst.ops)}),
127	},
128	}
129
130	return .{
131	.insts = lower.result_insts[0..lower.result_insts_len],
132	.relocs = lower.result_relocs[0..lower.result_relocs_len],
133	};
134	}
135
136	fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
137	const mnemonic = std.meta.stringToEnum(Encoding.Mnemonic, @tagName(inst.tag)) orelse {
138	return lower.fail("generic inst name {s}-{s} doesn't match with a mnemonic", .{
139	@tagName(inst.tag),
140	@tagName(inst.ops),
141	});
142	};
143	try lower.emit(mnemonic, switch (inst.ops) {
144	.none => &.{},
145	.ri => &.{
146	.{ .reg = inst.data.u_type.rd },
147	.{ .imm = inst.data.u_type.imm20 },
148	},
149	.rri => &.{
150	.{ .reg = inst.data.i_type.rd },
151	.{ .reg = inst.data.i_type.rs1 },
152	.{ .imm = inst.data.i_type.imm12 },
153	},
154	else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),
155	});
156	}
157
158	fn emit(lower: *Lower, mnemonic: Encoding.Mnemonic, ops: []const Instruction.Operand) !void {
159	lower.result_insts[lower.result_insts_len] =
160	try Instruction.new(mnemonic, ops);
161	lower.result_insts_len += 1;
162	}
163
164	fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
165	lower.result_relocs[lower.result_relocs_len] = .{
166	.lowered_inst_index = lower.result_insts_len,
167	.target = target,
168	};
169	lower.result_relocs_len += 1;
170	return Immediate.s(0);
171	}
172
173	fn pushPopRegList(lower: *Lower, comptime spilling: bool, reg_list: Mir.RegisterList) !void {
174	var it = reg_list.iterator(.{ .direction = if (spilling) .forward else .reverse });
175
176	var reg_i: u31 = 0;
177	while (it.next()) \|i\| {
178	const frame = lower.mir.frame_locs.get(@intFromEnum(bits.FrameIndex.spill_frame));
179
180	if (spilling) {
181	try lower.emit(.sd, &.{
182	.{ .reg = frame.base },
183	.{ .reg = abi.callee_preserved_regs[i] },
184	.{ .imm = Immediate.s(frame.disp + reg_i) },
185	});
186	} else {
187	try lower.emit(.ld, &.{
188	.{ .reg = abi.callee_preserved_regs[i] },
189	.{ .reg = frame.base },
190	.{ .imm = Immediate.s(frame.disp + reg_i) },
191	});
192	}
193
194	reg_i += 8;
195	}
196	}
197
198	pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
199	@setCold(true);
200	assert(lower.err_msg == null);
201	lower.err_msg = try ErrorMsg.create(lower.allocator, lower.src_loc, format, args);
202	return error.LowerFail;
203	}
204
205	const Lower = @This();
206
207	const abi = @import("abi.zig");
208	const assert = std.debug.assert;
209	const bits = @import("bits.zig");
210	const encoder = @import("encoder.zig");
211	const link = @import("../../link.zig");
212	const Encoding = @import("Encoding.zig");
213	const std = @import("std");
214	const log = std.log.scoped(.lower);
215
216	const Air = @import("../../Air.zig");
217	const Allocator = std.mem.Allocator;
218	const ErrorMsg = Module.ErrorMsg;
219	const Mir = @import("Mir.zig");
220	const Module = @import("../../Module.zig");
221	const Instruction = encoder.Instruction;
222	const Immediate = bits.Immediate;

src/arch/riscv64/Mir.zig+178-85

...	@@ -9,22 +9,32 @@	...	@@ -9,22 +9,32 @@
9	instructions: std.MultiArrayList(Inst).Slice,	9	instructions: std.MultiArrayList(Inst).Slice,
10	/// The meaning of this data is determined by `Inst.Tag` value.	10	/// The meaning of this data is determined by `Inst.Tag` value.
11	extra: []const u32,	11	extra: []const u32,
		12	frame_locs: std.MultiArrayList(FrameLoc).Slice,
12		13
13	pub const Inst = struct {	14	pub const Inst = struct {
14	tag: Tag,	15	tag: Tag,
15	/// The meaning of this depends on `tag`.
16	data: Data,	16	data: Data,
		17	ops: Ops,
		18
		19	/// The position of an MIR instruction within the `Mir` instructions array.
		20	pub const Index = u32;
17		21
18	pub const Tag = enum(u16) {	22	pub const Tag = enum(u16) {
		23	/// Add immediate. Uses i_type payload.
19	addi,	24	addi,
		25
		26	/// Add immediate and produce a sign-extended result.
		27	///
		28	/// Uses i-type payload.
20	addiw,	29	addiw,
		30
21	jalr,	31	jalr,
22	lui,	32	lui,
23	mv,	33	mv,
24		34
25	unimp,
26	ebreak,	35	ebreak,
27	ecall,	36	ecall,
		37	unimp,
28		38
29	/// OR instruction. Uses r_type payload.	39	/// OR instruction. Uses r_type payload.
30	@"or",	40	@"or",
...	@@ -48,9 +58,11 @@ pub const Inst = struct {	...	@@ -48,9 +58,11 @@ pub const Inst = struct {
48	/// Register Logical Right Shit, uses r_type payload	58	/// Register Logical Right Shit, uses r_type payload
49	srlw,	59	srlw,
50		60
		61	/// Jumps, but stores the address of the instruction following the
		62	/// jump in `rd`.
		63	///
		64	/// Uses j_type payload.
51	jal,	65	jal,
52	/// Jumps. Uses `inst` payload.
53	j,
54		66
55	/// Immediate AND, uses i_type payload	67	/// Immediate AND, uses i_type payload
56	andi,	68	andi,
...	@@ -93,55 +105,34 @@ pub const Inst = struct {	...	@@ -93,55 +105,34 @@ pub const Inst = struct {
93	/// Boolean NOT, Uses rr payload	105	/// Boolean NOT, Uses rr payload
94	not,	106	not,
95		107
		108	/// Generates a NO-OP, uses nop payload
96	nop,	109	nop,
97	ret,
98		110
99	/// Load double (64 bits)	111	/// Load double (64 bits), uses i_type payload
100	ld,	112	ld,
101	/// Store double (64 bits)	113	/// Load word (32 bits), uses i_type payload
102	sd,
103	/// Load word (32 bits)
104	lw,	114	lw,
105	/// Store word (32 bits)	115	/// Load half (16 bits), uses i_type payload
106	sw,
107	/// Load half (16 bits)
108	lh,	116	lh,
109	/// Store half (16 bits)	117	/// Load byte (8 bits), uses i_type payload
110	sh,
111	/// Load byte (8 bits)
112	lb,	118	lb,
113	/// Store byte (8 bits)
114	sb,
115
116	/// Pseudo-instruction: End of prologue
117	dbg_prologue_end,
118	/// Pseudo-instruction: Beginning of epilogue
119	dbg_epilogue_begin,
120	/// Pseudo-instruction: Update debug line
121	dbg_line,
122
123	/// Psuedo-instruction that will generate a backpatched
124	/// function prologue.
125	psuedo_prologue,
126	/// Psuedo-instruction that will generate a backpatched
127	/// function epilogue
128	psuedo_epilogue,
129		119
130	/// Loads the address of a value that hasn't yet been allocated in memory.	120	/// Store double (64 bits), uses s_type payload
131	///	121	sd,
132	/// uses the Mir.LoadSymbolPayload payload.	122	/// Store word (32 bits), uses s_type payload
133	load_symbol,	123	sw,
		124	/// Store half (16 bits), uses s_type payload
		125	sh,
		126	/// Store byte (8 bits), uses s_type payload
		127	sb,
134		128
135	// TODO: add description	129	/// A pseudo-instruction. Used for anything that isn't 1:1 with an
136	// this is bad, remove this	130	/// assembly instruction.
137	ldr_ptr_stack,	131	pseudo,
138	};	132	};
139		133
140	/// The position of an MIR instruction within the `Mir` instructions array.
141	pub const Index = u32;
142
143	/// All instructions have a 4-byte payload, which is contained within	134	/// All instructions have a 4-byte payload, which is contained within
144	/// this union. `Tag` determines which union field is active, as well as	135	/// this union. `Ops` determines which union field is active, as well as
145	/// how to interpret the data within.	136	/// how to interpret the data within.
146	pub const Data = union {	137	pub const Data = union {
147	/// No additional data	138	/// No additional data
...	@@ -152,74 +143,154 @@ pub const Inst = struct {	...	@@ -152,74 +143,154 @@ pub const Inst = struct {
152	///	143	///
153	/// Used by e.g. b	144	/// Used by e.g. b
154	inst: Index,	145	inst: Index,
155	/// A 16-bit immediate value.
156	///
157	/// Used by e.g. svc
158	imm16: i16,
159	/// A 12-bit immediate value.
160	///
161	/// Used by e.g. psuedo_prologue
162	imm12: i12,
163	/// Index into `extra`. Meaning of what can be found there is context-dependent.	146	/// Index into `extra`. Meaning of what can be found there is context-dependent.
164	///	147	///
165	/// Used by e.g. load_memory	148	/// Used by e.g. load_memory
166	payload: u32,	149	payload: u32,
167	/// A register	150
168	///	151	r_type: struct {
169	/// Used by e.g. blr
170	reg: Register,
171	/// Two registers
172	///
173	/// Used by e.g. mv
174	rr: struct {
175	rd: Register,	152	rd: Register,
176	rs: Register,	153	rs1: Register,
		154	rs2: Register,
177	},	155	},
178	/// I-Type	156
179	///
180	/// Used by e.g. jalr
181	i_type: struct {	157	i_type: struct {
182	rd: Register,	158	rd: Register,
183	rs1: Register,	159	rs1: Register,
184	imm12: i12,	160	imm12: Immediate,
185	},	161	},
186	/// R-Type	162
187	///	163	s_type: struct {
188	/// Used by e.g. add
189	r_type: struct {
190	rd: Register,
191	rs1: Register,	164	rs1: Register,
192	rs2: Register,	165	rs2: Register,
		166	imm5: Immediate,
		167	imm7: Immediate,
193	},	168	},
194	/// B-Type	169
195	///
196	/// Used by e.g. beq
197	b_type: struct {	170	b_type: struct {
198	rs1: Register,	171	rs1: Register,
199	rs2: Register,	172	rs2: Register,
200	inst: Inst.Index,	173	inst: Inst.Index,
201	},	174	},
202	/// J-Type	175
203	///	176	u_type: struct {
204	/// Used by e.g. jal
205	j_type: struct {
206	rd: Register,	177	rd: Register,
207	inst: Inst.Index,	178	imm20: Immediate,
208	},	179	},
209	/// U-Type	180
210	///	181	j_type: struct {
211	/// Used by e.g. lui
212	u_type: struct {
213	rd: Register,	182	rd: Register,
214	imm20: i20,	183	inst: Inst.Index,
215	},	184	},
		185
216	/// Debug info: line and column	186	/// Debug info: line and column
217	///	187	///
218	/// Used by e.g. dbg_line	188	/// Used by e.g. pseudo_dbg_line
219	dbg_line_column: struct {	189	pseudo_dbg_line_column: struct {
220	line: u32,	190	line: u32,
221	column: u32,	191	column: u32,
222	},	192	},
		193
		194	// Custom types to be lowered
		195
		196	/// Register + Memory
		197	rm: struct {
		198	r: Register,
		199	m: Memory,
		200	},
		201
		202	reg_list: Mir.RegisterList,
		203
		204	/// A register
		205	///
		206	/// Used by e.g. blr
		207	reg: Register,
		208
		209	/// Two registers
		210	///
		211	/// Used by e.g. mv
		212	rr: struct {
		213	rd: Register,
		214	rs: Register,
		215	},
		216	};
		217
		218	pub const Ops = enum {
		219	/// No data associated with this instruction (only mnemonic is used).
		220	none,
		221	/// Two registers
		222	rr,
		223	/// Three registers
		224	rrr,
		225
		226	/// Two registers + immediate, uses the i_type payload.
		227	rri,
		228	/// Two registers + Two Immediates
		229	rrii,
		230
		231	/// Two registers + another instruction.
		232	rr_inst,
		233
		234	/// Register + Memory
		235	rm,
		236
		237	/// Register + Immediate
		238	ri,
		239
		240	/// Another instruction.
		241	inst,
		242
		243	/// Pseudo-instruction that will generate a backpatched
		244	/// function prologue.
		245	pseudo_prologue,
		246	/// Pseudo-instruction that will generate a backpatched
		247	/// function epilogue
		248	pseudo_epilogue,
		249
		250	/// Pseudo-instruction: End of prologue
		251	pseudo_dbg_prologue_end,
		252	/// Pseudo-instruction: Beginning of epilogue
		253	pseudo_dbg_epilogue_begin,
		254	/// Pseudo-instruction: Update debug line
		255	pseudo_dbg_line_column,
		256
		257	/// Pseudo-instruction that loads from memory into a register.
		258	///
		259	/// Uses `rm` payload.
		260	pseudo_load_rm,
		261	/// Pseudo-instruction that stores from a register into memory
		262	///
		263	/// Uses `rm` payload.
		264	pseudo_store_rm,
		265
		266	/// Pseudo-instruction that loads the address of memory into a register.
		267	///
		268	/// Uses `rm` payload.
		269	pseudo_lea_rm,
		270
		271	/// Shorthand for returning, aka jumping to ra register.
		272	///
		273	/// Uses nop payload.
		274	pseudo_ret,
		275
		276	/// Jumps. Uses `inst` payload.
		277	pseudo_j,
		278
		279	/// Dead inst, ignored by the emitter.
		280	pseudo_dead,
		281
		282	/// Loads the address of a value that hasn't yet been allocated in memory.
		283	///
		284	/// uses the Mir.LoadSymbolPayload payload.
		285	pseudo_load_symbol,
		286
		287	/// Moves the value of rs1 to rd.
		288	///
		289	/// uses the `rr` payload.
		290	pseudo_mv,
		291
		292	pseudo_restore_regs,
		293	pseudo_spill_regs,
223	};	294	};
224		295
225	// Make sure we don't accidentally make instructions bigger than expected.	296	// Make sure we don't accidentally make instructions bigger than expected.
...	@@ -229,14 +300,32 @@ pub const Inst = struct {	...	@@ -229,14 +300,32 @@ pub const Inst = struct {
229	// assert(@sizeOf(Inst) == 8);	300	// assert(@sizeOf(Inst) == 8);
230	// }	301	// }
231	// }	302	// }
		303
		304	pub fn format(
		305	inst: Inst,
		306	comptime fmt: []const u8,
		307	options: std.fmt.FormatOptions,
		308	writer: anytype,
		309	) !void {
		310	assert(fmt.len == 0);
		311	_ = options;
		312
		313	try writer.print("Tag: {s}, Ops: {s}", .{ @tagName(inst.tag), @tagName(inst.ops) });
		314	}
232	};	315	};
233		316
234	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {	317	pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
235	mir.instructions.deinit(gpa);	318	mir.instructions.deinit(gpa);
		319	mir.frame_locs.deinit(gpa);
236	gpa.free(mir.extra);	320	gpa.free(mir.extra);
237	mir.* = undefined;	321	mir.* = undefined;
238	}	322	}
239		323
		324	pub const FrameLoc = struct {
		325	base: Register,
		326	disp: i32,
		327	};
		328
240	/// Returns the requested data, as well as the new index which is at the start of the	329	/// Returns the requested data, as well as the new index which is at the start of the
241	/// trailers for the object.	330	/// trailers for the object.
242	pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {	331	pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
...	@@ -291,11 +380,11 @@ pub const RegisterList = struct {	...	@@ -291,11 +380,11 @@ pub const RegisterList = struct {
291	return self.bitset.iterator(options);	380	return self.bitset.iterator(options);
292	}	381	}
293		382
294	pub fn count(self: Self) u32 {	383	pub fn count(self: Self) i32 {
295	return @intCast(self.bitset.count());	384	return @intCast(self.bitset.count());
296	}	385	}
297		386
298	pub fn size(self: Self) u32 {	387	pub fn size(self: Self) i32 {
299	return @intCast(self.bitset.count() * 8);	388	return @intCast(self.bitset.count() * 8);
300	}	389	}
301	};	390	};
...	@@ -307,4 +396,8 @@ const assert = std.debug.assert;	...	@@ -307,4 +396,8 @@ const assert = std.debug.assert;
307		396
308	const bits = @import("bits.zig");	397	const bits = @import("bits.zig");
309	const Register = bits.Register;	398	const Register = bits.Register;
		399	const Immediate = bits.Immediate;
		400	const Memory = bits.Memory;
		401	const FrameIndex = bits.FrameIndex;
		402	const FrameAddr = @import("CodeGen.zig").FrameAddr;
310	const IntegerBitSet = std.bit_set.IntegerBitSet;	403	const IntegerBitSet = std.bit_set.IntegerBitSet;

src/arch/riscv64/abi.zig+27-3

...	@@ -93,7 +93,7 @@ pub fn classifyType(ty: Type, mod: *Module) Class {	...	@@ -93,7 +93,7 @@ pub fn classifyType(ty: Type, mod: *Module) Class {
93		93
94	/// There are a maximum of 8 possible return slots. Returned values are in	94	/// There are a maximum of 8 possible return slots. Returned values are in
95	/// the beginning of the array; unused slots are filled with .none.	95	/// the beginning of the array; unused slots are filled with .none.
96	pub fn classifySystemV(ty: Type, mod: *Module) [8]Class {	96	pub fn classifySystem(ty: Type, mod: *Module) [8]Class {
97	var result = [1]Class{.none} ** 8;	97	var result = [1]Class{.none} ** 8;
98	switch (ty.zigTypeTag(mod)) {	98	switch (ty.zigTypeTag(mod)) {
99	.Pointer => switch (ty.ptrSize(mod)) {	99	.Pointer => switch (ty.ptrSize(mod)) {
...	@@ -109,18 +109,42 @@ pub fn classifySystemV(ty: Type, mod: *Module) [8]Class {	...	@@ -109,18 +109,42 @@ pub fn classifySystemV(ty: Type, mod: *Module) [8]Class {
109	},	109	},
110	.Optional => {	110	.Optional => {
111	if (ty.isPtrLikeOptional(mod)) {	111	if (ty.isPtrLikeOptional(mod)) {
		112	result[0] = .integer;
112	return result;	113	return result;
113	}	114	}
114	result[0] = .integer;	115	result[0] = .integer;
115	result[1] = .integer;	116	result[1] = .integer;
116	return result;	117	return result;
117	},	118	},
118	else => return result,	119	.Int, .Enum, .ErrorSet => {
		120	const int_bits = ty.intInfo(mod).bits;
		121	if (int_bits <= 64) {
		122	result[0] = .integer;
		123	return result;
		124	}
		125	if (int_bits <= 128) {
		126	result[0] = .integer;
		127	result[1] = .integer;
		128	return result;
		129	}
		130	unreachable; // support > 128 bit int arguments
		131	},
		132	.ErrorUnion => {
		133	const payload = ty.errorUnionPayload(mod);
		134	const payload_bits = payload.bitSize(mod);
		135	if (payload_bits <= 64) {
		136	result[0] = .integer;
		137	result[1] = .integer;
		138	}
		139	unreachable; // support > 64 bit error payloads
		140	},
		141	else => \|bad_ty\| std.debug.panic("classifySystem {s}", .{@tagName(bad_ty)}),
119	}	142	}
120	}	143	}
121		144
122	pub const callee_preserved_regs = [_]Register{	145	pub const callee_preserved_regs = [_]Register{
123	.s0, .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11,	146	// .s0 is ommited to be used as a frame pointer
		147	.s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11,
124	};	148	};
125		149
126	pub const function_arg_regs = [_]Register{	150	pub const function_arg_regs = [_]Register{

src/arch/riscv64/bits.zig+161-398

...	@@ -2,391 +2,141 @@ const std = @import("std");	...	@@ -2,391 +2,141 @@ const std = @import("std");
2	const DW = std.dwarf;	2	const DW = std.dwarf;
3	const assert = std.debug.assert;	3	const assert = std.debug.assert;
4	const testing = std.testing;	4	const testing = std.testing;
		5	const Encoding = @import("Encoding.zig");
		6	const Mir = @import("Mir.zig");
5		7
6	// TODO: this is only tagged to facilitate the monstrosity.	8	pub const Memory = struct {
7	// Once packed structs work make it packed.	9	base: Base,
8	pub const Instruction = union(enum) {	10	mod: Mod,
9	R: packed struct {
10	opcode: u7,
11	rd: u5,
12	funct3: u3,
13	rs1: u5,
14	rs2: u5,
15	funct7: u7,
16	},
17	I: packed struct {
18	opcode: u7,
19	rd: u5,
20	funct3: u3,
21	rs1: u5,
22	imm0_11: u12,
23	},
24	S: packed struct {
25	opcode: u7,
26	imm0_4: u5,
27	funct3: u3,
28	rs1: u5,
29	rs2: u5,
30	imm5_11: u7,
31	},
32	B: packed struct {
33	opcode: u7,
34	imm11: u1,
35	imm1_4: u4,
36	funct3: u3,
37	rs1: u5,
38	rs2: u5,
39	imm5_10: u6,
40	imm12: u1,
41	},
42	U: packed struct {
43	opcode: u7,
44	rd: u5,
45	imm12_31: u20,
46	},
47	J: packed struct {
48	opcode: u7,
49	rd: u5,
50	imm12_19: u8,
51	imm11: u1,
52	imm1_10: u10,
53	imm20: u1,
54	},
55		11
56	// TODO: once packed structs work we can remove this monstrosity.	12	pub const Base = union(enum) {
57	pub fn toU32(self: Instruction) u32 {	13	reg: Register,
58	return switch (self) {	14	frame: FrameIndex,
59	.R => \|v\| @as(u32, @bitCast(v)),	15	reloc: Symbol,
60	.I => \|v\| @as(u32, @bitCast(v)),	16	};
61	.S => \|v\| @as(u32, @bitCast(v)),
62	.B => \|v\| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
63	.U => \|v\| @as(u32, @bitCast(v)),
64	.J => \|v\| @as(u32, @bitCast(v)),
65	};
66	}
67		17
68	fn rType(op: u7, fn3: u3, fn7: u7, rd: Register, r1: Register, r2: Register) Instruction {	18	pub const Mod = union(enum(u1)) {
69	return Instruction{	19	rm: struct {
70	.R = .{	20	size: Size,
71	.opcode = op,	21	disp: i32 = 0,
72	.funct3 = fn3,	22	},
73	.funct7 = fn7,	23	off: u64,
74	.rd = rd.id(),	24	};
75	.rs1 = r1.id(),
76	.rs2 = r2.id(),
77	},
78	};
79	}
80		25
81	// RISC-V is all signed all the time -- convert immediates to unsigned for processing	26	pub const Size = enum(u4) {
82	fn iType(op: u7, fn3: u3, rd: Register, r1: Register, imm: i12) Instruction {	27	/// Byte, 1 byte
83	const umm = @as(u12, @bitCast(imm));	28	byte,
		29	/// Half word, 2 bytes
		30	hword,
		31	/// Word, 4 bytes
		32	word,
		33	/// Double word, 8 Bytes
		34	dword,
		35
		36	pub fn fromSize(size: u32) Size {
		37	return switch (size) {
		38	1 => .byte,
		39	2 => .hword,
		40	4 => .word,
		41	8 => .dword,
		42	else => unreachable,
		43	};
		44	}
		45
		46	pub fn fromBitSize(bit_size: u64) Size {
		47	return switch (bit_size) {
		48	8 => .byte,
		49	16 => .hword,
		50	32 => .word,
		51	64 => .dword,
		52	else => unreachable,
		53	};
		54	}
		55
		56	pub fn bitSize(s: Size) u64 {
		57	return switch (s) {
		58	.byte => 8,
		59	.hword => 16,
		60	.word => 32,
		61	.dword => 64,
		62	};
		63	}
		64	};
84		65
85	return Instruction{	66	/// Asserts `mem` can be represented as a `FrameLoc`.
86	.I = .{	67	pub fn toFrameLoc(mem: Memory, mir: Mir) Mir.FrameLoc {
87	.opcode = op,	68	switch (mem.base) {
88	.funct3 = fn3,	69	.reg => \|reg\| {
89	.rd = rd.id(),	70	return .{
90	.rs1 = r1.id(),	71	.base = reg,
91	.imm0_11 = umm,	72	.disp = switch (mem.mod) {
		73	.off => unreachable, // TODO: toFrameLoc disp.off
		74	.rm => \|rm\| rm.disp,
		75	},
		76	};
92	},	77	},
93	};	78	.frame => \|index\| return mir.frame_locs.get(@intFromEnum(index)),
		79	.reloc => unreachable,
		80	}
94	}	81	}
		82	};
95		83
96	fn sType(op: u7, fn3: u3, r1: Register, r2: Register, imm: i12) Instruction {	84	pub const Immediate = union(enum) {
97	const umm = @as(u12, @bitCast(imm));	85	signed: i32,
		86	unsigned: u32,
98		87
99	return Instruction{	88	pub fn u(x: u64) Immediate {
100	.S = .{	89	return .{ .unsigned = x };
101	.opcode = op,
102	.funct3 = fn3,
103	.rs1 = r1.id(),
104	.rs2 = r2.id(),
105	.imm0_4 = @as(u5, @truncate(umm)),
106	.imm5_11 = @as(u7, @truncate(umm >> 5)),
107	},
108	};
109	}	90	}
110		91
111	// Use significance value rather than bit value, same for J-type	92	pub fn s(x: i32) Immediate {
112	// -- less burden on callsite, bonus semantic checking	93	return .{ .signed = x };
113	fn bType(op: u7, fn3: u3, r1: Register, r2: Register, imm: i13) Instruction {
114	const umm = @as(u13, @bitCast(imm));
115	assert(umm % 4 == 0); // misaligned branch target
116
117	return Instruction{
118	.B = .{
119	.opcode = op,
120	.funct3 = fn3,
121	.rs1 = r1.id(),
122	.rs2 = r2.id(),
123	.imm1_4 = @as(u4, @truncate(umm >> 1)),
124	.imm5_10 = @as(u6, @truncate(umm >> 5)),
125	.imm11 = @as(u1, @truncate(umm >> 11)),
126	.imm12 = @as(u1, @truncate(umm >> 12)),
127	},
128	};
129	}	94	}
130		95
131	// We have to extract the 20 bits anyway -- let's not make it more painful	96	pub fn asSigned(imm: Immediate, bit_size: u64) i64 {
132	fn uType(op: u7, rd: Register, imm: i20) Instruction {	97	return switch (imm) {
133	const umm = @as(u20, @bitCast(imm));	98	.signed => \|x\| switch (bit_size) {
134		99	1, 8 => @as(i8, @intCast(x)),
135	return Instruction{	100	16 => @as(i16, @intCast(x)),
136	.U = .{	101	32, 64 => x,
137	.opcode = op,	102	else => unreachable,
138	.rd = rd.id(),	103	},
139	.imm12_31 = umm,	104	.unsigned => \|x\| switch (bit_size) {
		105	1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))),
		106	16 => @as(i16, @bitCast(@as(u16, @intCast(x)))),
		107	32 => @as(i32, @bitCast(@as(u32, @intCast(x)))),
		108	64 => @bitCast(x),
		109	else => unreachable,
140	},	110	},
141	};	111	};
142	}	112	}
143		113
144	fn jType(op: u7, rd: Register, imm: i21) Instruction {	114	pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 {
145	const umm = @as(u21, @bitCast(imm));	115	return switch (imm) {
146	assert(umm % 2 == 0); // misaligned jump target	116	.signed => \|x\| switch (bit_size) {
147		117	1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))),
148	return Instruction{	118	16 => @as(u16, @bitCast(@as(i16, @intCast(x)))),
149	.J = .{	119	32, 64 => @as(u32, @bitCast(x)),
150	.opcode = op,	120	else => unreachable,
151	.rd = rd.id(),	121	},
152	.imm1_10 = @as(u10, @truncate(umm >> 1)),	122	.unsigned => \|x\| switch (bit_size) {
153	.imm11 = @as(u1, @truncate(umm >> 11)),	123	1, 8 => @as(u8, @intCast(x)),
154	.imm12_19 = @as(u8, @truncate(umm >> 12)),	124	16 => @as(u16, @intCast(x)),
155	.imm20 = @as(u1, @truncate(umm >> 20)),	125	32 => @as(u32, @intCast(x)),
		126	64 => x,
		127	else => unreachable,
156	},	128	},
157	};	129	};
158	}	130	}
159		131
160	// The meat and potatoes. Arguments are in the order in which they would appear in assembly code.	132	pub fn asBits(imm: Immediate, comptime T: type) T {
161		133	const int_info = @typeInfo(T).Int;
162	// Arithmetic/Logical, Register-Register	134	if (int_info.signedness != .unsigned) @compileError("Immediate.asBits needs unsigned T");
163		135	return switch (imm) {
164	pub fn add(rd: Register, r1: Register, r2: Register) Instruction {	136	.signed => \|x\| @bitCast(@as(std.meta.Int(.signed, int_info.bits), @intCast(x))),
165	return rType(0b0110011, 0b000, 0b0000000, rd, r1, r2);	137	.unsigned => \|x\| @intCast(x),
166	}	138	};
167
168	pub fn sub(rd: Register, r1: Register, r2: Register) Instruction {
169	return rType(0b0110011, 0b000, 0b0100000, rd, r1, r2);
170	}
171
172	pub fn @"and"(rd: Register, r1: Register, r2: Register) Instruction {
173	return rType(0b0110011, 0b111, 0b0000000, rd, r1, r2);
174	}
175
176	pub fn @"or"(rd: Register, r1: Register, r2: Register) Instruction {
177	return rType(0b0110011, 0b110, 0b0000000, rd, r1, r2);
178	}
179
180	pub fn xor(rd: Register, r1: Register, r2: Register) Instruction {
181	return rType(0b0110011, 0b100, 0b0000000, rd, r1, r2);
182	}
183
184	pub fn sll(rd: Register, r1: Register, r2: Register) Instruction {
185	return rType(0b0110011, 0b001, 0b0000000, rd, r1, r2);
186	}
187
188	pub fn srl(rd: Register, r1: Register, r2: Register) Instruction {
189	return rType(0b0110011, 0b101, 0b0000000, rd, r1, r2);
190	}
191
192	pub fn sra(rd: Register, r1: Register, r2: Register) Instruction {
193	return rType(0b0110011, 0b101, 0b0100000, rd, r1, r2);
194	}
195
196	pub fn slt(rd: Register, r1: Register, r2: Register) Instruction {
197	return rType(0b0110011, 0b010, 0b0000000, rd, r1, r2);
198	}
199
200	pub fn sltu(rd: Register, r1: Register, r2: Register) Instruction {
201	return rType(0b0110011, 0b011, 0b0000000, rd, r1, r2);
202	}
203
204	// M extension operations
205
206	pub fn mul(rd: Register, r1: Register, r2: Register) Instruction {
207	return rType(0b0110011, 0b000, 0b0000001, rd, r1, r2);
208	}
209
210	// Arithmetic/Logical, Register-Register (32-bit)
211
212	pub fn addw(rd: Register, r1: Register, r2: Register) Instruction {
213	return rType(0b0111011, 0b000, rd, r1, r2);
214	}
215
216	pub fn subw(rd: Register, r1: Register, r2: Register) Instruction {
217	return rType(0b0111011, 0b000, 0b0100000, rd, r1, r2);
218	}
219
220	pub fn sllw(rd: Register, r1: Register, r2: Register) Instruction {
221	return rType(0b0111011, 0b001, 0b0000000, rd, r1, r2);
222	}
223
224	pub fn srlw(rd: Register, r1: Register, r2: Register) Instruction {
225	return rType(0b0111011, 0b101, 0b0000000, rd, r1, r2);
226	}
227
228	pub fn sraw(rd: Register, r1: Register, r2: Register) Instruction {
229	return rType(0b0111011, 0b101, 0b0100000, rd, r1, r2);
230	}
231
232	// Arithmetic/Logical, Register-Immediate
233
234	pub fn addi(rd: Register, r1: Register, imm: i12) Instruction {
235	return iType(0b0010011, 0b000, rd, r1, imm);
236	}
237
238	pub fn andi(rd: Register, r1: Register, imm: i12) Instruction {
239	return iType(0b0010011, 0b111, rd, r1, imm);
240	}
241
242	pub fn ori(rd: Register, r1: Register, imm: i12) Instruction {
243	return iType(0b0010011, 0b110, rd, r1, imm);
244	}
245
246	pub fn xori(rd: Register, r1: Register, imm: i12) Instruction {
247	return iType(0b0010011, 0b100, rd, r1, imm);
248	}
249
250	pub fn slli(rd: Register, r1: Register, shamt: u6) Instruction {
251	return iType(0b0010011, 0b001, rd, r1, shamt);
252	}
253
254	pub fn srli(rd: Register, r1: Register, shamt: u6) Instruction {
255	return iType(0b0010011, 0b101, rd, r1, shamt);
256	}
257
258	pub fn srai(rd: Register, r1: Register, shamt: u6) Instruction {
259	return iType(0b0010011, 0b101, rd, r1, (@as(i12, 1) << 10) + shamt);
260	}
261
262	pub fn slti(rd: Register, r1: Register, imm: i12) Instruction {
263	return iType(0b0010011, 0b010, rd, r1, imm);
264	}
265
266	pub fn sltiu(rd: Register, r1: Register, imm: u12) Instruction {
267	return iType(0b0010011, 0b011, rd, r1, @as(i12, @bitCast(imm)));
268	}
269
270	// Arithmetic/Logical, Register-Immediate (32-bit)
271
272	pub fn addiw(rd: Register, r1: Register, imm: i12) Instruction {
273	return iType(0b0011011, 0b000, rd, r1, imm);
274	}
275
276	pub fn slliw(rd: Register, r1: Register, shamt: u6) Instruction {
277	return iType(0b0011011, 0b001, rd, r1, shamt);
278	}
279
280	pub fn srliw(rd: Register, r1: Register, shamt: u6) Instruction {
281	return iType(0b0011011, 0b101, rd, r1, shamt);
282	}
283
284	pub fn sraiw(rd: Register, r1: Register, shamt: u6) Instruction {
285	return iType(0b0011011, 0b101, rd, r1, (@as(i12, 1) << 10) + shamt);
286	}
287
288	// Upper Immediate
289
290	pub fn lui(rd: Register, imm: i20) Instruction {
291	return uType(0b0110111, rd, imm);
292	}
293
294	pub fn auipc(rd: Register, imm: i20) Instruction {
295	return uType(0b0010111, rd, imm);
296	}
297
298	// Load
299
300	pub fn ld(rd: Register, offset: i12, base: Register) Instruction {
301	return iType(0b0000011, 0b011, rd, base, offset);
302	}
303
304	pub fn lw(rd: Register, offset: i12, base: Register) Instruction {
305	return iType(0b0000011, 0b010, rd, base, offset);
306	}
307
308	pub fn lwu(rd: Register, offset: i12, base: Register) Instruction {
309	return iType(0b0000011, 0b110, rd, base, offset);
310	}
311
312	pub fn lh(rd: Register, offset: i12, base: Register) Instruction {
313	return iType(0b0000011, 0b001, rd, base, offset);
314	}
315
316	pub fn lhu(rd: Register, offset: i12, base: Register) Instruction {
317	return iType(0b0000011, 0b101, rd, base, offset);
318	}
319
320	pub fn lb(rd: Register, offset: i12, base: Register) Instruction {
321	return iType(0b0000011, 0b000, rd, base, offset);
322	}
323
324	pub fn lbu(rd: Register, offset: i12, base: Register) Instruction {
325	return iType(0b0000011, 0b100, rd, base, offset);
326	}
327
328	// Store
329
330	pub fn sd(rs: Register, offset: i12, base: Register) Instruction {
331	return sType(0b0100011, 0b011, base, rs, offset);
332	}
333
334	pub fn sw(rs: Register, offset: i12, base: Register) Instruction {
335	return sType(0b0100011, 0b010, base, rs, offset);
336	}
337
338	pub fn sh(rs: Register, offset: i12, base: Register) Instruction {
339	return sType(0b0100011, 0b001, base, rs, offset);
340	}
341
342	pub fn sb(rs: Register, offset: i12, base: Register) Instruction {
343	return sType(0b0100011, 0b000, base, rs, offset);
344	}
345
346	// Fence
347	// TODO: implement fence
348
349	// Branch
350
351	pub fn beq(r1: Register, r2: Register, offset: i13) Instruction {
352	return bType(0b1100011, 0b000, r1, r2, offset);
353	}
354
355	pub fn bne(r1: Register, r2: Register, offset: i13) Instruction {
356	return bType(0b1100011, 0b001, r1, r2, offset);
357	}
358
359	pub fn blt(r1: Register, r2: Register, offset: i13) Instruction {
360	return bType(0b1100011, 0b100, r1, r2, offset);
361	}
362
363	pub fn bge(r1: Register, r2: Register, offset: i13) Instruction {
364	return bType(0b1100011, 0b101, r1, r2, offset);
365	}
366
367	pub fn bltu(r1: Register, r2: Register, offset: i13) Instruction {
368	return bType(0b1100011, 0b110, r1, r2, offset);
369	}
370
371	pub fn bgeu(r1: Register, r2: Register, offset: i13) Instruction {
372	return bType(0b1100011, 0b111, r1, r2, offset);
373	}
374
375	// Jump
376
377	pub fn jal(link: Register, offset: i21) Instruction {
378	return jType(0b1101111, link, offset);
379	}
380
381	pub fn jalr(link: Register, offset: i12, base: Register) Instruction {
382	return iType(0b1100111, 0b000, link, base, offset);
383	}	139	}
384
385	// System
386
387	pub const ecall = iType(0b1110011, 0b000, .zero, .zero, 0x000);
388	pub const ebreak = iType(0b1110011, 0b000, .zero, .zero, 0x001);
389	pub const unimp = iType(0, 0, .zero, .zero, 0);
390	};	140	};
391		141
392	pub const Register = enum(u6) {	142	pub const Register = enum(u6) {
...	@@ -421,39 +171,52 @@ pub const Register = enum(u6) {	...	@@ -421,39 +171,52 @@ pub const Register = enum(u6) {
421	}	171	}
422	};	172	};
423		173
424	// zig fmt: on	174	pub const FrameIndex = enum(u32) {
425		175	/// This index refers to the return address.
426	test "serialize instructions" {	176	ret_addr,
427	const Testcase = struct {	177	/// This index refers to the frame pointer.
428	inst: Instruction,	178	base_ptr,
429	expected: u32,	179	/// This index refers to the entire stack frame.
430	};	180	stack_frame,
431		181	/// This index referes to where in the stack frame the args are spilled to.
432	const testcases = [_]Testcase{	182	args_frame,
433	.{ // add t6, zero, zero	183	/// This index referes to a frame dedicated to setting up args for function called
434	.inst = Instruction.add(.t6, .zero, .zero),	184	/// in this function. Useful for aligning args separately.
435	.expected = 0b0000000_00000_00000_000_11111_0110011,	185	call_frame,
436	},	186	/// This index referes to the frame where callee saved registers are spilled and restore
437	.{ // sd s0, 0x7f(s0)	187	/// from.
438	.inst = Instruction.sd(.s0, 0x7f, .s0),	188	spill_frame,
439	.expected = 0b0000011_01000_01000_011_11111_0100011,	189	/// Other indices are used for local variable stack slots
440	},	190	_,
441	.{ // bne s0, s1, 0x42	191
442	.inst = Instruction.bne(.s0, .s1, 0x42),	192	pub const named_count = @typeInfo(FrameIndex).Enum.fields.len;
443	.expected = 0b0_000010_01001_01000_001_0001_0_1100011,	193
444	},	194	pub fn isNamed(fi: FrameIndex) bool {
445	.{ // j 0x1a	195	return @intFromEnum(fi) < named_count;
446	.inst = Instruction.jal(.zero, 0x1a),	196	}
447	.expected = 0b0_0000001101_0_00000000_00000_1101111,	197
448	},	198	pub fn format(
449	.{ // ebreak	199	fi: FrameIndex,
450	.inst = Instruction.ebreak,	200	comptime fmt: []const u8,
451	.expected = 0b000000000001_00000_000_00000_1110011,	201	options: std.fmt.FormatOptions,
452	},	202	writer: anytype,
453	};	203	) @TypeOf(writer).Error!void {
454		204	try writer.writeAll("FrameIndex");
455	for (testcases) \|case\| {	205	if (fi.isNamed()) {
456	const actual = case.inst.toU32();	206	try writer.writeByte('.');
457	try testing.expectEqual(case.expected, actual);	207	try writer.writeAll(@tagName(fi));
		208	} else {
		209	try writer.writeByte('(');
		210	try std.fmt.formatType(@intFromEnum(fi), fmt, options, writer, 0);
		211	try writer.writeByte(')');
		212	}
458	}	213	}
459	}	214	};
		215
		216	/// A linker symbol not yet allocated in VM.
		217	pub const Symbol = struct {
		218	/// Index of the containing atom.
		219	atom_index: u32,
		220	/// Index into the linker's symbol table.
		221	sym_index: u32,
		222	};

src/arch/riscv64/encoder.zig created+49

...	@@ -0,0 +1,49 @@
1	pub const Instruction = struct {
2	encoding: Encoding,
3	ops: [4]Operand = .{.none} ** 4,
4
5	pub const Operand = union(enum) {
6	none,
7	reg: Register,
8	mem: Memory,
9	imm: Immediate,
10	};
11
12	pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {
13	const encoding = (try Encoding.findByMnemonic(mnemonic, ops)) orelse {
14	log.err("no encoding found for: {s} {s} {s} {s} {s}", .{
15	@tagName(mnemonic),
16	@tagName(if (ops.len > 0) ops[0] else .none),
17	@tagName(if (ops.len > 1) ops[1] else .none),
18	@tagName(if (ops.len > 2) ops[2] else .none),
19	@tagName(if (ops.len > 3) ops[3] else .none),
20	});
21	return error.InvalidInstruction;
22	};
23
24	var result_ops: [4]Operand = .{.none} ** 4;
25	@memcpy(result_ops[0..ops.len], ops);
26
27	return .{
28	.encoding = encoding,
29	.ops = result_ops,
30	};
31	}
32
33	pub fn encode(inst: Instruction, writer: anytype) !void {
34	try writer.writeInt(u32, inst.encoding.data.toU32(), .little);
35	}
36	};
37
38	const std = @import("std");
39
40	const Lower = @import("Lower.zig");
41	const Mir = @import("Mir.zig");
42	const bits = @import("bits.zig");
43	const Encoding = @import("Encoding.zig");
44
45	const Register = bits.Register;
46	const Memory = bits.Memory;
47	const Immediate = bits.Immediate;
48
49	const log = std.log.scoped(.encode);

src/link/riscv.zig+33-18

 @@ -25,38 +25,52 @@ pub fn writeAddend(
 }
 pub fn writeInstU(code: *[4]u8, value: u32) void {
-    var inst = Instruction{
+    var data = Encoding.Data{
         .U = mem.bytesToValue(std.meta.TagPayload(
-            Instruction,
+            Encoding.Data,
-            Instruction.U,
+            Encoding.Data.U,
         ), code),
     };
     const compensated: u32 = @bitCast(@as(i32, @bitCast(value)) + 0x800);
-    inst.U.imm12_31 = bitSlice(compensated, 31, 12);
+    data.U.imm12_31 = bitSlice(compensated, 31, 12);
-    mem.writeInt(u32, code, inst.toU32(), .little);
+    mem.writeInt(u32, code, data.toU32(), .little);
 }
 pub fn writeInstI(code: *[4]u8, value: u32) void {
-    var inst = Instruction{
+    var data = Encoding.Data{
         .I = mem.bytesToValue(std.meta.TagPayload(
-            Instruction,
+            Encoding.Data,
-            Instruction.I,
+            Encoding.Data.I,
         ), code),
     };
-    inst.I.imm0_11 = bitSlice(value, 11, 0);
+    data.I.imm0_11 = bitSlice(value, 11, 0);
-    mem.writeInt(u32, code, inst.toU32(), .little);
+    mem.writeInt(u32, code, data.toU32(), .little);
 }
 pub fn writeInstS(code: *[4]u8, value: u32) void {
-    var inst = Instruction{
+    var data = Encoding.Data{
         .S = mem.bytesToValue(std.meta.TagPayload(
-            Instruction,
+            Encoding.Data,
-            Instruction.S,
+            Encoding.Data.S,
         ), code),
     };
-    inst.S.imm0_4 = bitSlice(value, 4, 0);
+    data.S.imm0_4 = bitSlice(value, 4, 0);
-    inst.S.imm5_11 = bitSlice(value, 11, 5);
+    data.S.imm5_11 = bitSlice(value, 11, 5);
-    mem.writeInt(u32, code, inst.toU32(), .little);
+    mem.writeInt(u32, code, data.toU32(), .little);
+}
+pub fn writeInstJ(code: *[4]u8, value: u32) void {
+    var data = Encoding.Data{
+        .J = mem.bytesToValue(std.meta.TagPayload(
+            Encoding.Data,
+            Encoding.Data.J,
+        ), code),
+    };
+    data.J.imm1_10 = bitSlice(value, 10, 1);
+    data.J.imm11 = bitSlice(value, 11, 11);
+    data.J.imm12_19 = bitSlice(value, 19, 12);
+    data.J.imm20 = bitSlice(value, 20, 20);
+    mem.writeInt(u32, code, data.toU32(), .little);
 }
 fn bitSlice(
 @@ -67,8 +81,9 @@ fn bitSlice(
     return @truncate((value >> low) & (1 << (high - low + 1)) - 1);
 }
-const bits = @import("../arch/riscv64/bits.zig");
+const encoder = @import("../arch/riscv64/encoder.zig");
+const Encoding = @import("../arch/riscv64/Encoding.zig");
 const mem = std.mem;
 const std = @import("std");
-pub const Instruction = bits.Instruction;
+pub const Instruction = encoder.Instruction;

src/register_manager.zig+1

...	@@ -360,6 +360,7 @@ pub fn RegisterManager(	...	@@ -360,6 +360,7 @@ pub fn RegisterManager(
360	} else self.getRegIndexAssumeFree(tracked_index, inst);	360	} else self.getRegIndexAssumeFree(tracked_index, inst);
361	}	361	}
362	pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {	362	pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
		363	log.debug("getting reg: {}", .{reg});
363	return self.getRegIndex(indexOfRegIntoTracked(reg) orelse return, inst);	364	return self.getRegIndex(indexOfRegIntoTracked(reg) orelse return, inst);
364	}	365	}
365	pub fn getKnownReg(	366	pub fn getKnownReg(

src/target.zig+1-1

...	@@ -526,7 +526,7 @@ pub fn backendSupportsFeature(	...	@@ -526,7 +526,7 @@ pub fn backendSupportsFeature(
526	feature: Feature,	526	feature: Feature,
527	) bool {	527	) bool {
528	return switch (feature) {	528	return switch (feature) {
529	.panic_fn => ofmt == .c or use_llvm or cpu_arch == .x86_64 or cpu_arch == .riscv64,	529	.panic_fn => ofmt == .c or use_llvm or cpu_arch == .x86_64,
530	.panic_unwrap_error => ofmt == .c or use_llvm,	530	.panic_unwrap_error => ofmt == .c or use_llvm,
531	.safety_check_formatted => ofmt == .c or use_llvm,	531	.safety_check_formatted => ofmt == .c or use_llvm,
532	.error_return_trace => use_llvm,	532	.error_return_trace => use_llvm,

test/behavior/align.zig+17

test/behavior/array.zig+95

...	@@ -16,6 +16,7 @@ test "global variable alignment" {	...	@@ -16,6 +16,7 @@ test "global variable alignment" {
16	}	16	}
17		17
18	test "large alignment of local constant" {	18	test "large alignment of local constant" {
		19	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
19	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	20	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
20	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	21	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
21	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // flaky	22	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // flaky
...	@@ -25,6 +26,7 @@ test "large alignment of local constant" {	...	@@ -25,6 +26,7 @@ test "large alignment of local constant" {
25	}	26	}
26		27
27	test "slicing array of length 1 can not assume runtime index is always zero" {	28	test "slicing array of length 1 can not assume runtime index is always zero" {
		29	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
28	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	30	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
29	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	31	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
30	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // flaky	32	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // flaky
...	@@ -42,6 +44,7 @@ test "default alignment allows unspecified in type syntax" {	...	@@ -42,6 +44,7 @@ test "default alignment allows unspecified in type syntax" {
42	}	44	}
43		45
44	test "implicitly decreasing pointer alignment" {	46	test "implicitly decreasing pointer alignment" {
		47	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
45	const a: u32 align(4) = 3;	48	const a: u32 align(4) = 3;
46	const b: u32 align(8) = 4;	49	const b: u32 align(8) = 4;
47	try expect(addUnaligned(&a, &b) == 7);	50	try expect(addUnaligned(&a, &b) == 7);
...	@@ -52,6 +55,7 @@ fn addUnaligned(a: align(1) const u32, b: align(1) const u32) u32 {	...	@@ -52,6 +55,7 @@ fn addUnaligned(a: align(1) const u32, b: align(1) const u32) u32 {
52	}	55	}
53		56
54	test "@alignCast pointers" {	57	test "@alignCast pointers" {
		58	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
55	var x: u32 align(4) = 1;	59	var x: u32 align(4) = 1;
56	expectsOnly1(&x);	60	expectsOnly1(&x);
57	try expect(x == 2);	61	try expect(x == 2);
...	@@ -223,6 +227,7 @@ fn fnWithAlignedStack() i32 {	...	@@ -223,6 +227,7 @@ fn fnWithAlignedStack() i32 {
223	}	227	}
224		228
225	test "implicitly decreasing slice alignment" {	229	test "implicitly decreasing slice alignment" {
		230	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
226	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	231	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
227	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	232	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
228		233
...	@@ -235,6 +240,7 @@ fn addUnalignedSlice(a: []align(1) const u32, b: []align(1) const u32) u32 {	...	@@ -235,6 +240,7 @@ fn addUnalignedSlice(a: []align(1) const u32, b: []align(1) const u32) u32 {
235	}	240	}
236		241
237	test "specifying alignment allows pointer cast" {	242	test "specifying alignment allows pointer cast" {
		243	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
238	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	244	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
239	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	245	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
240		246
...	@@ -247,6 +253,7 @@ fn testBytesAlign(b: u8) !void {	...	@@ -247,6 +253,7 @@ fn testBytesAlign(b: u8) !void {
247	}	253	}
248		254
249	test "@alignCast slices" {	255	test "@alignCast slices" {
		256	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
250	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	257	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
251	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	258	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
252	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	259	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -265,6 +272,7 @@ fn sliceExpects4(slice: []align(4) u32) void {	...	@@ -265,6 +272,7 @@ fn sliceExpects4(slice: []align(4) u32) void {
265	}	272	}
266		273
267	test "return error union with 128-bit integer" {	274	test "return error union with 128-bit integer" {
		275	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
268	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	276	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
269	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	277	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
270	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	278	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -277,6 +285,7 @@ fn give() anyerror!u128 {	...	@@ -277,6 +285,7 @@ fn give() anyerror!u128 {
277	}	285	}
278		286
279	test "page aligned array on stack" {	287	test "page aligned array on stack" {
		288	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
280	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	289	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
281	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	290	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
282	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	291	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -418,6 +427,7 @@ test "function callconv expression depends on generic parameter" {	...	@@ -418,6 +427,7 @@ test "function callconv expression depends on generic parameter" {
418	}	427	}
419		428
420	test "runtime-known array index has best alignment possible" {	429	test "runtime-known array index has best alignment possible" {
		430	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
421	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	431	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
422		432
423	// take full advantage of over-alignment	433	// take full advantage of over-alignment
...	@@ -478,6 +488,7 @@ const DefaultAligned = struct {	...	@@ -478,6 +488,7 @@ const DefaultAligned = struct {
478	};	488	};
479		489
480	test "read 128-bit field from default aligned struct in stack memory" {	490	test "read 128-bit field from default aligned struct in stack memory" {
		491	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
481	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	492	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
482	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	493	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
483	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	494	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -497,6 +508,7 @@ var default_aligned_global = DefaultAligned{	...	@@ -497,6 +508,7 @@ var default_aligned_global = DefaultAligned{
497	};	508	};
498		509
499	test "read 128-bit field from default aligned struct in global memory" {	510	test "read 128-bit field from default aligned struct in global memory" {
		511	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
500	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	512	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
501	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	513	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
502	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	514	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -506,6 +518,7 @@ test "read 128-bit field from default aligned struct in global memory" {	...	@@ -506,6 +518,7 @@ test "read 128-bit field from default aligned struct in global memory" {
506	}	518	}
507		519
508	test "struct field explicit alignment" {	520	test "struct field explicit alignment" {
		521	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
509	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	522	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
510	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	523	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
511	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	524	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -550,6 +563,7 @@ test "align(@alignOf(T)) T does not force resolution of T" {	...	@@ -550,6 +563,7 @@ test "align(@alignOf(T)) T does not force resolution of T" {
550	}	563	}
551		564
552	test "align(N) on functions" {	565	test "align(N) on functions" {
		566	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
553	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	567	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
554	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	568	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
555	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	569	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -595,6 +609,7 @@ test "comptime alloc alignment" {	...	@@ -595,6 +609,7 @@ test "comptime alloc alignment" {
595	}	609	}
596		610
597	test "@alignCast null" {	611	test "@alignCast null" {
		612	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
598	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	613	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
599	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	614	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
600		615
...	@@ -610,6 +625,7 @@ test "alignment of slice element" {	...	@@ -610,6 +625,7 @@ test "alignment of slice element" {
610	}	625	}
611		626
612	test "sub-aligned pointer field access" {	627	test "sub-aligned pointer field access" {
		628	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
613	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	629	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
614	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	630	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
615	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;	631	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
...	@@ -658,6 +674,7 @@ test "alignment of zero-bit types is respected" {	...	@@ -658,6 +674,7 @@ test "alignment of zero-bit types is respected" {
658	}	674	}
659		675
660	test "zero-bit fields in extern struct pad fields appropriately" {	676	test "zero-bit fields in extern struct pad fields appropriately" {
		677	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
661	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	678	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
662	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	679	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
663	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;	680	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;