Commit 8ac239ebcea0eacfd99680d51489371e28266ec3 | zig

author	David Rubin <david@vortan.dev> 2024-03-30 05:49:16-07:00
committer	David Rubin <david@vortan.dev> 2024-05-11 02:17:11-07:00
log	8ac239ebcea0eacfd99680d51489371e28266ec3
tree	c5deabdaab7895a7753c054346ea515b4f5d8213
parent	c0629c3539f1c944636b6cc9cb531113759b089a

riscv: add enough components to get a test runner working

6 files changed, 782 insertions(+), 284 deletions(-)

lib
- compiler
  - test_runner.zig
- std
  - start.zig
src
- arch
  - riscv64
    CodeGen.zig
    Emit.zig
    Mir.zig
    bits.zig

lib/compiler/test_runner.zig+17-3

...	@@ -12,9 +12,9 @@ var cmdline_buffer: [4096]u8 = undefined;	...	@@ -12,9 +12,9 @@ var cmdline_buffer: [4096]u8 = undefined;
12	var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);	12	var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);
13		13
14	pub fn main() void {	14	pub fn main() void {
15	if (builtin.zig_backend == .stage2_aarch64 or	15	if (builtin.zig_backend == .stage2_riscv64) return mainExtraSimple() catch @panic("test failure");
16	builtin.zig_backend == .stage2_riscv64)	16
17	{	17	if (builtin.zig_backend == .stage2_aarch64) {
18	return mainSimple() catch @panic("test failure");	18	return mainSimple() catch @panic("test failure");
19	}	19	}
20		20
...	@@ -249,3 +249,17 @@ pub fn mainSimple() anyerror!void {	...	@@ -249,3 +249,17 @@ pub fn mainSimple() anyerror!void {
249	if (failed != 0) std.process.exit(1);	249	if (failed != 0) std.process.exit(1);
250	}	250	}
251	}	251	}
		252
		253	pub fn mainExtraSimple() !void {
		254	var pass_count: u8 = 0;
		255
		256	for (builtin.test_functions) \|test_fn\| {
		257	test_fn.func() catch \|err\| {
		258	if (err != error.SkipZigTest) {
		259	@panic(test_fn.name);
		260	}
		261	continue;
		262	};
		263	pass_count += 1;
		264	}
		265	}

lib/std/start.zig+9-9

 @@ -20,7 +20,6 @@ pub const simplified_logic =
     builtin.zig_backend == .stage2_x86 or
     builtin.zig_backend == .stage2_aarch64 or
     builtin.zig_backend == .stage2_arm or
-    builtin.zig_backend == .stage2_riscv64 or
     builtin.zig_backend == .stage2_sparc64 or
     builtin.cpu.arch == .spirv32 or
     builtin.cpu.arch == .spirv64;
 @@ -61,6 +60,10 @@ comptime {
                 } else if (@typeInfo(@TypeOf(root.main)).Fn.calling_convention != .C) {
                     @export(main, .{ .name = "main" });
                 }
+            } else if (native_arch.isRISCV()) {
+                if (!@hasDecl(root, "_start")) {
+                    @export(riscv_start, .{ .name = "_start" });
+                }
             } else if (native_os == .windows) {
                 if (!@hasDecl(root, "WinMain") and !@hasDecl(root, "WinMainCRTStartup") and
                     !@hasDecl(root, "wWinMain") and !@hasDecl(root, "wWinMainCRTStartup"))
 @@ -151,14 +154,6 @@ fn exit2(code: usize) noreturn {
                     : "memory", "cc"
                 );
             },
-            .riscv64 => {
-                asm volatile ("ecall"
-                    :
-                    : [number] "{a7}" (94),
-                      [arg1] "{a0}" (code),
-                    : "rcx", "r11", "memory"
-                );
-            },
             .sparc64 => {
                 asm volatile ("ta 0x6d"
                     :
 @@ -212,6 +207,11 @@ fn wasi_start() callconv(.C) void {
     }
 }
+fn riscv_start() callconv(.C) noreturn {
+    const code = @call(.always_inline, callMain, .{});
+    std.process.exit(code);
+}
 fn EfiMain(handle: uefi.Handle, system_table: *uefi.tables.SystemTable) callconv(.C) usize {
     uefi.handle = handle;
     uefi.system_table = system_table;

src/arch/riscv64/CodeGen.zig+727-265

...	@@ -167,9 +167,7 @@ const MCValue = union(enum) {	...	@@ -167,9 +167,7 @@ const MCValue = union(enum) {
167	.immediate,	167	.immediate,
168	.memory,	168	.memory,
169	.ptr_stack_offset,	169	.ptr_stack_offset,
170	.indirect,
171	.undef,	170	.undef,
172	.load_symbol,
173	.addr_symbol,	171	.addr_symbol,
174	.air_ref,	172	.air_ref,
175	=> false,	173	=> false,
...	@@ -178,6 +176,8 @@ const MCValue = union(enum) {	...	@@ -178,6 +176,8 @@ const MCValue = union(enum) {
178	.register_pair,	176	.register_pair,
179	.register_offset,	177	.register_offset,
180	.stack_offset,	178	.stack_offset,
		179	.load_symbol,
		180	.indirect,
181	=> true,	181	=> true,
182	};	182	};
183	}	183	}
...	@@ -265,7 +265,7 @@ const Branch = struct {	...	@@ -265,7 +265,7 @@ const Branch = struct {
265	};	265	};
266		266
267	const StackAllocation = struct {	267	const StackAllocation = struct {
268	inst: Air.Inst.Index,	268	inst: ?Air.Inst.Index,
269	/// TODO: make the size inferred from the bits of the inst	269	/// TODO: make the size inferred from the bits of the inst
270	size: u32,	270	size: u32,
271	};	271	};
...	@@ -410,6 +410,8 @@ pub fn generate(	...	@@ -410,6 +410,8 @@ pub fn generate(
410	// need to at least decrease the sp by -8	410	// need to at least decrease the sp by -8
411	.stack_size = @max(8, mem.alignForward(u32, function.max_end_stack, 16)),	411	.stack_size = @max(8, mem.alignForward(u32, function.max_end_stack, 16)),
412	.save_reg_list = save_reg_list,	412	.save_reg_list = save_reg_list,
		413	.output_mode = lf.comp.config.output_mode,
		414	.link_mode = lf.comp.config.link_mode,
413	};	415	};
414	defer emit.deinit();	416	defer emit.deinit();
415		417
...	@@ -633,7 +635,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -633,7 +635,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
633	.mul_add => try self.airMulAdd(inst),	635	.mul_add => try self.airMulAdd(inst),
634	.addrspace_cast => return self.fail("TODO: addrspace_cast", .{}),	636	.addrspace_cast => return self.fail("TODO: addrspace_cast", .{}),
635		637
636	.@"try" => return self.fail("TODO: try", .{}),	638	.@"try" => try self.airTry(inst),
637	.try_ptr => return self.fail("TODO: try_ptr", .{}),	639	.try_ptr => return self.fail("TODO: try_ptr", .{}),
638		640
639	.dbg_var_ptr,	641	.dbg_var_ptr,
...	@@ -846,7 +848,7 @@ fn symbolIndex(self: *Self) !u32 {	...	@@ -846,7 +848,7 @@ fn symbolIndex(self: *Self) !u32 {
846	};	848	};
847	}	849	}
848		850
849	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: Alignment) !u32 {	851	fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: Alignment) !u32 {
850	self.stack_align = self.stack_align.max(abi_align);	852	self.stack_align = self.stack_align.max(abi_align);
851	// TODO find a free slot instead of always appending	853	// TODO find a free slot instead of always appending
852	const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset));	854	const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset));
...	@@ -905,6 +907,36 @@ fn allocReg(self: *Self) !struct { Register, RegisterLock } {	...	@@ -905,6 +907,36 @@ fn allocReg(self: *Self) !struct { Register, RegisterLock } {
905	return .{ reg, lock };	907	return .{ reg, lock };
906	}	908	}
907		909
		910	fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {
		911	log.debug("elemOffset: {}", .{index});
		912	const reg: Register = blk: {
		913	switch (index) {
		914	.immediate => \|imm\| {
		915	// Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
		916	// and set the register directly to the scaled offset as an immediate.
		917	const reg = try self.register_manager.allocReg(null, gp);
		918	try self.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });
		919	break :blk reg;
		920	},
		921	else => {
		922	const reg = try self.copyToTmpRegister(index_ty, index);
		923	const lock = self.register_manager.lockRegAssumeUnused(reg);
		924	defer self.register_manager.unlockReg(lock);
		925
		926	try self.binOpMir(
		927	.mul,
		928	null,
		929	index_ty,
		930	.{ .register = reg },
		931	.{ .immediate = elem_size },
		932	);
		933	break :blk reg;
		934	},
		935	}
		936	};
		937	return reg;
		938	}
		939
908	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {	940	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
909	const mod = self.bin_file.comp.module.?;	941	const mod = self.bin_file.comp.module.?;
910	const elem_ty = self.typeOfIndex(inst);	942	const elem_ty = self.typeOfIndex(inst);
...	@@ -1104,6 +1136,22 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	...	@@ -1104,6 +1136,22 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1104	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1136	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1105	}	1137	}
1106		1138
		1139	fn supportImmediate(tag: Air.Inst.Tag) bool {
		1140	return switch (tag) {
		1141	.add,
		1142	.sub,
		1143	.cmp_eq,
		1144	.cmp_neq,
		1145	.cmp_gt,
		1146	.cmp_gte,
		1147	.cmp_lt,
		1148	.cmp_lte,
		1149	=> true,
		1150
		1151	else => false,
		1152	};
		1153	}
		1154
1107	/// For all your binary operation needs, this function will generate	1155	/// For all your binary operation needs, this function will generate
1108	/// the corresponding Mir instruction(s). Returns the location of the	1156	/// the corresponding Mir instruction(s). Returns the location of the
1109	/// result.	1157	/// result.
...	@@ -1132,6 +1180,7 @@ fn binOp(	...	@@ -1132,6 +1180,7 @@ fn binOp(
1132	// Arithmetic operations on integers and floats	1180	// Arithmetic operations on integers and floats
1133	.add,	1181	.add,
1134	.sub,	1182	.sub,
		1183	.mul,
1135	.cmp_eq,	1184	.cmp_eq,
1136	.cmp_neq,	1185	.cmp_neq,
1137	.cmp_gt,	1186	.cmp_gt,
...	@@ -1146,7 +1195,7 @@ fn binOp(	...	@@ -1146,7 +1195,7 @@ fn binOp(
1146	assert(lhs_ty.eql(rhs_ty, mod));	1195	assert(lhs_ty.eql(rhs_ty, mod));
1147	const int_info = lhs_ty.intInfo(mod);	1196	const int_info = lhs_ty.intInfo(mod);
1148	if (int_info.bits <= 64) {	1197	if (int_info.bits <= 64) {
1149	if (rhs == .immediate) {	1198	if (rhs == .immediate and supportImmediate(tag)) {
1150	return self.binOpImm(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);	1199	return self.binOpImm(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
1151	}	1200	}
1152	return self.binOpRegister(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);	1201	return self.binOpRegister(tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
...	@@ -1154,9 +1203,10 @@ fn binOp(	...	@@ -1154,9 +1203,10 @@ fn binOp(
1154	return self.fail("TODO binary operations on int with bits > 64", .{});	1203	return self.fail("TODO binary operations on int with bits > 64", .{});
1155	}	1204	}
1156	},	1205	},
1157	else => unreachable,	1206	else => \|x\| return self.fail("TOOD: binOp {s}", .{@tagName(x)}),
1158	}	1207	}
1159	},	1208	},
		1209
1160	.ptr_add,	1210	.ptr_add,
1161	.ptr_sub,	1211	.ptr_sub,
1162	=> {	1212	=> {
...	@@ -1178,7 +1228,24 @@ fn binOp(	...	@@ -1178,7 +1228,24 @@ fn binOp(
1178		1228
1179	return try self.binOpRegister(base_tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);	1229	return try self.binOpRegister(base_tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
1180	} else {	1230	} else {
1181	return self.fail("TODO ptr_add with elem_size > 1", .{});	1231	const offset = try self.binOp(
		1232	.mul,
		1233	null,
		1234	rhs,
		1235	.{ .immediate = elem_size },
		1236	Type.usize,
		1237	Type.usize,
		1238	);
		1239
		1240	const addr = try self.binOp(
		1241	tag,
		1242	null,
		1243	lhs,
		1244	offset,
		1245	Type.manyptr_u8,
		1246	Type.usize,
		1247	);
		1248	return addr;
1182	}	1249	}
1183	},	1250	},
1184	else => unreachable,	1251	else => unreachable,
...	@@ -1206,7 +1273,7 @@ fn binOp(	...	@@ -1206,7 +1273,7 @@ fn binOp(
1206	else => unreachable,	1273	else => unreachable,
1207	}	1274	}
1208	},	1275	},
1209	else => unreachable,	1276	else => return self.fail("TODO binOp {}", .{tag}),
1210	}	1277	}
1211	}	1278	}
1212	/// Don't call this function directly. Use binOp instead.	1279	/// Don't call this function directly. Use binOp instead.
...	@@ -1252,6 +1319,7 @@ fn binOpRegister(	...	@@ -1252,6 +1319,7 @@ fn binOpRegister(
1252	const mir_tag: Mir.Inst.Tag = switch (tag) {	1319	const mir_tag: Mir.Inst.Tag = switch (tag) {
1253	.add => .add,	1320	.add => .add,
1254	.sub => .sub,	1321	.sub => .sub,
		1322	.mul => .mul,
1255	.cmp_eq => .cmp_eq,	1323	.cmp_eq => .cmp_eq,
1256	.cmp_neq => .cmp_neq,	1324	.cmp_neq => .cmp_neq,
1257	.cmp_gt => .cmp_gt,	1325	.cmp_gt => .cmp_gt,
...	@@ -1314,7 +1382,9 @@ fn binOpImm(	...	@@ -1314,7 +1382,9 @@ fn binOpImm(
1314	.shr => .srli,	1382	.shr => .srli,
1315	.cmp_gte => .cmp_imm_gte,	1383	.cmp_gte => .cmp_imm_gte,
1316	.cmp_eq => .cmp_imm_eq,	1384	.cmp_eq => .cmp_imm_eq,
		1385	.cmp_neq => .cmp_imm_neq,
1317	.cmp_lte => .cmp_imm_lte,	1386	.cmp_lte => .cmp_imm_lte,
		1387	.cmp_lt => .cmp_imm_lt,
1318	.add => .addi,	1388	.add => .addi,
1319	.sub => .addiw,	1389	.sub => .addiw,
1320	else => return self.fail("TODO: binOpImm {s}", .{@tagName(tag)}),	1390	else => return self.fail("TODO: binOpImm {s}", .{@tagName(tag)}),
...	@@ -1326,7 +1396,9 @@ fn binOpImm(	...	@@ -1326,7 +1396,9 @@ fn binOpImm(
1326	.srli,	1396	.srli,
1327	.addi,	1397	.addi,
1328	.cmp_imm_eq,	1398	.cmp_imm_eq,
		1399	.cmp_imm_neq,
1329	.cmp_imm_lte,	1400	.cmp_imm_lte,
		1401	.cmp_imm_lt,
1330	=> {	1402	=> {
1331	_ = try self.addInst(.{	1403	_ = try self.addInst(.{
1332	.tag = mir_tag,	1404	.tag = mir_tag,
...	@@ -1369,6 +1441,40 @@ fn binOpImm(	...	@@ -1369,6 +1441,40 @@ fn binOpImm(
1369	return MCValue{ .register = dest_reg };	1441	return MCValue{ .register = dest_reg };
1370	}	1442	}
1371		1443
		1444	fn binOpMir(
		1445	self: *Self,
		1446	mir_tag: Mir.Inst.Tag,
		1447	maybe_inst: ?Air.Inst.Index,
		1448	ty: Type,
		1449	dst_mcv: MCValue,
		1450	src_mcv: MCValue,
		1451	) !void {
		1452	const mod = self.bin_file.comp.module.?;
		1453	const abi_size: u32 = @intCast(ty.abiSize(mod));
		1454
		1455	_ = abi_size;
		1456	_ = maybe_inst;
		1457
		1458	switch (dst_mcv) {
		1459	.register => \|dst_reg\| {
		1460	const src_reg = try self.copyToTmpRegister(ty, src_mcv);
		1461
		1462	_ = try self.addInst(.{
		1463	.tag = mir_tag,
		1464	.data = .{
		1465	.r_type = .{
		1466	.rd = dst_reg,
		1467	.rs1 = dst_reg,
		1468	.rs2 = src_reg,
		1469	},
		1470	},
		1471	});
		1472	},
		1473
		1474	else => return self.fail("TODO: binOpMir {s}", .{@tagName(dst_mcv)}),
		1475	}
		1476	}
		1477
1372	fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {	1478	fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1373	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	1479	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1374	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;	1480	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
...	@@ -1520,8 +1626,101 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1520,8 +1626,101 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1520	}	1626	}
1521		1627
1522	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	1628	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1523	_ = inst;	1629	//const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
1524	return self.fail("TODO implement airMulWithOverflow for {}", .{self.target.cpu.arch});	1630	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
		1631	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
		1632	const mod = self.bin_file.comp.module.?;
		1633	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
		1634	const lhs = try self.resolveInst(extra.lhs);
		1635	const rhs = try self.resolveInst(extra.rhs);
		1636	const lhs_ty = self.typeOf(extra.lhs);
		1637	const rhs_ty = self.typeOf(extra.rhs);
		1638
		1639	switch (lhs_ty.zigTypeTag(mod)) {
		1640	else => \|x\| return self.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}),
		1641	.Int => {
		1642	assert(lhs_ty.eql(rhs_ty, mod));
		1643	const int_info = lhs_ty.intInfo(mod);
		1644	switch (int_info.bits) {
		1645	1...32 => {
		1646	if (self.hasFeature(.m)) {
		1647	const dest = try self.binOp(.mul, null, lhs, rhs, lhs_ty, rhs_ty);
		1648
		1649	const add_result_lock = self.register_manager.lockRegAssumeUnused(dest.register);
		1650	defer self.register_manager.unlockReg(add_result_lock);
		1651
		1652	const tuple_ty = self.typeOfIndex(inst);
		1653
		1654	// TODO: optimization, set this to true. needs the other struct access stuff to support
		1655	// accessing registers.
		1656	const result_mcv = try self.allocRegOrMem(inst, false);
		1657	const offset = result_mcv.stack_offset;
		1658
		1659	const result_offset = tuple_ty.structFieldOffset(0, mod) + offset;
		1660
		1661	try self.genSetStack(lhs_ty, @intCast(result_offset), dest);
		1662
		1663	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
		1664	if (int_info.signedness == .unsigned) {
		1665	switch (int_info.bits) {
		1666	1...8 => {
		1667	const max_val = std.math.pow(u16, 2, int_info.bits) - 1;
		1668
		1669	const overflow_reg, const overflow_lock = try self.allocReg();
		1670	defer self.register_manager.unlockReg(overflow_lock);
		1671
		1672	const add_reg, const add_lock = blk: {
		1673	if (dest == .register) break :blk .{ dest.register, null };
		1674
		1675	const add_reg, const add_lock = try self.allocReg();
		1676	try self.genSetReg(lhs_ty, add_reg, dest);
		1677	break :blk .{ add_reg, add_lock };
		1678	};
		1679	defer if (add_lock) \|lock\| self.register_manager.unlockReg(lock);
		1680
		1681	_ = try self.addInst(.{
		1682	.tag = .andi,
		1683	.data = .{ .i_type = .{
		1684	.rd = overflow_reg,
		1685	.rs1 = add_reg,
		1686	.imm12 = @intCast(max_val),
		1687	} },
		1688	});
		1689
		1690	const overflow_mcv = try self.binOp(
		1691	.cmp_neq,
		1692	null,
		1693	.{ .register = overflow_reg },
		1694	.{ .register = add_reg },
		1695	lhs_ty,
		1696	lhs_ty,
		1697	);
		1698
		1699	const overflow_offset = tuple_ty.structFieldOffset(1, mod) + offset;
		1700	try self.genSetStack(Type.u1, @intCast(overflow_offset), overflow_mcv);
		1701
		1702	break :result result_mcv;
		1703	},
		1704
		1705	else => return self.fail("TODO: airMulWithOverflow check for size {d}", .{int_info.bits}),
		1706	}
		1707	} else {
		1708	return self.fail("TODO: airMulWithOverflow calculate carry for signed addition", .{});
		1709	}
		1710	} else {
		1711	return self.fail("TODO: airMulWithOverflow with < 8 bits or non-pow of 2", .{});
		1712	}
		1713	} else {
		1714	return self.fail("TODO: emulate mul for targets without M feature", .{});
		1715	}
		1716	},
		1717	else => return self.fail("TODO: airMulWithOverflow larger than 32-bit mul", .{}),
		1718	}
		1719	},
		1720	}
		1721	};
		1722
		1723	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1525	}	1724	}
1526		1725
1527	fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	1726	fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -1610,16 +1809,98 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1610,16 +1809,98 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
1610		1809
1611	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {	1810	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
1612	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1811	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1613	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error for {}", .{self.target.cpu.arch});	1812	const mod = self.bin_file.comp.module.?;
		1813	const err_union_ty = self.typeOf(ty_op.operand);
		1814	const err_ty = err_union_ty.errorUnionSet(mod);
		1815	const payload_ty = err_union_ty.errorUnionPayload(mod);
		1816	const operand = try self.resolveInst(ty_op.operand);
		1817
		1818	const result: MCValue = result: {
		1819	if (err_ty.errorSetIsEmpty(mod)) {
		1820	break :result .{ .immediate = 0 };
		1821	}
		1822
		1823	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
		1824	break :result operand;
		1825	}
		1826
		1827	const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, mod));
		1828
		1829	switch (operand) {
		1830	.register => \|reg\| {
		1831	const eu_lock = self.register_manager.lockReg(reg);
		1832	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
		1833
		1834	var result = try self.copyToNewRegister(inst, operand);
		1835
		1836	if (err_off > 0) {
		1837	result = try self.binOp(
		1838	.shr,
		1839	null,
		1840	result,
		1841	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
		1842	err_union_ty,
		1843	Type.u8,
		1844	);
		1845	}
		1846	break :result result;
		1847	},
		1848	else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),
		1849	}
		1850	};
		1851
1614	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	1852	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1615	}	1853	}
1616		1854
1617	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {	1855	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
1618	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1856	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1619	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload for {}", .{self.target.cpu.arch});	1857	const operand_ty = self.typeOf(ty_op.operand);
		1858	const operand = try self.resolveInst(ty_op.operand);
		1859	const result = try self.genUnwrapErrUnionPayloadMir(operand_ty, operand);
1620	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	1860	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1621	}	1861	}
1622		1862
		1863	fn genUnwrapErrUnionPayloadMir(
		1864	self: *Self,
		1865	err_union_ty: Type,
		1866	err_union: MCValue,
		1867	) !MCValue {
		1868	const mod = self.bin_file.comp.module.?;
		1869
		1870	const payload_ty = err_union_ty.errorUnionPayload(mod);
		1871
		1872	const result: MCValue = result: {
		1873	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;
		1874
		1875	const payload_off: u32 = @intCast(errUnionPayloadOffset(payload_ty, mod));
		1876	switch (err_union) {
		1877	.stack_offset => \|off\| break :result .{ .stack_offset = off + payload_off },
		1878	.register => \|reg\| {
		1879	const eu_lock = self.register_manager.lockReg(reg);
		1880	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
		1881
		1882	var result: MCValue = .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
		1883
		1884	if (payload_off > 0) {
		1885	result = try self.binOp(
		1886	.shr,
		1887	null,
		1888	result,
		1889	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },
		1890	err_union_ty,
		1891	Type.u8,
		1892	);
		1893	}
		1894
		1895	break :result result;
		1896	},
		1897	else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
		1898	}
		1899	};
		1900
		1901	return result;
		1902	}
		1903
1623	// *(E!T) -> E	1904	// *(E!T) -> E
1624	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {	1905	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1625	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1906	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	@@ -1682,11 +1963,108 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1682,11 +1963,108 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
1682		1963
1683	/// E to E!T	1964	/// E to E!T
1684	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	1965	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
		1966	const mod = self.bin_file.comp.module.?;
1685	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	1967	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1686	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion error for {}", .{self.target.cpu.arch});	1968
		1969	const eu_ty = ty_op.ty.toType();
		1970	const pl_ty = eu_ty.errorUnionPayload(mod);
		1971	const err_ty = eu_ty.errorUnionSet(mod);
		1972
		1973	const result: MCValue = result: {
		1974	if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);
		1975
		1976	const stack_off = try self.allocMem(null, @intCast(eu_ty.abiSize(mod)), eu_ty.abiAlignment(mod));
		1977	const pl_off: u32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
		1978	const err_off: u32 = @intCast(errUnionErrorOffset(pl_ty, mod));
		1979	try self.genSetStack(pl_ty, stack_off + pl_off, .undef);
		1980	const operand = try self.resolveInst(ty_op.operand);
		1981	try self.genSetStack(err_ty, stack_off + err_off, operand);
		1982	break :result .{ .stack_offset = stack_off };
		1983	};
1687	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	1984	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1688	}	1985	}
1689		1986
		1987	fn airTry(self: *Self, inst: Air.Inst.Index) !void {
		1988	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
		1989	const extra = self.air.extraData(Air.Try, pl_op.payload);
		1990	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
		1991	const operand_ty = self.typeOf(pl_op.operand);
		1992	const result = try self.genTry(inst, pl_op.operand, body, operand_ty, false);
		1993	return self.finishAir(inst, result, .{ .none, .none, .none });
		1994	}
		1995
		1996	fn genTry(
		1997	self: *Self,
		1998	inst: Air.Inst.Index,
		1999	operand: Air.Inst.Ref,
		2000	body: []const Air.Inst.Index,
		2001	operand_ty: Type,
		2002	operand_is_ptr: bool,
		2003	) !MCValue {
		2004	const liveness_condbr = self.liveness.getCondBr(inst);
		2005
		2006	_ = operand_is_ptr;
		2007
		2008	const operand_mcv = try self.resolveInst(operand);
		2009	const is_err_mcv = try self.isErr(null, operand_ty, operand_mcv);
		2010
		2011	const cond_reg = try self.register_manager.allocReg(inst, gp);
		2012	const cond_reg_lock = self.register_manager.lockRegAssumeUnused(cond_reg);
		2013	defer self.register_manager.unlockReg(cond_reg_lock);
		2014
		2015	// A branch to the false section. Uses beq. 1 is the default "true" state.
		2016	const reloc = try self.condBr(Type.anyerror, is_err_mcv, cond_reg);
		2017
		2018	if (self.liveness.operandDies(inst, 0)) {
		2019	if (operand.toIndex()) \|op_inst\| self.processDeath(op_inst);
		2020	}
		2021
		2022	// Save state
		2023	const parent_next_stack_offset = self.next_stack_offset;
		2024	const parent_free_registers = self.register_manager.free_registers;
		2025	var parent_stack = try self.stack.clone(self.gpa);
		2026	defer parent_stack.deinit(self.gpa);
		2027	const parent_registers = self.register_manager.registers;
		2028
		2029	try self.branch_stack.append(.{});
		2030	errdefer {
		2031	_ = self.branch_stack.pop();
		2032	}
		2033
		2034	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
		2035	for (liveness_condbr.else_deaths) \|op\| {
		2036	self.processDeath(op);
		2037	}
		2038
		2039	try self.genBody(body);
		2040
		2041	// Restore state
		2042	var saved_then_branch = self.branch_stack.pop();
		2043	defer saved_then_branch.deinit(self.gpa);
		2044
		2045	self.register_manager.registers = parent_registers;
		2046
		2047	self.stack.deinit(self.gpa);
		2048	self.stack = parent_stack;
		2049	parent_stack = .{};
		2050
		2051	self.next_stack_offset = parent_next_stack_offset;
		2052	self.register_manager.free_registers = parent_free_registers;
		2053
		2054	try self.performReloc(reloc, @intCast(self.mir_instructions.len));
		2055
		2056	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
		2057	for (liveness_condbr.then_deaths) \|op\| {
		2058	self.processDeath(op);
		2059	}
		2060
		2061	const result = if (self.liveness.isUnused(inst))
		2062	.unreach
		2063	else
		2064	try self.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv);
		2065	return result;
		2066	}
		2067
1690	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {	2068	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
1691	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2069	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1692	const result = result: {	2070	const result = result: {
...	@@ -1742,9 +2120,36 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1742,9 +2120,36 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
1742	}	2120	}
1743		2121
1744	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	2122	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
		2123	const mod = self.bin_file.comp.module.?;
1745	const is_volatile = false; // TODO	2124	const is_volatile = false; // TODO
1746	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2125	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1747	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_val for {}", .{self.target.cpu.arch});	2126
		2127	if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
		2128	const result: MCValue = result: {
		2129	const slice_mcv = try self.resolveInst(bin_op.lhs);
		2130	const index_mcv = try self.resolveInst(bin_op.rhs);
		2131
		2132	const slice_ty = self.typeOf(bin_op.lhs);
		2133
		2134	const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
		2135
		2136	const index_lock: ?RegisterLock = if (index_mcv == .register)
		2137	self.register_manager.lockRegAssumeUnused(index_mcv.register)
		2138	else
		2139	null;
		2140	defer if (index_lock) \|reg\| self.register_manager.unlockReg(reg);
		2141
		2142	const base_mcv: MCValue = switch (slice_mcv) {
		2143	.stack_offset => \|off\| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
		2144	else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
		2145	};
		2146
		2147	const dest = try self.allocRegOrMem(inst, true);
		2148	const addr = try self.binOp(.ptr_add, null, base_mcv, index_mcv, slice_ptr_field_type, Type.usize);
		2149	try self.load(dest, addr, slice_ptr_field_type);
		2150
		2151	break :result dest;
		2152	};
1748	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2153	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1749	}	2154	}
1750		2155
...	@@ -1763,21 +2168,44 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1763,21 +2168,44 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
1763	const array_mcv = try self.resolveInst(bin_op.lhs);	2168	const array_mcv = try self.resolveInst(bin_op.lhs);
1764		2169
1765	const index_mcv = try self.resolveInst(bin_op.rhs);	2170	const index_mcv = try self.resolveInst(bin_op.rhs);
		2171	const index_ty = self.typeOf(bin_op.rhs);
1766		2172
1767	const elem_ty = array_ty.childType(mod);	2173	const elem_ty = array_ty.childType(mod);
1768	const elem_abi_size = elem_ty.abiSize(mod);	2174	const elem_abi_size = elem_ty.abiSize(mod);
1769		2175
		2176	const addr_reg, const addr_reg_lock = try self.allocReg();
		2177	defer self.register_manager.unlockReg(addr_reg_lock);
		2178
1770	switch (array_mcv) {	2179	switch (array_mcv) {
1771	// all we need to do is calculate the offset that the elem exits at.	2180	.register => {
		2181	const stack_offset = try self.allocMem(
		2182	null,
		2183	@intCast(array_ty.abiSize(mod)),
		2184	array_ty.abiAlignment(mod),
		2185	);
		2186	try self.genSetStack(array_ty, stack_offset, array_mcv);
		2187	try self.genSetReg(Type.usize, addr_reg, .{ .ptr_stack_offset = stack_offset });
		2188	},
1772	.stack_offset => \|off\| {	2189	.stack_offset => \|off\| {
1773	if (index_mcv == .immediate) {	2190	try self.genSetReg(Type.usize, addr_reg, .{ .ptr_stack_offset = off });
1774	const true_offset: u32 = @intCast(index_mcv.immediate * elem_abi_size);
1775	break :result MCValue{ .stack_offset = off + true_offset };
1776	}
1777	return self.fail("TODO: airArrayElemVal with runtime index", .{});
1778	},	2191	},
1779	else => return self.fail("TODO: airArrayElemVal {s}", .{@tagName(array_mcv)}),	2192	else => try self.genSetReg(Type.usize, addr_reg, array_mcv.address()),
1780	}	2193	}
		2194
		2195	const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size);
		2196	const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
		2197	defer self.register_manager.unlockReg(offset_lock);
		2198
		2199	const dst_mcv = try self.allocRegOrMem(inst, false);
		2200	try self.binOpMir(
		2201	.add,
		2202	null,
		2203	Type.usize,
		2204	.{ .register = addr_reg },
		2205	.{ .register = offset_reg },
		2206	);
		2207	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
		2208	break :result dst_mcv;
1781	};	2209	};
1782	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2210	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1783	}	2211	}
...	@@ -2094,54 +2522,37 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -2094,54 +2522,37 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
2094	}	2522	}
2095		2523
2096	/// Loads `value` into the "payload" of `pointer`.	2524	/// Loads `value` into the "payload" of `pointer`.
2097	fn store(self: *Self, pointer: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) !void {	2525	fn store(self: *Self, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty: Type) !void {
2098	const mod = self.bin_file.comp.module.?;	2526	const mod = self.bin_file.comp.module.?;
2099	const value_abi_size = value_ty.abiSize(mod);
2100		2527
2101	log.debug("storing {}:{} in {}:{}", .{ value, value_ty.fmt(mod), pointer, ptr_ty.fmt(mod) });	2528	log.debug("storing {}:{} in {}:{}", .{ src_mcv, src_ty.fmt(mod), ptr_mcv, ptr_ty.fmt(mod) });
2102		2529
2103	switch (pointer) {	2530	switch (ptr_mcv) {
2104	.none => unreachable,	2531	.none => unreachable,
2105	.undef => unreachable,	2532	.undef => unreachable,
2106	.unreach => unreachable,	2533	.unreach => unreachable,
2107	.dead => unreachable,	2534	.dead => unreachable,
2108	.ptr_stack_offset => \|off\| try self.genSetStack(value_ty, off, value),	2535	.register_pair => unreachable,
2109
2110	.stack_offset => {
2111	const pointer_reg, const lock = try self.allocReg();
2112	defer self.register_manager.unlockReg(lock);
2113
2114	try self.genSetReg(ptr_ty, pointer_reg, pointer);
2115
2116	return self.store(.{ .register = pointer_reg }, value, ptr_ty, value_ty);
2117	},
2118		2536
2119	.register => \|reg\| {	2537	.immediate,
2120	const value_reg = try self.copyToTmpRegister(value_ty, value);	2538	.register,
		2539	.register_offset,
		2540	.addr_symbol,
		2541	.ptr_stack_offset,
		2542	=> try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
2121		2543
2122	switch (value_abi_size) {	2544	.memory,
2123	1, 2, 4, 8 => {	2545	.indirect,
2124	const tag: Mir.Inst.Tag = switch (value_abi_size) {	2546	.load_symbol,
2125	1 => .sb,	2547	.stack_offset,
2126	2 => .sh,	2548	=> {
2127	4 => .sw,	2549	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
2128	8 => .sd,	2550	const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
2129	else => unreachable,	2551	defer self.register_manager.unlockReg(addr_lock);
2130	};
2131		2552
2132	_ = try self.addInst(.{	2553	try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
2133	.tag = tag,
2134	.data = .{ .i_type = .{
2135	.rd = value_reg,
2136	.rs1 = reg,
2137	.imm12 = 0,
2138	} },
2139	});
2140	},
2141	else => return self.fail("TODO: genSetStack for size={d}", .{value_abi_size}),
2142	}
2143	},	2554	},
2144	else => return self.fail("TODO implement storing to MCValue.{s}", .{@tagName(pointer)}),	2555	.air_ref => \|ptr_ref\| try self.store(try self.resolveInst(ptr_ref), src_mcv, ptr_ty, src_ty),
2145	}	2556	}
2146	}	2557	}
2147		2558
...	@@ -2405,45 +2816,58 @@ fn genCall(	...	@@ -2405,45 +2816,58 @@ fn genCall(
2405	// Due to incremental compilation, how function calls are generated depends	2816	// Due to incremental compilation, how function calls are generated depends
2406	// on linking.	2817	// on linking.
2407	switch (info) {	2818	switch (info) {
2408	.air => \|callee\| if (try self.air.value(callee, mod)) \|func_value\| {	2819	.air => \|callee\| {
2409	const func_key = mod.intern_pool.indexToKey(func_value.ip_index);	2820	if (try self.air.value(callee, mod)) \|func_value\| {
2410	switch (switch (func_key) {	2821	const func_key = mod.intern_pool.indexToKey(func_value.ip_index);
2411	else => func_key,	2822	switch (switch (func_key) {
2412	.ptr => \|ptr\| switch (ptr.addr) {
2413	.decl => \|decl\| mod.intern_pool.indexToKey(mod.declPtr(decl).val.toIntern()),
2414	else => func_key,	2823	else => func_key,
2415	},	2824	.ptr => \|ptr\| switch (ptr.addr) {
2416	}) {	2825	.decl => \|decl\| mod.intern_pool.indexToKey(mod.declPtr(decl).val.toIntern()),
2417	.func => \|func\| {	2826	else => func_key,
2418	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {	2827	},
2419	const sym_index = try elf_file.zigObjectPtr().?.getOrCreateMetadataForDecl(elf_file, func.owner_decl);	2828	}) {
2420	const sym = elf_file.symbol(sym_index);	2829	.func => \|func\| {
2421	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);	2830	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
2422	const got_addr = sym.zigGotAddress(elf_file);	2831	const sym_index = try elf_file.zigObjectPtr().?.getOrCreateMetadataForDecl(elf_file, func.owner_decl);
2423	try self.genSetReg(Type.usize, .ra, .{ .memory = got_addr });	2832	const sym = elf_file.symbol(sym_index);
2424	_ = try self.addInst(.{	2833	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
2425	.tag = .jalr,	2834	const got_addr = sym.zigGotAddress(elf_file);
2426	.data = .{ .i_type = .{	2835	try self.genSetReg(Type.usize, .ra, .{ .memory = got_addr });
2427	.rd = .ra,	2836	_ = try self.addInst(.{
2428	.rs1 = .ra,	2837	.tag = .jalr,
2429	.imm12 = 0,	2838	.data = .{ .i_type = .{
2430	} },	2839	.rd = .ra,
2431	});	2840	.rs1 = .ra,
2432	} else if (self.bin_file.cast(link.File.Coff)) \|_\| {	2841	.imm12 = 0,
2433	return self.fail("TODO implement calling in COFF for {}", .{self.target.cpu.arch});	2842	} },
2434	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {	2843	});
2435	unreachable; // unsupported architecture for MachO	2844	} else if (self.bin_file.cast(link.File.Coff)) \|_\| {
2436	} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {	2845	return self.fail("TODO implement calling in COFF for {}", .{self.target.cpu.arch});
2437	return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});	2846	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
2438	} else unreachable;	2847	unreachable; // unsupported architecture for MachO
2439	},	2848	} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {
2440	.extern_func => {	2849	return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});
2441	return self.fail("TODO: extern func calls", .{});	2850	} else unreachable;
2442	},	2851	},
2443	else => return self.fail("TODO implement calling bitcasted functions", .{}),	2852	.extern_func => {
		2853	return self.fail("TODO: extern func calls", .{});
		2854	},
		2855	else => return self.fail("TODO implement calling bitcasted functions", .{}),
		2856	}
		2857	} else {
		2858	assert(self.typeOf(callee).zigTypeTag(mod) == .Pointer);
		2859	const addr_reg, const addr_lock = try self.allocReg();
		2860	defer self.register_manager.unlockReg(addr_lock);
		2861	try self.genSetReg(Type.usize, addr_reg, .{ .air_ref = callee });
		2862	_ = try self.addInst(.{
		2863	.tag = .jalr,
		2864	.data = .{ .i_type = .{
		2865	.rd = .ra,
		2866	.rs1 = addr_reg,
		2867	.imm12 = 0,
		2868	} },
		2869	});
2444	}	2870	}
2445	} else {
2446	return self.fail("TODO: call function pointers", .{});
2447	},	2871	},
2448	.lib => return self.fail("TODO: lib func calls", .{}),	2872	.lib => return self.fail("TODO: lib func calls", .{}),
2449	}	2873	}
...	@@ -2506,21 +2930,41 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2506,21 +2930,41 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
2506	fn airCmp(self: *Self, inst: Air.Inst.Index) !void {	2930	fn airCmp(self: *Self, inst: Air.Inst.Index) !void {
2507	const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];	2931	const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
2508	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	2932	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2509	if (self.liveness.isUnused(inst))
2510	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2511	const ty = self.typeOf(bin_op.lhs);
2512	const mod = self.bin_file.comp.module.?;	2933	const mod = self.bin_file.comp.module.?;
2513	assert(ty.eql(self.typeOf(bin_op.rhs), mod));
2514		2934
2515	if (ty.zigTypeTag(mod) == .ErrorSet)	2935	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2516	return self.fail("TODO implement cmp for errors", .{});	2936	const lhs = try self.resolveInst(bin_op.lhs);
		2937	const rhs = try self.resolveInst(bin_op.rhs);
		2938	const lhs_ty = self.typeOf(bin_op.lhs);
2517		2939
2518	const lhs = try self.resolveInst(bin_op.lhs);	2940	const int_ty = switch (lhs_ty.zigTypeTag(mod)) {
2519	const rhs = try self.resolveInst(bin_op.rhs);	2941	.Vector => unreachable, // Handled by cmp_vector.
2520	const lhs_ty = self.typeOf(bin_op.lhs);	2942	.Enum => lhs_ty.intTagType(mod),
2521	const rhs_ty = self.typeOf(bin_op.rhs);	2943	.Int => lhs_ty,
		2944	.Bool => Type.u1,
		2945	.Pointer => Type.usize,
		2946	.ErrorSet => Type.u16,
		2947	.Optional => blk: {
		2948	const payload_ty = lhs_ty.optionalChild(mod);
		2949	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
		2950	break :blk Type.u1;
		2951	} else if (lhs_ty.isPtrLikeOptional(mod)) {
		2952	break :blk Type.usize;
		2953	} else {
		2954	return self.fail("TODO riscv cmp non-pointer optionals", .{});
		2955	}
		2956	},
		2957	.Float => return self.fail("TODO riscv cmp floats", .{}),
		2958	else => unreachable,
		2959	};
2522		2960
2523	const result = try self.binOp(tag, null, lhs, rhs, lhs_ty, rhs_ty);	2961	const int_info = int_ty.intInfo(mod);
		2962	if (int_info.bits <= 64) {
		2963	break :result try self.binOp(tag, null, lhs, rhs, int_ty, int_ty);
		2964	} else {
		2965	return self.fail("TODO riscv cmp for ints > 64 bits", .{});
		2966	}
		2967	};
2524		2968
2525	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2969	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2526	}	2970	}
...	@@ -2555,9 +2999,7 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2555,9 +2999,7 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
2555	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {	2999	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
2556	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	3000	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2557	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);	3001	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
2558	_ = extra;	3002	try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
2559	// TODO: emit debug info for this block
2560	return self.finishAir(inst, .dead, .{ .none, .none, .none });
2561	}	3003	}
2562		3004
2563	fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {	3005	fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -2632,9 +3074,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2632,9 +3074,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2632	// that death now instead of later as this has an effect on	3074	// that death now instead of later as this has an effect on
2633	// whether it needs to be spilled in the branches	3075	// whether it needs to be spilled in the branches
2634	if (self.liveness.operandDies(inst, 0)) {	3076	if (self.liveness.operandDies(inst, 0)) {
2635	if (pl_op.operand.toIndex()) \|op_index\| {	3077	if (pl_op.operand.toIndex()) \|op_inst\| self.processDeath(op_inst);
2636	self.processDeath(op_index);
2637	}
2638	}	3078	}
2639		3079
2640	// Save state	3080	// Save state
...	@@ -2654,11 +3094,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2654,11 +3094,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2654	self.processDeath(operand);	3094	self.processDeath(operand);
2655	}	3095	}
2656	try self.genBody(then_body);	3096	try self.genBody(then_body);
2657	// point at the to-be-generated else case
2658	try self.performReloc(reloc, @intCast(self.mir_instructions.len));
2659
2660	// Revert to the previous register and stack allocation state.
2661		3097
		3098	// Restore state
2662	var saved_then_branch = self.branch_stack.pop();	3099	var saved_then_branch = self.branch_stack.pop();
2663	defer saved_then_branch.deinit(self.gpa);	3100	defer saved_then_branch.deinit(self.gpa);
2664		3101
...	@@ -2674,6 +3111,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2674,6 +3111,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2674	const else_branch = self.branch_stack.addOneAssumeCapacity();	3111	const else_branch = self.branch_stack.addOneAssumeCapacity();
2675	else_branch.* = .{};	3112	else_branch.* = .{};
2676		3113
		3114	try self.performReloc(reloc, @intCast(self.mir_instructions.len));
		3115
2677	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);	3116	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
2678	for (liveness_condbr.else_deaths) \|operand\| {	3117	for (liveness_condbr.else_deaths) \|operand\| {
2679	self.processDeath(operand);	3118	self.processDeath(operand);
...	@@ -2772,34 +3211,6 @@ fn condBr(self: *Self, cond_ty: Type, condition: MCValue, cond_reg: Register) !M	...	@@ -2772,34 +3211,6 @@ fn condBr(self: *Self, cond_ty: Type, condition: MCValue, cond_reg: Register) !M
2772	});	3211	});
2773	}	3212	}
2774		3213
2775	fn isNull(self: *Self, operand: MCValue) !MCValue {
2776	_ = operand;
2777	// Here you can specialize this instruction if it makes sense to, otherwise the default
2778	// will call isNonNull and invert the result.
2779	return self.fail("TODO call isNonNull and invert the result", .{});
2780	}
2781
2782	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
2783	_ = operand;
2784	// Here you can specialize this instruction if it makes sense to, otherwise the default
2785	// will call isNull and invert the result.
2786	return self.fail("TODO call isNull and invert the result", .{});
2787	}
2788
2789	fn isErr(self: *Self, operand: MCValue) !MCValue {
2790	_ = operand;
2791	// Here you can specialize this instruction if it makes sense to, otherwise the default
2792	// will call isNonNull and invert the result.
2793	return self.fail("TODO call isNonErr and invert the result", .{});
2794	}
2795
2796	fn isNonErr(self: *Self, operand: MCValue) !MCValue {
2797	_ = operand;
2798	// Here you can specialize this instruction if it makes sense to, otherwise the default
2799	// will call isNull and invert the result.
2800	return self.fail("TODO call isErr and invert the result", .{});
2801	}
2802
2803	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {	3214	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
2804	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3215	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2805	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3216	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
...	@@ -2827,6 +3238,13 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2827,6 +3238,13 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2827	return self.finishAir(inst, result, .{ un_op, .none, .none });	3238	return self.finishAir(inst, result, .{ un_op, .none, .none });
2828	}	3239	}
2829		3240
		3241	fn isNull(self: *Self, operand: MCValue) !MCValue {
		3242	_ = operand;
		3243	// Here you can specialize this instruction if it makes sense to, otherwise the default
		3244	// will call isNonNull and invert the result.
		3245	return self.fail("TODO call isNonNull and invert the result", .{});
		3246	}
		3247
2830	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {	3248	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
2831	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3249	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2832	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3250	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
...	@@ -2836,6 +3254,13 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2836,6 +3254,13 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
2836	return self.finishAir(inst, result, .{ un_op, .none, .none });	3254	return self.finishAir(inst, result, .{ un_op, .none, .none });
2837	}	3255	}
2838		3256
		3257	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
		3258	_ = operand;
		3259	// Here you can specialize this instruction if it makes sense to, otherwise the default
		3260	// will call isNull and invert the result.
		3261	return self.fail("TODO call isNull and invert the result", .{});
		3262	}
		3263
2839	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {	3264	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2840	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3265	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2841	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3266	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
...	@@ -2858,12 +3283,14 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2858,12 +3283,14 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
2858	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3283	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2859	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3284	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2860	const operand = try self.resolveInst(un_op);	3285	const operand = try self.resolveInst(un_op);
2861	break :result try self.isErr(operand);	3286	const operand_ty = self.typeOf(un_op);
		3287	break :result try self.isErr(inst, operand_ty, operand);
2862	};	3288	};
2863	return self.finishAir(inst, result, .{ un_op, .none, .none });	3289	return self.finishAir(inst, result, .{ un_op, .none, .none });
2864	}	3290	}
2865		3291
2866	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	3292	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
		3293	const mod = self.bin_file.comp.module.?;
2867	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3294	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2868	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3295	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2869	const operand_ptr = try self.resolveInst(un_op);	3296	const operand_ptr = try self.resolveInst(un_op);
...	@@ -2876,21 +3303,98 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2876,21 +3303,98 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2876	}	3303	}
2877	};	3304	};
2878	try self.load(operand, operand_ptr, self.typeOf(un_op));	3305	try self.load(operand, operand_ptr, self.typeOf(un_op));
2879	break :result try self.isErr(operand);	3306	const operand_ptr_ty = self.typeOf(un_op);
		3307	const operand_ty = operand_ptr_ty.childType(mod);
		3308
		3309	break :result try self.isErr(inst, operand_ty, operand);
2880	};	3310	};
2881	return self.finishAir(inst, result, .{ un_op, .none, .none });	3311	return self.finishAir(inst, result, .{ un_op, .none, .none });
2882	}	3312	}
2883		3313
		3314	/// Generates a compare instruction which will indicate if `eu_mcv` is an error.
		3315	///
		3316	/// Result is in the return register.
		3317	fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
		3318	const mod = self.bin_file.comp.module.?;
		3319	const err_ty = eu_ty.errorUnionSet(mod);
		3320	if (err_ty.errorSetIsEmpty(mod)) return MCValue{ .immediate = 0 }; // always false
		3321
		3322	_ = maybe_inst;
		3323
		3324	const err_off = errUnionErrorOffset(eu_ty.errorUnionPayload(mod), mod);
		3325
		3326	switch (eu_mcv) {
		3327	.register => \|reg\| {
		3328	const eu_lock = self.register_manager.lockReg(reg);
		3329	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
		3330
		3331	const return_reg = try self.copyToTmpRegister(eu_ty, eu_mcv);
		3332	const return_lock = self.register_manager.lockRegAssumeUnused(return_reg);
		3333	defer self.register_manager.unlockReg(return_lock);
		3334
		3335	var return_mcv: MCValue = .{ .register = return_reg };
		3336
		3337	if (err_off > 0) {
		3338	return_mcv = try self.binOp(
		3339	.shr,
		3340	null,
		3341	return_mcv,
		3342	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
		3343	eu_ty,
		3344	Type.u8,
		3345	);
		3346	}
		3347
		3348	try self.binOpMir(
		3349	.cmp_neq,
		3350	null,
		3351	Type.anyerror,
		3352	return_mcv,
		3353	.{ .immediate = 0 },
		3354	);
		3355
		3356	return return_mcv;
		3357	},
		3358	else => return self.fail("TODO implement isErr for {}", .{eu_mcv}),
		3359	}
		3360	}
		3361
2884	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {	3362	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
2885	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3363	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2886	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3364	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2887	const operand = try self.resolveInst(un_op);	3365	const operand = try self.resolveInst(un_op);
2888	break :result try self.isNonErr(operand);	3366	const ty = self.typeOf(un_op);
		3367	break :result try self.isNonErr(inst, ty, operand);
2889	};	3368	};
2890	return self.finishAir(inst, result, .{ un_op, .none, .none });	3369	return self.finishAir(inst, result, .{ un_op, .none, .none });
2891	}	3370	}
2892		3371
		3372	fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
		3373	const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);
		3374	switch (is_err_res) {
		3375	.register => \|reg\| {
		3376	_ = try self.addInst(.{
		3377	.tag = .not,
		3378	.data = .{
		3379	.rr = .{
		3380	.rd = reg,
		3381	.rs = reg,
		3382	},
		3383	},
		3384	});
		3385	return is_err_res;
		3386	},
		3387	// always false case
		3388	.immediate => \|imm\| {
		3389	assert(imm == 0);
		3390	return MCValue{ .immediate = @intFromBool(imm == 0) };
		3391	},
		3392	else => unreachable,
		3393	}
		3394	}
		3395
2893	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	3396	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
		3397	const mod = self.bin_file.comp.module.?;
2894	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;	3398	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2895	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	3399	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2896	const operand_ptr = try self.resolveInst(un_op);	3400	const operand_ptr = try self.resolveInst(un_op);
...	@@ -2902,8 +3406,11 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2902,8 +3406,11 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2902	break :blk try self.allocRegOrMem(inst, true);	3406	break :blk try self.allocRegOrMem(inst, true);
2903	}	3407	}
2904	};	3408	};
		3409	const operand_ptr_ty = self.typeOf(un_op);
		3410	const operand_ty = operand_ptr_ty.childType(mod);
		3411
2905	try self.load(operand, operand_ptr, self.typeOf(un_op));	3412	try self.load(operand, operand_ptr, self.typeOf(un_op));
2906	break :result try self.isNonErr(operand);	3413	break :result try self.isNonErr(inst, operand_ty, operand);
2907	};	3414	};
2908	return self.finishAir(inst, result, .{ un_op, .none, .none });	3415	return self.finishAir(inst, result, .{ un_op, .none, .none });
2909	}	3416	}
...	@@ -2914,7 +3421,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2914,7 +3421,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
2914	const loop = self.air.extraData(Air.Block, ty_pl.payload);	3421	const loop = self.air.extraData(Air.Block, ty_pl.payload);
2915	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);	3422	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
2916		3423
2917	const start_index: Mir.Inst.Index = @intCast(self.code.items.len);	3424	const start_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
2918		3425
2919	try self.genBody(body);	3426	try self.genBody(body);
2920	try self.jump(start_index);	3427	try self.jump(start_index);
...	@@ -2933,6 +3440,12 @@ fn jump(self: *Self, index: Mir.Inst.Index) !void {	...	@@ -2933,6 +3440,12 @@ fn jump(self: *Self, index: Mir.Inst.Index) !void {
2933	}	3440	}
2934		3441
2935	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {	3442	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
		3443	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
		3444	const extra = self.air.extraData(Air.Block, ty_pl.payload);
		3445	try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
		3446	}
		3447
		3448	fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
2936	try self.blocks.putNoClobber(self.gpa, inst, .{	3449	try self.blocks.putNoClobber(self.gpa, inst, .{
2937	// A block is a setup to be able to jump to the end.	3450	// A block is a setup to be able to jump to the end.
2938	.relocs = .{},	3451	.relocs = .{},
...	@@ -2945,9 +3458,6 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2945,9 +3458,6 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
2945	});	3458	});
2946	defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);	3459	defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);
2947		3460
2948	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2949	const extra = self.air.extraData(Air.Block, ty_pl.payload);
2950	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
2951	// TODO emit debug info lexical block	3461	// TODO emit debug info lexical block
2952	try self.genBody(body);	3462	try self.genBody(body);
2953		3463
...	@@ -2982,6 +3492,8 @@ fn performReloc(self: *Self, inst: Mir.Inst.Index, target: Mir.Inst.Index) !void	...	@@ -2982,6 +3492,8 @@ fn performReloc(self: *Self, inst: Mir.Inst.Index, target: Mir.Inst.Index) !void
2982	=> self.mir_instructions.items(.data)[inst].b_type.inst = target,	3492	=> self.mir_instructions.items(.data)[inst].b_type.inst = target,
2983	.jal,	3493	.jal,
2984	=> self.mir_instructions.items(.data)[inst].j_type.inst = target,	3494	=> self.mir_instructions.items(.data)[inst].j_type.inst = target,
		3495	.j,
		3496	=> self.mir_instructions.items(.data)[inst].inst = target,
2985	else => return self.fail("TODO: performReloc {s}", .{@tagName(tag)}),	3497	else => return self.fail("TODO: performReloc {s}", .{@tagName(tag)}),
2986	}	3498	}
2987	}	3499	}
...	@@ -3023,13 +3535,8 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {	...	@@ -3023,13 +3535,8 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
3023	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);	3535	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
3024		3536
3025	block_data.relocs.appendAssumeCapacity(try self.addInst(.{	3537	block_data.relocs.appendAssumeCapacity(try self.addInst(.{
3026	.tag = .jal,	3538	.tag = .j,
3027	.data = .{	3539	.data = .{ .inst = undefined },
3028	.j_type = .{
3029	.rd = .ra,
3030	.inst = undefined,
3031	},
3032	},
3033	}));	3540	}));
3034	}	3541	}
3035		3542
...	@@ -3273,59 +3780,22 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) Inner	...	@@ -3273,59 +3780,22 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, src_mcv: MCValue) Inner
3273	else => unreachable, // register can hold a max of 8 bytes	3780	else => unreachable, // register can hold a max of 8 bytes
3274	}	3781	}
3275	},	3782	},
3276	.stack_offset, .load_symbol => {	3783	.stack_offset,
3277	switch (src_mcv) {	3784	.indirect,
3278	.stack_offset => \|off\| if (off == stack_offset) return,	3785	.load_symbol,
3279	else => {},	3786	=> {
3280	}	3787	if (src_mcv == .stack_offset and src_mcv.stack_offset == stack_offset) return;
3281		3788
3282	if (abi_size <= 8) {	3789	if (abi_size <= 8) {
3283	const reg = try self.copyToTmpRegister(ty, src_mcv);	3790	const reg = try self.copyToTmpRegister(ty, src_mcv);
3284	return self.genSetStack(ty, stack_offset, .{ .register = reg });	3791	return self.genSetStack(ty, stack_offset, .{ .register = reg });
3285	}	3792	}
3286		3793
3287	const ptr_ty = try mod.singleMutPtrType(ty);	3794	try self.genInlineMemcpy(
3288		3795	.{ .ptr_stack_offset = stack_offset },
3289	// TODO call extern memcpy	3796	src_mcv.address(),
3290	const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);	3797	.{ .immediate = abi_size },
3291	const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);	3798	);
3292	defer for (regs_locks) \|reg\| {
3293	self.register_manager.unlockReg(reg);
3294	};
3295
3296	const src_reg = regs[0];
3297	const dst_reg = regs[1];
3298	const len_reg = regs[2];
3299	const count_reg = regs[3];
3300	const tmp_reg = regs[4];
3301
3302	switch (src_mcv) {
3303	.stack_offset => \|offset\| {
3304	try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = offset });
3305	},
3306	.load_symbol => \|sym_off\| {
3307	const atom_index = try self.symbolIndex();
3308
3309	// setup the src pointer
3310	_ = try self.addInst(.{
3311	.tag = .load_symbol,
3312	.data = .{
3313	.payload = try self.addExtra(Mir.LoadSymbolPayload{
3314	.register = src_reg.id(),
3315	.atom_index = atom_index,
3316	.sym_index = sym_off.sym,
3317	}),
3318	},
3319	});
3320	},
3321	else => return self.fail("TODO: genSetStack unreachable {s}", .{@tagName(src_mcv)}),
3322	}
3323
3324	try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset });
3325	try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
3326
3327	// memcpy(src, dst, len)
3328	try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
3329	},	3799	},
3330	.air_ref => \|ref\| try self.genSetStack(ty, stack_offset, try self.resolveInst(ref)),	3800	.air_ref => \|ref\| try self.genSetStack(ty, stack_offset, try self.resolveInst(ref)),
3331	else => return self.fail("TODO: genSetStack {s}", .{@tagName(src_mcv)}),	3801	else => return self.fail("TODO: genSetStack {s}", .{@tagName(src_mcv)}),
...	@@ -3344,13 +3814,22 @@ fn genSetMem(self: *Self, ty: Type, addr: u64, src_mcv: MCValue) InnerError!void	...	@@ -3344,13 +3814,22 @@ fn genSetMem(self: *Self, ty: Type, addr: u64, src_mcv: MCValue) InnerError!void
3344		3814
3345	fn genInlineMemcpy(	3815	fn genInlineMemcpy(
3346	self: *Self,	3816	self: *Self,
3347	src: Register,	3817	dst_ptr: MCValue,
3348	dst: Register,	3818	src_ptr: MCValue,
3349	len: Register,	3819	len: MCValue,
3350	count: Register,
3351	tmp: Register,
3352	) !void {	3820	) !void {
3353	try self.genSetReg(Type.usize, count, .{ .register = len });	3821	const regs = try self.register_manager.allocRegs(4, .{null} ** 4, tp);
		3822	const locks = self.register_manager.lockRegsAssumeUnused(4, regs);
		3823	defer for (locks) \|lock\| self.register_manager.unlockReg(lock);
		3824
		3825	const count = regs[0];
		3826	const tmp = regs[1];
		3827	const src = regs[2];
		3828	const dst = regs[3];
		3829
		3830	try self.genSetReg(Type.usize, count, len);
		3831	try self.genSetReg(Type.usize, src, src_ptr);
		3832	try self.genSetReg(Type.usize, dst, dst_ptr);
3354		3833
3355	// lb tmp, 0(src)	3834	// lb tmp, 0(src)
3356	const first_inst = try self.addInst(.{	3835	const first_inst = try self.addInst(.{
...	@@ -3437,6 +3916,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -3437,6 +3916,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
3437	const mod = self.bin_file.comp.module.?;	3916	const mod = self.bin_file.comp.module.?;
3438	const abi_size: u32 = @intCast(ty.abiSize(mod));	3917	const abi_size: u32 = @intCast(ty.abiSize(mod));
3439		3918
		3919	const load_tag: Mir.Inst.Tag = switch (abi_size) {
		3920	1 => .lb,
		3921	2 => .lh,
		3922	4 => .lw,
		3923	8 => .ld,
		3924	else => return self.fail("TODO: genSetReg for size {d}", .{abi_size}),
		3925	};
		3926
3440	switch (src_mcv) {	3927	switch (src_mcv) {
3441	.dead => unreachable,	3928	.dead => unreachable,
3442	.ptr_stack_offset => \|off\| {	3929	.ptr_stack_offset => \|off\| {
...	@@ -3550,16 +4037,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -3550,16 +4037,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
3550	});	4037	});
3551	},	4038	},
3552	.stack_offset => \|off\| {	4039	.stack_offset => \|off\| {
3553	const tag: Mir.Inst.Tag = switch (abi_size) {
3554	1 => .lb,
3555	2 => .lh,
3556	4 => .lw,
3557	8 => .ld,
3558	else => return self.fail("TODO: genSetReg for size {d}", .{abi_size}),
3559	};
3560
3561	_ = try self.addInst(.{	4040	_ = try self.addInst(.{
3562	.tag = tag,	4041	.tag = load_tag,
3563	.data = .{ .i_type = .{	4042	.data = .{ .i_type = .{
3564	.rd = reg,	4043	.rd = reg,
3565	.rs1 = .sp,	4044	.rs1 = .sp,
...	@@ -3569,53 +4048,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!	...	@@ -3569,53 +4048,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
3569	} },	4048	} },
3570	});	4049	});
3571	},	4050	},
3572	.load_symbol => \|sym_off\| {	4051	.load_symbol => {
3573	assert(sym_off.off == 0);	4052	try self.genSetReg(ty, reg, src_mcv.address());
3574		4053	try self.genSetReg(ty, reg, .{ .indirect = .{ .reg = reg } });
3575	const atom_index = try self.symbolIndex();
3576
3577	_ = try self.addInst(.{
3578	.tag = .load_symbol,
3579	.data = .{
3580	.payload = try self.addExtra(Mir.LoadSymbolPayload{
3581	.register = reg.id(),
3582	.atom_index = atom_index,
3583	.sym_index = sym_off.sym,
3584	}),
3585	},
3586	});
3587
3588	const tag: Mir.Inst.Tag = switch (abi_size) {
3589	1 => .lb,
3590	2 => .lh,
3591	4 => .lw,
3592	8 => .ld,
3593	else => return self.fail("TODO: genSetReg for size {d}", .{abi_size}),
3594	};
3595
3596	_ = try self.addInst(.{
3597	.tag = tag,
3598	.data = .{
3599	.i_type = .{
3600	.rd = reg,
3601	.rs1 = reg,
3602	.imm12 = 0,
3603	},
3604	},
3605	});
3606	},	4054	},
3607	.air_ref => \|ref\| try self.genSetReg(ty, reg, try self.resolveInst(ref)),	4055	.air_ref => \|ref\| try self.genSetReg(ty, reg, try self.resolveInst(ref)),
3608	.indirect => \|reg_off\| {	4056	.indirect => \|reg_off\| {
3609	const tag: Mir.Inst.Tag = switch (abi_size) {
3610	1 => .lb,
3611	2 => .lh,
3612	4 => .lw,
3613	8 => .ld,
3614	else => return self.fail("TODO: genSetReg for size {d}", .{abi_size}),
3615	};
3616
3617	_ = try self.addInst(.{	4057	_ = try self.addInst(.{
3618	.tag = tag,	4058	.tag = load_tag,
3619	.data = .{	4059	.data = .{
3620	.i_type = .{	4060	.i_type = .{
3621	.rd = reg,	4061	.rd = reg,
...	@@ -4026,3 +4466,25 @@ fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {	...	@@ -4026,3 +4466,25 @@ fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
4026	fn hasFeature(self: *Self, feature: Target.riscv.Feature) bool {	4466	fn hasFeature(self: *Self, feature: Target.riscv.Feature) bool {
4027	return Target.riscv.featureSetHas(self.target.cpu.features, feature);	4467	return Target.riscv.featureSetHas(self.target.cpu.features, feature);
4028	}	4468	}
		4469
		4470	pub fn errUnionPayloadOffset(payload_ty: Type, mod: *Module) u64 {
		4471	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) return 0;
		4472	const payload_align = payload_ty.abiAlignment(mod);
		4473	const error_align = Type.anyerror.abiAlignment(mod);
		4474	if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
		4475	return 0;
		4476	} else {
		4477	return payload_align.forward(Type.anyerror.abiSize(mod));
		4478	}
		4479	}
		4480
		4481	pub fn errUnionErrorOffset(payload_ty: Type, mod: *Module) u64 {
		4482	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) return 0;
		4483	const payload_align = payload_ty.abiAlignment(mod);
		4484	const error_align = Type.anyerror.abiAlignment(mod);
		4485	if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
		4486	return error_align.forward(payload_ty.abiSize(mod));
		4487	} else {
		4488	return 0;
		4489	}
		4490	}

src/arch/riscv64/Emit.zig+16-6

...	@@ -4,6 +4,8 @@	...	@@ -4,6 +4,8 @@
4	mir: Mir,	4	mir: Mir,
5	bin_file: *link.File,	5	bin_file: *link.File,
6	debug_output: DebugInfoOutput,	6	debug_output: DebugInfoOutput,
		7	output_mode: std.builtin.OutputMode,
		8	link_mode: std.builtin.LinkMode,
7	target: *const std.Target,	9	target: *const std.Target,
8	err_msg: ?*ErrorMsg = null,	10	err_msg: ?*ErrorMsg = null,
9	src_loc: Module.SrcLoc,	11	src_loc: Module.SrcLoc,
...	@@ -47,6 +49,7 @@ pub fn emitMir(	...	@@ -47,6 +49,7 @@ pub fn emitMir(
47	switch (tag) {	49	switch (tag) {
48	.add => try emit.mirRType(inst),	50	.add => try emit.mirRType(inst),
49	.sub => try emit.mirRType(inst),	51	.sub => try emit.mirRType(inst),
		52	.mul => try emit.mirRType(inst),
50	.@"or" => try emit.mirRType(inst),	53	.@"or" => try emit.mirRType(inst),
51		54
52	.cmp_eq => try emit.mirRType(inst),	55	.cmp_eq => try emit.mirRType(inst),
...	@@ -56,7 +59,9 @@ pub fn emitMir(	...	@@ -56,7 +59,9 @@ pub fn emitMir(
56	.cmp_lt => try emit.mirRType(inst),	59	.cmp_lt => try emit.mirRType(inst),
57	.cmp_imm_gte => try emit.mirRType(inst),	60	.cmp_imm_gte => try emit.mirRType(inst),
58	.cmp_imm_eq => try emit.mirIType(inst),	61	.cmp_imm_eq => try emit.mirIType(inst),
		62	.cmp_imm_neq => try emit.mirIType(inst),
59	.cmp_imm_lte => try emit.mirIType(inst),	63	.cmp_imm_lte => try emit.mirIType(inst),
		64	.cmp_imm_lt => try emit.mirIType(inst),
60		65
61	.beq => try emit.mirBType(inst),	66	.beq => try emit.mirBType(inst),
62	.bne => try emit.mirBType(inst),	67	.bne => try emit.mirBType(inst),
...	@@ -186,6 +191,7 @@ fn mirRType(emit: *Emit, inst: Mir.Inst.Index) !void {	...	@@ -186,6 +191,7 @@ fn mirRType(emit: *Emit, inst: Mir.Inst.Index) !void {
186	switch (tag) {	191	switch (tag) {
187	.add => try emit.writeInstruction(Instruction.add(rd, rs1, rs2)),	192	.add => try emit.writeInstruction(Instruction.add(rd, rs1, rs2)),
188	.sub => try emit.writeInstruction(Instruction.sub(rd, rs1, rs2)),	193	.sub => try emit.writeInstruction(Instruction.sub(rd, rs1, rs2)),
		194	.mul => try emit.writeInstruction(Instruction.mul(rd, rs1, rs2)),
189	.cmp_gt => {	195	.cmp_gt => {
190	// rs1 > rs2	196	// rs1 > rs2
191	try emit.writeInstruction(Instruction.sltu(rd, rs2, rs1));	197	try emit.writeInstruction(Instruction.sltu(rd, rs2, rs1));
...	@@ -284,6 +290,14 @@ fn mirIType(emit: *Emit, inst: Mir.Inst.Index) !void {	...	@@ -284,6 +290,14 @@ fn mirIType(emit: *Emit, inst: Mir.Inst.Index) !void {
284	try emit.writeInstruction(Instruction.xori(rd, rs1, imm12));	290	try emit.writeInstruction(Instruction.xori(rd, rs1, imm12));
285	try emit.writeInstruction(Instruction.sltiu(rd, rd, 1));	291	try emit.writeInstruction(Instruction.sltiu(rd, rd, 1));
286	},	292	},
		293	.cmp_imm_neq => {
		294	try emit.writeInstruction(Instruction.xori(rd, rs1, imm12));
		295	try emit.writeInstruction(Instruction.sltu(rd, .x0, rd));
		296	},
		297
		298	.cmp_imm_lt => {
		299	try emit.writeInstruction(Instruction.slti(rd, rs1, imm12));
		300	},
287		301
288	.cmp_imm_lte => {	302	.cmp_imm_lte => {
289	try emit.writeInstruction(Instruction.sltiu(rd, rs1, @bitCast(imm12)));	303	try emit.writeInstruction(Instruction.sltiu(rd, rs1, @bitCast(imm12)));
...	@@ -447,6 +461,7 @@ fn mirLoadSymbol(emit: *Emit, inst: Mir.Inst.Index) !void {	...	@@ -447,6 +461,7 @@ fn mirLoadSymbol(emit: *Emit, inst: Mir.Inst.Index) !void {
447		461
448	const start_offset = @as(u32, @intCast(emit.code.items.len));	462	const start_offset = @as(u32, @intCast(emit.code.items.len));
449	try emit.writeInstruction(Instruction.lui(reg, 0));	463	try emit.writeInstruction(Instruction.lui(reg, 0));
		464	try emit.writeInstruction(Instruction.addi(reg, reg, 0));
450		465
451	switch (emit.bin_file.tag) {	466	switch (emit.bin_file.tag) {
452	.elf => {	467	.elf => {
...	@@ -463,12 +478,6 @@ fn mirLoadSymbol(emit: *Emit, inst: Mir.Inst.Index) !void {	...	@@ -463,12 +478,6 @@ fn mirLoadSymbol(emit: *Emit, inst: Mir.Inst.Index) !void {
463		478
464	hi_r_type = Elf.R_ZIG_GOT_HI20;	479	hi_r_type = Elf.R_ZIG_GOT_HI20;
465	lo_r_type = Elf.R_ZIG_GOT_LO12;	480	lo_r_type = Elf.R_ZIG_GOT_LO12;
466
467	// we need to deref once if we are getting from zig_got, as itll
468	// reloc an address of the address in the got.
469	try emit.writeInstruction(Instruction.ld(reg, 0, reg));
470	} else {
471	try emit.writeInstruction(Instruction.addi(reg, reg, 0));
472	}	481	}
473		482
474	try atom_ptr.addReloc(elf_file, .{	483	try atom_ptr.addReloc(elf_file, .{
...	@@ -544,6 +553,7 @@ fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {	...	@@ -544,6 +553,7 @@ fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
544	.cmp_eq,	553	.cmp_eq,
545	.cmp_neq,	554	.cmp_neq,
546	.cmp_imm_eq,	555	.cmp_imm_eq,
		556	.cmp_imm_neq,
547	.cmp_gte,	557	.cmp_gte,
548	.load_symbol,	558	.load_symbol,
549	.abs,	559	.abs,

src/arch/riscv64/Mir.zig+6

 @@ -33,6 +33,8 @@ pub const Inst = struct {
         add,
         /// Subtraction
         sub,
+        /// Multiply, uses r_type. Needs the M extension.
+        mul,
         /// Absolute Value, uses i_type payload.
         abs,
 @@ -76,8 +78,12 @@ pub const Inst = struct {
         /// Immediate `==`, uses i_type
         cmp_imm_eq,
+        /// Immediate `!=`, uses i_type.
+        cmp_imm_neq,
         /// Immediate `<=`, uses i_type
         cmp_imm_lte,
+        /// Immediate `<`, uses i_type
+        cmp_imm_lt,
         /// Branch if equal, Uses b_type
         beq,

src/arch/riscv64/bits.zig+7-1

 @@ -112,7 +112,7 @@ pub const Instruction = union(enum) {
     // -- less burden on callsite, bonus semantic checking
     fn bType(op: u7, fn3: u3, r1: Register, r2: Register, imm: i13) Instruction {
         const umm = @as(u13, @bitCast(imm));
-        assert(umm % 2 == 0); // misaligned branch target
+        assert(umm % 4 == 0); // misaligned branch target
         return Instruction{
             .B = .{
 @@ -201,6 +201,12 @@ pub const Instruction = union(enum) {
         return rType(0b0110011, 0b011, 0b0000000, rd, r1, r2);
     }
+    // M extension operations
+    pub fn mul(rd: Register, r1: Register, r2: Register) Instruction {
+        return rType(0b0110011, 0b000, 0b0000001, rd, r1, r2);
+    }
     // Arithmetic/Logical, Register-Register (32-bit)
     pub fn addw(rd: Register, r1: Register, r2: Register) Instruction {