Commit 6b5e403e5dcdd55baf318e8734b77ce4bc635fe9 | zig

author	Joachim Schmidt <joachim.schmidt557@outlook.com> 2021-11-07 09:38:04+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2021-11-07 21:20:58+01:00
log	6b5e403e5dcdd55baf318e8734b77ce4bc635fe9
tree	b3272899580f248c901a6de1570c84528e4b096d
parent	0fb26b03690eb327164bbc7a21ba2d69d3b1a5b8

stage2 ARM: move codegen to separate file

This also removes i386 codegen code, which was unused and untested

2 files changed, 3041 insertions(+), 3395 deletions(-)

src
- arch
  - arm
    CodeGen.zig
- codegen.zig

src/arch/arm/CodeGen.zig created+3038

...	...	@@ -0,0 +1,3038 @@
	1	const std = @import("std");
	2	const builtin = @import("builtin");
	3	const mem = std.mem;
	4	const math = std.math;
	5	const assert = std.debug.assert;
	6	const Air = @import("../../Air.zig");
	7	const Zir = @import("../../Zir.zig");
	8	const Liveness = @import("../../Liveness.zig");
	9	const Type = @import("../../type.zig").Type;
	10	const Value = @import("../../value.zig").Value;
	11	const TypedValue = @import("../../TypedValue.zig");
	12	const link = @import("../../link.zig");
	13	const Module = @import("../../Module.zig");
	14	const Compilation = @import("../../Compilation.zig");
	15	const ErrorMsg = Module.ErrorMsg;
	16	const Target = std.Target;
	17	const Allocator = mem.Allocator;
	18	const trace = @import("../../tracy.zig").trace;
	19	const DW = std.dwarf;
	20	const leb128 = std.leb;
	21	const log = std.log.scoped(.codegen);
	22	const build_options = @import("build_options");
	23	const RegisterManager = @import("../../register_manager.zig").RegisterManager;
	24
	25	pub const FnResult = @import("../../codegen.zig").FnResult;
	26	pub const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
	27	pub const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
	28
	29	const InnerError = error{
	30	OutOfMemory,
	31	CodegenFail,
	32	};
	33
	34	gpa: *Allocator,
	35	air: Air,
	36	liveness: Liveness,
	37	bin_file: *link.File,
	38	target: *const std.Target,
	39	mod_fn: *const Module.Fn,
	40	code: *std.ArrayList(u8),
	41	debug_output: DebugInfoOutput,
	42	err_msg: ?*ErrorMsg,
	43	args: []MCValue,
	44	ret_mcv: MCValue,
	45	fn_type: Type,
	46	arg_index: usize,
	47	src_loc: Module.SrcLoc,
	48	stack_align: u32,
	49
	50	prev_di_line: u32,
	51	prev_di_column: u32,
	52	/// Byte offset within the source file of the ending curly.
	53	end_di_line: u32,
	54	end_di_column: u32,
	55	/// Relative to the beginning of `code`.
	56	prev_di_pc: usize,
	57
	58	/// The value is an offset into the `Function` `code` from the beginning.
	59	/// To perform the reloc, write 32-bit signed little-endian integer
	60	/// which is a relative jump, based on the address following the reloc.
	61	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{},
	62
	63	/// Whenever there is a runtime branch, we push a Branch onto this stack,
	64	/// and pop it off when the runtime branch joins. This provides an "overlay"
	65	/// of the table of mappings from instructions to `MCValue` from within the branch.
	66	/// This way we can modify the `MCValue` for an instruction in different ways
	67	/// within different branches. Special consideration is needed when a branch
	68	/// joins with its parent, to make sure all instructions have the same MCValue
	69	/// across each runtime branch upon joining.
	70	branch_stack: *std.ArrayList(Branch),
	71
	72	// Key is the block instruction
	73	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
	74
	75	register_manager: RegisterManager(Self, Register, &callee_preserved_regs) = .{},
	76	/// Maps offset to what is stored there.
	77	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
	78
	79	/// Offset from the stack base, representing the end of the stack frame.
	80	max_end_stack: u32 = 0,
	81	/// Represents the current end stack offset. If there is no existing slot
	82	/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
	83	next_stack_offset: u32 = 0,
	84
	85	/// Debug field, used to find bugs in the compiler.
	86	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
	87
	88	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
	89
	90	const MCValue = union(enum) {
	91	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
	92	/// TODO Look into deleting this tag and using `dead` instead, since every use
	93	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
	94	none,
	95	/// Control flow will not allow this value to be observed.
	96	unreach,
	97	/// No more references to this value remain.
	98	dead,
	99	/// The value is undefined.
	100	undef,
	101	/// A pointer-sized integer that fits in a register.
	102	/// If the type is a pointer, this is the pointer address in virtual address space.
	103	immediate: u64,
	104	/// The constant was emitted into the code, at this offset.
	105	/// If the type is a pointer, it means the pointer address is embedded in the code.
	106	embedded_in_code: usize,
	107	/// The value is a pointer to a constant which was emitted into the code, at this offset.
	108	ptr_embedded_in_code: usize,
	109	/// The value is in a target-specific register.
	110	register: Register,
	111	/// The value is in memory at a hard-coded address.
	112	/// If the type is a pointer, it means the pointer address is at this memory location.
	113	memory: u64,
	114	/// The value is one of the stack variables.
	115	/// If the type is a pointer, it means the pointer address is in the stack at this offset.
	116	stack_offset: u32,
	117	/// The value is a pointer to one of the stack variables (payload is stack offset).
	118	ptr_stack_offset: u32,
	119	/// The value is in the compare flags assuming an unsigned operation,
	120	/// with this operator applied on top of it.
	121	compare_flags_unsigned: math.CompareOperator,
	122	/// The value is in the compare flags assuming a signed operation,
	123	/// with this operator applied on top of it.
	124	compare_flags_signed: math.CompareOperator,
	125
	126	fn isMemory(mcv: MCValue) bool {
	127	return switch (mcv) {
	128	.embedded_in_code, .memory, .stack_offset => true,
	129	else => false,
	130	};
	131	}
	132
	133	fn isImmediate(mcv: MCValue) bool {
	134	return switch (mcv) {
	135	.immediate => true,
	136	else => false,
	137	};
	138	}
	139
	140	fn isMutable(mcv: MCValue) bool {
	141	return switch (mcv) {
	142	.none => unreachable,
	143	.unreach => unreachable,
	144	.dead => unreachable,
	145
	146	.immediate,
	147	.embedded_in_code,
	148	.memory,
	149	.compare_flags_unsigned,
	150	.compare_flags_signed,
	151	.ptr_stack_offset,
	152	.ptr_embedded_in_code,
	153	.undef,
	154	=> false,
	155
	156	.register,
	157	.stack_offset,
	158	=> true,
	159	};
	160	}
	161	};
	162
	163	const Branch = struct {
	164	inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .{},
	165
	166	fn deinit(self: Branch, gpa: Allocator) void {
	167	self.inst_table.deinit(gpa);
	168	self.* = undefined;
	169	}
	170	};
	171
	172	const StackAllocation = struct {
	173	inst: Air.Inst.Index,
	174	/// TODO do we need size? should be determined by inst.ty.abiSize()
	175	size: u32,
	176	};
	177
	178	const BlockData = struct {
	179	relocs: std.ArrayListUnmanaged(Reloc),
	180	/// The first break instruction encounters `null` here and chooses a
	181	/// machine code value for the block result, populating this field.
	182	/// Following break instructions encounter that value and use it for
	183	/// the location to store their block results.
	184	mcv: MCValue,
	185	};
	186
	187	const Reloc = union(enum) {
	188	/// The value is an offset into the `Function` `code` from the beginning.
	189	/// To perform the reloc, write 32-bit signed little-endian integer
	190	/// which is a relative jump, based on the address following the reloc.
	191	rel32: usize,
	192	/// A branch in the ARM instruction set
	193	arm_branch: struct {
	194	pos: usize,
	195	cond: @import("bits.zig").Condition,
	196	},
	197	};
	198
	199	const BigTomb = struct {
	200	function: *Self,
	201	inst: Air.Inst.Index,
	202	tomb_bits: Liveness.Bpi,
	203	big_tomb_bits: u32,
	204	bit_index: usize,
	205
	206	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
	207	const this_bit_index = bt.bit_index;
	208	bt.bit_index += 1;
	209
	210	const op_int = @enumToInt(op_ref);
	211	if (op_int < Air.Inst.Ref.typed_value_map.len) return;
	212	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
	213
	214	if (this_bit_index < Liveness.bpi - 1) {
	215	const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
	216	if (!dies) return;
	217	} else {
	218	const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
	219	const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
	220	if (!dies) return;
	221	}
	222	bt.function.processDeath(op_index);
	223	}
	224
	225	fn finishAir(bt: *BigTomb, result: MCValue) void {
	226	const is_used = !bt.function.liveness.isUnused(bt.inst);
	227	if (is_used) {
	228	log.debug("%{d} => {}", .{ bt.inst, result });
	229	const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
	230	branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
	231	}
	232	bt.function.finishAirBookkeeping();
	233	}
	234	};
	235
	236	const Self = @This();
	237
	238	pub fn generate(
	239	bin_file: *link.File,
	240	src_loc: Module.SrcLoc,
	241	module_fn: *Module.Fn,
	242	air: Air,
	243	liveness: Liveness,
	244	code: *std.ArrayList(u8),
	245	debug_output: DebugInfoOutput,
	246	) GenerateSymbolError!FnResult {
	247	if (build_options.skip_non_native and builtin.cpu.arch != bin_file.options.target.cpu.arch) {
	248	@panic("Attempted to compile for architecture that was disabled by build configuration");
	249	}
	250
	251	assert(module_fn.owner_decl.has_tv);
	252	const fn_type = module_fn.owner_decl.ty;
	253
	254	var branch_stack = std.ArrayList(Branch).init(bin_file.allocator);
	255	defer {
	256	assert(branch_stack.items.len == 1);
	257	branch_stack.items[0].deinit(bin_file.allocator);
	258	branch_stack.deinit();
	259	}
	260	try branch_stack.append(.{});
	261
	262	var function = Self{
	263	.gpa = bin_file.allocator,
	264	.air = air,
	265	.liveness = liveness,
	266	.target = &bin_file.options.target,
	267	.bin_file = bin_file,
	268	.mod_fn = module_fn,
	269	.code = code,
	270	.debug_output = debug_output,
	271	.err_msg = null,
	272	.args = undefined, // populated after `resolveCallingConventionValues`
	273	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
	274	.fn_type = fn_type,
	275	.arg_index = 0,
	276	.branch_stack = &branch_stack,
	277	.src_loc = src_loc,
	278	.stack_align = undefined,
	279	.prev_di_pc = 0,
	280	.prev_di_line = module_fn.lbrace_line,
	281	.prev_di_column = module_fn.lbrace_column,
	282	.end_di_line = module_fn.rbrace_line,
	283	.end_di_column = module_fn.rbrace_column,
	284	};
	285	defer function.stack.deinit(bin_file.allocator);
	286	defer function.blocks.deinit(bin_file.allocator);
	287	defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
	288
	289	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
	290	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
	291	else => \|e\| return e,
	292	};
	293	defer call_info.deinit(&function);
	294
	295	function.args = call_info.args;
	296	function.ret_mcv = call_info.return_value;
	297	function.stack_align = call_info.stack_align;
	298	function.max_end_stack = call_info.stack_byte_count;
	299
	300	function.gen() catch \|err\| switch (err) {
	301	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
	302	else => \|e\| return e,
	303	};
	304
	305	if (function.err_msg) \|em\| {
	306	return FnResult{ .fail = em };
	307	} else {
	308	return FnResult{ .appended = {} };
	309	}
	310	}
	311
	312	fn gen(self: *Self) !void {
	313	const cc = self.fn_type.fnCallingConvention();
	314	if (cc != .Naked) {
	315	// push {fp, lr}
	316	// mov fp, sp
	317	// sub sp, sp, #reloc
	318	const prologue_reloc = self.code.items.len;
	319	try self.code.resize(prologue_reloc + 12);
	320	self.writeInt(u32, self.code.items[prologue_reloc + 4 ..][0..4], Instruction.mov(.al, .fp, Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none)).toU32());
	321
	322	try self.dbgSetPrologueEnd();
	323
	324	try self.genBody(self.air.getMainBody());
	325
	326	// Backpatch push callee saved regs
	327	var saved_regs = Instruction.RegisterList{
	328	.r11 = true, // fp
	329	.r14 = true, // lr
	330	};
	331	inline for (callee_preserved_regs) \|reg\| {
	332	if (self.register_manager.isRegAllocated(reg)) {
	333	@field(saved_regs, @tagName(reg)) = true;
	334	}
	335	}
	336	self.writeInt(u32, self.code.items[prologue_reloc..][0..4], Instruction.stmdb(.al, .sp, true, saved_regs).toU32());
	337
	338	// Backpatch stack offset
	339	const stack_end = self.max_end_stack;
	340	const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
	341	if (Instruction.Operand.fromU32(@intCast(u32, aligned_stack_end))) \|op\| {
	342	self.writeInt(u32, self.code.items[prologue_reloc + 8 ..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32());
	343	} else {
	344	return self.failSymbol("TODO ARM: allow larger stacks", .{});
	345	}
	346
	347	try self.dbgSetEpilogueBegin();
	348
	349	// exitlude jumps
	350	if (self.exitlude_jump_relocs.items.len == 1) {
	351	// There is only one relocation. Hence,
	352	// this relocation must be at the end of
	353	// the code. Therefore, we can just delete
	354	// the space initially reserved for the
	355	// jump
	356	self.code.items.len -= 4;
	357	} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
	358	const amt = @intCast(i32, self.code.items.len) - @intCast(i32, jmp_reloc + 8);
	359	if (amt == -4) {
	360	// This return is at the end of the
	361	// code block. We can't just delete
	362	// the space because there may be
	363	// other jumps we already relocated to
	364	// the address. Instead, insert a nop
	365	self.writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.nop().toU32());
	366	} else {
	367	if (math.cast(i26, amt)) \|offset\| {
	368	self.writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(.al, offset).toU32());
	369	} else \|_\| {
	370	return self.failSymbol("exitlude jump is too large", .{});
	371	}
	372	}
	373	}
	374
	375	// Epilogue: pop callee saved registers (swap lr with pc in saved_regs)
	376	saved_regs.r14 = false; // lr
	377	saved_regs.r15 = true; // pc
	378
	379	// mov sp, fp
	380	// pop {fp, pc}
	381	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .sp, Instruction.Operand.reg(.fp, Instruction.Operand.Shift.none)).toU32());
	382	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldm(.al, .sp, true, saved_regs).toU32());
	383	} else {
	384	try self.dbgSetPrologueEnd();
	385	try self.genBody(self.air.getMainBody());
	386	try self.dbgSetEpilogueBegin();
	387	}
	388
	389	// Drop them off at the rbrace.
	390	try self.dbgAdvancePCAndLine(self.end_di_line, self.end_di_column);
	391	}
	392
	393	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
	394	const air_tags = self.air.instructions.items(.tag);
	395
	396	for (body) \|inst\| {
	397	const old_air_bookkeeping = self.air_bookkeeping;
	398	try self.ensureProcessDeathCapacity(Liveness.bpi);
	399
	400	switch (air_tags[inst]) {
	401	// zig fmt: off
	402	.add, .ptr_add => try self.airAdd(inst),
	403	.addwrap => try self.airAddWrap(inst),
	404	.add_sat => try self.airAddSat(inst),
	405	.sub, .ptr_sub => try self.airSub(inst),
	406	.subwrap => try self.airSubWrap(inst),
	407	.sub_sat => try self.airSubSat(inst),
	408	.mul => try self.airMul(inst),
	409	.mulwrap => try self.airMulWrap(inst),
	410	.mul_sat => try self.airMulSat(inst),
	411	.rem => try self.airRem(inst),
	412	.mod => try self.airMod(inst),
	413	.shl, .shl_exact => try self.airShl(inst),
	414	.shl_sat => try self.airShlSat(inst),
	415	.min => try self.airMin(inst),
	416	.max => try self.airMax(inst),
	417	.slice => try self.airSlice(inst),
	418
	419	.div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
	420
	421	.cmp_lt => try self.airCmp(inst, .lt),
	422	.cmp_lte => try self.airCmp(inst, .lte),
	423	.cmp_eq => try self.airCmp(inst, .eq),
	424	.cmp_gte => try self.airCmp(inst, .gte),
	425	.cmp_gt => try self.airCmp(inst, .gt),
	426	.cmp_neq => try self.airCmp(inst, .neq),
	427
	428	.bool_and => try self.airBoolOp(inst),
	429	.bool_or => try self.airBoolOp(inst),
	430	.bit_and => try self.airBitAnd(inst),
	431	.bit_or => try self.airBitOr(inst),
	432	.xor => try self.airXor(inst),
	433	.shr => try self.airShr(inst),
	434
	435	.alloc => try self.airAlloc(inst),
	436	.ret_ptr => try self.airRetPtr(inst),
	437	.arg => try self.airArg(inst),
	438	.assembly => try self.airAsm(inst),
	439	.bitcast => try self.airBitCast(inst),
	440	.block => try self.airBlock(inst),
	441	.br => try self.airBr(inst),
	442	.breakpoint => try self.airBreakpoint(),
	443	.fence => try self.airFence(),
	444	.call => try self.airCall(inst),
	445	.cond_br => try self.airCondBr(inst),
	446	.dbg_stmt => try self.airDbgStmt(inst),
	447	.fptrunc => try self.airFptrunc(inst),
	448	.fpext => try self.airFpext(inst),
	449	.intcast => try self.airIntCast(inst),
	450	.trunc => try self.airTrunc(inst),
	451	.bool_to_int => try self.airBoolToInt(inst),
	452	.is_non_null => try self.airIsNonNull(inst),
	453	.is_non_null_ptr => try self.airIsNonNullPtr(inst),
	454	.is_null => try self.airIsNull(inst),
	455	.is_null_ptr => try self.airIsNullPtr(inst),
	456	.is_non_err => try self.airIsNonErr(inst),
	457	.is_non_err_ptr => try self.airIsNonErrPtr(inst),
	458	.is_err => try self.airIsErr(inst),
	459	.is_err_ptr => try self.airIsErrPtr(inst),
	460	.load => try self.airLoad(inst),
	461	.loop => try self.airLoop(inst),
	462	.not => try self.airNot(inst),
	463	.ptrtoint => try self.airPtrToInt(inst),
	464	.ret => try self.airRet(inst),
	465	.ret_load => try self.airRetLoad(inst),
	466	.store => try self.airStore(inst),
	467	.struct_field_ptr=> try self.airStructFieldPtr(inst),
	468	.struct_field_val=> try self.airStructFieldVal(inst),
	469	.array_to_slice => try self.airArrayToSlice(inst),
	470	.int_to_float => try self.airIntToFloat(inst),
	471	.float_to_int => try self.airFloatToInt(inst),
	472	.cmpxchg_strong => try self.airCmpxchg(inst),
	473	.cmpxchg_weak => try self.airCmpxchg(inst),
	474	.atomic_rmw => try self.airAtomicRmw(inst),
	475	.atomic_load => try self.airAtomicLoad(inst),
	476	.memcpy => try self.airMemcpy(inst),
	477	.memset => try self.airMemset(inst),
	478	.set_union_tag => try self.airSetUnionTag(inst),
	479	.get_union_tag => try self.airGetUnionTag(inst),
	480	.clz => try self.airClz(inst),
	481	.ctz => try self.airCtz(inst),
	482	.popcount => try self.airPopcount(inst),
	483
	484	.atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
	485	.atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
	486	.atomic_store_release => try self.airAtomicStore(inst, .Release),
	487	.atomic_store_seq_cst => try self.airAtomicStore(inst, .SeqCst),
	488
	489	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
	490	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
	491	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
	492	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
	493
	494	.switch_br => try self.airSwitch(inst),
	495	.slice_ptr => try self.airSlicePtr(inst),
	496	.slice_len => try self.airSliceLen(inst),
	497
	498	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
	499	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
	500
	501	.array_elem_val => try self.airArrayElemVal(inst),
	502	.slice_elem_val => try self.airSliceElemVal(inst),
	503	.slice_elem_ptr => try self.airSliceElemPtr(inst),
	504	.ptr_elem_val => try self.airPtrElemVal(inst),
	505	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
	506
	507	.constant => unreachable, // excluded from function bodies
	508	.const_ty => unreachable, // excluded from function bodies
	509	.unreach => self.finishAirBookkeeping(),
	510
	511	.optional_payload => try self.airOptionalPayload(inst),
	512	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
	513	.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
	514	.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
	515	.unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
	516	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
	517
	518	.wrap_optional => try self.airWrapOptional(inst),
	519	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
	520	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
	521	// zig fmt: on
	522	}
	523	if (std.debug.runtime_safety) {
	524	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
	525	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] });
	526	}
	527	}
	528	}
	529	}
	530
	531	fn writeInt(self: Self, comptime T: type, buf: [@divExact(@typeInfo(T).Int.bits, 8)]u8, value: T) void {
	532	const endian = self.target.cpu.arch.endian();
	533	std.mem.writeInt(T, buf, value, endian);
	534	}
	535
	536	fn dbgSetPrologueEnd(self: *Self) InnerError!void {
	537	switch (self.debug_output) {
	538	.dwarf => \|dbg_out\| {
	539	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
	540	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
	541	},
	542	.plan9 => {},
	543	.none => {},
	544	}
	545	}
	546
	547	fn dbgSetEpilogueBegin(self: *Self) InnerError!void {
	548	switch (self.debug_output) {
	549	.dwarf => \|dbg_out\| {
	550	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
	551	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
	552	},
	553	.plan9 => {},
	554	.none => {},
	555	}
	556	}
	557
	558	fn dbgAdvancePCAndLine(self: *Self, line: u32, column: u32) InnerError!void {
	559	const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
	560	const delta_pc: usize = self.code.items.len - self.prev_di_pc;
	561	switch (self.debug_output) {
	562	.dwarf => \|dbg_out\| {
	563	// TODO Look into using the DWARF special opcodes to compress this data.
	564	// It lets you emit single-byte opcodes that add different numbers to
	565	// both the PC and the line number at the same time.
	566	try dbg_out.dbg_line.ensureUnusedCapacity(11);
	567	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
	568	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
	569	if (delta_line != 0) {
	570	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
	571	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	572	}
	573	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
	574	self.prev_di_pc = self.code.items.len;
	575	self.prev_di_line = line;
	576	self.prev_di_column = column;
	577	self.prev_di_pc = self.code.items.len;
	578	},
	579	.plan9 => \|dbg_out\| {
	580	if (delta_pc <= 0) return; // only do this when the pc changes
	581	// we have already checked the target in the linker to make sure it is compatable
	582	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;
	583
	584	// increasing the line number
	585	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
	586	// increasing the pc
	587	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
	588	if (d_pc_p9 > 0) {
	589	// minus one because if its the last one, we want to leave space to change the line which is one quanta
	590	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
	591	if (dbg_out.pcop_change_index.*) \|pci\|
	592	dbg_out.dbg_line.items[pci] += 1;
	593	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
	594	} else if (d_pc_p9 == 0) {
	595	// we don't need to do anything, because adding the quant does it for us
	596	} else unreachable;
	597	if (dbg_out.start_line.* == null)
	598	dbg_out.start_line.* = self.prev_di_line;
	599	dbg_out.end_line.* = line;
	600	// only do this if the pc changed
	601	self.prev_di_line = line;
	602	self.prev_di_column = column;
	603	self.prev_di_pc = self.code.items.len;
	604	},
	605	.none => {},
	606	}
	607	}
	608
	609	/// Asserts there is already capacity to insert into top branch inst_table.
	610	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
	611	const air_tags = self.air.instructions.items(.tag);
	612	if (air_tags[inst] == .constant) return; // Constants are immortal.
	613	// When editing this function, note that the logic must synchronize with `reuseOperand`.
	614	const prev_value = self.getResolvedInstValue(inst);
	615	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	616	branch.inst_table.putAssumeCapacity(inst, .dead);
	617	switch (prev_value) {
	618	.register => \|reg\| {
	619	self.register_manager.freeReg(reg);
	620	},
	621	else => {}, // TODO process stack allocation death
	622	}
	623	}
	624
	625	/// Called when there are no operands, and the instruction is always unreferenced.
	626	fn finishAirBookkeeping(self: *Self) void {
	627	if (std.debug.runtime_safety) {
	628	self.air_bookkeeping += 1;
	629	}
	630	}
	631
	632	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
	633	var tomb_bits = self.liveness.getTombBits(inst);
	634	for (operands) \|op\| {
	635	const dies = @truncate(u1, tomb_bits) != 0;
	636	tomb_bits >>= 1;
	637	if (!dies) continue;
	638	const op_int = @enumToInt(op);
	639	if (op_int < Air.Inst.Ref.typed_value_map.len) continue;
	640	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
	641	self.processDeath(op_index);
	642	}
	643	const is_used = @truncate(u1, tomb_bits) == 0;
	644	if (is_used) {
	645	log.debug("%{d} => {}", .{ inst, result });
	646	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	647	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
	648
	649	switch (result) {
	650	.register => \|reg\| {
	651	// In some cases (such as bitcast), an operand
	652	// may be the same MCValue as the result. If
	653	// that operand died and was a register, it
	654	// was freed by processDeath. We have to
	655	// "re-allocate" the register.
	656	if (self.register_manager.isRegFree(reg)) {
	657	self.register_manager.getRegAssumeFree(reg, inst);
	658	}
	659	},
	660	else => {},
	661	}
	662	}
	663	self.finishAirBookkeeping();
	664	}
	665
	666	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
	667	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
	668	try table.ensureUnusedCapacity(self.gpa, additional_count);
	669	}
	670
	671	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
	672	/// after codegen for this symbol is done.
	673	fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void {
	674	switch (self.debug_output) {
	675	.dwarf => \|dbg_out\| {
	676	assert(ty.hasCodeGenBits());
	677	const index = dbg_out.dbg_info.items.len;
	678	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
	679
	680	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(self.gpa, ty);
	681	if (!gop.found_existing) {
	682	gop.value_ptr.* = .{
	683	.off = undefined,
	684	.relocs = .{},
	685	};
	686	}
	687	try gop.value_ptr.relocs.append(self.gpa, @intCast(u32, index));
	688	},
	689	.plan9 => {},
	690	.none => {},
	691	}
	692	}
	693
	694	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
	695	if (abi_align > self.stack_align)
	696	self.stack_align = abi_align;
	697	// TODO find a free slot instead of always appending
	698	const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align);
	699	self.next_stack_offset = offset + abi_size;
	700	if (self.next_stack_offset > self.max_end_stack)
	701	self.max_end_stack = self.next_stack_offset;
	702	try self.stack.putNoClobber(self.gpa, offset, .{
	703	.inst = inst,
	704	.size = abi_size,
	705	});
	706	return offset;
	707	}
	708
	709	/// Use a pointer instruction as the basis for allocating stack memory.
	710	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
	711	const elem_ty = self.air.typeOfIndex(inst).elemType();
	712	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
	713	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
	714	};
	715	// TODO swap this for inst.ty.ptrAlign
	716	const abi_align = elem_ty.abiAlignment(self.target.*);
	717	return self.allocMem(inst, abi_size, abi_align);
	718	}
	719
	720	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
	721	const elem_ty = self.air.typeOfIndex(inst);
	722	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
	723	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
	724	};
	725	const abi_align = elem_ty.abiAlignment(self.target.*);
	726	if (abi_align > self.stack_align)
	727	self.stack_align = abi_align;
	728
	729	if (reg_ok) {
	730	// Make sure the type can fit in a register before we try to allocate one.
	731	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
	732	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
	733	if (abi_size <= ptr_bytes) {
	734	if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {
	735	return MCValue{ .register = reg };
	736	}
	737	}
	738	}
	739	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
	740	return MCValue{ .stack_offset = stack_offset };
	741	}
	742
	743	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
	744	const stack_mcv = try self.allocRegOrMem(inst, false);
	745	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
	746	const reg_mcv = self.getResolvedInstValue(inst);
	747	assert(reg == reg_mcv.register);
	748	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	749	try branch.inst_table.put(self.gpa, inst, stack_mcv);
	750	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
	751	}
	752
	753	/// Copies a value to a register without tracking the register. The register is not considered
	754	/// allocated. A second call to `copyToTmpRegister` may return the same register.
	755	/// This can have a side effect of spilling instructions to the stack to free up a register.
	756	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
	757	const reg = try self.register_manager.allocReg(null, &.{});
	758	try self.genSetReg(ty, reg, mcv);
	759	return reg;
	760	}
	761
	762	/// Allocates a new register and copies `mcv` into it.
	763	/// `reg_owner` is the instruction that gets associated with the register in the register table.
	764	/// This can have a side effect of spilling instructions to the stack to free up a register.
	765	fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
	766	const reg = try self.register_manager.allocReg(reg_owner, &.{});
	767	try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
	768	return MCValue{ .register = reg };
	769	}
	770
	771	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
	772	const stack_offset = try self.allocMemPtr(inst);
	773	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
	774	}
	775
	776	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
	777	const stack_offset = try self.allocMemPtr(inst);
	778	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
	779	}
	780
	781	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
	782	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	783	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
	784	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	785	}
	786
	787	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
	788	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	789	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
	790	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	791	}
	792
	793	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
	794	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	795	if (self.liveness.isUnused(inst))
	796	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	797
	798	const operand_ty = self.air.typeOf(ty_op.operand);
	799	const operand = try self.resolveInst(ty_op.operand);
	800	const info_a = operand_ty.intInfo(self.target.*);
	801	const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);
	802	if (info_a.signedness != info_b.signedness)
	803	return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
	804
	805	if (info_a.bits == info_b.bits)
	806	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
	807
	808	return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch});
	809	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	810	}
	811
	812	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
	813	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	814	if (self.liveness.isUnused(inst))
	815	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
	816
	817	const operand = try self.resolveInst(ty_op.operand);
	818	_ = operand;
	819
	820	return self.fail("TODO implement trunc for {}", .{self.target.cpu.arch});
	821	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	822	}
	823
	824	fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
	825	const un_op = self.air.instructions.items(.data)[inst].un_op;
	826	const operand = try self.resolveInst(un_op);
	827	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;
	828	return self.finishAir(inst, result, .{ un_op, .none, .none });
	829	}
	830
	831	fn airNot(self: *Self, inst: Air.Inst.Index) !void {
	832	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	833	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	834	const operand = try self.resolveInst(ty_op.operand);
	835	switch (operand) {
	836	.dead => unreachable,
	837	.unreach => unreachable,
	838	.compare_flags_unsigned => \|op\| {
	839	const r = MCValue{
	840	.compare_flags_unsigned = switch (op) {
	841	.gte => .lt,
	842	.gt => .lte,
	843	.neq => .eq,
	844	.lt => .gte,
	845	.lte => .gt,
	846	.eq => .neq,
	847	},
	848	};
	849	break :result r;
	850	},
	851	.compare_flags_signed => \|op\| {
	852	const r = MCValue{
	853	.compare_flags_signed = switch (op) {
	854	.gte => .lt,
	855	.gt => .lte,
	856	.neq => .eq,
	857	.lt => .gte,
	858	.lte => .gt,
	859	.eq => .neq,
	860	},
	861	};
	862	break :result r;
	863	},
	864	else => {},
	865	}
	866
	867	break :result try self.genArmBinOp(inst, ty_op.operand, .bool_true, .not);
	868	};
	869	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	870	}
	871
	872	fn airMin(self: *Self, inst: Air.Inst.Index) !void {
	873	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	874	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement min for {}", .{self.target.cpu.arch});
	875	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	876	}
	877
	878	fn airMax(self: *Self, inst: Air.Inst.Index) !void {
	879	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	880	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement max for {}", .{self.target.cpu.arch});
	881	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	882	}
	883
	884	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
	885	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	886	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	887	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice for {}", .{self.target.cpu.arch});
	888	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	889	}
	890
	891	fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
	892	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	893	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .add);
	894	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	895	}
	896
	897	fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {
	898	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	899	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch});
	900	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	901	}
	902
	903	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
	904	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	905	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
	906	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	907	}
	908
	909	fn airSub(self: *Self, inst: Air.Inst.Index) !void {
	910	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	911	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .sub);
	912	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	913	}
	914
	915	fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
	916	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	917	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement subwrap for {}", .{self.target.cpu.arch});
	918	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	919	}
	920
	921	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
	922	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	923	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
	924	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	925	}
	926
	927	fn airMul(self: *Self, inst: Air.Inst.Index) !void {
	928	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	929	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmMul(inst, bin_op.lhs, bin_op.rhs);
	930	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	931	}
	932
	933	fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {
	934	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	935	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch});
	936	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	937	}
	938
	939	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
	940	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	941	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
	942	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	943	}
	944
	945	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
	946	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	947	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
	948	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	949	}
	950
	951	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
	952	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	953	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});
	954	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	955	}
	956
	957	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
	958	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	959	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mod for {}", .{self.target.cpu.arch});
	960	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	961	}
	962
	963	fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
	964	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	965	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_and);
	966	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	967	}
	968
	969	fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
	970	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	971	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_or);
	972	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	973	}
	974
	975	fn airXor(self: *Self, inst: Air.Inst.Index) !void {
	976	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	977	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .xor);
	978	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	979	}
	980
	981	fn airShl(self: *Self, inst: Air.Inst.Index) !void {
	982	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	983	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shl);
	984	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	985	}
	986
	987	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
	988	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	989	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
	990	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	991	}
	992
	993	fn airShr(self: *Self, inst: Air.Inst.Index) !void {
	994	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	995	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shr);
	996	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	997	}
	998
	999	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
	1000	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1001	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});
	1002	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1003	}
	1004
	1005	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
	1006	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1007	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
	1008	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1009	}
	1010
	1011	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
	1012	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1013	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error for {}", .{self.target.cpu.arch});
	1014	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1015	}
	1016
	1017	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
	1018	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1019	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload for {}", .{self.target.cpu.arch});
	1020	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1021	}
	1022
	1023	// *(E!T) -> E
	1024	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
	1025	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1026	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
	1027	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1028	}
	1029
	1030	// (E!T) -> T
	1031	fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
	1032	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1033	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
	1034	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1035	}
	1036
	1037	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
	1038	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1039	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1040	const optional_ty = self.air.typeOfIndex(inst);
	1041
	1042	// Optional with a zero-bit payload type is just a boolean true
	1043	if (optional_ty.abiSize(self.target.*) == 1)
	1044	break :result MCValue{ .immediate = 1 };
	1045
	1046	return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});
	1047	};
	1048	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1049	}
	1050
	1051	/// T to E!T
	1052	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
	1053	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1054	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});
	1055	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1056	}
	1057
	1058	/// E to E!T
	1059	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
	1060	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1061	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion error for {}", .{self.target.cpu.arch});
	1062	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1063	}
	1064
	1065	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
	1066	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1067	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_ptr for {}", .{self.target.cpu.arch});
	1068	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1069	}
	1070
	1071	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
	1072	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1073	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_len for {}", .{self.target.cpu.arch});
	1074	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1075	}
	1076
	1077	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
	1078	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1079	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});
	1080	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1081	}
	1082
	1083	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
	1084	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1085	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});
	1086	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1087	}
	1088
	1089	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
	1090	const is_volatile = false; // TODO
	1091	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1092	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_val for {}", .{self.target.cpu.arch});
	1093	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1094	}
	1095
	1096	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
	1097	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	1098	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
	1099	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});
	1100	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
	1101	}
	1102
	1103	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
	1104	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1105	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch});
	1106	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1107	}
	1108
	1109	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
	1110	const is_volatile = false; // TODO
	1111	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1112	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch});
	1113	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1114	}
	1115
	1116	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
	1117	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	1118	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
	1119	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});
	1120	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
	1121	}
	1122
	1123	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
	1124	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1125	_ = bin_op;
	1126	return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch});
	1127	// return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1128	}
	1129
	1130	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
	1131	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1132	_ = ty_op;
	1133	return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
	1134	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1135	}
	1136
	1137	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
	1138	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1139	_ = ty_op;
	1140	return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
	1141	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1142	}
	1143
	1144	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
	1145	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1146	_ = ty_op;
	1147	return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
	1148	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1149	}
	1150
	1151	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
	1152	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1153	_ = ty_op;
	1154	return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
	1155	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1156	}
	1157
	1158	fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {
	1159	if (!self.liveness.operandDies(inst, op_index))
	1160	return false;
	1161
	1162	switch (mcv) {
	1163	.register => \|reg\| {
	1164	// If it's in the registers table, need to associate the register with the
	1165	// new instruction.
	1166	if (reg.allocIndex()) \|index\| {
	1167	if (!self.register_manager.isRegFree(reg)) {
	1168	self.register_manager.registers[index] = inst;
	1169	}
	1170	}
	1171	log.debug("%{d} => {} (reused)", .{ inst, reg });
	1172	},
	1173	.stack_offset => \|off\| {
	1174	log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
	1175	},
	1176	else => return false,
	1177	}
	1178
	1179	// Prevent the operand deaths processing code from deallocating it.
	1180	self.liveness.clearOperandDeath(inst, op_index);
	1181
	1182	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
	1183	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	1184	branch.inst_table.putAssumeCapacity(Air.refToIndex(operand).?, .dead);
	1185
	1186	return true;
	1187	}
	1188
	1189	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
	1190	const elem_ty = ptr_ty.elemType();
	1191	switch (ptr) {
	1192	.none => unreachable,
	1193	.undef => unreachable,
	1194	.unreach => unreachable,
	1195	.dead => unreachable,
	1196	.compare_flags_unsigned => unreachable,
	1197	.compare_flags_signed => unreachable,
	1198	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
	1199	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
	1200	.ptr_embedded_in_code => \|off\| {
	1201	try self.setRegOrMem(elem_ty, dst_mcv, .{ .embedded_in_code = off });
	1202	},
	1203	.embedded_in_code => {
	1204	return self.fail("TODO implement loading from MCValue.embedded_in_code", .{});
	1205	},
	1206	.register => \|reg\| {
	1207	switch (dst_mcv) {
	1208	.dead => unreachable,
	1209	.undef => unreachable,
	1210	.compare_flags_signed, .compare_flags_unsigned => unreachable,
	1211	.embedded_in_code => unreachable,
	1212	.register => \|dst_reg\| {
	1213	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, dst_reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
	1214	},
	1215	else => return self.fail("TODO load from register into {}", .{dst_mcv}),
	1216	}
	1217	},
	1218	.memory => \|addr\| {
	1219	const reg = try self.register_manager.allocReg(null, &.{});
	1220	try self.genSetReg(ptr_ty, reg, .{ .memory = addr });
	1221	try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
	1222	},
	1223	.stack_offset => {
	1224	return self.fail("TODO implement loading from MCValue.stack_offset", .{});
	1225	},
	1226	}
	1227	}
	1228
	1229	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
	1230	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1231	const elem_ty = self.air.typeOfIndex(inst);
	1232	const result: MCValue = result: {
	1233	if (!elem_ty.hasCodeGenBits())
	1234	break :result MCValue.none;
	1235
	1236	const ptr = try self.resolveInst(ty_op.operand);
	1237	const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
	1238	if (self.liveness.isUnused(inst) and !is_volatile)
	1239	break :result MCValue.dead;
	1240
	1241	const dst_mcv: MCValue = blk: {
	1242	if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
	1243	// The MCValue that holds the pointer can be re-used as the value.
	1244	break :blk ptr;
	1245	} else {
	1246	break :blk try self.allocRegOrMem(inst, true);
	1247	}
	1248	};
	1249	try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
	1250	break :result dst_mcv;
	1251	};
	1252	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1253	}
	1254
	1255	fn airStore(self: *Self, inst: Air.Inst.Index) !void {
	1256	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1257	const ptr = try self.resolveInst(bin_op.lhs);
	1258	const value = try self.resolveInst(bin_op.rhs);
	1259	const elem_ty = self.air.typeOf(bin_op.rhs);
	1260	switch (ptr) {
	1261	.none => unreachable,
	1262	.undef => unreachable,
	1263	.unreach => unreachable,
	1264	.dead => unreachable,
	1265	.compare_flags_unsigned => unreachable,
	1266	.compare_flags_signed => unreachable,
	1267	.immediate => \|imm\| {
	1268	try self.setRegOrMem(elem_ty, .{ .memory = imm }, value);
	1269	},
	1270	.ptr_stack_offset => \|off\| {
	1271	try self.genSetStack(elem_ty, off, value);
	1272	},
	1273	.ptr_embedded_in_code => \|off\| {
	1274	try self.setRegOrMem(elem_ty, .{ .embedded_in_code = off }, value);
	1275	},
	1276	.embedded_in_code => {
	1277	return self.fail("TODO implement storing to MCValue.embedded_in_code", .{});
	1278	},
	1279	.register => {
	1280	return self.fail("TODO implement storing to MCValue.register", .{});
	1281	},
	1282	.memory => {
	1283	return self.fail("TODO implement storing to MCValue.memory", .{});
	1284	},
	1285	.stack_offset => {
	1286	return self.fail("TODO implement storing to MCValue.stack_offset", .{});
	1287	},
	1288	}
	1289	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	1290	}
	1291
	1292	fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
	1293	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	1294	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
	1295	return self.structFieldPtr(extra.struct_operand, ty_pl.ty, extra.field_index);
	1296	}
	1297
	1298	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
	1299	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1300	return self.structFieldPtr(ty_op.operand, ty_op.ty, index);
	1301	}
	1302	fn structFieldPtr(self: *Self, operand: Air.Inst.Ref, ty: Air.Inst.Ref, index: u32) !void {
	1303	_ = self;
	1304	_ = operand;
	1305	_ = ty;
	1306	_ = index;
	1307	return self.fail("TODO implement codegen struct_field_ptr", .{});
	1308	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
	1309	}
	1310
	1311	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
	1312	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	1313	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
	1314	_ = extra;
	1315	return self.fail("TODO implement codegen struct_field_val", .{});
	1316	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
	1317	}
	1318
	1319	fn armOperandShouldBeRegister(self: *Self, mcv: MCValue) !bool {
	1320	return switch (mcv) {
	1321	.none => unreachable,
	1322	.undef => unreachable,
	1323	.dead, .unreach => unreachable,
	1324	.compare_flags_unsigned => unreachable,
	1325	.compare_flags_signed => unreachable,
	1326	.ptr_stack_offset => unreachable,
	1327	.ptr_embedded_in_code => unreachable,
	1328	.immediate => \|imm\| blk: {
	1329	if (imm > std.math.maxInt(u32)) return self.fail("TODO ARM binary arithmetic immediate larger than u32", .{});
	1330
	1331	// Load immediate into register if it doesn't fit
	1332	// in an operand
	1333	break :blk Instruction.Operand.fromU32(@intCast(u32, imm)) == null;
	1334	},
	1335	.register => true,
	1336	.stack_offset,
	1337	.embedded_in_code,
	1338	.memory,
	1339	=> true,
	1340	};
	1341	}
	1342
	1343	fn genArmBinOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref, op: Air.Inst.Tag) !MCValue {
	1344	// In the case of bitshifts, the type of rhs is different
	1345	// from the resulting type
	1346	const ty = self.air.typeOf(op_lhs);
	1347
	1348	switch (ty.zigTypeTag()) {
	1349	.Float => return self.fail("TODO ARM binary operations on floats", .{}),
	1350	.Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
	1351	.Bool => {
	1352	return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, 1, .unsigned);
	1353	},
	1354	.Int => {
	1355	const int_info = ty.intInfo(self.target.*);
	1356	return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, int_info.bits, int_info.signedness);
	1357	},
	1358	else => unreachable,
	1359	}
	1360	}
	1361
	1362	fn genArmBinIntOp(
	1363	self: *Self,
	1364	inst: Air.Inst.Index,
	1365	op_lhs: Air.Inst.Ref,
	1366	op_rhs: Air.Inst.Ref,
	1367	op: Air.Inst.Tag,
	1368	bits: u16,
	1369	signedness: std.builtin.Signedness,
	1370	) !MCValue {
	1371	if (bits > 32) {
	1372	return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
	1373	}
	1374
	1375	const lhs = try self.resolveInst(op_lhs);
	1376	const rhs = try self.resolveInst(op_rhs);
	1377
	1378	const lhs_is_register = lhs == .register;
	1379	const rhs_is_register = rhs == .register;
	1380	const lhs_should_be_register = switch (op) {
	1381	.shr, .shl => true,
	1382	else => try self.armOperandShouldBeRegister(lhs),
	1383	};
	1384	const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
	1385	const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
	1386	const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
	1387	const can_swap_lhs_and_rhs = switch (op) {
	1388	.shr, .shl => false,
	1389	else => true,
	1390	};
	1391
	1392	// Destination must be a register
	1393	var dst_mcv: MCValue = undefined;
	1394	var lhs_mcv = lhs;
	1395	var rhs_mcv = rhs;
	1396	var swap_lhs_and_rhs = false;
	1397
	1398	// Allocate registers for operands and/or destination
	1399	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	1400	if (reuse_lhs) {
	1401	// Allocate 0 or 1 registers
	1402	if (!rhs_is_register and rhs_should_be_register) {
	1403	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
	1404	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
	1405	}
	1406	dst_mcv = lhs;
	1407	} else if (reuse_rhs and can_swap_lhs_and_rhs) {
	1408	// Allocate 0 or 1 registers
	1409	if (!lhs_is_register and lhs_should_be_register) {
	1410	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
	1411	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
	1412	}
	1413	dst_mcv = rhs;
	1414
	1415	swap_lhs_and_rhs = true;
	1416	} else {
	1417	// Allocate 1 or 2 registers
	1418	if (lhs_should_be_register and rhs_should_be_register) {
	1419	if (lhs_is_register and rhs_is_register) {
	1420	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
	1421	} else if (lhs_is_register) {
	1422	// Move RHS to register
	1423	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
	1424	rhs_mcv = dst_mcv;
	1425	} else if (rhs_is_register) {
	1426	// Move LHS to register
	1427	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
	1428	lhs_mcv = dst_mcv;
	1429	} else {
	1430	// Move LHS and RHS to register
	1431	const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
	1432	lhs_mcv = MCValue{ .register = regs[0] };
	1433	rhs_mcv = MCValue{ .register = regs[1] };
	1434	dst_mcv = lhs_mcv;
	1435
	1436	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
	1437	}
	1438	} else if (lhs_should_be_register) {
	1439	// RHS is immediate
	1440	if (lhs_is_register) {
	1441	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
	1442	} else {
	1443	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
	1444	lhs_mcv = dst_mcv;
	1445	}
	1446	} else if (rhs_should_be_register and can_swap_lhs_and_rhs) {
	1447	// LHS is immediate
	1448	if (rhs_is_register) {
	1449	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
	1450	} else {
	1451	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
	1452	rhs_mcv = dst_mcv;
	1453	}
	1454
	1455	swap_lhs_and_rhs = true;
	1456	} else unreachable; // binary operation on two immediates
	1457	}
	1458
	1459	// Move the operands to the newly allocated registers
	1460	if (lhs_mcv == .register and !lhs_is_register) {
	1461	try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
	1462	}
	1463	if (rhs_mcv == .register and !rhs_is_register) {
	1464	try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
	1465	}
	1466
	1467	try self.genArmBinOpCode(
	1468	dst_mcv.register,
	1469	lhs_mcv,
	1470	rhs_mcv,
	1471	swap_lhs_and_rhs,
	1472	op,
	1473	signedness,
	1474	);
	1475	return dst_mcv;
	1476	}
	1477
	1478	fn genArmBinOpCode(
	1479	self: *Self,
	1480	dst_reg: Register,
	1481	lhs_mcv: MCValue,
	1482	rhs_mcv: MCValue,
	1483	swap_lhs_and_rhs: bool,
	1484	op: Air.Inst.Tag,
	1485	signedness: std.builtin.Signedness,
	1486	) !void {
	1487	assert(lhs_mcv == .register or rhs_mcv == .register);
	1488
	1489	const op1 = if (swap_lhs_and_rhs) rhs_mcv.register else lhs_mcv.register;
	1490	const op2 = if (swap_lhs_and_rhs) lhs_mcv else rhs_mcv;
	1491
	1492	const operand = switch (op2) {
	1493	.none => unreachable,
	1494	.undef => unreachable,
	1495	.dead, .unreach => unreachable,
	1496	.compare_flags_unsigned => unreachable,
	1497	.compare_flags_signed => unreachable,
	1498	.ptr_stack_offset => unreachable,
	1499	.ptr_embedded_in_code => unreachable,
	1500	.immediate => \|imm\| Instruction.Operand.fromU32(@intCast(u32, imm)).?,
	1501	.register => \|reg\| Instruction.Operand.reg(reg, Instruction.Operand.Shift.none),
	1502	.stack_offset,
	1503	.embedded_in_code,
	1504	.memory,
	1505	=> unreachable,
	1506	};
	1507
	1508	switch (op) {
	1509	.add => {
	1510	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.al, dst_reg, op1, operand).toU32());
	1511	},
	1512	.sub => {
	1513	if (swap_lhs_and_rhs) {
	1514	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.rsb(.al, dst_reg, op1, operand).toU32());
	1515	} else {
	1516	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.sub(.al, dst_reg, op1, operand).toU32());
	1517	}
	1518	},
	1519	.bool_and, .bit_and => {
	1520	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.@"and"(.al, dst_reg, op1, operand).toU32());
	1521	},
	1522	.bool_or, .bit_or => {
	1523	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, dst_reg, op1, operand).toU32());
	1524	},
	1525	.not, .xor => {
	1526	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.eor(.al, dst_reg, op1, operand).toU32());
	1527	},
	1528	.cmp_eq => {
	1529	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, op1, operand).toU32());
	1530	},
	1531	.shl => {
	1532	assert(!swap_lhs_and_rhs);
	1533	const shift_amount = switch (operand) {
	1534	.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
	1535	.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
	1536	};
	1537	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.lsl(.al, dst_reg, op1, shift_amount).toU32());
	1538	},
	1539	.shr => {
	1540	assert(!swap_lhs_and_rhs);
	1541	const shift_amount = switch (operand) {
	1542	.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
	1543	.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
	1544	};
	1545
	1546	const shr = switch (signedness) {
	1547	.signed => Instruction.asr,
	1548	.unsigned => Instruction.lsr,
	1549	};
	1550	self.writeInt(u32, try self.code.addManyAsArray(4), shr(.al, dst_reg, op1, shift_amount).toU32());
	1551	},
	1552	else => unreachable, // not a binary instruction
	1553	}
	1554	}
	1555
	1556	fn genArmMul(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
	1557	const lhs = try self.resolveInst(op_lhs);
	1558	const rhs = try self.resolveInst(op_rhs);
	1559
	1560	const lhs_is_register = lhs == .register;
	1561	const rhs_is_register = rhs == .register;
	1562	const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
	1563	const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
	1564
	1565	// Destination must be a register
	1566	// LHS must be a register
	1567	// RHS must be a register
	1568	var dst_mcv: MCValue = undefined;
	1569	var lhs_mcv: MCValue = lhs;
	1570	var rhs_mcv: MCValue = rhs;
	1571
	1572	// Allocate registers for operands and/or destination
	1573	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	1574	if (reuse_lhs) {
	1575	// Allocate 0 or 1 registers
	1576	if (!rhs_is_register) {
	1577	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
	1578	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
	1579	}
	1580	dst_mcv = lhs;
	1581	} else if (reuse_rhs) {
	1582	// Allocate 0 or 1 registers
	1583	if (!lhs_is_register) {
	1584	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
	1585	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
	1586	}
	1587	dst_mcv = rhs;
	1588	} else {
	1589	// Allocate 1 or 2 registers
	1590	if (lhs_is_register and rhs_is_register) {
	1591	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
	1592	} else if (lhs_is_register) {
	1593	// Move RHS to register
	1594	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
	1595	rhs_mcv = dst_mcv;
	1596	} else if (rhs_is_register) {
	1597	// Move LHS to register
	1598	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
	1599	lhs_mcv = dst_mcv;
	1600	} else {
	1601	// Move LHS and RHS to register
	1602	const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
	1603	lhs_mcv = MCValue{ .register = regs[0] };
	1604	rhs_mcv = MCValue{ .register = regs[1] };
	1605	dst_mcv = lhs_mcv;
	1606
	1607	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
	1608	}
	1609	}
	1610
	1611	// Move the operands to the newly allocated registers
	1612	if (!lhs_is_register) {
	1613	try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
	1614	}
	1615	if (!rhs_is_register) {
	1616	try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
	1617	}
	1618
	1619	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mul(.al, dst_mcv.register, lhs_mcv.register, rhs_mcv.register).toU32());
	1620	return dst_mcv;
	1621	}
	1622
	1623	fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
	1624	const ty_str = self.air.instructions.items(.data)[inst].ty_str;
	1625	const zir = &self.mod_fn.owner_decl.getFileScope().zir;
	1626	const name = zir.nullTerminatedString(ty_str.str);
	1627	const name_with_null = name.ptr[0 .. name.len + 1];
	1628	const ty = self.air.getRefType(ty_str.ty);
	1629
	1630	switch (mcv) {
	1631	.register => \|reg\| {
	1632	switch (self.debug_output) {
	1633	.dwarf => \|dbg_out\| {
	1634	try dbg_out.dbg_info.ensureUnusedCapacity(3);
	1635	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
	1636	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
	1637	1, // ULEB128 dwarf expression length
	1638	reg.dwarfLocOp(),
	1639	});
	1640	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	1641	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	1642	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	1643	},
	1644	.plan9 => {},
	1645	.none => {},
	1646	}
	1647	},
	1648	.stack_offset => \|offset\| {
	1649	switch (self.debug_output) {
	1650	.dwarf => \|dbg_out\| {
	1651	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
	1652	return self.fail("type '{}' too big to fit into stack frame", .{ty});
	1653	};
	1654	const adjusted_stack_offset = math.negateCast(offset + abi_size) catch {
	1655	return self.fail("Stack offset too large for arguments", .{});
	1656	};
	1657
	1658	try dbg_out.dbg_info.append(link.File.Elf.abbrev_parameter);
	1659
	1660	// Get length of the LEB128 stack offset
	1661	var counting_writer = std.io.countingWriter(std.io.null_writer);
	1662	leb128.writeILEB128(counting_writer.writer(), adjusted_stack_offset) catch unreachable;
	1663
	1664	// DW.AT.location, DW.FORM.exprloc
	1665	// ULEB128 dwarf expression length
	1666	try leb128.writeULEB128(dbg_out.dbg_info.writer(), counting_writer.bytes_written + 1);
	1667	try dbg_out.dbg_info.append(DW.OP.breg11);
	1668	try leb128.writeILEB128(dbg_out.dbg_info.writer(), adjusted_stack_offset);
	1669
	1670	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	1671	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	1672	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	1673	},
	1674	.plan9 => {},
	1675	.none => {},
	1676	}
	1677	},
	1678	else => {},
	1679	}
	1680	}
	1681
	1682	fn airArg(self: *Self, inst: Air.Inst.Index) !void {
	1683	const arg_index = self.arg_index;
	1684	self.arg_index += 1;
	1685
	1686	const ty = self.air.typeOfIndex(inst);
	1687
	1688	const result = self.args[arg_index];
	1689	const mcv = switch (result) {
	1690	// Copy registers to the stack
	1691	.register => \|reg\| blk: {
	1692	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
	1693	return self.fail("type '{}' too big to fit into stack frame", .{ty});
	1694	};
	1695	const abi_align = ty.abiAlignment(self.target.*);
	1696	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
	1697	try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
	1698
	1699	break :blk MCValue{ .stack_offset = stack_offset };
	1700	},
	1701	else => result,
	1702	};
	1703	try self.genArgDbgInfo(inst, mcv);
	1704
	1705	if (self.liveness.isUnused(inst))
	1706	return self.finishAirBookkeeping();
	1707
	1708	switch (mcv) {
	1709	.register => \|reg\| {
	1710	self.register_manager.getRegAssumeFree(reg, inst);
	1711	},
	1712	else => {},
	1713	}
	1714
	1715	return self.finishAir(inst, mcv, .{ .none, .none, .none });
	1716	}
	1717
	1718	fn airBreakpoint(self: *Self) !void {
	1719	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.bkpt(0).toU32());
	1720	return self.finishAirBookkeeping();
	1721	}
	1722
	1723	fn airFence(self: *Self) !void {
	1724	return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
	1725	//return self.finishAirBookkeeping();
	1726	}
	1727
	1728	fn airCall(self: *Self, inst: Air.Inst.Index) !void {
	1729	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
	1730	const fn_ty = self.air.typeOf(pl_op.operand);
	1731	const callee = pl_op.operand;
	1732	const extra = self.air.extraData(Air.Call, pl_op.payload);
	1733	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[extra.end..][0..extra.data.args_len]);
	1734
	1735	var info = try self.resolveCallingConventionValues(fn_ty);
	1736	defer info.deinit(self);
	1737
	1738	// Due to incremental compilation, how function calls are generated depends
	1739	// on linking.
	1740	if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {
	1741	for (info.args) \|mc_arg, arg_i\| {
	1742	const arg = args[arg_i];
	1743	const arg_ty = self.air.typeOf(arg);
	1744	const arg_mcv = try self.resolveInst(args[arg_i]);
	1745
	1746	switch (mc_arg) {
	1747	.none => continue,
	1748	.undef => unreachable,
	1749	.immediate => unreachable,
	1750	.unreach => unreachable,
	1751	.dead => unreachable,
	1752	.embedded_in_code => unreachable,
	1753	.memory => unreachable,
	1754	.compare_flags_signed => unreachable,
	1755	.compare_flags_unsigned => unreachable,
	1756	.register => \|reg\| {
	1757	try self.register_manager.getReg(reg, null);
	1758	try self.genSetReg(arg_ty, reg, arg_mcv);
	1759	},
	1760	.stack_offset => {
	1761	return self.fail("TODO implement calling with parameters in memory", .{});
	1762	},
	1763	.ptr_stack_offset => {
	1764	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
	1765	},
	1766	.ptr_embedded_in_code => {
	1767	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
	1768	},
	1769	}
	1770	}
	1771
	1772	if (self.air.value(callee)) \|func_value\| {
	1773	if (func_value.castTag(.function)) \|func_payload\| {
	1774	const func = func_payload.data;
	1775	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
	1776	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
	1777	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
	1778	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
	1779	break :blk @intCast(u32, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * ptr_bytes);
	1780	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\|
	1781	coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes
	1782	else
	1783	unreachable;
	1784
	1785	try self.genSetReg(Type.initTag(.usize), .lr, .{ .memory = got_addr });
	1786
	1787	// TODO: add Instruction.supportedOn
	1788	// function for ARM
	1789	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v5t)) {
	1790	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.blx(.al, .lr).toU32());
	1791	} else {
	1792	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .lr, Instruction.Operand.reg(.pc, Instruction.Operand.Shift.none)).toU32());
	1793	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.bx(.al, .lr).toU32());
	1794	}
	1795	} else if (func_value.castTag(.extern_fn)) \|_\| {
	1796	return self.fail("TODO implement calling extern functions", .{});
	1797	} else {
	1798	return self.fail("TODO implement calling bitcasted functions", .{});
	1799	}
	1800	} else {
	1801	return self.fail("TODO implement calling runtime known function pointer", .{});
	1802	}
	1803	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
	1804	unreachable; // unsupported architecture for MachO
	1805	} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {
	1806	return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});
	1807	} else unreachable;
	1808
	1809	const result: MCValue = result: {
	1810	switch (info.return_value) {
	1811	.register => \|reg\| {
	1812	if (Register.allocIndex(reg) == null) {
	1813	// Save function return value in a callee saved register
	1814	break :result try self.copyToNewRegister(inst, info.return_value);
	1815	}
	1816	},
	1817	else => {},
	1818	}
	1819	break :result info.return_value;
	1820	};
	1821
	1822	if (args.len <= Liveness.bpi - 2) {
	1823	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
	1824	buf[0] = callee;
	1825	std.mem.copy(Air.Inst.Ref, buf[1..], args);
	1826	return self.finishAir(inst, result, buf);
	1827	}
	1828	var bt = try self.iterateBigTomb(inst, 1 + args.len);
	1829	bt.feed(callee);
	1830	for (args) \|arg\| {
	1831	bt.feed(arg);
	1832	}
	1833	return bt.finishAir(result);
	1834	}
	1835
	1836	fn ret(self: *Self, mcv: MCValue) !void {
	1837	const ret_ty = self.fn_type.fnReturnType();
	1838	try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
	1839
	1840	// Just add space for an instruction, patch this later
	1841	try self.code.resize(self.code.items.len + 4);
	1842	try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
	1843	}
	1844
	1845	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
	1846	const un_op = self.air.instructions.items(.data)[inst].un_op;
	1847	const operand = try self.resolveInst(un_op);
	1848	try self.ret(operand);
	1849	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
	1850	}
	1851
	1852	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
	1853	const un_op = self.air.instructions.items(.data)[inst].un_op;
	1854	const ptr = try self.resolveInst(un_op);
	1855	_ = ptr;
	1856	return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch});
	1857	//return self.finishAir(inst, .dead, .{ un_op, .none, .none });
	1858	}
	1859
	1860	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
	1861	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1862	if (self.liveness.isUnused(inst))
	1863	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	1864	const ty = self.air.typeOf(bin_op.lhs);
	1865	assert(ty.eql(self.air.typeOf(bin_op.rhs)));
	1866	if (ty.zigTypeTag() == .ErrorSet)
	1867	return self.fail("TODO implement cmp for errors", .{});
	1868
	1869	const lhs = try self.resolveInst(bin_op.lhs);
	1870	const rhs = try self.resolveInst(bin_op.rhs);
	1871	const result: MCValue = result: {
	1872	const lhs_is_register = lhs == .register;
	1873	const rhs_is_register = rhs == .register;
	1874	// lhs should always be a register
	1875	const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
	1876
	1877	var lhs_mcv = lhs;
	1878	var rhs_mcv = rhs;
	1879
	1880	// Allocate registers
	1881	if (rhs_should_be_register) {
	1882	if (!lhs_is_register and !rhs_is_register) {
	1883	const regs = try self.register_manager.allocRegs(2, .{
	1884	Air.refToIndex(bin_op.rhs).?, Air.refToIndex(bin_op.lhs).?,
	1885	}, &.{});
	1886	lhs_mcv = MCValue{ .register = regs[0] };
	1887	rhs_mcv = MCValue{ .register = regs[1] };
	1888	} else if (!rhs_is_register) {
	1889	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.rhs).?, &.{}) };
	1890	}
	1891	}
	1892	if (!lhs_is_register) {
	1893	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.lhs).?, &.{}) };
	1894	}
	1895
	1896	// Move the operands to the newly allocated registers
	1897	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	1898	if (lhs_mcv == .register and !lhs_is_register) {
	1899	try self.genSetReg(ty, lhs_mcv.register, lhs);
	1900	branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.lhs).?, lhs);
	1901	}
	1902	if (rhs_mcv == .register and !rhs_is_register) {
	1903	try self.genSetReg(ty, rhs_mcv.register, rhs);
	1904	branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.rhs).?, rhs);
	1905	}
	1906
	1907	// The destination register is not present in the cmp instruction
	1908	// The signedness of the integer does not matter for the cmp instruction
	1909	try self.genArmBinOpCode(undefined, lhs_mcv, rhs_mcv, false, .cmp_eq, undefined);
	1910
	1911	break :result switch (ty.isSignedInt()) {
	1912	true => MCValue{ .compare_flags_signed = op },
	1913	false => MCValue{ .compare_flags_unsigned = op },
	1914	};
	1915	};
	1916	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1917	}
	1918
	1919	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
	1920	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
	1921	try self.dbgAdvancePCAndLine(dbg_stmt.line, dbg_stmt.column);
	1922	return self.finishAirBookkeeping();
	1923	}
	1924
	1925	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
	1926	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
	1927	const cond = try self.resolveInst(pl_op.operand);
	1928	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
	1929	const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
	1930	const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
	1931	const liveness_condbr = self.liveness.getCondBr(inst);
	1932
	1933	const reloc: Reloc = reloc: {
	1934	const condition: Condition = switch (cond) {
	1935	.compare_flags_signed => \|cmp_op\| blk: {
	1936	// Here we map to the opposite condition because the jump is to the false branch.
	1937	const condition = Condition.fromCompareOperatorSigned(cmp_op);
	1938	break :blk condition.negate();
	1939	},
	1940	.compare_flags_unsigned => \|cmp_op\| blk: {
	1941	// Here we map to the opposite condition because the jump is to the false branch.
	1942	const condition = Condition.fromCompareOperatorUnsigned(cmp_op);
	1943	break :blk condition.negate();
	1944	},
	1945	.register => \|reg\| blk: {
	1946	// cmp reg, 1
	1947	// bne ...
	1948	const op = Instruction.Operand.imm(1, 0);
	1949	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, reg, op).toU32());
	1950	break :blk .ne;
	1951	},
	1952	else => return self.fail("TODO implement condbr {} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
	1953	};
	1954
	1955	const reloc = Reloc{
	1956	.arm_branch = .{
	1957	.pos = self.code.items.len,
	1958	.cond = condition,
	1959	},
	1960	};
	1961	try self.code.resize(self.code.items.len + 4);
	1962	break :reloc reloc;
	1963	};
	1964
	1965	// Capture the state of register and stack allocation state so that we can revert to it.
	1966	const parent_next_stack_offset = self.next_stack_offset;
	1967	const parent_free_registers = self.register_manager.free_registers;
	1968	var parent_stack = try self.stack.clone(self.gpa);
	1969	defer parent_stack.deinit(self.gpa);
	1970	const parent_registers = self.register_manager.registers;
	1971
	1972	try self.branch_stack.append(.{});
	1973
	1974	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
	1975	for (liveness_condbr.then_deaths) \|operand\| {
	1976	self.processDeath(operand);
	1977	}
	1978	try self.genBody(then_body);
	1979
	1980	// Revert to the previous register and stack allocation state.
	1981
	1982	var saved_then_branch = self.branch_stack.pop();
	1983	defer saved_then_branch.deinit(self.gpa);
	1984
	1985	self.register_manager.registers = parent_registers;
	1986
	1987	self.stack.deinit(self.gpa);
	1988	self.stack = parent_stack;
	1989	parent_stack = .{};
	1990
	1991	self.next_stack_offset = parent_next_stack_offset;
	1992	self.register_manager.free_registers = parent_free_registers;
	1993
	1994	try self.performReloc(reloc);
	1995	const else_branch = self.branch_stack.addOneAssumeCapacity();
	1996	else_branch.* = .{};
	1997
	1998	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
	1999	for (liveness_condbr.else_deaths) \|operand\| {
	2000	self.processDeath(operand);
	2001	}
	2002	try self.genBody(else_body);
	2003
	2004	// At this point, each branch will possibly have conflicting values for where
	2005	// each instruction is stored. They agree, however, on which instructions are alive/dead.
	2006	// We use the first ("then") branch as canonical, and here emit
	2007	// instructions into the second ("else") branch to make it conform.
	2008	// We continue respect the data structure semantic guarantees of the else_branch so
	2009	// that we can use all the code emitting abstractions. This is why at the bottom we
	2010	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
	2011	// rather than assigning it.
	2012	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
	2013	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
	2014
	2015	const else_slice = else_branch.inst_table.entries.slice();
	2016	const else_keys = else_slice.items(.key);
	2017	const else_values = else_slice.items(.value);
	2018	for (else_keys) \|else_key, else_idx\| {
	2019	const else_value = else_values[else_idx];
	2020	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
	2021	// The instruction's MCValue is overridden in both branches.
	2022	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
	2023	if (else_value == .dead) {
	2024	assert(then_entry.value == .dead);
	2025	continue;
	2026	}
	2027	break :blk then_entry.value;
	2028	} else blk: {
	2029	if (else_value == .dead)
	2030	continue;
	2031	// The instruction is only overridden in the else branch.
	2032	var i: usize = self.branch_stack.items.len - 2;
	2033	while (true) {
	2034	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
	2035	if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
	2036	assert(mcv != .dead);
	2037	break :blk mcv;
	2038	}
	2039	}
	2040	};
	2041	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
	2042	// TODO make sure the destination stack offset / register does not already have something
	2043	// going on there.
	2044	try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
	2045	// TODO track the new register / stack allocation
	2046	}
	2047	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
	2048	const then_slice = saved_then_branch.inst_table.entries.slice();
	2049	const then_keys = then_slice.items(.key);
	2050	const then_values = then_slice.items(.value);
	2051	for (then_keys) \|then_key, then_idx\| {
	2052	const then_value = then_values[then_idx];
	2053	// We already deleted the items from this table that matched the else_branch.
	2054	// So these are all instructions that are only overridden in the then branch.
	2055	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
	2056	if (then_value == .dead)
	2057	continue;
	2058	const parent_mcv = blk: {
	2059	var i: usize = self.branch_stack.items.len - 2;
	2060	while (true) {
	2061	i -= 1;
	2062	if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
	2063	assert(mcv != .dead);
	2064	break :blk mcv;
	2065	}
	2066	}
	2067	};
	2068	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
	2069	// TODO make sure the destination stack offset / register does not already have something
	2070	// going on there.
	2071	try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
	2072	// TODO track the new register / stack allocation
	2073	}
	2074
	2075	self.branch_stack.pop().deinit(self.gpa);
	2076
	2077	return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
	2078	}
	2079
	2080	fn isNull(self: *Self, operand: MCValue) !MCValue {
	2081	_ = operand;
	2082	// Here you can specialize this instruction if it makes sense to, otherwise the default
	2083	// will call isNonNull and invert the result.
	2084	return self.fail("TODO call isNonNull and invert the result", .{});
	2085	}
	2086
	2087	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
	2088	_ = operand;
	2089	// Here you can specialize this instruction if it makes sense to, otherwise the default
	2090	// will call isNull and invert the result.
	2091	return self.fail("TODO call isNull and invert the result", .{});
	2092	}
	2093
	2094	fn isErr(self: *Self, operand: MCValue) !MCValue {
	2095	_ = operand;
	2096	// Here you can specialize this instruction if it makes sense to, otherwise the default
	2097	// will call isNonNull and invert the result.
	2098	return self.fail("TODO call isNonErr and invert the result", .{});
	2099	}
	2100
	2101	fn isNonErr(self: *Self, operand: MCValue) !MCValue {
	2102	_ = operand;
	2103	// Here you can specialize this instruction if it makes sense to, otherwise the default
	2104	// will call isNull and invert the result.
	2105	return self.fail("TODO call isErr and invert the result", .{});
	2106	}
	2107
	2108	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
	2109	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2110	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2111	const operand = try self.resolveInst(un_op);
	2112	break :result try self.isNull(operand);
	2113	};
	2114	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2115	}
	2116
	2117	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
	2118	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2119	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2120	const operand_ptr = try self.resolveInst(un_op);
	2121	const operand: MCValue = blk: {
	2122	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
	2123	// The MCValue that holds the pointer can be re-used as the value.
	2124	break :blk operand_ptr;
	2125	} else {
	2126	break :blk try self.allocRegOrMem(inst, true);
	2127	}
	2128	};
	2129	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
	2130	break :result try self.isNull(operand);
	2131	};
	2132	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2133	}
	2134
	2135	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
	2136	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2137	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2138	const operand = try self.resolveInst(un_op);
	2139	break :result try self.isNonNull(operand);
	2140	};
	2141	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2142	}
	2143
	2144	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
	2145	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2146	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2147	const operand_ptr = try self.resolveInst(un_op);
	2148	const operand: MCValue = blk: {
	2149	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
	2150	// The MCValue that holds the pointer can be re-used as the value.
	2151	break :blk operand_ptr;
	2152	} else {
	2153	break :blk try self.allocRegOrMem(inst, true);
	2154	}
	2155	};
	2156	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
	2157	break :result try self.isNonNull(operand);
	2158	};
	2159	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2160	}
	2161
	2162	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
	2163	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2164	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2165	const operand = try self.resolveInst(un_op);
	2166	break :result try self.isErr(operand);
	2167	};
	2168	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2169	}
	2170
	2171	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
	2172	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2173	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2174	const operand_ptr = try self.resolveInst(un_op);
	2175	const operand: MCValue = blk: {
	2176	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
	2177	// The MCValue that holds the pointer can be re-used as the value.
	2178	break :blk operand_ptr;
	2179	} else {
	2180	break :blk try self.allocRegOrMem(inst, true);
	2181	}
	2182	};
	2183	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
	2184	break :result try self.isErr(operand);
	2185	};
	2186	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2187	}
	2188
	2189	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
	2190	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2191	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2192	const operand = try self.resolveInst(un_op);
	2193	break :result try self.isNonErr(operand);
	2194	};
	2195	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2196	}
	2197
	2198	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
	2199	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2200	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	2201	const operand_ptr = try self.resolveInst(un_op);
	2202	const operand: MCValue = blk: {
	2203	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
	2204	// The MCValue that holds the pointer can be re-used as the value.
	2205	break :blk operand_ptr;
	2206	} else {
	2207	break :blk try self.allocRegOrMem(inst, true);
	2208	}
	2209	};
	2210	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
	2211	break :result try self.isNonErr(operand);
	2212	};
	2213	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2214	}
	2215
	2216	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
	2217	// A loop is a setup to be able to jump back to the beginning.
	2218	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	2219	const loop = self.air.extraData(Air.Block, ty_pl.payload);
	2220	const body = self.air.extra[loop.end..][0..loop.data.body_len];
	2221	const start_index = self.code.items.len;
	2222	try self.genBody(body);
	2223	try self.jump(start_index);
	2224	return self.finishAirBookkeeping();
	2225	}
	2226
	2227	/// Send control flow to the `index` of `self.code`.
	2228	fn jump(self: *Self, index: usize) !void {
	2229	if (math.cast(i26, @intCast(i32, index) - @intCast(i32, self.code.items.len + 8))) \|delta\| {
	2230	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.b(.al, delta).toU32());
	2231	} else \|_\| {
	2232	return self.fail("TODO: enable larger branch offset", .{});
	2233	}
	2234	}
	2235
	2236	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
	2237	try self.blocks.putNoClobber(self.gpa, inst, .{
	2238	// A block is a setup to be able to jump to the end.
	2239	.relocs = .{},
	2240	// It also acts as a receptacle for break operands.
	2241	// Here we use `MCValue.none` to represent a null value so that the first
	2242	// break instruction will choose a MCValue for the block result and overwrite
	2243	// this field. Following break instructions will use that MCValue to put their
	2244	// block results.
	2245	.mcv = MCValue{ .none = {} },
	2246	});
	2247	const block_data = self.blocks.getPtr(inst).?;
	2248	defer block_data.relocs.deinit(self.gpa);
	2249
	2250	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	2251	const extra = self.air.extraData(Air.Block, ty_pl.payload);
	2252	const body = self.air.extra[extra.end..][0..extra.data.body_len];
	2253	try self.genBody(body);
	2254
	2255	for (block_data.relocs.items) \|reloc\| try self.performReloc(reloc);
	2256
	2257	const result = @bitCast(MCValue, block_data.mcv);
	2258	return self.finishAir(inst, result, .{ .none, .none, .none });
	2259	}
	2260
	2261	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
	2262	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
	2263	const condition = pl_op.operand;
	2264	_ = condition;
	2265	return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch});
	2266	// return self.finishAir(inst, .dead, .{ condition, .none, .none });
	2267	}
	2268
	2269	fn performReloc(self: *Self, reloc: Reloc) !void {
	2270	switch (reloc) {
	2271	.rel32 => \|pos\| {
	2272	const amt = self.code.items.len - (pos + 4);
	2273	// Here it would be tempting to implement testing for amt == 0 and then elide the
	2274	// jump. However, that will cause a problem because other jumps may assume that they
	2275	// can jump to this code. Or maybe I didn't understand something when I was debugging.
	2276	// It could be worth another look. Anyway, that's why that isn't done here. Probably the
	2277	// best place to elide jumps will be in semantic analysis, by inlining blocks that only
	2278	// only have 1 break instruction.
	2279	const s32_amt = math.cast(i32, amt) catch
	2280	return self.fail("unable to perform relocation: jump too far", .{});
	2281	mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt);
	2282	},
	2283	.arm_branch => \|info\| {
	2284	const amt = @intCast(i32, self.code.items.len) - @intCast(i32, info.pos + 8);
	2285	if (math.cast(i26, amt)) \|delta\| {
	2286	self.writeInt(u32, self.code.items[info.pos..][0..4], Instruction.b(info.cond, delta).toU32());
	2287	} else \|_\| {
	2288	return self.fail("TODO: enable larger branch offset", .{});
	2289	}
	2290	},
	2291	}
	2292	}
	2293
	2294	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
	2295	const branch = self.air.instructions.items(.data)[inst].br;
	2296	try self.br(branch.block_inst, branch.operand);
	2297	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
	2298	}
	2299
	2300	fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
	2301	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	2302	const air_tags = self.air.instructions.items(.tag);
	2303	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (air_tags[inst]) {
	2304	.bool_and => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_and),
	2305	.bool_or => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_or),
	2306	else => unreachable, // Not a boolean operation
	2307	};
	2308	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	2309	}
	2310
	2311	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
	2312	const block_data = self.blocks.getPtr(block).?;
	2313
	2314	if (self.air.typeOf(operand).hasCodeGenBits()) {
	2315	const operand_mcv = try self.resolveInst(operand);
	2316	const block_mcv = block_data.mcv;
	2317	if (block_mcv == .none) {
	2318	block_data.mcv = operand_mcv;
	2319	} else {
	2320	try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv);
	2321	}
	2322	}
	2323	return self.brVoid(block);
	2324	}
	2325
	2326	fn brVoid(self: *Self, block: Air.Inst.Index) !void {
	2327	const block_data = self.blocks.getPtr(block).?;
	2328
	2329	// Emit a jump with a relocation. It will be patched up after the block ends.
	2330	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
	2331
	2332	try self.code.resize(self.code.items.len + 4);
	2333	block_data.relocs.appendAssumeCapacity(.{
	2334	.arm_branch = .{
	2335	.pos = self.code.items.len - 4,
	2336	.cond = .al,
	2337	},
	2338	});
	2339	}
	2340
	2341	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
	2342	const air_datas = self.air.instructions.items(.data);
	2343	const air_extra = self.air.extraData(Air.Asm, air_datas[inst].ty_pl.payload);
	2344	const zir = self.mod_fn.owner_decl.getFileScope().zir;
	2345	const extended = zir.instructions.items(.data)[air_extra.data.zir_index].extended;
	2346	const zir_extra = zir.extraData(Zir.Inst.Asm, extended.operand);
	2347	const asm_source = zir.nullTerminatedString(zir_extra.data.asm_source);
	2348	const outputs_len = @truncate(u5, extended.small);
	2349	const args_len = @truncate(u5, extended.small >> 5);
	2350	const clobbers_len = @truncate(u5, extended.small >> 10);
	2351	_ = clobbers_len; // TODO honor these
	2352	const is_volatile = @truncate(u1, extended.small >> 15) != 0;
	2353	const outputs = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end..][0..outputs_len]);
	2354	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end + outputs.len ..][0..args_len]);
	2355
	2356	if (outputs_len > 1) {
	2357	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
	2358	}
	2359	var extra_i: usize = zir_extra.end;
	2360	const output_constraint: ?[]const u8 = out: {
	2361	var i: usize = 0;
	2362	while (i < outputs_len) : (i += 1) {
	2363	const output = zir.extraData(Zir.Inst.Asm.Output, extra_i);
	2364	extra_i = output.end;
	2365	break :out zir.nullTerminatedString(output.data.constraint);
	2366	}
	2367	break :out null;
	2368	};
	2369
	2370	const dead = !is_volatile and self.liveness.isUnused(inst);
	2371	const result: MCValue = if (dead) .dead else result: {
	2372	for (args) \|arg\| {
	2373	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
	2374	extra_i = input.end;
	2375	const constraint = zir.nullTerminatedString(input.data.constraint);
	2376
	2377	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
	2378	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
	2379	}
	2380	const reg_name = constraint[1 .. constraint.len - 1];
	2381	const reg = parseRegName(reg_name) orelse
	2382	return self.fail("unrecognized register: '{s}'", .{reg_name});
	2383
	2384	const arg_mcv = try self.resolveInst(arg);
	2385	try self.register_manager.getReg(reg, null);
	2386	try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
	2387	}
	2388
	2389	if (mem.eql(u8, asm_source, "svc #0")) {
	2390	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.svc(.al, 0).toU32());
	2391	} else {
	2392	return self.fail("TODO implement support for more arm assembly instructions", .{});
	2393	}
	2394
	2395	if (output_constraint) \|output\| {
	2396	if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
	2397	return self.fail("unrecognized asm output constraint: '{s}'", .{output});
	2398	}
	2399	const reg_name = output[2 .. output.len - 1];
	2400	const reg = parseRegName(reg_name) orelse
	2401	return self.fail("unrecognized register: '{s}'", .{reg_name});
	2402
	2403	break :result MCValue{ .register = reg };
	2404	} else {
	2405	break :result MCValue{ .none = {} };
	2406	}
	2407	};
	2408	if (outputs.len + args.len <= Liveness.bpi - 1) {
	2409	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
	2410	std.mem.copy(Air.Inst.Ref, &buf, outputs);
	2411	std.mem.copy(Air.Inst.Ref, buf[outputs.len..], args);
	2412	return self.finishAir(inst, result, buf);
	2413	}
	2414	var bt = try self.iterateBigTomb(inst, outputs.len + args.len);
	2415	for (outputs) \|output\| {
	2416	bt.feed(output);
	2417	}
	2418	for (args) \|arg\| {
	2419	bt.feed(arg);
	2420	}
	2421	return bt.finishAir(result);
	2422	}
	2423
	2424	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
	2425	try self.ensureProcessDeathCapacity(operand_count + 1);
	2426	return BigTomb{
	2427	.function = self,
	2428	.inst = inst,
	2429	.tomb_bits = self.liveness.getTombBits(inst),
	2430	.big_tomb_bits = self.liveness.special.get(inst) orelse 0,
	2431	.bit_index = 0,
	2432	};
	2433	}
	2434
	2435	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
	2436	fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
	2437	switch (loc) {
	2438	.none => return,
	2439	.register => \|reg\| return self.genSetReg(ty, reg, val),
	2440	.stack_offset => \|off\| return self.genSetStack(ty, off, val),
	2441	.memory => {
	2442	return self.fail("TODO implement setRegOrMem for memory", .{});
	2443	},
	2444	else => unreachable,
	2445	}
	2446	}
	2447
	2448	fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
	2449	switch (mcv) {
	2450	.dead => unreachable,
	2451	.ptr_stack_offset => unreachable,
	2452	.ptr_embedded_in_code => unreachable,
	2453	.unreach, .none => return, // Nothing to do.
	2454	.undef => {
	2455	if (!self.wantSafety())
	2456	return; // The already existing value will do just fine.
	2457	// TODO Upgrade this to a memset call when we have that available.
	2458	switch (ty.abiSize(self.target.*)) {
	2459	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
	2460	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
	2461	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
	2462	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
	2463	else => return self.fail("TODO implement memset", .{}),
	2464	}
	2465	},
	2466	.compare_flags_unsigned,
	2467	.compare_flags_signed,
	2468	.immediate,
	2469	=> {
	2470	const reg = try self.copyToTmpRegister(ty, mcv);
	2471	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
	2472	},
	2473	.embedded_in_code => \|code_offset\| {
	2474	_ = code_offset;
	2475	return self.fail("TODO implement set stack variable from embedded_in_code", .{});
	2476	},
	2477	.register => \|reg\| {
	2478	const abi_size = ty.abiSize(self.target.*);
	2479	const adj_off = stack_offset + abi_size;
	2480
	2481	switch (abi_size) {
	2482	1, 4 => {
	2483	const offset = if (math.cast(u12, adj_off)) \|imm\| blk: {
	2484	break :blk Instruction.Offset.imm(imm);
	2485	} else \|_\| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
	2486	const str = switch (abi_size) {
	2487	1 => Instruction.strb,
	2488	4 => Instruction.str,
	2489	else => unreachable,
	2490	};
	2491
	2492	self.writeInt(u32, try self.code.addManyAsArray(4), str(.al, reg, .fp, .{
	2493	.offset = offset,
	2494	.positive = false,
	2495	}).toU32());
	2496	},
	2497	2 => {
	2498	const offset = if (adj_off <= math.maxInt(u8)) blk: {
	2499	break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
	2500	} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
	2501
	2502	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.strh(.al, reg, .fp, .{
	2503	.offset = offset,
	2504	.positive = false,
	2505	}).toU32());
	2506	},
	2507	else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
	2508	}
	2509	},
	2510	.memory => \|vaddr\| {
	2511	_ = vaddr;
	2512	return self.fail("TODO implement set stack variable from memory vaddr", .{});
	2513	},
	2514	.stack_offset => \|off\| {
	2515	if (stack_offset == off)
	2516	return; // Copy stack variable to itself; nothing to do.
	2517
	2518	const reg = try self.copyToTmpRegister(ty, mcv);
	2519	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
	2520	},
	2521	}
	2522	}
	2523
	2524	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
	2525	switch (mcv) {
	2526	.dead => unreachable,
	2527	.ptr_stack_offset => unreachable,
	2528	.ptr_embedded_in_code => unreachable,
	2529	.unreach, .none => return, // Nothing to do.
	2530	.undef => {
	2531	if (!self.wantSafety())
	2532	return; // The already existing value will do just fine.
	2533	// Write the debug undefined value.
	2534	return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa });
	2535	},
	2536	.compare_flags_unsigned,
	2537	.compare_flags_signed,
	2538	=> \|op\| {
	2539	const condition = switch (mcv) {
	2540	.compare_flags_unsigned => Condition.fromCompareOperatorUnsigned(op),
	2541	.compare_flags_signed => Condition.fromCompareOperatorSigned(op),
	2542	else => unreachable,
	2543	};
	2544
	2545	// mov reg, 0
	2546	// moveq reg, 1
	2547	const zero = Instruction.Operand.imm(0, 0);
	2548	const one = Instruction.Operand.imm(1, 0);
	2549	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, zero).toU32());
	2550	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(condition, reg, one).toU32());
	2551	},
	2552	.immediate => \|x\| {
	2553	if (x > math.maxInt(u32)) return self.fail("ARM registers are 32-bit wide", .{});
	2554
	2555	if (Instruction.Operand.fromU32(@intCast(u32, x))) \|op\| {
	2556	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, op).toU32());
	2557	} else if (Instruction.Operand.fromU32(~@intCast(u32, x))) \|op\| {
	2558	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mvn(.al, reg, op).toU32());
	2559	} else if (x <= math.maxInt(u16)) {
	2560	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
	2561	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @intCast(u16, x)).toU32());
	2562	} else {
	2563	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
	2564	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
	2565	}
	2566	} else {
	2567	// TODO write constant to code and load
	2568	// relative to pc
	2569	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
	2570	// immediate: 0xaaaabbbb
	2571	// movw reg, #0xbbbb
	2572	// movt reg, #0xaaaa
	2573	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @truncate(u16, x)).toU32());
	2574	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.movt(.al, reg, @truncate(u16, x >> 16)).toU32());
	2575	} else {
	2576	// immediate: 0xaabbccdd
	2577	// mov reg, #0xaa
	2578	// orr reg, reg, #0xbb, 24
	2579	// orr reg, reg, #0xcc, 16
	2580	// orr reg, reg, #0xdd, 8
	2581	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
	2582	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
	2583	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 16), 8)).toU32());
	2584	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 24), 4)).toU32());
	2585	}
	2586	}
	2587	},
	2588	.register => \|src_reg\| {
	2589	// If the registers are the same, nothing to do.
	2590	if (src_reg.id() == reg.id())
	2591	return;
	2592
	2593	// mov reg, src_reg
	2594	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.reg(src_reg, Instruction.Operand.Shift.none)).toU32());
	2595	},
	2596	.memory => \|addr\| {
	2597	// The value is in memory at a hard-coded address.
	2598	// If the type is a pointer, it means the pointer address is at this memory location.
	2599	try self.genSetReg(ty, reg, .{ .immediate = addr });
	2600	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
	2601	},
	2602	.stack_offset => \|unadjusted_off\| {
	2603	// TODO: maybe addressing from sp instead of fp
	2604	const abi_size = ty.abiSize(self.target.*);
	2605	const adj_off = unadjusted_off + abi_size;
	2606
	2607	switch (abi_size) {
	2608	1, 4 => {
	2609	const offset = if (adj_off <= math.maxInt(u12)) blk: {
	2610	break :blk Instruction.Offset.imm(@intCast(u12, adj_off));
	2611	} else Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
	2612	const ldr = switch (abi_size) {
	2613	1 => Instruction.ldrb,
	2614	4 => Instruction.ldr,
	2615	else => unreachable,
	2616	};
	2617
	2618	self.writeInt(u32, try self.code.addManyAsArray(4), ldr(.al, reg, .fp, .{
	2619	.offset = offset,
	2620	.positive = false,
	2621	}).toU32());
	2622	},
	2623	2 => {
	2624	const offset = if (adj_off <= math.maxInt(u8)) blk: {
	2625	break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
	2626	} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
	2627
	2628	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldrh(.al, reg, .fp, .{
	2629	.offset = offset,
	2630	.positive = false,
	2631	}).toU32());
	2632	},
	2633	else => return self.fail("TODO a type of size {} is not allowed in a register", .{abi_size}),
	2634	}
	2635	},
	2636	else => return self.fail("TODO implement getSetReg for arm {}", .{mcv}),
	2637	}
	2638	}
	2639
	2640	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
	2641	const un_op = self.air.instructions.items(.data)[inst].un_op;
	2642	const result = try self.resolveInst(un_op);
	2643	return self.finishAir(inst, result, .{ un_op, .none, .none });
	2644	}
	2645
	2646	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
	2647	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	2648	const result = try self.resolveInst(ty_op.operand);
	2649	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	2650	}
	2651
	2652	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
	2653	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	2654	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airArrayToSlice for {}", .{
	2655	self.target.cpu.arch,
	2656	});
	2657	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	2658	}
	2659
	2660	fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
	2661	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	2662	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntToFloat for {}", .{
	2663	self.target.cpu.arch,
	2664	});
	2665	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	2666	}
	2667
	2668	fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
	2669	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	2670	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatToInt for {}", .{
	2671	self.target.cpu.arch,
	2672	});
	2673	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	2674	}
	2675
	2676	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
	2677	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	2678	const extra = self.air.extraData(Air.Block, ty_pl.payload);
	2679	_ = extra;
	2680
	2681	return self.fail("TODO implement airCmpxchg for {}", .{
	2682	self.target.cpu.arch,
	2683	});
	2684	}
	2685
	2686	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
	2687	_ = inst;
	2688	return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
	2689	}
	2690
	2691	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
	2692	_ = inst;
	2693	return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
	2694	}
	2695
	2696	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
	2697	_ = inst;
	2698	_ = order;
	2699	return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
	2700	}
	2701
	2702	fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
	2703	_ = inst;
	2704	return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
	2705	}
	2706
	2707	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
	2708	_ = inst;
	2709	return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
	2710	}
	2711
	2712	fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
	2713	// First section of indexes correspond to a set number of constant values.
	2714	const ref_int = @enumToInt(inst);
	2715	if (ref_int < Air.Inst.Ref.typed_value_map.len) {
	2716	const tv = Air.Inst.Ref.typed_value_map[ref_int];
	2717	if (!tv.ty.hasCodeGenBits()) {
	2718	return MCValue{ .none = {} };
	2719	}
	2720	return self.genTypedValue(tv);
	2721	}
	2722
	2723	// If the type has no codegen bits, no need to store it.
	2724	const inst_ty = self.air.typeOf(inst);
	2725	if (!inst_ty.hasCodeGenBits())
	2726	return MCValue{ .none = {} };
	2727
	2728	const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
	2729	switch (self.air.instructions.items(.tag)[inst_index]) {
	2730	.constant => {
	2731	// Constants have static lifetimes, so they are always memoized in the outer most table.
	2732	const branch = &self.branch_stack.items[0];
	2733	const gop = try branch.inst_table.getOrPut(self.gpa, inst_index);
	2734	if (!gop.found_existing) {
	2735	const ty_pl = self.air.instructions.items(.data)[inst_index].ty_pl;
	2736	gop.value_ptr.* = try self.genTypedValue(.{
	2737	.ty = inst_ty,
	2738	.val = self.air.values[ty_pl.payload],
	2739	});
	2740	}
	2741	return gop.value_ptr.*;
	2742	},
	2743	.const_ty => unreachable,
	2744	else => return self.getResolvedInstValue(inst_index),
	2745	}
	2746	}
	2747
	2748	fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
	2749	// Treat each stack item as a "layer" on top of the previous one.
	2750	var i: usize = self.branch_stack.items.len;
	2751	while (true) {
	2752	i -= 1;
	2753	if (self.branch_stack.items[i].inst_table.get(inst)) \|mcv\| {
	2754	assert(mcv != .dead);
	2755	return mcv;
	2756	}
	2757	}
	2758	}
	2759
	2760	/// If the MCValue is an immediate, and it does not fit within this type,
	2761	/// we put it in a register.
	2762	/// A potential opportunity for future optimization here would be keeping track
	2763	/// of the fact that the instruction is available both as an immediate
	2764	/// and as a register.
	2765	fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
	2766	const mcv = try self.resolveInst(operand);
	2767	const ti = @typeInfo(T).Int;
	2768	switch (mcv) {
	2769	.immediate => \|imm\| {
	2770	// This immediate is unsigned.
	2771	const U = std.meta.Int(.unsigned, ti.bits - @boolToInt(ti.signedness == .signed));
	2772	if (imm >= math.maxInt(U)) {
	2773	return MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.usize), mcv) };
	2774	}
	2775	},
	2776	else => {},
	2777	}
	2778	return mcv;
	2779	}
	2780
	2781	fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
	2782	if (typed_value.val.isUndef())
	2783	return MCValue{ .undef = {} };
	2784	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
	2785	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
	2786	switch (typed_value.ty.zigTypeTag()) {
	2787	.Pointer => switch (typed_value.ty.ptrSize()) {
	2788	.Slice => {
	2789	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
	2790	const ptr_type = typed_value.ty.slicePtrFieldType(&buf);
	2791	const ptr_mcv = try self.genTypedValue(.{ .ty = ptr_type, .val = typed_value.val });
	2792	const slice_len = typed_value.val.sliceLen();
	2793	// Codegen can't handle some kinds of indirection. If the wrong union field is accessed here it may mean
	2794	// the Sema code needs to use anonymous Decls or alloca instructions to store data.
	2795	const ptr_imm = ptr_mcv.memory;
	2796	_ = slice_len;
	2797	_ = ptr_imm;
	2798	// We need more general support for const data being stored in memory to make this work.
	2799	return self.fail("TODO codegen for const slices", .{});
	2800	},
	2801	else => {
	2802	if (typed_value.val.castTag(.decl_ref)) \|payload\| {
	2803	const decl = payload.data;
	2804	decl.alive = true;
	2805	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
	2806	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
	2807	const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes;
	2808	return MCValue{ .memory = got_addr };
	2809	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
	2810	// TODO I'm hacking my way through here by repurposing .memory for storing
	2811	// index to the GOT target symbol index.
	2812	return MCValue{ .memory = decl.link.macho.local_sym_index };
	2813	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
	2814	const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes;
	2815	return MCValue{ .memory = got_addr };
	2816	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
	2817	try p9.seeDecl(decl);
	2818	const got_addr = p9.bases.data + decl.link.plan9.got_index.? * ptr_bytes;
	2819	return MCValue{ .memory = got_addr };
	2820	} else {
	2821	return self.fail("TODO codegen non-ELF const Decl pointer", .{});
	2822	}
	2823	}
	2824	if (typed_value.val.tag() == .int_u64) {
	2825	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
	2826	}
	2827	return self.fail("TODO codegen more kinds of const pointers", .{});
	2828	},
	2829	},
	2830	.Int => {
	2831	const info = typed_value.ty.intInfo(self.target.*);
	2832	if (info.bits > ptr_bits or info.signedness == .signed) {
	2833	return self.fail("TODO const int bigger than ptr and signed int", .{});
	2834	}
	2835	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
	2836	},
	2837	.Bool => {
	2838	return MCValue{ .immediate = @boolToInt(typed_value.val.toBool()) };
	2839	},
	2840	.ComptimeInt => unreachable, // semantic analysis prevents this
	2841	.ComptimeFloat => unreachable, // semantic analysis prevents this
	2842	.Optional => {
	2843	if (typed_value.ty.isPtrLikeOptional()) {
	2844	if (typed_value.val.isNull())
	2845	return MCValue{ .immediate = 0 };
	2846
	2847	var buf: Type.Payload.ElemType = undefined;
	2848	return self.genTypedValue(.{
	2849	.ty = typed_value.ty.optionalChild(&buf),
	2850	.val = typed_value.val,
	2851	});
	2852	} else if (typed_value.ty.abiSize(self.target.*) == 1) {
	2853	return MCValue{ .immediate = @boolToInt(typed_value.val.isNull()) };
	2854	}
	2855	return self.fail("TODO non pointer optionals", .{});
	2856	},
	2857	.Enum => {
	2858	if (typed_value.val.castTag(.enum_field_index)) \|field_index\| {
	2859	switch (typed_value.ty.tag()) {
	2860	.enum_simple => {
	2861	return MCValue{ .immediate = field_index.data };
	2862	},
	2863	.enum_full, .enum_nonexhaustive => {
	2864	const enum_full = typed_value.ty.cast(Type.Payload.EnumFull).?.data;
	2865	if (enum_full.values.count() != 0) {
	2866	const tag_val = enum_full.values.keys()[field_index.data];
	2867	return self.genTypedValue(.{ .ty = enum_full.tag_ty, .val = tag_val });
	2868	} else {
	2869	return MCValue{ .immediate = field_index.data };
	2870	}
	2871	},
	2872	else => unreachable,
	2873	}
	2874	} else {
	2875	var int_tag_buffer: Type.Payload.Bits = undefined;
	2876	const int_tag_ty = typed_value.ty.intTagType(&int_tag_buffer);
	2877	return self.genTypedValue(.{ .ty = int_tag_ty, .val = typed_value.val });
	2878	}
	2879	},
	2880	.ErrorSet => {
	2881	switch (typed_value.val.tag()) {
	2882	.@"error" => {
	2883	const err_name = typed_value.val.castTag(.@"error").?.data.name;
	2884	const module = self.bin_file.options.module.?;
	2885	const global_error_set = module.global_error_set;
	2886	const error_index = global_error_set.get(err_name).?;
	2887	return MCValue{ .immediate = error_index };
	2888	},
	2889	else => {
	2890	// In this case we are rendering an error union which has a 0 bits payload.
	2891	return MCValue{ .immediate = 0 };
	2892	},
	2893	}
	2894	},
	2895	.ErrorUnion => {
	2896	const error_type = typed_value.ty.errorUnionSet();
	2897	const payload_type = typed_value.ty.errorUnionPayload();
	2898	const sub_val = typed_value.val.castTag(.eu_payload).?.data;
	2899
	2900	if (!payload_type.hasCodeGenBits()) {
	2901	// We use the error type directly as the type.
	2902	return self.genTypedValue(.{ .ty = error_type, .val = sub_val });
	2903	}
	2904
	2905	return self.fail("TODO implement error union const of type '{}'", .{typed_value.ty});
	2906	},
	2907	else => return self.fail("TODO implement const of type '{}'", .{typed_value.ty}),
	2908	}
	2909	}
	2910
	2911	const CallMCValues = struct {
	2912	args: []MCValue,
	2913	return_value: MCValue,
	2914	stack_byte_count: u32,
	2915	stack_align: u32,
	2916
	2917	fn deinit(self: CallMCValues, func: Self) void {
	2918	func.gpa.free(self.args);
	2919	self.* = undefined;
	2920	}
	2921	};
	2922
	2923	/// Caller must call `CallMCValues.deinit`.
	2924	fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
	2925	const cc = fn_ty.fnCallingConvention();
	2926	const param_types = try self.gpa.alloc(Type, fn_ty.fnParamLen());
	2927	defer self.gpa.free(param_types);
	2928	fn_ty.fnParamTypes(param_types);
	2929	var result: CallMCValues = .{
	2930	.args = try self.gpa.alloc(MCValue, param_types.len),
	2931	// These undefined values must be populated before returning from this function.
	2932	.return_value = undefined,
	2933	.stack_byte_count = undefined,
	2934	.stack_align = undefined,
	2935	};
	2936	errdefer self.gpa.free(result.args);
	2937
	2938	const ret_ty = fn_ty.fnReturnType();
	2939
	2940	switch (cc) {
	2941	.Naked => {
	2942	assert(result.args.len == 0);
	2943	result.return_value = .{ .unreach = {} };
	2944	result.stack_byte_count = 0;
	2945	result.stack_align = 1;
	2946	return result;
	2947	},
	2948	.Unspecified, .C => {
	2949	// ARM Procedure Call Standard, Chapter 6.5
	2950	var ncrn: usize = 0; // Next Core Register Number
	2951	var nsaa: u32 = 0; // Next stacked argument address
	2952
	2953	for (param_types) \|ty, i\| {
	2954	if (ty.abiAlignment(self.target.*) == 8)
	2955	ncrn = std.mem.alignForwardGeneric(usize, ncrn, 2);
	2956
	2957	const param_size = @intCast(u32, ty.abiSize(self.target.*));
	2958	if (std.math.divCeil(u32, param_size, 4) catch unreachable <= 4 - ncrn) {
	2959	if (param_size <= 4) {
	2960	result.args[i] = .{ .register = c_abi_int_param_regs[ncrn] };
	2961	ncrn += 1;
	2962	} else {
	2963	return self.fail("TODO MCValues with multiple registers", .{});
	2964	}
	2965	} else if (ncrn < 4 and nsaa == 0) {
	2966	return self.fail("TODO MCValues split between registers and stack", .{});
	2967	} else {
	2968	ncrn = 4;
	2969	if (ty.abiAlignment(self.target.*) == 8)
	2970	nsaa = std.mem.alignForwardGeneric(u32, nsaa, 8);
	2971
	2972	result.args[i] = .{ .stack_offset = nsaa };
	2973	nsaa += param_size;
	2974	}
	2975	}
	2976
	2977	result.stack_byte_count = nsaa;
	2978	result.stack_align = 8;
	2979	},
	2980	else => return self.fail("TODO implement function parameters for {} on arm", .{cc}),
	2981	}
	2982
	2983	if (ret_ty.zigTypeTag() == .NoReturn) {
	2984	result.return_value = .{ .unreach = {} };
	2985	} else if (!ret_ty.hasCodeGenBits()) {
	2986	result.return_value = .{ .none = {} };
	2987	} else switch (cc) {
	2988	.Naked => unreachable,
	2989	.Unspecified, .C => {
	2990	const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
	2991	if (ret_ty_size <= 4) {
	2992	result.return_value = .{ .register = c_abi_int_return_regs[0] };
	2993	} else {
	2994	return self.fail("TODO support more return types for ARM backend", .{});
	2995	}
	2996	},
	2997	else => return self.fail("TODO implement function return values for {}", .{cc}),
	2998	}
	2999	return result;
	3000	}
	3001
	3002	/// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`.
	3003	fn wantSafety(self: *Self) bool {
	3004	return switch (self.bin_file.options.optimize_mode) {
	3005	.Debug => true,
	3006	.ReleaseSafe => true,
	3007	.ReleaseFast => false,
	3008	.ReleaseSmall => false,
	3009	};
	3010	}
	3011
	3012	fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError {
	3013	@setCold(true);
	3014	assert(self.err_msg == null);
	3015	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
	3016	return error.CodegenFail;
	3017	}
	3018
	3019	fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError {
	3020	@setCold(true);
	3021	assert(self.err_msg == null);
	3022	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
	3023	return error.CodegenFail;
	3024	}
	3025
	3026	const Register = @import("bits.zig").Register;
	3027	const Instruction = @import("bits.zig").Instruction;
	3028	const Condition = @import("bits.zig").Condition;
	3029	const callee_preserved_regs = @import("bits.zig").callee_preserved_regs;
	3030	const c_abi_int_param_regs = @import("bits.zig").c_abi_int_param_regs;
	3031	const c_abi_int_return_regs = @import("bits.zig").c_abi_int_return_regs;
	3032
	3033	fn parseRegName(name: []const u8) ?Register {
	3034	if (@hasDecl(Register, "parseRegName")) {
	3035	return Register.parseRegName(name);
	3036	}
	3037	return std.meta.stringToEnum(Register, name);
	3038	}

src/codegen.zig+3-3395

...	...	@@ -84,8 +84,9 @@ pub fn generateFunction(
84	84	switch (bin_file.options.target.cpu.arch) {
85	85	.wasm32 => unreachable, // has its own code path
86	86	.wasm64 => unreachable, // has its own code path
87		.arm => return Function(.arm).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
88		.armeb => return Function(.armeb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
	87	.arm,
	88	.armeb,
	89	=> return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
89	90	.aarch64,
90	91	.aarch64_be,
91	92	.aarch64_32,
...	...	@@ -303,3396 +304,3 @@ pub fn generateSymbol(
303	304	},
304	305	}
305	306	}
306
307		const InnerError = error{
308		OutOfMemory,
309		CodegenFail,
310		};
311
312		fn Function(comptime arch: std.Target.Cpu.Arch) type {
313		const writeInt = switch (arch.endian()) {
314		.Little => mem.writeIntLittle,
315		.Big => mem.writeIntBig,
316		};
317
318		return struct {
319		gpa: *Allocator,
320		air: Air,
321		liveness: Liveness,
322		bin_file: *link.File,
323		target: *const std.Target,
324		mod_fn: *const Module.Fn,
325		code: *std.ArrayList(u8),
326		debug_output: DebugInfoOutput,
327		err_msg: ?*ErrorMsg,
328		args: []MCValue,
329		ret_mcv: MCValue,
330		fn_type: Type,
331		arg_index: usize,
332		src_loc: Module.SrcLoc,
333		stack_align: u32,
334
335		prev_di_line: u32,
336		prev_di_column: u32,
337		/// Byte offset within the source file of the ending curly.
338		end_di_line: u32,
339		end_di_column: u32,
340		/// Relative to the beginning of `code`.
341		prev_di_pc: usize,
342
343		/// The value is an offset into the `Function` `code` from the beginning.
344		/// To perform the reloc, write 32-bit signed little-endian integer
345		/// which is a relative jump, based on the address following the reloc.
346		exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{},
347
348		/// Whenever there is a runtime branch, we push a Branch onto this stack,
349		/// and pop it off when the runtime branch joins. This provides an "overlay"
350		/// of the table of mappings from instructions to `MCValue` from within the branch.
351		/// This way we can modify the `MCValue` for an instruction in different ways
352		/// within different branches. Special consideration is needed when a branch
353		/// joins with its parent, to make sure all instructions have the same MCValue
354		/// across each runtime branch upon joining.
355		branch_stack: *std.ArrayList(Branch),
356
357		// Key is the block instruction
358		blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
359
360		register_manager: RegisterManager(Self, Register, &callee_preserved_regs) = .{},
361		/// Maps offset to what is stored there.
362		stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
363
364		/// Offset from the stack base, representing the end of the stack frame.
365		max_end_stack: u32 = 0,
366		/// Represents the current end stack offset. If there is no existing slot
367		/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
368		next_stack_offset: u32 = 0,
369
370		/// Debug field, used to find bugs in the compiler.
371		air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
372
373		const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
374
375		const MCValue = union(enum) {
376		/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
377		/// TODO Look into deleting this tag and using `dead` instead, since every use
378		/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
379		none,
380		/// Control flow will not allow this value to be observed.
381		unreach,
382		/// No more references to this value remain.
383		dead,
384		/// The value is undefined.
385		undef,
386		/// A pointer-sized integer that fits in a register.
387		/// If the type is a pointer, this is the pointer address in virtual address space.
388		immediate: u64,
389		/// The constant was emitted into the code, at this offset.
390		/// If the type is a pointer, it means the pointer address is embedded in the code.
391		embedded_in_code: usize,
392		/// The value is a pointer to a constant which was emitted into the code, at this offset.
393		ptr_embedded_in_code: usize,
394		/// The value is in a target-specific register.
395		register: Register,
396		/// The value is in memory at a hard-coded address.
397		/// If the type is a pointer, it means the pointer address is at this memory location.
398		memory: u64,
399		/// The value is one of the stack variables.
400		/// If the type is a pointer, it means the pointer address is in the stack at this offset.
401		stack_offset: u32,
402		/// The value is a pointer to one of the stack variables (payload is stack offset).
403		ptr_stack_offset: u32,
404		/// The value is in the compare flags assuming an unsigned operation,
405		/// with this operator applied on top of it.
406		compare_flags_unsigned: math.CompareOperator,
407		/// The value is in the compare flags assuming a signed operation,
408		/// with this operator applied on top of it.
409		compare_flags_signed: math.CompareOperator,
410
411		fn isMemory(mcv: MCValue) bool {
412		return switch (mcv) {
413		.embedded_in_code, .memory, .stack_offset => true,
414		else => false,
415		};
416		}
417
418		fn isImmediate(mcv: MCValue) bool {
419		return switch (mcv) {
420		.immediate => true,
421		else => false,
422		};
423		}
424
425		fn isMutable(mcv: MCValue) bool {
426		return switch (mcv) {
427		.none => unreachable,
428		.unreach => unreachable,
429		.dead => unreachable,
430
431		.immediate,
432		.embedded_in_code,
433		.memory,
434		.compare_flags_unsigned,
435		.compare_flags_signed,
436		.ptr_stack_offset,
437		.ptr_embedded_in_code,
438		.undef,
439		=> false,
440
441		.register,
442		.stack_offset,
443		=> true,
444		};
445		}
446		};
447
448		const Branch = struct {
449		inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .{},
450
451		fn deinit(self: Branch, gpa: Allocator) void {
452		self.inst_table.deinit(gpa);
453		self.* = undefined;
454		}
455		};
456
457		const StackAllocation = struct {
458		inst: Air.Inst.Index,
459		/// TODO do we need size? should be determined by inst.ty.abiSize()
460		size: u32,
461		};
462
463		const BlockData = struct {
464		relocs: std.ArrayListUnmanaged(Reloc),
465		/// The first break instruction encounters `null` here and chooses a
466		/// machine code value for the block result, populating this field.
467		/// Following break instructions encounter that value and use it for
468		/// the location to store their block results.
469		mcv: MCValue,
470		};
471
472		const Reloc = union(enum) {
473		/// The value is an offset into the `Function` `code` from the beginning.
474		/// To perform the reloc, write 32-bit signed little-endian integer
475		/// which is a relative jump, based on the address following the reloc.
476		rel32: usize,
477		/// A branch in the ARM instruction set
478		arm_branch: struct {
479		pos: usize,
480		cond: @import("arch/arm/bits.zig").Condition,
481		},
482		};
483
484		const BigTomb = struct {
485		function: *Self,
486		inst: Air.Inst.Index,
487		tomb_bits: Liveness.Bpi,
488		big_tomb_bits: u32,
489		bit_index: usize,
490
491		fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
492		const this_bit_index = bt.bit_index;
493		bt.bit_index += 1;
494
495		const op_int = @enumToInt(op_ref);
496		if (op_int < Air.Inst.Ref.typed_value_map.len) return;
497		const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
498
499		if (this_bit_index < Liveness.bpi - 1) {
500		const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
501		if (!dies) return;
502		} else {
503		const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
504		const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
505		if (!dies) return;
506		}
507		bt.function.processDeath(op_index);
508		}
509
510		fn finishAir(bt: *BigTomb, result: MCValue) void {
511		const is_used = !bt.function.liveness.isUnused(bt.inst);
512		if (is_used) {
513		log.debug("%{d} => {}", .{ bt.inst, result });
514		const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
515		branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
516		}
517		bt.function.finishAirBookkeeping();
518		}
519		};
520
521		const Self = @This();
522
523		fn generate(
524		bin_file: *link.File,
525		src_loc: Module.SrcLoc,
526		module_fn: *Module.Fn,
527		air: Air,
528		liveness: Liveness,
529		code: *std.ArrayList(u8),
530		debug_output: DebugInfoOutput,
531		) GenerateSymbolError!FnResult {
532		if (build_options.skip_non_native and builtin.cpu.arch != arch) {
533		@panic("Attempted to compile for architecture that was disabled by build configuration");
534		}
535
536		assert(module_fn.owner_decl.has_tv);
537		const fn_type = module_fn.owner_decl.ty;
538
539		var branch_stack = std.ArrayList(Branch).init(bin_file.allocator);
540		defer {
541		assert(branch_stack.items.len == 1);
542		branch_stack.items[0].deinit(bin_file.allocator);
543		branch_stack.deinit();
544		}
545		try branch_stack.append(.{});
546
547		var function = Self{
548		.gpa = bin_file.allocator,
549		.air = air,
550		.liveness = liveness,
551		.target = &bin_file.options.target,
552		.bin_file = bin_file,
553		.mod_fn = module_fn,
554		.code = code,
555		.debug_output = debug_output,
556		.err_msg = null,
557		.args = undefined, // populated after `resolveCallingConventionValues`
558		.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
559		.fn_type = fn_type,
560		.arg_index = 0,
561		.branch_stack = &branch_stack,
562		.src_loc = src_loc,
563		.stack_align = undefined,
564		.prev_di_pc = 0,
565		.prev_di_line = module_fn.lbrace_line,
566		.prev_di_column = module_fn.lbrace_column,
567		.end_di_line = module_fn.rbrace_line,
568		.end_di_column = module_fn.rbrace_column,
569		};
570		defer function.stack.deinit(bin_file.allocator);
571		defer function.blocks.deinit(bin_file.allocator);
572		defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
573
574		var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
575		error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
576		else => \|e\| return e,
577		};
578		defer call_info.deinit(&function);
579
580		function.args = call_info.args;
581		function.ret_mcv = call_info.return_value;
582		function.stack_align = call_info.stack_align;
583		function.max_end_stack = call_info.stack_byte_count;
584
585		function.gen() catch \|err\| switch (err) {
586		error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
587		else => \|e\| return e,
588		};
589
590		if (function.err_msg) \|em\| {
591		return FnResult{ .fail = em };
592		} else {
593		return FnResult{ .appended = {} };
594		}
595		}
596
597		fn gen(self: *Self) !void {
598		switch (arch) {
599		.arm, .armeb => {
600		const cc = self.fn_type.fnCallingConvention();
601		if (cc != .Naked) {
602		// push {fp, lr}
603		// mov fp, sp
604		// sub sp, sp, #reloc
605		const prologue_reloc = self.code.items.len;
606		try self.code.resize(prologue_reloc + 12);
607		writeInt(u32, self.code.items[prologue_reloc + 4 ..][0..4], Instruction.mov(.al, .fp, Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none)).toU32());
608
609		try self.dbgSetPrologueEnd();
610
611		try self.genBody(self.air.getMainBody());
612
613		// Backpatch push callee saved regs
614		var saved_regs = Instruction.RegisterList{
615		.r11 = true, // fp
616		.r14 = true, // lr
617		};
618		inline for (callee_preserved_regs) \|reg\| {
619		if (self.register_manager.isRegAllocated(reg)) {
620		@field(saved_regs, @tagName(reg)) = true;
621		}
622		}
623		writeInt(u32, self.code.items[prologue_reloc..][0..4], Instruction.stmdb(.al, .sp, true, saved_regs).toU32());
624
625		// Backpatch stack offset
626		const stack_end = self.max_end_stack;
627		const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
628		if (Instruction.Operand.fromU32(@intCast(u32, aligned_stack_end))) \|op\| {
629		writeInt(u32, self.code.items[prologue_reloc + 8 ..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32());
630		} else {
631		return self.failSymbol("TODO ARM: allow larger stacks", .{});
632		}
633
634		try self.dbgSetEpilogueBegin();
635
636		// exitlude jumps
637		if (self.exitlude_jump_relocs.items.len == 1) {
638		// There is only one relocation. Hence,
639		// this relocation must be at the end of
640		// the code. Therefore, we can just delete
641		// the space initially reserved for the
642		// jump
643		self.code.items.len -= 4;
644		} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
645		const amt = @intCast(i32, self.code.items.len) - @intCast(i32, jmp_reloc + 8);
646		if (amt == -4) {
647		// This return is at the end of the
648		// code block. We can't just delete
649		// the space because there may be
650		// other jumps we already relocated to
651		// the address. Instead, insert a nop
652		writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.nop().toU32());
653		} else {
654		if (math.cast(i26, amt)) \|offset\| {
655		writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(.al, offset).toU32());
656		} else \|_\| {
657		return self.failSymbol("exitlude jump is too large", .{});
658		}
659		}
660		}
661
662		// Epilogue: pop callee saved registers (swap lr with pc in saved_regs)
663		saved_regs.r14 = false; // lr
664		saved_regs.r15 = true; // pc
665
666		// mov sp, fp
667		// pop {fp, pc}
668		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .sp, Instruction.Operand.reg(.fp, Instruction.Operand.Shift.none)).toU32());
669		writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldm(.al, .sp, true, saved_regs).toU32());
670		} else {
671		try self.dbgSetPrologueEnd();
672		try self.genBody(self.air.getMainBody());
673		try self.dbgSetEpilogueBegin();
674		}
675		},
676		else => {
677		try self.dbgSetPrologueEnd();
678		try self.genBody(self.air.getMainBody());
679		try self.dbgSetEpilogueBegin();
680		},
681		}
682		// Drop them off at the rbrace.
683		try self.dbgAdvancePCAndLine(self.end_di_line, self.end_di_column);
684		}
685
686		fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
687		const air_tags = self.air.instructions.items(.tag);
688
689		for (body) \|inst\| {
690		const old_air_bookkeeping = self.air_bookkeeping;
691		try self.ensureProcessDeathCapacity(Liveness.bpi);
692
693		switch (air_tags[inst]) {
694		// zig fmt: off
695		.add, .ptr_add => try self.airAdd(inst),
696		.addwrap => try self.airAddWrap(inst),
697		.add_sat => try self.airAddSat(inst),
698		.sub, .ptr_sub => try self.airSub(inst),
699		.subwrap => try self.airSubWrap(inst),
700		.sub_sat => try self.airSubSat(inst),
701		.mul => try self.airMul(inst),
702		.mulwrap => try self.airMulWrap(inst),
703		.mul_sat => try self.airMulSat(inst),
704		.rem => try self.airRem(inst),
705		.mod => try self.airMod(inst),
706		.shl, .shl_exact => try self.airShl(inst),
707		.shl_sat => try self.airShlSat(inst),
708		.min => try self.airMin(inst),
709		.max => try self.airMax(inst),
710		.slice => try self.airSlice(inst),
711
712		.div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
713
714		.cmp_lt => try self.airCmp(inst, .lt),
715		.cmp_lte => try self.airCmp(inst, .lte),
716		.cmp_eq => try self.airCmp(inst, .eq),
717		.cmp_gte => try self.airCmp(inst, .gte),
718		.cmp_gt => try self.airCmp(inst, .gt),
719		.cmp_neq => try self.airCmp(inst, .neq),
720
721		.bool_and => try self.airBoolOp(inst),
722		.bool_or => try self.airBoolOp(inst),
723		.bit_and => try self.airBitAnd(inst),
724		.bit_or => try self.airBitOr(inst),
725		.xor => try self.airXor(inst),
726		.shr => try self.airShr(inst),
727
728		.alloc => try self.airAlloc(inst),
729		.ret_ptr => try self.airRetPtr(inst),
730		.arg => try self.airArg(inst),
731		.assembly => try self.airAsm(inst),
732		.bitcast => try self.airBitCast(inst),
733		.block => try self.airBlock(inst),
734		.br => try self.airBr(inst),
735		.breakpoint => try self.airBreakpoint(),
736		.fence => try self.airFence(),
737		.call => try self.airCall(inst),
738		.cond_br => try self.airCondBr(inst),
739		.dbg_stmt => try self.airDbgStmt(inst),
740		.fptrunc => try self.airFptrunc(inst),
741		.fpext => try self.airFpext(inst),
742		.intcast => try self.airIntCast(inst),
743		.trunc => try self.airTrunc(inst),
744		.bool_to_int => try self.airBoolToInt(inst),
745		.is_non_null => try self.airIsNonNull(inst),
746		.is_non_null_ptr => try self.airIsNonNullPtr(inst),
747		.is_null => try self.airIsNull(inst),
748		.is_null_ptr => try self.airIsNullPtr(inst),
749		.is_non_err => try self.airIsNonErr(inst),
750		.is_non_err_ptr => try self.airIsNonErrPtr(inst),
751		.is_err => try self.airIsErr(inst),
752		.is_err_ptr => try self.airIsErrPtr(inst),
753		.load => try self.airLoad(inst),
754		.loop => try self.airLoop(inst),
755		.not => try self.airNot(inst),
756		.ptrtoint => try self.airPtrToInt(inst),
757		.ret => try self.airRet(inst),
758		.ret_load => try self.airRetLoad(inst),
759		.store => try self.airStore(inst),
760		.struct_field_ptr=> try self.airStructFieldPtr(inst),
761		.struct_field_val=> try self.airStructFieldVal(inst),
762		.array_to_slice => try self.airArrayToSlice(inst),
763		.int_to_float => try self.airIntToFloat(inst),
764		.float_to_int => try self.airFloatToInt(inst),
765		.cmpxchg_strong => try self.airCmpxchg(inst),
766		.cmpxchg_weak => try self.airCmpxchg(inst),
767		.atomic_rmw => try self.airAtomicRmw(inst),
768		.atomic_load => try self.airAtomicLoad(inst),
769		.memcpy => try self.airMemcpy(inst),
770		.memset => try self.airMemset(inst),
771		.set_union_tag => try self.airSetUnionTag(inst),
772		.get_union_tag => try self.airGetUnionTag(inst),
773		.clz => try self.airClz(inst),
774		.ctz => try self.airCtz(inst),
775		.popcount => try self.airPopcount(inst),
776
777		.atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
778		.atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
779		.atomic_store_release => try self.airAtomicStore(inst, .Release),
780		.atomic_store_seq_cst => try self.airAtomicStore(inst, .SeqCst),
781
782		.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
783		.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
784		.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
785		.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
786
787		.switch_br => try self.airSwitch(inst),
788		.slice_ptr => try self.airSlicePtr(inst),
789		.slice_len => try self.airSliceLen(inst),
790
791		.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
792		.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
793
794		.array_elem_val => try self.airArrayElemVal(inst),
795		.slice_elem_val => try self.airSliceElemVal(inst),
796		.slice_elem_ptr => try self.airSliceElemPtr(inst),
797		.ptr_elem_val => try self.airPtrElemVal(inst),
798		.ptr_elem_ptr => try self.airPtrElemPtr(inst),
799
800		.constant => unreachable, // excluded from function bodies
801		.const_ty => unreachable, // excluded from function bodies
802		.unreach => self.finishAirBookkeeping(),
803
804		.optional_payload => try self.airOptionalPayload(inst),
805		.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
806		.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
807		.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
808		.unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
809		.unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
810
811		.wrap_optional => try self.airWrapOptional(inst),
812		.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
813		.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
814		// zig fmt: on
815		}
816		if (std.debug.runtime_safety) {
817		if (self.air_bookkeeping < old_air_bookkeeping + 1) {
818		std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] });
819		}
820		}
821		}
822		}
823
824		fn dbgSetPrologueEnd(self: *Self) InnerError!void {
825		switch (self.debug_output) {
826		.dwarf => \|dbg_out\| {
827		try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
828		try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
829		},
830		.plan9 => {},
831		.none => {},
832		}
833		}
834
835		fn dbgSetEpilogueBegin(self: *Self) InnerError!void {
836		switch (self.debug_output) {
837		.dwarf => \|dbg_out\| {
838		try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
839		try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
840		},
841		.plan9 => {},
842		.none => {},
843		}
844		}
845
846		fn dbgAdvancePCAndLine(self: *Self, line: u32, column: u32) InnerError!void {
847		const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
848		const delta_pc: usize = self.code.items.len - self.prev_di_pc;
849		switch (self.debug_output) {
850		.dwarf => \|dbg_out\| {
851		// TODO Look into using the DWARF special opcodes to compress this data.
852		// It lets you emit single-byte opcodes that add different numbers to
853		// both the PC and the line number at the same time.
854		try dbg_out.dbg_line.ensureUnusedCapacity(11);
855		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
856		leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
857		if (delta_line != 0) {
858		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
859		leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
860		}
861		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
862		self.prev_di_pc = self.code.items.len;
863		self.prev_di_line = line;
864		self.prev_di_column = column;
865		self.prev_di_pc = self.code.items.len;
866		},
867		.plan9 => \|dbg_out\| {
868		if (delta_pc <= 0) return; // only do this when the pc changes
869		// we have already checked the target in the linker to make sure it is compatable
870		const quant = @import("link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;
871
872		// increasing the line number
873		try @import("link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
874		// increasing the pc
875		const d_pc_p9 = @intCast(i64, delta_pc) - quant;
876		if (d_pc_p9 > 0) {
877		// minus one because if its the last one, we want to leave space to change the line which is one quanta
878		try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
879		if (dbg_out.pcop_change_index.*) \|pci\|
880		dbg_out.dbg_line.items[pci] += 1;
881		dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
882		} else if (d_pc_p9 == 0) {
883		// we don't need to do anything, because adding the quant does it for us
884		} else unreachable;
885		if (dbg_out.start_line.* == null)
886		dbg_out.start_line.* = self.prev_di_line;
887		dbg_out.end_line.* = line;
888		// only do this if the pc changed
889		self.prev_di_line = line;
890		self.prev_di_column = column;
891		self.prev_di_pc = self.code.items.len;
892		},
893		.none => {},
894		}
895		}
896
897		/// Asserts there is already capacity to insert into top branch inst_table.
898		fn processDeath(self: *Self, inst: Air.Inst.Index) void {
899		const air_tags = self.air.instructions.items(.tag);
900		if (air_tags[inst] == .constant) return; // Constants are immortal.
901		// When editing this function, note that the logic must synchronize with `reuseOperand`.
902		const prev_value = self.getResolvedInstValue(inst);
903		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
904		branch.inst_table.putAssumeCapacity(inst, .dead);
905		switch (prev_value) {
906		.register => \|reg\| {
907		self.register_manager.freeReg(reg);
908		},
909		else => {}, // TODO process stack allocation death
910		}
911		}
912
913		/// Called when there are no operands, and the instruction is always unreferenced.
914		fn finishAirBookkeeping(self: *Self) void {
915		if (std.debug.runtime_safety) {
916		self.air_bookkeeping += 1;
917		}
918		}
919
920		fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
921		var tomb_bits = self.liveness.getTombBits(inst);
922		for (operands) \|op\| {
923		const dies = @truncate(u1, tomb_bits) != 0;
924		tomb_bits >>= 1;
925		if (!dies) continue;
926		const op_int = @enumToInt(op);
927		if (op_int < Air.Inst.Ref.typed_value_map.len) continue;
928		const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
929		self.processDeath(op_index);
930		}
931		const is_used = @truncate(u1, tomb_bits) == 0;
932		if (is_used) {
933		log.debug("%{d} => {}", .{ inst, result });
934		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
935		branch.inst_table.putAssumeCapacityNoClobber(inst, result);
936
937		switch (result) {
938		.register => \|reg\| {
939		// In some cases (such as bitcast), an operand
940		// may be the same MCValue as the result. If
941		// that operand died and was a register, it
942		// was freed by processDeath. We have to
943		// "re-allocate" the register.
944		if (self.register_manager.isRegFree(reg)) {
945		self.register_manager.getRegAssumeFree(reg, inst);
946		}
947		},
948		else => {},
949		}
950		}
951		self.finishAirBookkeeping();
952		}
953
954		fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
955		const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
956		try table.ensureUnusedCapacity(self.gpa, additional_count);
957		}
958
959		/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
960		/// after codegen for this symbol is done.
961		fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void {
962		switch (self.debug_output) {
963		.dwarf => \|dbg_out\| {
964		assert(ty.hasCodeGenBits());
965		const index = dbg_out.dbg_info.items.len;
966		try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
967
968		const gop = try dbg_out.dbg_info_type_relocs.getOrPut(self.gpa, ty);
969		if (!gop.found_existing) {
970		gop.value_ptr.* = .{
971		.off = undefined,
972		.relocs = .{},
973		};
974		}
975		try gop.value_ptr.relocs.append(self.gpa, @intCast(u32, index));
976		},
977		.plan9 => {},
978		.none => {},
979		}
980		}
981
982		fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
983		if (abi_align > self.stack_align)
984		self.stack_align = abi_align;
985		// TODO find a free slot instead of always appending
986		const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align);
987		self.next_stack_offset = offset + abi_size;
988		if (self.next_stack_offset > self.max_end_stack)
989		self.max_end_stack = self.next_stack_offset;
990		try self.stack.putNoClobber(self.gpa, offset, .{
991		.inst = inst,
992		.size = abi_size,
993		});
994		return offset;
995		}
996
997		/// Use a pointer instruction as the basis for allocating stack memory.
998		fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
999		const elem_ty = self.air.typeOfIndex(inst).elemType();
1000		const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
1001		return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
1002		};
1003		// TODO swap this for inst.ty.ptrAlign
1004		const abi_align = elem_ty.abiAlignment(self.target.*);
1005		return self.allocMem(inst, abi_size, abi_align);
1006		}
1007
1008		fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
1009		const elem_ty = self.air.typeOfIndex(inst);
1010		const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
1011		return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
1012		};
1013		const abi_align = elem_ty.abiAlignment(self.target.*);
1014		if (abi_align > self.stack_align)
1015		self.stack_align = abi_align;
1016
1017		if (reg_ok) {
1018		// Make sure the type can fit in a register before we try to allocate one.
1019		const ptr_bits = arch.ptrBitWidth();
1020		const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1021		if (abi_size <= ptr_bytes) {
1022		if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {
1023		return MCValue{ .register = reg };
1024		}
1025		}
1026		}
1027		const stack_offset = try self.allocMem(inst, abi_size, abi_align);
1028		return MCValue{ .stack_offset = stack_offset };
1029		}
1030
1031		pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1032		const stack_mcv = try self.allocRegOrMem(inst, false);
1033		log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
1034		const reg_mcv = self.getResolvedInstValue(inst);
1035		assert(reg == reg_mcv.register);
1036		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1037		try branch.inst_table.put(self.gpa, inst, stack_mcv);
1038		try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
1039		}
1040
1041		/// Copies a value to a register without tracking the register. The register is not considered
1042		/// allocated. A second call to `copyToTmpRegister` may return the same register.
1043		/// This can have a side effect of spilling instructions to the stack to free up a register.
1044		fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1045		const reg = try self.register_manager.allocReg(null, &.{});
1046		try self.genSetReg(ty, reg, mcv);
1047		return reg;
1048		}
1049
1050		/// Allocates a new register and copies `mcv` into it.
1051		/// `reg_owner` is the instruction that gets associated with the register in the register table.
1052		/// This can have a side effect of spilling instructions to the stack to free up a register.
1053		fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
1054		const reg = try self.register_manager.allocReg(reg_owner, &.{});
1055		try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
1056		return MCValue{ .register = reg };
1057		}
1058
1059		fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1060		const stack_offset = try self.allocMemPtr(inst);
1061		return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1062		}
1063
1064		fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1065		const stack_offset = try self.allocMemPtr(inst);
1066		return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1067		}
1068
1069		fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
1070		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1071		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1072		else => return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch}),
1073		};
1074		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1075		}
1076
1077		fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
1078		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1079		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1080		else => return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch}),
1081		};
1082		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1083		}
1084
1085		fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1086		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1087		if (self.liveness.isUnused(inst))
1088		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1089
1090		const operand_ty = self.air.typeOf(ty_op.operand);
1091		const operand = try self.resolveInst(ty_op.operand);
1092		const info_a = operand_ty.intInfo(self.target.*);
1093		const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);
1094		if (info_a.signedness != info_b.signedness)
1095		return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
1096
1097		if (info_a.bits == info_b.bits)
1098		return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
1099
1100		const result: MCValue = switch (arch) {
1101		else => return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch}),
1102		};
1103		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1104		}
1105
1106		fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1107		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1108		if (self.liveness.isUnused(inst))
1109		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1110
1111		const operand = try self.resolveInst(ty_op.operand);
1112		_ = operand;
1113		const result: MCValue = switch (arch) {
1114		else => return self.fail("TODO implement trunc for {}", .{self.target.cpu.arch}),
1115		};
1116		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1117		}
1118
1119		fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
1120		const un_op = self.air.instructions.items(.data)[inst].un_op;
1121		const operand = try self.resolveInst(un_op);
1122		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;
1123		return self.finishAir(inst, result, .{ un_op, .none, .none });
1124		}
1125
1126		fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1127		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1128		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1129		const operand = try self.resolveInst(ty_op.operand);
1130		switch (operand) {
1131		.dead => unreachable,
1132		.unreach => unreachable,
1133		.compare_flags_unsigned => \|op\| {
1134		const r = MCValue{
1135		.compare_flags_unsigned = switch (op) {
1136		.gte => .lt,
1137		.gt => .lte,
1138		.neq => .eq,
1139		.lt => .gte,
1140		.lte => .gt,
1141		.eq => .neq,
1142		},
1143		};
1144		break :result r;
1145		},
1146		.compare_flags_signed => \|op\| {
1147		const r = MCValue{
1148		.compare_flags_signed = switch (op) {
1149		.gte => .lt,
1150		.gt => .lte,
1151		.neq => .eq,
1152		.lt => .gte,
1153		.lte => .gt,
1154		.eq => .neq,
1155		},
1156		};
1157		break :result r;
1158		},
1159		else => {},
1160		}
1161
1162		switch (arch) {
1163		.arm, .armeb => {
1164		break :result try self.genArmBinOp(inst, ty_op.operand, .bool_true, .not);
1165		},
1166		else => return self.fail("TODO implement NOT for {}", .{self.target.cpu.arch}),
1167		}
1168		};
1169		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1170		}
1171
1172		fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1173		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1174		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1175		else => return self.fail("TODO implement min for {}", .{self.target.cpu.arch}),
1176		};
1177		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1178		}
1179
1180		fn airMax(self: *Self, inst: Air.Inst.Index) !void {
1181		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1182		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1183		else => return self.fail("TODO implement max for {}", .{self.target.cpu.arch}),
1184		};
1185		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1186		}
1187
1188		fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1189		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1190		const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1191		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1192		else => return self.fail("TODO implement slice for {}", .{self.target.cpu.arch}),
1193		};
1194		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1195		}
1196
1197		fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
1198		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1199		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1200		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .add),
1201		else => return self.fail("TODO implement add for {}", .{self.target.cpu.arch}),
1202		};
1203		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1204		}
1205
1206		fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {
1207		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1208		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1209		else => return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch}),
1210		};
1211		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1212		}
1213
1214		fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1215		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1216		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1217		else => return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch}),
1218		};
1219		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1220		}
1221
1222		fn airSub(self: *Self, inst: Air.Inst.Index) !void {
1223		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1224		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1225		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .sub),
1226		else => return self.fail("TODO implement sub for {}", .{self.target.cpu.arch}),
1227		};
1228		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1229		}
1230
1231		fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
1232		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1233		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1234		else => return self.fail("TODO implement subwrap for {}", .{self.target.cpu.arch}),
1235		};
1236		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1237		}
1238
1239		fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1240		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1241		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1242		else => return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch}),
1243		};
1244		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1245		}
1246
1247		fn airMul(self: *Self, inst: Air.Inst.Index) !void {
1248		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1249		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1250		.arm, .armeb => try self.genArmMul(inst, bin_op.lhs, bin_op.rhs),
1251		else => return self.fail("TODO implement mul for {}", .{self.target.cpu.arch}),
1252		};
1253		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1254		}
1255
1256		fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {
1257		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1258		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1259		else => return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch}),
1260		};
1261		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1262		}
1263
1264		fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1265		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1266		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1267		else => return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch}),
1268		};
1269		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1270		}
1271
1272		fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1273		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1274		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1275		else => return self.fail("TODO implement div for {}", .{self.target.cpu.arch}),
1276		};
1277		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1278		}
1279
1280		fn airRem(self: *Self, inst: Air.Inst.Index) !void {
1281		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1282		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1283		else => return self.fail("TODO implement rem for {}", .{self.target.cpu.arch}),
1284		};
1285		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1286		}
1287
1288		fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1289		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1290		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1291		else => return self.fail("TODO implement mod for {}", .{self.target.cpu.arch}),
1292		};
1293		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1294		}
1295
1296		fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
1297		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1298		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1299		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_and),
1300		else => return self.fail("TODO implement bitwise and for {}", .{self.target.cpu.arch}),
1301		};
1302		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1303		}
1304
1305		fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
1306		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1307		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1308		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_or),
1309		else => return self.fail("TODO implement bitwise or for {}", .{self.target.cpu.arch}),
1310		};
1311		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1312		}
1313
1314		fn airXor(self: *Self, inst: Air.Inst.Index) !void {
1315		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1316		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1317		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .xor),
1318		else => return self.fail("TODO implement xor for {}", .{self.target.cpu.arch}),
1319		};
1320		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1321		}
1322
1323		fn airShl(self: *Self, inst: Air.Inst.Index) !void {
1324		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1325		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1326		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shl),
1327		else => return self.fail("TODO implement shl for {}", .{self.target.cpu.arch}),
1328		};
1329		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1330		}
1331
1332		fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
1333		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1334		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1335		else => return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch}),
1336		};
1337		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1338		}
1339
1340		fn airShr(self: *Self, inst: Air.Inst.Index) !void {
1341		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1342		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1343		.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shr),
1344		else => return self.fail("TODO implement shr for {}", .{self.target.cpu.arch}),
1345		};
1346		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1347		}
1348
1349		fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1350		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1351		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1352		else => return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch}),
1353		};
1354		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1355		}
1356
1357		fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1358		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1359		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1360		else => return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch}),
1361		};
1362		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1363		}
1364
1365		fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
1366		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1367		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1368		else => return self.fail("TODO implement unwrap error union error for {}", .{self.target.cpu.arch}),
1369		};
1370		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1371		}
1372
1373		fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
1374		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1375		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1376		else => return self.fail("TODO implement unwrap error union payload for {}", .{self.target.cpu.arch}),
1377		};
1378		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1379		}
1380
1381		// *(E!T) -> E
1382		fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1383		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1384		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1385		else => return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch}),
1386		};
1387		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1388		}
1389
1390		// (E!T) -> T
1391		fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1392		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1393		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1394		else => return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch}),
1395		};
1396		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1397		}
1398
1399		fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
1400		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1401		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1402		const optional_ty = self.air.typeOfIndex(inst);
1403
1404		// Optional with a zero-bit payload type is just a boolean true
1405		if (optional_ty.abiSize(self.target.*) == 1)
1406		break :result MCValue{ .immediate = 1 };
1407
1408		switch (arch) {
1409		else => return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch}),
1410		}
1411		};
1412		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1413		}
1414
1415		/// T to E!T
1416		fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
1417		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1418		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1419		else => return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch}),
1420		};
1421		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1422		}
1423
1424		/// E to E!T
1425		fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
1426		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1427		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1428		else => return self.fail("TODO implement wrap errunion error for {}", .{self.target.cpu.arch}),
1429		};
1430		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1431		}
1432
1433		fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
1434		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1435		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1436		else => return self.fail("TODO implement slice_ptr for {}", .{self.target.cpu.arch}),
1437		};
1438		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1439		}
1440
1441		fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
1442		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1443		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1444		else => return self.fail("TODO implement slice_len for {}", .{self.target.cpu.arch}),
1445		};
1446		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1447		}
1448
1449		fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
1450		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1451		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1452		else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch}),
1453		};
1454		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1455		}
1456
1457		fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
1458		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1459		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1460		else => return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch}),
1461		};
1462		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1463		}
1464
1465		fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1466		const is_volatile = false; // TODO
1467		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1468		const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else switch (arch) {
1469		else => return self.fail("TODO implement slice_elem_val for {}", .{self.target.cpu.arch}),
1470		};
1471		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1472		}
1473
1474		fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1475		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1476		const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1477		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1478		else => return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch}),
1479		};
1480		return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1481		}
1482
1483		fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
1484		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1485		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1486		else => return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch}),
1487		};
1488		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1489		}
1490
1491		fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
1492		const is_volatile = false; // TODO
1493		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1494		const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else switch (arch) {
1495		else => return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch}),
1496		};
1497		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1498		}
1499
1500		fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1501		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1502		const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1503		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1504		else => return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch}),
1505		};
1506		return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1507		}
1508
1509		fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1510		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1511		const result: MCValue = switch (arch) {
1512		else => return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch}),
1513		};
1514		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1515		}
1516
1517		fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1518		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1519		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1520		else => return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch}),
1521		};
1522		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1523		}
1524
1525		fn airClz(self: *Self, inst: Air.Inst.Index) !void {
1526		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1527		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1528		else => return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch}),
1529		};
1530		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1531		}
1532
1533		fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
1534		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1535		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1536		else => return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch}),
1537		};
1538		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1539		}
1540
1541		fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
1542		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1543		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1544		else => return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch}),
1545		};
1546		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1547		}
1548
1549		fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {
1550		if (!self.liveness.operandDies(inst, op_index))
1551		return false;
1552
1553		switch (mcv) {
1554		.register => \|reg\| {
1555		// If it's in the registers table, need to associate the register with the
1556		// new instruction.
1557		if (reg.allocIndex()) \|index\| {
1558		if (!self.register_manager.isRegFree(reg)) {
1559		self.register_manager.registers[index] = inst;
1560		}
1561		}
1562		log.debug("%{d} => {} (reused)", .{ inst, reg });
1563		},
1564		.stack_offset => \|off\| {
1565		log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
1566		},
1567		else => return false,
1568		}
1569
1570		// Prevent the operand deaths processing code from deallocating it.
1571		self.liveness.clearOperandDeath(inst, op_index);
1572
1573		// That makes us responsible for doing the rest of the stuff that processDeath would have done.
1574		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1575		branch.inst_table.putAssumeCapacity(Air.refToIndex(operand).?, .dead);
1576
1577		return true;
1578		}
1579
1580		fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
1581		const elem_ty = ptr_ty.elemType();
1582		switch (ptr) {
1583		.none => unreachable,
1584		.undef => unreachable,
1585		.unreach => unreachable,
1586		.dead => unreachable,
1587		.compare_flags_unsigned => unreachable,
1588		.compare_flags_signed => unreachable,
1589		.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
1590		.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
1591		.ptr_embedded_in_code => \|off\| {
1592		try self.setRegOrMem(elem_ty, dst_mcv, .{ .embedded_in_code = off });
1593		},
1594		.embedded_in_code => {
1595		return self.fail("TODO implement loading from MCValue.embedded_in_code", .{});
1596		},
1597		.register => \|reg\| {
1598		switch (arch) {
1599		.arm, .armeb => switch (dst_mcv) {
1600		.dead => unreachable,
1601		.undef => unreachable,
1602		.compare_flags_signed, .compare_flags_unsigned => unreachable,
1603		.embedded_in_code => unreachable,
1604		.register => \|dst_reg\| {
1605		writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, dst_reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
1606		},
1607		else => return self.fail("TODO load from register into {}", .{dst_mcv}),
1608		},
1609		else => return self.fail("TODO implement loading from MCValue.register for {}", .{arch}),
1610		}
1611		},
1612		.memory => \|addr\| {
1613		const reg = try self.register_manager.allocReg(null, &.{});
1614		try self.genSetReg(ptr_ty, reg, .{ .memory = addr });
1615		try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
1616		},
1617		.stack_offset => {
1618		return self.fail("TODO implement loading from MCValue.stack_offset", .{});
1619		},
1620		}
1621		}
1622
1623		fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
1624		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1625		const elem_ty = self.air.typeOfIndex(inst);
1626		const result: MCValue = result: {
1627		if (!elem_ty.hasCodeGenBits())
1628		break :result MCValue.none;
1629
1630		const ptr = try self.resolveInst(ty_op.operand);
1631		const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
1632		if (self.liveness.isUnused(inst) and !is_volatile)
1633		break :result MCValue.dead;
1634
1635		const dst_mcv: MCValue = blk: {
1636		if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
1637		// The MCValue that holds the pointer can be re-used as the value.
1638		break :blk ptr;
1639		} else {
1640		break :blk try self.allocRegOrMem(inst, true);
1641		}
1642		};
1643		try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
1644		break :result dst_mcv;
1645		};
1646		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1647		}
1648
1649		fn airStore(self: *Self, inst: Air.Inst.Index) !void {
1650		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1651		const ptr = try self.resolveInst(bin_op.lhs);
1652		const value = try self.resolveInst(bin_op.rhs);
1653		const elem_ty = self.air.typeOf(bin_op.rhs);
1654		switch (ptr) {
1655		.none => unreachable,
1656		.undef => unreachable,
1657		.unreach => unreachable,
1658		.dead => unreachable,
1659		.compare_flags_unsigned => unreachable,
1660		.compare_flags_signed => unreachable,
1661		.immediate => \|imm\| {
1662		try self.setRegOrMem(elem_ty, .{ .memory = imm }, value);
1663		},
1664		.ptr_stack_offset => \|off\| {
1665		try self.genSetStack(elem_ty, off, value);
1666		},
1667		.ptr_embedded_in_code => \|off\| {
1668		try self.setRegOrMem(elem_ty, .{ .embedded_in_code = off }, value);
1669		},
1670		.embedded_in_code => {
1671		return self.fail("TODO implement storing to MCValue.embedded_in_code", .{});
1672		},
1673		.register => {
1674		return self.fail("TODO implement storing to MCValue.register", .{});
1675		},
1676		.memory => {
1677		return self.fail("TODO implement storing to MCValue.memory", .{});
1678		},
1679		.stack_offset => {
1680		return self.fail("TODO implement storing to MCValue.stack_offset", .{});
1681		},
1682		}
1683		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1684		}
1685
1686		fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
1687		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1688		const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1689		return self.structFieldPtr(extra.struct_operand, ty_pl.ty, extra.field_index);
1690		}
1691
1692		fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
1693		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1694		return self.structFieldPtr(ty_op.operand, ty_op.ty, index);
1695		}
1696		fn structFieldPtr(self: *Self, operand: Air.Inst.Ref, ty: Air.Inst.Ref, index: u32) !void {
1697		_ = self;
1698		_ = operand;
1699		_ = ty;
1700		_ = index;
1701		return self.fail("TODO implement codegen struct_field_ptr", .{});
1702		//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1703		}
1704
1705		fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
1706		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1707		const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1708		_ = extra;
1709		return self.fail("TODO implement codegen struct_field_val", .{});
1710		//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1711		}
1712
1713		fn armOperandShouldBeRegister(self: *Self, mcv: MCValue) !bool {
1714		return switch (mcv) {
1715		.none => unreachable,
1716		.undef => unreachable,
1717		.dead, .unreach => unreachable,
1718		.compare_flags_unsigned => unreachable,
1719		.compare_flags_signed => unreachable,
1720		.ptr_stack_offset => unreachable,
1721		.ptr_embedded_in_code => unreachable,
1722		.immediate => \|imm\| blk: {
1723		if (imm > std.math.maxInt(u32)) return self.fail("TODO ARM binary arithmetic immediate larger than u32", .{});
1724
1725		// Load immediate into register if it doesn't fit
1726		// in an operand
1727		break :blk Instruction.Operand.fromU32(@intCast(u32, imm)) == null;
1728		},
1729		.register => true,
1730		.stack_offset,
1731		.embedded_in_code,
1732		.memory,
1733		=> true,
1734		};
1735		}
1736
1737		fn genArmBinOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref, op: Air.Inst.Tag) !MCValue {
1738		// In the case of bitshifts, the type of rhs is different
1739		// from the resulting type
1740		const ty = self.air.typeOf(op_lhs);
1741
1742		switch (ty.zigTypeTag()) {
1743		.Float => return self.fail("TODO ARM binary operations on floats", .{}),
1744		.Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
1745		.Bool => {
1746		return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, 1, .unsigned);
1747		},
1748		.Int => {
1749		const int_info = ty.intInfo(self.target.*);
1750		return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, int_info.bits, int_info.signedness);
1751		},
1752		else => unreachable,
1753		}
1754		}
1755
1756		fn genArmBinIntOp(
1757		self: *Self,
1758		inst: Air.Inst.Index,
1759		op_lhs: Air.Inst.Ref,
1760		op_rhs: Air.Inst.Ref,
1761		op: Air.Inst.Tag,
1762		bits: u16,
1763		signedness: std.builtin.Signedness,
1764		) !MCValue {
1765		if (bits > 32) {
1766		return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
1767		}
1768
1769		const lhs = try self.resolveInst(op_lhs);
1770		const rhs = try self.resolveInst(op_rhs);
1771
1772		const lhs_is_register = lhs == .register;
1773		const rhs_is_register = rhs == .register;
1774		const lhs_should_be_register = switch (op) {
1775		.shr, .shl => true,
1776		else => try self.armOperandShouldBeRegister(lhs),
1777		};
1778		const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
1779		const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
1780		const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
1781		const can_swap_lhs_and_rhs = switch (op) {
1782		.shr, .shl => false,
1783		else => true,
1784		};
1785
1786		// Destination must be a register
1787		var dst_mcv: MCValue = undefined;
1788		var lhs_mcv = lhs;
1789		var rhs_mcv = rhs;
1790		var swap_lhs_and_rhs = false;
1791
1792		// Allocate registers for operands and/or destination
1793		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1794		if (reuse_lhs) {
1795		// Allocate 0 or 1 registers
1796		if (!rhs_is_register and rhs_should_be_register) {
1797		rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
1798		branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1799		}
1800		dst_mcv = lhs;
1801		} else if (reuse_rhs and can_swap_lhs_and_rhs) {
1802		// Allocate 0 or 1 registers
1803		if (!lhs_is_register and lhs_should_be_register) {
1804		lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
1805		branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
1806		}
1807		dst_mcv = rhs;
1808
1809		swap_lhs_and_rhs = true;
1810		} else {
1811		// Allocate 1 or 2 registers
1812		if (lhs_should_be_register and rhs_should_be_register) {
1813		if (lhs_is_register and rhs_is_register) {
1814		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
1815		} else if (lhs_is_register) {
1816		// Move RHS to register
1817		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1818		rhs_mcv = dst_mcv;
1819		} else if (rhs_is_register) {
1820		// Move LHS to register
1821		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1822		lhs_mcv = dst_mcv;
1823		} else {
1824		// Move LHS and RHS to register
1825		const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
1826		lhs_mcv = MCValue{ .register = regs[0] };
1827		rhs_mcv = MCValue{ .register = regs[1] };
1828		dst_mcv = lhs_mcv;
1829
1830		branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1831		}
1832		} else if (lhs_should_be_register) {
1833		// RHS is immediate
1834		if (lhs_is_register) {
1835		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1836		} else {
1837		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
1838		lhs_mcv = dst_mcv;
1839		}
1840		} else if (rhs_should_be_register and can_swap_lhs_and_rhs) {
1841		// LHS is immediate
1842		if (rhs_is_register) {
1843		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1844		} else {
1845		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
1846		rhs_mcv = dst_mcv;
1847		}
1848
1849		swap_lhs_and_rhs = true;
1850		} else unreachable; // binary operation on two immediates
1851		}
1852
1853		// Move the operands to the newly allocated registers
1854		if (lhs_mcv == .register and !lhs_is_register) {
1855		try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
1856		}
1857		if (rhs_mcv == .register and !rhs_is_register) {
1858		try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
1859		}
1860
1861		try self.genArmBinOpCode(
1862		dst_mcv.register,
1863		lhs_mcv,
1864		rhs_mcv,
1865		swap_lhs_and_rhs,
1866		op,
1867		signedness,
1868		);
1869		return dst_mcv;
1870		}
1871
1872		fn genArmBinOpCode(
1873		self: *Self,
1874		dst_reg: Register,
1875		lhs_mcv: MCValue,
1876		rhs_mcv: MCValue,
1877		swap_lhs_and_rhs: bool,
1878		op: Air.Inst.Tag,
1879		signedness: std.builtin.Signedness,
1880		) !void {
1881		assert(lhs_mcv == .register or rhs_mcv == .register);
1882
1883		const op1 = if (swap_lhs_and_rhs) rhs_mcv.register else lhs_mcv.register;
1884		const op2 = if (swap_lhs_and_rhs) lhs_mcv else rhs_mcv;
1885
1886		const operand = switch (op2) {
1887		.none => unreachable,
1888		.undef => unreachable,
1889		.dead, .unreach => unreachable,
1890		.compare_flags_unsigned => unreachable,
1891		.compare_flags_signed => unreachable,
1892		.ptr_stack_offset => unreachable,
1893		.ptr_embedded_in_code => unreachable,
1894		.immediate => \|imm\| Instruction.Operand.fromU32(@intCast(u32, imm)).?,
1895		.register => \|reg\| Instruction.Operand.reg(reg, Instruction.Operand.Shift.none),
1896		.stack_offset,
1897		.embedded_in_code,
1898		.memory,
1899		=> unreachable,
1900		};
1901
1902		switch (op) {
1903		.add => {
1904		writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.al, dst_reg, op1, operand).toU32());
1905		},
1906		.sub => {
1907		if (swap_lhs_and_rhs) {
1908		writeInt(u32, try self.code.addManyAsArray(4), Instruction.rsb(.al, dst_reg, op1, operand).toU32());
1909		} else {
1910		writeInt(u32, try self.code.addManyAsArray(4), Instruction.sub(.al, dst_reg, op1, operand).toU32());
1911		}
1912		},
1913		.bool_and, .bit_and => {
1914		writeInt(u32, try self.code.addManyAsArray(4), Instruction.@"and"(.al, dst_reg, op1, operand).toU32());
1915		},
1916		.bool_or, .bit_or => {
1917		writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, dst_reg, op1, operand).toU32());
1918		},
1919		.not, .xor => {
1920		writeInt(u32, try self.code.addManyAsArray(4), Instruction.eor(.al, dst_reg, op1, operand).toU32());
1921		},
1922		.cmp_eq => {
1923		writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, op1, operand).toU32());
1924		},
1925		.shl => {
1926		assert(!swap_lhs_and_rhs);
1927		const shift_amount = switch (operand) {
1928		.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
1929		.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
1930		};
1931		writeInt(u32, try self.code.addManyAsArray(4), Instruction.lsl(.al, dst_reg, op1, shift_amount).toU32());
1932		},
1933		.shr => {
1934		assert(!swap_lhs_and_rhs);
1935		const shift_amount = switch (operand) {
1936		.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
1937		.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
1938		};
1939
1940		const shr = switch (signedness) {
1941		.signed => Instruction.asr,
1942		.unsigned => Instruction.lsr,
1943		};
1944		writeInt(u32, try self.code.addManyAsArray(4), shr(.al, dst_reg, op1, shift_amount).toU32());
1945		},
1946		else => unreachable, // not a binary instruction
1947		}
1948		}
1949
1950		fn genArmMul(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
1951		const lhs = try self.resolveInst(op_lhs);
1952		const rhs = try self.resolveInst(op_rhs);
1953
1954		const lhs_is_register = lhs == .register;
1955		const rhs_is_register = rhs == .register;
1956		const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
1957		const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
1958
1959		// Destination must be a register
1960		// LHS must be a register
1961		// RHS must be a register
1962		var dst_mcv: MCValue = undefined;
1963		var lhs_mcv: MCValue = lhs;
1964		var rhs_mcv: MCValue = rhs;
1965
1966		// Allocate registers for operands and/or destination
1967		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1968		if (reuse_lhs) {
1969		// Allocate 0 or 1 registers
1970		if (!rhs_is_register) {
1971		rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
1972		branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1973		}
1974		dst_mcv = lhs;
1975		} else if (reuse_rhs) {
1976		// Allocate 0 or 1 registers
1977		if (!lhs_is_register) {
1978		lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
1979		branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
1980		}
1981		dst_mcv = rhs;
1982		} else {
1983		// Allocate 1 or 2 registers
1984		if (lhs_is_register and rhs_is_register) {
1985		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
1986		} else if (lhs_is_register) {
1987		// Move RHS to register
1988		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1989		rhs_mcv = dst_mcv;
1990		} else if (rhs_is_register) {
1991		// Move LHS to register
1992		dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1993		lhs_mcv = dst_mcv;
1994		} else {
1995		// Move LHS and RHS to register
1996		const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
1997		lhs_mcv = MCValue{ .register = regs[0] };
1998		rhs_mcv = MCValue{ .register = regs[1] };
1999		dst_mcv = lhs_mcv;
2000
2001		branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
2002		}
2003		}
2004
2005		// Move the operands to the newly allocated registers
2006		if (!lhs_is_register) {
2007		try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
2008		}
2009		if (!rhs_is_register) {
2010		try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
2011		}
2012
2013		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mul(.al, dst_mcv.register, lhs_mcv.register, rhs_mcv.register).toU32());
2014		return dst_mcv;
2015		}
2016
2017		fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
2018		const ty_str = self.air.instructions.items(.data)[inst].ty_str;
2019		const zir = &self.mod_fn.owner_decl.getFileScope().zir;
2020		const name = zir.nullTerminatedString(ty_str.str);
2021		const name_with_null = name.ptr[0 .. name.len + 1];
2022		const ty = self.air.getRefType(ty_str.ty);
2023
2024		switch (mcv) {
2025		.register => \|reg\| {
2026		switch (self.debug_output) {
2027		.dwarf => \|dbg_out\| {
2028		try dbg_out.dbg_info.ensureUnusedCapacity(3);
2029		dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
2030		dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
2031		1, // ULEB128 dwarf expression length
2032		reg.dwarfLocOp(),
2033		});
2034		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
2035		try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
2036		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
2037		},
2038		.plan9 => {},
2039		.none => {},
2040		}
2041		},
2042		.stack_offset => \|offset\| {
2043		switch (self.debug_output) {
2044		.dwarf => \|dbg_out\| {
2045		switch (arch) {
2046		.arm, .armeb => {
2047		const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
2048		return self.fail("type '{}' too big to fit into stack frame", .{ty});
2049		};
2050		const adjusted_stack_offset = math.negateCast(offset + abi_size) catch {
2051		return self.fail("Stack offset too large for arguments", .{});
2052		};
2053
2054		try dbg_out.dbg_info.append(link.File.Elf.abbrev_parameter);
2055
2056		// Get length of the LEB128 stack offset
2057		var counting_writer = std.io.countingWriter(std.io.null_writer);
2058		leb128.writeILEB128(counting_writer.writer(), adjusted_stack_offset) catch unreachable;
2059
2060		// DW.AT.location, DW.FORM.exprloc
2061		// ULEB128 dwarf expression length
2062		try leb128.writeULEB128(dbg_out.dbg_info.writer(), counting_writer.bytes_written + 1);
2063		try dbg_out.dbg_info.append(DW.OP.breg11);
2064		try leb128.writeILEB128(dbg_out.dbg_info.writer(), adjusted_stack_offset);
2065
2066		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
2067		try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
2068		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
2069		},
2070		else => {},
2071		}
2072		},
2073		.plan9 => {},
2074		.none => {},
2075		}
2076		},
2077		else => {},
2078		}
2079		}
2080
2081		fn airArg(self: *Self, inst: Air.Inst.Index) !void {
2082		const arg_index = self.arg_index;
2083		self.arg_index += 1;
2084
2085		const ty = self.air.typeOfIndex(inst);
2086
2087		const result = self.args[arg_index];
2088		const mcv = switch (arch) {
2089		// TODO support stack-only arguments on all target architectures
2090		.arm, .armeb => switch (result) {
2091		// Copy registers to the stack
2092		.register => \|reg\| blk: {
2093		const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
2094		return self.fail("type '{}' too big to fit into stack frame", .{ty});
2095		};
2096		const abi_align = ty.abiAlignment(self.target.*);
2097		const stack_offset = try self.allocMem(inst, abi_size, abi_align);
2098		try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2099
2100		break :blk MCValue{ .stack_offset = stack_offset };
2101		},
2102		else => result,
2103		},
2104		else => result,
2105		};
2106		try self.genArgDbgInfo(inst, mcv);
2107
2108		if (self.liveness.isUnused(inst))
2109		return self.finishAirBookkeeping();
2110
2111		switch (mcv) {
2112		.register => \|reg\| {
2113		self.register_manager.getRegAssumeFree(reg, inst);
2114		},
2115		else => {},
2116		}
2117
2118		return self.finishAir(inst, mcv, .{ .none, .none, .none });
2119		}
2120
2121		fn airBreakpoint(self: *Self) !void {
2122		switch (arch) {
2123		.i386 => {
2124		try self.code.append(0xcc); // int3
2125		},
2126		.arm, .armeb => {
2127		writeInt(u32, try self.code.addManyAsArray(4), Instruction.bkpt(0).toU32());
2128		},
2129		else => return self.fail("TODO implement @breakpoint() for {}", .{self.target.cpu.arch}),
2130		}
2131		return self.finishAirBookkeeping();
2132		}
2133
2134		fn airFence(self: *Self) !void {
2135		return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
2136		//return self.finishAirBookkeeping();
2137		}
2138
2139		fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2140		const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2141		const fn_ty = self.air.typeOf(pl_op.operand);
2142		const callee = pl_op.operand;
2143		const extra = self.air.extraData(Air.Call, pl_op.payload);
2144		const args = @bitCast([]const Air.Inst.Ref, self.air.extra[extra.end..][0..extra.data.args_len]);
2145
2146		var info = try self.resolveCallingConventionValues(fn_ty);
2147		defer info.deinit(self);
2148
2149		// Due to incremental compilation, how function calls are generated depends
2150		// on linking.
2151		if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {
2152		switch (arch) {
2153		.arm, .armeb => {
2154		for (info.args) \|mc_arg, arg_i\| {
2155		const arg = args[arg_i];
2156		const arg_ty = self.air.typeOf(arg);
2157		const arg_mcv = try self.resolveInst(args[arg_i]);
2158
2159		switch (mc_arg) {
2160		.none => continue,
2161		.undef => unreachable,
2162		.immediate => unreachable,
2163		.unreach => unreachable,
2164		.dead => unreachable,
2165		.embedded_in_code => unreachable,
2166		.memory => unreachable,
2167		.compare_flags_signed => unreachable,
2168		.compare_flags_unsigned => unreachable,
2169		.register => \|reg\| {
2170		try self.register_manager.getReg(reg, null);
2171		try self.genSetReg(arg_ty, reg, arg_mcv);
2172		},
2173		.stack_offset => {
2174		return self.fail("TODO implement calling with parameters in memory", .{});
2175		},
2176		.ptr_stack_offset => {
2177		return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2178		},
2179		.ptr_embedded_in_code => {
2180		return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2181		},
2182		}
2183		}
2184
2185		if (self.air.value(callee)) \|func_value\| {
2186		if (func_value.castTag(.function)) \|func_payload\| {
2187		const func = func_payload.data;
2188		const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2189		const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2190		const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
2191		const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
2192		break :blk @intCast(u32, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * ptr_bytes);
2193		} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\|
2194		coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes
2195		else
2196		unreachable;
2197
2198		try self.genSetReg(Type.initTag(.usize), .lr, .{ .memory = got_addr });
2199
2200		// TODO: add Instruction.supportedOn
2201		// function for ARM
2202		if (Target.arm.featureSetHas(self.target.cpu.features, .has_v5t)) {
2203		writeInt(u32, try self.code.addManyAsArray(4), Instruction.blx(.al, .lr).toU32());
2204		} else {
2205		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .lr, Instruction.Operand.reg(.pc, Instruction.Operand.Shift.none)).toU32());
2206		writeInt(u32, try self.code.addManyAsArray(4), Instruction.bx(.al, .lr).toU32());
2207		}
2208		} else if (func_value.castTag(.extern_fn)) \|_\| {
2209		return self.fail("TODO implement calling extern functions", .{});
2210		} else {
2211		return self.fail("TODO implement calling bitcasted functions", .{});
2212		}
2213		} else {
2214		return self.fail("TODO implement calling runtime known function pointer", .{});
2215		}
2216		},
2217		else => return self.fail("TODO implement call for {}", .{self.target.cpu.arch}),
2218		}
2219		} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
2220		unreachable; // unsupported architecture for MachO
2221		} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {
2222		return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});
2223		} else unreachable;
2224
2225		const result: MCValue = result: {
2226		switch (info.return_value) {
2227		.register => \|reg\| {
2228		if (Register.allocIndex(reg) == null) {
2229		// Save function return value in a callee saved register
2230		break :result try self.copyToNewRegister(inst, info.return_value);
2231		}
2232		},
2233		else => {},
2234		}
2235		break :result info.return_value;
2236		};
2237
2238		if (args.len <= Liveness.bpi - 2) {
2239		var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
2240		buf[0] = callee;
2241		std.mem.copy(Air.Inst.Ref, buf[1..], args);
2242		return self.finishAir(inst, result, buf);
2243		}
2244		var bt = try self.iterateBigTomb(inst, 1 + args.len);
2245		bt.feed(callee);
2246		for (args) \|arg\| {
2247		bt.feed(arg);
2248		}
2249		return bt.finishAir(result);
2250		}
2251
2252		fn ret(self: *Self, mcv: MCValue) !void {
2253		const ret_ty = self.fn_type.fnReturnType();
2254		try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
2255		switch (arch) {
2256		.i386 => {
2257		try self.code.append(0xc3); // ret
2258		},
2259		.arm, .armeb => {
2260		// Just add space for an instruction, patch this later
2261		try self.code.resize(self.code.items.len + 4);
2262		try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
2263		},
2264		else => return self.fail("TODO implement return for {}", .{self.target.cpu.arch}),
2265		}
2266		}
2267
2268		fn airRet(self: *Self, inst: Air.Inst.Index) !void {
2269		const un_op = self.air.instructions.items(.data)[inst].un_op;
2270		const operand = try self.resolveInst(un_op);
2271		try self.ret(operand);
2272		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2273		}
2274
2275		fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
2276		const un_op = self.air.instructions.items(.data)[inst].un_op;
2277		const ptr = try self.resolveInst(un_op);
2278		_ = ptr;
2279		return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch});
2280		//return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2281		}
2282
2283		fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2284		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2285		if (self.liveness.isUnused(inst))
2286		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2287		const ty = self.air.typeOf(bin_op.lhs);
2288		assert(ty.eql(self.air.typeOf(bin_op.rhs)));
2289		if (ty.zigTypeTag() == .ErrorSet)
2290		return self.fail("TODO implement cmp for errors", .{});
2291
2292		const lhs = try self.resolveInst(bin_op.lhs);
2293		const rhs = try self.resolveInst(bin_op.rhs);
2294		const result: MCValue = switch (arch) {
2295		.arm, .armeb => result: {
2296		const lhs_is_register = lhs == .register;
2297		const rhs_is_register = rhs == .register;
2298		// lhs should always be a register
2299		const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
2300
2301		var lhs_mcv = lhs;
2302		var rhs_mcv = rhs;
2303
2304		// Allocate registers
2305		if (rhs_should_be_register) {
2306		if (!lhs_is_register and !rhs_is_register) {
2307		const regs = try self.register_manager.allocRegs(2, .{
2308		Air.refToIndex(bin_op.rhs).?, Air.refToIndex(bin_op.lhs).?,
2309		}, &.{});
2310		lhs_mcv = MCValue{ .register = regs[0] };
2311		rhs_mcv = MCValue{ .register = regs[1] };
2312		} else if (!rhs_is_register) {
2313		rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.rhs).?, &.{}) };
2314		}
2315		}
2316		if (!lhs_is_register) {
2317		lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.lhs).?, &.{}) };
2318		}
2319
2320		// Move the operands to the newly allocated registers
2321		const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
2322		if (lhs_mcv == .register and !lhs_is_register) {
2323		try self.genSetReg(ty, lhs_mcv.register, lhs);
2324		branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.lhs).?, lhs);
2325		}
2326		if (rhs_mcv == .register and !rhs_is_register) {
2327		try self.genSetReg(ty, rhs_mcv.register, rhs);
2328		branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.rhs).?, rhs);
2329		}
2330
2331		// The destination register is not present in the cmp instruction
2332		// The signedness of the integer does not matter for the cmp instruction
2333		try self.genArmBinOpCode(undefined, lhs_mcv, rhs_mcv, false, .cmp_eq, undefined);
2334
2335		break :result switch (ty.isSignedInt()) {
2336		true => MCValue{ .compare_flags_signed = op },
2337		false => MCValue{ .compare_flags_unsigned = op },
2338		};
2339		},
2340		else => return self.fail("TODO implement cmp for {}", .{self.target.cpu.arch}),
2341		};
2342		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2343		}
2344
2345		fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
2346		const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
2347		try self.dbgAdvancePCAndLine(dbg_stmt.line, dbg_stmt.column);
2348		return self.finishAirBookkeeping();
2349		}
2350
2351		fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2352		const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2353		const cond = try self.resolveInst(pl_op.operand);
2354		const extra = self.air.extraData(Air.CondBr, pl_op.payload);
2355		const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
2356		const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
2357		const liveness_condbr = self.liveness.getCondBr(inst);
2358
2359		const reloc: Reloc = switch (arch) {
2360		.i386 => reloc: {
2361		try self.code.ensureUnusedCapacity(6);
2362
2363		const opcode: u8 = switch (cond) {
2364		.compare_flags_signed => \|cmp_op\| blk: {
2365		// Here we map to the opposite opcode because the jump is to the false branch.
2366		const opcode: u8 = switch (cmp_op) {
2367		.gte => 0x8c,
2368		.gt => 0x8e,
2369		.neq => 0x84,
2370		.lt => 0x8d,
2371		.lte => 0x8f,
2372		.eq => 0x85,
2373		};
2374		break :blk opcode;
2375		},
2376		.compare_flags_unsigned => \|cmp_op\| blk: {
2377		// Here we map to the opposite opcode because the jump is to the false branch.
2378		const opcode: u8 = switch (cmp_op) {
2379		.gte => 0x82,
2380		.gt => 0x86,
2381		.neq => 0x84,
2382		.lt => 0x83,
2383		.lte => 0x87,
2384		.eq => 0x85,
2385		};
2386		break :blk opcode;
2387		},
2388		.register => \|reg\| blk: {
2389		// test reg, 1
2390		// TODO detect al, ax, eax
2391		const Encoder = @import("arch/x86_64/bits.zig").Encoder;
2392		const encoder = try Encoder.init(self.code, 4);
2393		encoder.rex(.{
2394		// TODO audit this codegen: we force w = true here to make
2395		// the value affect the big register
2396		.w = true,
2397		.b = reg.isExtended(),
2398		});
2399		encoder.opcode_1byte(0xf6);
2400		encoder.modRm_direct(
2401		0,
2402		reg.low_id(),
2403		);
2404		encoder.disp8(1);
2405		break :blk 0x84;
2406		},
2407		else => return self.fail("TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
2408		};
2409		self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode });
2410		const reloc = Reloc{ .rel32 = self.code.items.len };
2411		self.code.items.len += 4;
2412		break :reloc reloc;
2413		},
2414		.arm, .armeb => reloc: {
2415		const condition: Condition = switch (cond) {
2416		.compare_flags_signed => \|cmp_op\| blk: {
2417		// Here we map to the opposite condition because the jump is to the false branch.
2418		const condition = Condition.fromCompareOperatorSigned(cmp_op);
2419		break :blk condition.negate();
2420		},
2421		.compare_flags_unsigned => \|cmp_op\| blk: {
2422		// Here we map to the opposite condition because the jump is to the false branch.
2423		const condition = Condition.fromCompareOperatorUnsigned(cmp_op);
2424		break :blk condition.negate();
2425		},
2426		.register => \|reg\| blk: {
2427		// cmp reg, 1
2428		// bne ...
2429		const op = Instruction.Operand.imm(1, 0);
2430		writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, reg, op).toU32());
2431		break :blk .ne;
2432		},
2433		else => return self.fail("TODO implement condbr {} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
2434		};
2435
2436		const reloc = Reloc{
2437		.arm_branch = .{
2438		.pos = self.code.items.len,
2439		.cond = condition,
2440		},
2441		};
2442		try self.code.resize(self.code.items.len + 4);
2443		break :reloc reloc;
2444		},
2445		else => return self.fail("TODO implement condbr {}", .{self.target.cpu.arch}),
2446		};
2447
2448		// Capture the state of register and stack allocation state so that we can revert to it.
2449		const parent_next_stack_offset = self.next_stack_offset;
2450		const parent_free_registers = self.register_manager.free_registers;
2451		var parent_stack = try self.stack.clone(self.gpa);
2452		defer parent_stack.deinit(self.gpa);
2453		const parent_registers = self.register_manager.registers;
2454
2455		try self.branch_stack.append(.{});
2456
2457		try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
2458		for (liveness_condbr.then_deaths) \|operand\| {
2459		self.processDeath(operand);
2460		}
2461		try self.genBody(then_body);
2462
2463		// Revert to the previous register and stack allocation state.
2464
2465		var saved_then_branch = self.branch_stack.pop();
2466		defer saved_then_branch.deinit(self.gpa);
2467
2468		self.register_manager.registers = parent_registers;
2469
2470		self.stack.deinit(self.gpa);
2471		self.stack = parent_stack;
2472		parent_stack = .{};
2473
2474		self.next_stack_offset = parent_next_stack_offset;
2475		self.register_manager.free_registers = parent_free_registers;
2476
2477		try self.performReloc(reloc);
2478		const else_branch = self.branch_stack.addOneAssumeCapacity();
2479		else_branch.* = .{};
2480
2481		try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
2482		for (liveness_condbr.else_deaths) \|operand\| {
2483		self.processDeath(operand);
2484		}
2485		try self.genBody(else_body);
2486
2487		// At this point, each branch will possibly have conflicting values for where
2488		// each instruction is stored. They agree, however, on which instructions are alive/dead.
2489		// We use the first ("then") branch as canonical, and here emit
2490		// instructions into the second ("else") branch to make it conform.
2491		// We continue respect the data structure semantic guarantees of the else_branch so
2492		// that we can use all the code emitting abstractions. This is why at the bottom we
2493		// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
2494		// rather than assigning it.
2495		const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
2496		try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
2497
2498		const else_slice = else_branch.inst_table.entries.slice();
2499		const else_keys = else_slice.items(.key);
2500		const else_values = else_slice.items(.value);
2501		for (else_keys) \|else_key, else_idx\| {
2502		const else_value = else_values[else_idx];
2503		const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
2504		// The instruction's MCValue is overridden in both branches.
2505		parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
2506		if (else_value == .dead) {
2507		assert(then_entry.value == .dead);
2508		continue;
2509		}
2510		break :blk then_entry.value;
2511		} else blk: {
2512		if (else_value == .dead)
2513		continue;
2514		// The instruction is only overridden in the else branch.
2515		var i: usize = self.branch_stack.items.len - 2;
2516		while (true) {
2517		i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
2518		if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
2519		assert(mcv != .dead);
2520		break :blk mcv;
2521		}
2522		}
2523		};
2524		log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
2525		// TODO make sure the destination stack offset / register does not already have something
2526		// going on there.
2527		try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
2528		// TODO track the new register / stack allocation
2529		}
2530		try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
2531		const then_slice = saved_then_branch.inst_table.entries.slice();
2532		const then_keys = then_slice.items(.key);
2533		const then_values = then_slice.items(.value);
2534		for (then_keys) \|then_key, then_idx\| {
2535		const then_value = then_values[then_idx];
2536		// We already deleted the items from this table that matched the else_branch.
2537		// So these are all instructions that are only overridden in the then branch.
2538		parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
2539		if (then_value == .dead)
2540		continue;
2541		const parent_mcv = blk: {
2542		var i: usize = self.branch_stack.items.len - 2;
2543		while (true) {
2544		i -= 1;
2545		if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
2546		assert(mcv != .dead);
2547		break :blk mcv;
2548		}
2549		}
2550		};
2551		log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
2552		// TODO make sure the destination stack offset / register does not already have something
2553		// going on there.
2554		try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
2555		// TODO track the new register / stack allocation
2556		}
2557
2558		self.branch_stack.pop().deinit(self.gpa);
2559
2560		return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
2561		}
2562
2563		fn isNull(self: *Self, operand: MCValue) !MCValue {
2564		_ = operand;
2565		// Here you can specialize this instruction if it makes sense to, otherwise the default
2566		// will call isNonNull and invert the result.
2567		switch (arch) {
2568		else => return self.fail("TODO call isNonNull and invert the result", .{}),
2569		}
2570		}
2571
2572		fn isNonNull(self: *Self, operand: MCValue) !MCValue {
2573		_ = operand;
2574		// Here you can specialize this instruction if it makes sense to, otherwise the default
2575		// will call isNull and invert the result.
2576		switch (arch) {
2577		else => return self.fail("TODO call isNull and invert the result", .{}),
2578		}
2579		}
2580
2581		fn isErr(self: *Self, operand: MCValue) !MCValue {
2582		_ = operand;
2583		// Here you can specialize this instruction if it makes sense to, otherwise the default
2584		// will call isNonNull and invert the result.
2585		switch (arch) {
2586		else => return self.fail("TODO call isNonErr and invert the result", .{}),
2587		}
2588		}
2589
2590		fn isNonErr(self: *Self, operand: MCValue) !MCValue {
2591		_ = operand;
2592		// Here you can specialize this instruction if it makes sense to, otherwise the default
2593		// will call isNull and invert the result.
2594		switch (arch) {
2595		else => return self.fail("TODO call isErr and invert the result", .{}),
2596		}
2597		}
2598
2599		fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
2600		const un_op = self.air.instructions.items(.data)[inst].un_op;
2601		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2602		const operand = try self.resolveInst(un_op);
2603		break :result try self.isNull(operand);
2604		};
2605		return self.finishAir(inst, result, .{ un_op, .none, .none });
2606		}
2607
2608		fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2609		const un_op = self.air.instructions.items(.data)[inst].un_op;
2610		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2611		const operand_ptr = try self.resolveInst(un_op);
2612		const operand: MCValue = blk: {
2613		if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2614		// The MCValue that holds the pointer can be re-used as the value.
2615		break :blk operand_ptr;
2616		} else {
2617		break :blk try self.allocRegOrMem(inst, true);
2618		}
2619		};
2620		try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2621		break :result try self.isNull(operand);
2622		};
2623		return self.finishAir(inst, result, .{ un_op, .none, .none });
2624		}
2625
2626		fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
2627		const un_op = self.air.instructions.items(.data)[inst].un_op;
2628		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2629		const operand = try self.resolveInst(un_op);
2630		break :result try self.isNonNull(operand);
2631		};
2632		return self.finishAir(inst, result, .{ un_op, .none, .none });
2633		}
2634
2635		fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2636		const un_op = self.air.instructions.items(.data)[inst].un_op;
2637		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2638		const operand_ptr = try self.resolveInst(un_op);
2639		const operand: MCValue = blk: {
2640		if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2641		// The MCValue that holds the pointer can be re-used as the value.
2642		break :blk operand_ptr;
2643		} else {
2644		break :blk try self.allocRegOrMem(inst, true);
2645		}
2646		};
2647		try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2648		break :result try self.isNonNull(operand);
2649		};
2650		return self.finishAir(inst, result, .{ un_op, .none, .none });
2651		}
2652
2653		fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
2654		const un_op = self.air.instructions.items(.data)[inst].un_op;
2655		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2656		const operand = try self.resolveInst(un_op);
2657		break :result try self.isErr(operand);
2658		};
2659		return self.finishAir(inst, result, .{ un_op, .none, .none });
2660		}
2661
2662		fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2663		const un_op = self.air.instructions.items(.data)[inst].un_op;
2664		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2665		const operand_ptr = try self.resolveInst(un_op);
2666		const operand: MCValue = blk: {
2667		if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2668		// The MCValue that holds the pointer can be re-used as the value.
2669		break :blk operand_ptr;
2670		} else {
2671		break :blk try self.allocRegOrMem(inst, true);
2672		}
2673		};
2674		try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2675		break :result try self.isErr(operand);
2676		};
2677		return self.finishAir(inst, result, .{ un_op, .none, .none });
2678		}
2679
2680		fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
2681		const un_op = self.air.instructions.items(.data)[inst].un_op;
2682		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2683		const operand = try self.resolveInst(un_op);
2684		break :result try self.isNonErr(operand);
2685		};
2686		return self.finishAir(inst, result, .{ un_op, .none, .none });
2687		}
2688
2689		fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2690		const un_op = self.air.instructions.items(.data)[inst].un_op;
2691		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2692		const operand_ptr = try self.resolveInst(un_op);
2693		const operand: MCValue = blk: {
2694		if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2695		// The MCValue that holds the pointer can be re-used as the value.
2696		break :blk operand_ptr;
2697		} else {
2698		break :blk try self.allocRegOrMem(inst, true);
2699		}
2700		};
2701		try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2702		break :result try self.isNonErr(operand);
2703		};
2704		return self.finishAir(inst, result, .{ un_op, .none, .none });
2705		}
2706
2707		fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
2708		// A loop is a setup to be able to jump back to the beginning.
2709		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2710		const loop = self.air.extraData(Air.Block, ty_pl.payload);
2711		const body = self.air.extra[loop.end..][0..loop.data.body_len];
2712		const start_index = self.code.items.len;
2713		try self.genBody(body);
2714		try self.jump(start_index);
2715		return self.finishAirBookkeeping();
2716		}
2717
2718		/// Send control flow to the `index` of `self.code`.
2719		fn jump(self: *Self, index: usize) !void {
2720		switch (arch) {
2721		.i386 => {
2722		try self.code.ensureUnusedCapacity(5);
2723		if (math.cast(i8, @intCast(i32, index) - (@intCast(i32, self.code.items.len + 2)))) \|delta\| {
2724		self.code.appendAssumeCapacity(0xeb); // jmp rel8
2725		self.code.appendAssumeCapacity(@bitCast(u8, delta));
2726		} else \|_\| {
2727		const delta = @intCast(i32, index) - (@intCast(i32, self.code.items.len + 5));
2728		self.code.appendAssumeCapacity(0xe9); // jmp rel32
2729		mem.writeIntLittle(i32, self.code.addManyAsArrayAssumeCapacity(4), delta);
2730		}
2731		},
2732		.arm, .armeb => {
2733		if (math.cast(i26, @intCast(i32, index) - @intCast(i32, self.code.items.len + 8))) \|delta\| {
2734		writeInt(u32, try self.code.addManyAsArray(4), Instruction.b(.al, delta).toU32());
2735		} else \|_\| {
2736		return self.fail("TODO: enable larger branch offset", .{});
2737		}
2738		},
2739		else => return self.fail("TODO implement jump for {}", .{self.target.cpu.arch}),
2740		}
2741		}
2742
2743		fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
2744		try self.blocks.putNoClobber(self.gpa, inst, .{
2745		// A block is a setup to be able to jump to the end.
2746		.relocs = .{},
2747		// It also acts as a receptacle for break operands.
2748		// Here we use `MCValue.none` to represent a null value so that the first
2749		// break instruction will choose a MCValue for the block result and overwrite
2750		// this field. Following break instructions will use that MCValue to put their
2751		// block results.
2752		.mcv = MCValue{ .none = {} },
2753		});
2754		const block_data = self.blocks.getPtr(inst).?;
2755		defer block_data.relocs.deinit(self.gpa);
2756
2757		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2758		const extra = self.air.extraData(Air.Block, ty_pl.payload);
2759		const body = self.air.extra[extra.end..][0..extra.data.body_len];
2760		try self.genBody(body);
2761
2762		for (block_data.relocs.items) \|reloc\| try self.performReloc(reloc);
2763
2764		const result = @bitCast(MCValue, block_data.mcv);
2765		return self.finishAir(inst, result, .{ .none, .none, .none });
2766		}
2767
2768		fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
2769		const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2770		const condition = pl_op.operand;
2771		switch (arch) {
2772		else => return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch}),
2773		}
2774		return self.finishAir(inst, .dead, .{ condition, .none, .none });
2775		}
2776
2777		fn performReloc(self: *Self, reloc: Reloc) !void {
2778		switch (reloc) {
2779		.rel32 => \|pos\| {
2780		const amt = self.code.items.len - (pos + 4);
2781		// Here it would be tempting to implement testing for amt == 0 and then elide the
2782		// jump. However, that will cause a problem because other jumps may assume that they
2783		// can jump to this code. Or maybe I didn't understand something when I was debugging.
2784		// It could be worth another look. Anyway, that's why that isn't done here. Probably the
2785		// best place to elide jumps will be in semantic analysis, by inlining blocks that only
2786		// only have 1 break instruction.
2787		const s32_amt = math.cast(i32, amt) catch
2788		return self.fail("unable to perform relocation: jump too far", .{});
2789		mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt);
2790		},
2791		.arm_branch => \|info\| {
2792		switch (arch) {
2793		.arm, .armeb => {
2794		const amt = @intCast(i32, self.code.items.len) - @intCast(i32, info.pos + 8);
2795		if (math.cast(i26, amt)) \|delta\| {
2796		writeInt(u32, self.code.items[info.pos..][0..4], Instruction.b(info.cond, delta).toU32());
2797		} else \|_\| {
2798		return self.fail("TODO: enable larger branch offset", .{});
2799		}
2800		},
2801		else => unreachable, // attempting to perform an ARM relocation on a non-ARM target arch
2802		}
2803		},
2804		}
2805		}
2806
2807		fn airBr(self: *Self, inst: Air.Inst.Index) !void {
2808		const branch = self.air.instructions.items(.data)[inst].br;
2809		try self.br(branch.block_inst, branch.operand);
2810		return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
2811		}
2812
2813		fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
2814		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2815		const air_tags = self.air.instructions.items(.tag);
2816		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
2817		.arm, .armeb => switch (air_tags[inst]) {
2818		.bool_and => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_and),
2819		.bool_or => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_or),
2820		else => unreachable, // Not a boolean operation
2821		},
2822		else => return self.fail("TODO implement boolean operations for {}", .{self.target.cpu.arch}),
2823		};
2824		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2825		}
2826
2827		fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
2828		const block_data = self.blocks.getPtr(block).?;
2829
2830		if (self.air.typeOf(operand).hasCodeGenBits()) {
2831		const operand_mcv = try self.resolveInst(operand);
2832		const block_mcv = block_data.mcv;
2833		if (block_mcv == .none) {
2834		block_data.mcv = operand_mcv;
2835		} else {
2836		try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv);
2837		}
2838		}
2839		return self.brVoid(block);
2840		}
2841
2842		fn brVoid(self: *Self, block: Air.Inst.Index) !void {
2843		const block_data = self.blocks.getPtr(block).?;
2844
2845		// Emit a jump with a relocation. It will be patched up after the block ends.
2846		try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
2847
2848		switch (arch) {
2849		.i386 => {
2850		// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
2851		// which is available if the jump is 127 bytes or less forward.
2852		try self.code.resize(self.code.items.len + 5);
2853		self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
2854		// Leave the jump offset undefined
2855		block_data.relocs.appendAssumeCapacity(.{ .rel32 = self.code.items.len - 4 });
2856		},
2857		.arm, .armeb => {
2858		try self.code.resize(self.code.items.len + 4);
2859		block_data.relocs.appendAssumeCapacity(.{
2860		.arm_branch = .{
2861		.pos = self.code.items.len - 4,
2862		.cond = .al,
2863		},
2864		});
2865		},
2866		else => return self.fail("TODO implement brvoid for {}", .{self.target.cpu.arch}),
2867		}
2868		}
2869
2870		fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
2871		const air_datas = self.air.instructions.items(.data);
2872		const air_extra = self.air.extraData(Air.Asm, air_datas[inst].ty_pl.payload);
2873		const zir = self.mod_fn.owner_decl.getFileScope().zir;
2874		const extended = zir.instructions.items(.data)[air_extra.data.zir_index].extended;
2875		const zir_extra = zir.extraData(Zir.Inst.Asm, extended.operand);
2876		const asm_source = zir.nullTerminatedString(zir_extra.data.asm_source);
2877		const outputs_len = @truncate(u5, extended.small);
2878		const args_len = @truncate(u5, extended.small >> 5);
2879		const clobbers_len = @truncate(u5, extended.small >> 10);
2880		_ = clobbers_len; // TODO honor these
2881		const is_volatile = @truncate(u1, extended.small >> 15) != 0;
2882		const outputs = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end..][0..outputs_len]);
2883		const args = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end + outputs.len ..][0..args_len]);
2884
2885		if (outputs_len > 1) {
2886		return self.fail("TODO implement codegen for asm with more than 1 output", .{});
2887		}
2888		var extra_i: usize = zir_extra.end;
2889		const output_constraint: ?[]const u8 = out: {
2890		var i: usize = 0;
2891		while (i < outputs_len) : (i += 1) {
2892		const output = zir.extraData(Zir.Inst.Asm.Output, extra_i);
2893		extra_i = output.end;
2894		break :out zir.nullTerminatedString(output.data.constraint);
2895		}
2896		break :out null;
2897		};
2898
2899		const dead = !is_volatile and self.liveness.isUnused(inst);
2900		const result: MCValue = if (dead) .dead else switch (arch) {
2901		.arm, .armeb => result: {
2902		for (args) \|arg\| {
2903		const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
2904		extra_i = input.end;
2905		const constraint = zir.nullTerminatedString(input.data.constraint);
2906
2907		if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
2908		return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
2909		}
2910		const reg_name = constraint[1 .. constraint.len - 1];
2911		const reg = parseRegName(reg_name) orelse
2912		return self.fail("unrecognized register: '{s}'", .{reg_name});
2913
2914		const arg_mcv = try self.resolveInst(arg);
2915		try self.register_manager.getReg(reg, null);
2916		try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
2917		}
2918
2919		if (mem.eql(u8, asm_source, "svc #0")) {
2920		writeInt(u32, try self.code.addManyAsArray(4), Instruction.svc(.al, 0).toU32());
2921		} else {
2922		return self.fail("TODO implement support for more arm assembly instructions", .{});
2923		}
2924
2925		if (output_constraint) \|output\| {
2926		if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
2927		return self.fail("unrecognized asm output constraint: '{s}'", .{output});
2928		}
2929		const reg_name = output[2 .. output.len - 1];
2930		const reg = parseRegName(reg_name) orelse
2931		return self.fail("unrecognized register: '{s}'", .{reg_name});
2932
2933		break :result MCValue{ .register = reg };
2934		} else {
2935		break :result MCValue{ .none = {} };
2936		}
2937		},
2938		.i386 => result: {
2939		for (args) \|arg\| {
2940		const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
2941		extra_i = input.end;
2942		const constraint = zir.nullTerminatedString(input.data.constraint);
2943
2944		if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
2945		return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
2946		}
2947		const reg_name = constraint[1 .. constraint.len - 1];
2948		const reg = parseRegName(reg_name) orelse
2949		return self.fail("unrecognized register: '{s}'", .{reg_name});
2950
2951		const arg_mcv = try self.resolveInst(arg);
2952		try self.register_manager.getReg(reg, null);
2953		try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
2954		}
2955
2956		{
2957		var iter = std.mem.tokenize(u8, asm_source, "\n\r");
2958		while (iter.next()) \|ins\| {
2959		if (mem.eql(u8, ins, "syscall")) {
2960		try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });
2961		} else if (mem.indexOf(u8, ins, "push")) \|_\| {
2962		const arg = ins[4..];
2963		if (mem.indexOf(u8, arg, "$")) \|l\| {
2964		const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch return self.fail("TODO implement more inline asm int parsing", .{});
2965		try self.code.appendSlice(&.{ 0x6a, n });
2966		} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
2967		const reg_name = ins[4 + l + 2 ..];
2968		const reg = parseRegName(reg_name) orelse
2969		return self.fail("unrecognized register: '{s}'", .{reg_name});
2970		const low_id: u8 = reg.low_id();
2971		if (reg.isExtended()) {
2972		try self.code.appendSlice(&.{ 0x41, 0b1010000 \| low_id });
2973		} else {
2974		try self.code.append(0b1010000 \| low_id);
2975		}
2976		} else return self.fail("TODO more push operands", .{});
2977		} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
2978		const arg = ins[3..];
2979		if (mem.indexOf(u8, arg, "%%")) \|l\| {
2980		const reg_name = ins[3 + l + 2 ..];
2981		const reg = parseRegName(reg_name) orelse
2982		return self.fail("unrecognized register: '{s}'", .{reg_name});
2983		const low_id: u8 = reg.low_id();
2984		if (reg.isExtended()) {
2985		try self.code.appendSlice(&.{ 0x41, 0b1011000 \| low_id });
2986		} else {
2987		try self.code.append(0b1011000 \| low_id);
2988		}
2989		} else return self.fail("TODO more pop operands", .{});
2990		} else {
2991		return self.fail("TODO implement support for more x86 assembly instructions", .{});
2992		}
2993		}
2994		}
2995
2996		if (output_constraint) \|output\| {
2997		if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
2998		return self.fail("unrecognized asm output constraint: '{s}'", .{output});
2999		}
3000		const reg_name = output[2 .. output.len - 1];
3001		const reg = parseRegName(reg_name) orelse
3002		return self.fail("unrecognized register: '{s}'", .{reg_name});
3003		break :result MCValue{ .register = reg };
3004		} else {
3005		break :result MCValue{ .none = {} };
3006		}
3007		},
3008		else => return self.fail("TODO implement inline asm support for more architectures", .{}),
3009		};
3010		if (outputs.len + args.len <= Liveness.bpi - 1) {
3011		var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
3012		std.mem.copy(Air.Inst.Ref, &buf, outputs);
3013		std.mem.copy(Air.Inst.Ref, buf[outputs.len..], args);
3014		return self.finishAir(inst, result, buf);
3015		}
3016		var bt = try self.iterateBigTomb(inst, outputs.len + args.len);
3017		for (outputs) \|output\| {
3018		bt.feed(output);
3019		}
3020		for (args) \|arg\| {
3021		bt.feed(arg);
3022		}
3023		return bt.finishAir(result);
3024		}
3025
3026		fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
3027		try self.ensureProcessDeathCapacity(operand_count + 1);
3028		return BigTomb{
3029		.function = self,
3030		.inst = inst,
3031		.tomb_bits = self.liveness.getTombBits(inst),
3032		.big_tomb_bits = self.liveness.special.get(inst) orelse 0,
3033		.bit_index = 0,
3034		};
3035		}
3036
3037		/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
3038		fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
3039		switch (loc) {
3040		.none => return,
3041		.register => \|reg\| return self.genSetReg(ty, reg, val),
3042		.stack_offset => \|off\| return self.genSetStack(ty, off, val),
3043		.memory => {
3044		return self.fail("TODO implement setRegOrMem for memory", .{});
3045		},
3046		else => unreachable,
3047		}
3048		}
3049
3050		fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
3051		switch (arch) {
3052		.arm, .armeb => switch (mcv) {
3053		.dead => unreachable,
3054		.ptr_stack_offset => unreachable,
3055		.ptr_embedded_in_code => unreachable,
3056		.unreach, .none => return, // Nothing to do.
3057		.undef => {
3058		if (!self.wantSafety())
3059		return; // The already existing value will do just fine.
3060		// TODO Upgrade this to a memset call when we have that available.
3061		switch (ty.abiSize(self.target.*)) {
3062		1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
3063		2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
3064		4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
3065		8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
3066		else => return self.fail("TODO implement memset", .{}),
3067		}
3068		},
3069		.compare_flags_unsigned,
3070		.compare_flags_signed,
3071		.immediate,
3072		=> {
3073		const reg = try self.copyToTmpRegister(ty, mcv);
3074		return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3075		},
3076		.embedded_in_code => \|code_offset\| {
3077		_ = code_offset;
3078		return self.fail("TODO implement set stack variable from embedded_in_code", .{});
3079		},
3080		.register => \|reg\| {
3081		const abi_size = ty.abiSize(self.target.*);
3082		const adj_off = stack_offset + abi_size;
3083
3084		switch (abi_size) {
3085		1, 4 => {
3086		const offset = if (math.cast(u12, adj_off)) \|imm\| blk: {
3087		break :blk Instruction.Offset.imm(imm);
3088		} else \|_\| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
3089		const str = switch (abi_size) {
3090		1 => Instruction.strb,
3091		4 => Instruction.str,
3092		else => unreachable,
3093		};
3094
3095		writeInt(u32, try self.code.addManyAsArray(4), str(.al, reg, .fp, .{
3096		.offset = offset,
3097		.positive = false,
3098		}).toU32());
3099		},
3100		2 => {
3101		const offset = if (adj_off <= math.maxInt(u8)) blk: {
3102		break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
3103		} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
3104
3105		writeInt(u32, try self.code.addManyAsArray(4), Instruction.strh(.al, reg, .fp, .{
3106		.offset = offset,
3107		.positive = false,
3108		}).toU32());
3109		},
3110		else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
3111		}
3112		},
3113		.memory => \|vaddr\| {
3114		_ = vaddr;
3115		return self.fail("TODO implement set stack variable from memory vaddr", .{});
3116		},
3117		.stack_offset => \|off\| {
3118		if (stack_offset == off)
3119		return; // Copy stack variable to itself; nothing to do.
3120
3121		const reg = try self.copyToTmpRegister(ty, mcv);
3122		return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3123		},
3124		},
3125		else => return self.fail("TODO implement getSetStack for {}", .{self.target.cpu.arch}),
3126		}
3127		}
3128
3129		fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
3130		switch (arch) {
3131		.arm, .armeb => switch (mcv) {
3132		.dead => unreachable,
3133		.ptr_stack_offset => unreachable,
3134		.ptr_embedded_in_code => unreachable,
3135		.unreach, .none => return, // Nothing to do.
3136		.undef => {
3137		if (!self.wantSafety())
3138		return; // The already existing value will do just fine.
3139		// Write the debug undefined value.
3140		return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa });
3141		},
3142		.compare_flags_unsigned,
3143		.compare_flags_signed,
3144		=> \|op\| {
3145		const condition = switch (mcv) {
3146		.compare_flags_unsigned => Condition.fromCompareOperatorUnsigned(op),
3147		.compare_flags_signed => Condition.fromCompareOperatorSigned(op),
3148		else => unreachable,
3149		};
3150
3151		// mov reg, 0
3152		// moveq reg, 1
3153		const zero = Instruction.Operand.imm(0, 0);
3154		const one = Instruction.Operand.imm(1, 0);
3155		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, zero).toU32());
3156		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(condition, reg, one).toU32());
3157		},
3158		.immediate => \|x\| {
3159		if (x > math.maxInt(u32)) return self.fail("ARM registers are 32-bit wide", .{});
3160
3161		if (Instruction.Operand.fromU32(@intCast(u32, x))) \|op\| {
3162		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, op).toU32());
3163		} else if (Instruction.Operand.fromU32(~@intCast(u32, x))) \|op\| {
3164		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mvn(.al, reg, op).toU32());
3165		} else if (x <= math.maxInt(u16)) {
3166		if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
3167		writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @intCast(u16, x)).toU32());
3168		} else {
3169		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
3170		writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
3171		}
3172		} else {
3173		// TODO write constant to code and load
3174		// relative to pc
3175		if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
3176		// immediate: 0xaaaabbbb
3177		// movw reg, #0xbbbb
3178		// movt reg, #0xaaaa
3179		writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @truncate(u16, x)).toU32());
3180		writeInt(u32, try self.code.addManyAsArray(4), Instruction.movt(.al, reg, @truncate(u16, x >> 16)).toU32());
3181		} else {
3182		// immediate: 0xaabbccdd
3183		// mov reg, #0xaa
3184		// orr reg, reg, #0xbb, 24
3185		// orr reg, reg, #0xcc, 16
3186		// orr reg, reg, #0xdd, 8
3187		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
3188		writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
3189		writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 16), 8)).toU32());
3190		writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 24), 4)).toU32());
3191		}
3192		}
3193		},
3194		.register => \|src_reg\| {
3195		// If the registers are the same, nothing to do.
3196		if (src_reg.id() == reg.id())
3197		return;
3198
3199		// mov reg, src_reg
3200		writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.reg(src_reg, Instruction.Operand.Shift.none)).toU32());
3201		},
3202		.memory => \|addr\| {
3203		// The value is in memory at a hard-coded address.
3204		// If the type is a pointer, it means the pointer address is at this memory location.
3205		try self.genSetReg(ty, reg, .{ .immediate = addr });
3206		writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
3207		},
3208		.stack_offset => \|unadjusted_off\| {
3209		// TODO: maybe addressing from sp instead of fp
3210		const abi_size = ty.abiSize(self.target.*);
3211		const adj_off = unadjusted_off + abi_size;
3212
3213		switch (abi_size) {
3214		1, 4 => {
3215		const offset = if (adj_off <= math.maxInt(u12)) blk: {
3216		break :blk Instruction.Offset.imm(@intCast(u12, adj_off));
3217		} else Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
3218		const ldr = switch (abi_size) {
3219		1 => Instruction.ldrb,
3220		4 => Instruction.ldr,
3221		else => unreachable,
3222		};
3223
3224		writeInt(u32, try self.code.addManyAsArray(4), ldr(.al, reg, .fp, .{
3225		.offset = offset,
3226		.positive = false,
3227		}).toU32());
3228		},
3229		2 => {
3230		const offset = if (adj_off <= math.maxInt(u8)) blk: {
3231		break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
3232		} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
3233
3234		writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldrh(.al, reg, .fp, .{
3235		.offset = offset,
3236		.positive = false,
3237		}).toU32());
3238		},
3239		else => return self.fail("TODO a type of size {} is not allowed in a register", .{abi_size}),
3240		}
3241		},
3242		else => return self.fail("TODO implement getSetReg for arm {}", .{mcv}),
3243		},
3244		else => return self.fail("TODO implement getSetReg for {}", .{self.target.cpu.arch}),
3245		}
3246		}
3247
3248		fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
3249		const un_op = self.air.instructions.items(.data)[inst].un_op;
3250		const result = try self.resolveInst(un_op);
3251		return self.finishAir(inst, result, .{ un_op, .none, .none });
3252		}
3253
3254		fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
3255		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3256		const result = try self.resolveInst(ty_op.operand);
3257		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3258		}
3259
3260		fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
3261		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3262		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3263		else => return self.fail("TODO implement airArrayToSlice for {}", .{
3264		self.target.cpu.arch,
3265		}),
3266		};
3267		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3268		}
3269
3270		fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
3271		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3272		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3273		else => return self.fail("TODO implement airIntToFloat for {}", .{
3274		self.target.cpu.arch,
3275		}),
3276		};
3277		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3278		}
3279
3280		fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
3281		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3282		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3283		else => return self.fail("TODO implement airFloatToInt for {}", .{
3284		self.target.cpu.arch,
3285		}),
3286		};
3287		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3288		}
3289
3290		fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
3291		const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
3292		const extra = self.air.extraData(Air.Block, ty_pl.payload);
3293		const result: MCValue = switch (arch) {
3294		else => return self.fail("TODO implement airCmpxchg for {}", .{
3295		self.target.cpu.arch,
3296		}),
3297		};
3298		return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
3299		}
3300
3301		fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
3302		_ = inst;
3303		return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
3304		}
3305
3306		fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
3307		_ = inst;
3308		return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
3309		}
3310
3311		fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
3312		_ = inst;
3313		_ = order;
3314		return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
3315		}
3316
3317		fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
3318		_ = inst;
3319		return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
3320		}
3321
3322		fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
3323		_ = inst;
3324		return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
3325		}
3326
3327		fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3328		// First section of indexes correspond to a set number of constant values.
3329		const ref_int = @enumToInt(inst);
3330		if (ref_int < Air.Inst.Ref.typed_value_map.len) {
3331		const tv = Air.Inst.Ref.typed_value_map[ref_int];
3332		if (!tv.ty.hasCodeGenBits()) {
3333		return MCValue{ .none = {} };
3334		}
3335		return self.genTypedValue(tv);
3336		}
3337
3338		// If the type has no codegen bits, no need to store it.
3339		const inst_ty = self.air.typeOf(inst);
3340		if (!inst_ty.hasCodeGenBits())
3341		return MCValue{ .none = {} };
3342
3343		const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
3344		switch (self.air.instructions.items(.tag)[inst_index]) {
3345		.constant => {
3346		// Constants have static lifetimes, so they are always memoized in the outer most table.
3347		const branch = &self.branch_stack.items[0];
3348		const gop = try branch.inst_table.getOrPut(self.gpa, inst_index);
3349		if (!gop.found_existing) {
3350		const ty_pl = self.air.instructions.items(.data)[inst_index].ty_pl;
3351		gop.value_ptr.* = try self.genTypedValue(.{
3352		.ty = inst_ty,
3353		.val = self.air.values[ty_pl.payload],
3354		});
3355		}
3356		return gop.value_ptr.*;
3357		},
3358		.const_ty => unreachable,
3359		else => return self.getResolvedInstValue(inst_index),
3360		}
3361		}
3362
3363		fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
3364		// Treat each stack item as a "layer" on top of the previous one.
3365		var i: usize = self.branch_stack.items.len;
3366		while (true) {
3367		i -= 1;
3368		if (self.branch_stack.items[i].inst_table.get(inst)) \|mcv\| {
3369		assert(mcv != .dead);
3370		return mcv;
3371		}
3372		}
3373		}
3374
3375		/// If the MCValue is an immediate, and it does not fit within this type,
3376		/// we put it in a register.
3377		/// A potential opportunity for future optimization here would be keeping track
3378		/// of the fact that the instruction is available both as an immediate
3379		/// and as a register.
3380		fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
3381		const mcv = try self.resolveInst(operand);
3382		const ti = @typeInfo(T).Int;
3383		switch (mcv) {
3384		.immediate => \|imm\| {
3385		// This immediate is unsigned.
3386		const U = std.meta.Int(.unsigned, ti.bits - @boolToInt(ti.signedness == .signed));
3387		if (imm >= math.maxInt(U)) {
3388		return MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.usize), mcv) };
3389		}
3390		},
3391		else => {},
3392		}
3393		return mcv;
3394		}
3395
3396		fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
3397		if (typed_value.val.isUndef())
3398		return MCValue{ .undef = {} };
3399		const ptr_bits = self.target.cpu.arch.ptrBitWidth();
3400		const ptr_bytes: u64 = @divExact(ptr_bits, 8);
3401		switch (typed_value.ty.zigTypeTag()) {
3402		.Pointer => switch (typed_value.ty.ptrSize()) {
3403		.Slice => {
3404		var buf: Type.SlicePtrFieldTypeBuffer = undefined;
3405		const ptr_type = typed_value.ty.slicePtrFieldType(&buf);
3406		const ptr_mcv = try self.genTypedValue(.{ .ty = ptr_type, .val = typed_value.val });
3407		const slice_len = typed_value.val.sliceLen();
3408		// Codegen can't handle some kinds of indirection. If the wrong union field is accessed here it may mean
3409		// the Sema code needs to use anonymous Decls or alloca instructions to store data.
3410		const ptr_imm = ptr_mcv.memory;
3411		_ = slice_len;
3412		_ = ptr_imm;
3413		// We need more general support for const data being stored in memory to make this work.
3414		return self.fail("TODO codegen for const slices", .{});
3415		},
3416		else => {
3417		if (typed_value.val.castTag(.decl_ref)) \|payload\| {
3418		const decl = payload.data;
3419		decl.alive = true;
3420		if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
3421		const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
3422		const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes;
3423		return MCValue{ .memory = got_addr };
3424		} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
3425		// TODO I'm hacking my way through here by repurposing .memory for storing
3426		// index to the GOT target symbol index.
3427		return MCValue{ .memory = decl.link.macho.local_sym_index };
3428		} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
3429		const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes;
3430		return MCValue{ .memory = got_addr };
3431		} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
3432		try p9.seeDecl(decl);
3433		const got_addr = p9.bases.data + decl.link.plan9.got_index.? * ptr_bytes;
3434		return MCValue{ .memory = got_addr };
3435		} else {
3436		return self.fail("TODO codegen non-ELF const Decl pointer", .{});
3437		}
3438		}
3439		if (typed_value.val.tag() == .int_u64) {
3440		return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
3441		}
3442		return self.fail("TODO codegen more kinds of const pointers", .{});
3443		},
3444		},
3445		.Int => {
3446		const info = typed_value.ty.intInfo(self.target.*);
3447		if (info.bits > ptr_bits or info.signedness == .signed) {
3448		return self.fail("TODO const int bigger than ptr and signed int", .{});
3449		}
3450		return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
3451		},
3452		.Bool => {
3453		return MCValue{ .immediate = @boolToInt(typed_value.val.toBool()) };
3454		},
3455		.ComptimeInt => unreachable, // semantic analysis prevents this
3456		.ComptimeFloat => unreachable, // semantic analysis prevents this
3457		.Optional => {
3458		if (typed_value.ty.isPtrLikeOptional()) {
3459		if (typed_value.val.isNull())
3460		return MCValue{ .immediate = 0 };
3461
3462		var buf: Type.Payload.ElemType = undefined;
3463		return self.genTypedValue(.{
3464		.ty = typed_value.ty.optionalChild(&buf),
3465		.val = typed_value.val,
3466		});
3467		} else if (typed_value.ty.abiSize(self.target.*) == 1) {
3468		return MCValue{ .immediate = @boolToInt(typed_value.val.isNull()) };
3469		}
3470		return self.fail("TODO non pointer optionals", .{});
3471		},
3472		.Enum => {
3473		if (typed_value.val.castTag(.enum_field_index)) \|field_index\| {
3474		switch (typed_value.ty.tag()) {
3475		.enum_simple => {
3476		return MCValue{ .immediate = field_index.data };
3477		},
3478		.enum_full, .enum_nonexhaustive => {
3479		const enum_full = typed_value.ty.cast(Type.Payload.EnumFull).?.data;
3480		if (enum_full.values.count() != 0) {
3481		const tag_val = enum_full.values.keys()[field_index.data];
3482		return self.genTypedValue(.{ .ty = enum_full.tag_ty, .val = tag_val });
3483		} else {
3484		return MCValue{ .immediate = field_index.data };
3485		}
3486		},
3487		else => unreachable,
3488		}
3489		} else {
3490		var int_tag_buffer: Type.Payload.Bits = undefined;
3491		const int_tag_ty = typed_value.ty.intTagType(&int_tag_buffer);
3492		return self.genTypedValue(.{ .ty = int_tag_ty, .val = typed_value.val });
3493		}
3494		},
3495		.ErrorSet => {
3496		switch (typed_value.val.tag()) {
3497		.@"error" => {
3498		const err_name = typed_value.val.castTag(.@"error").?.data.name;
3499		const module = self.bin_file.options.module.?;
3500		const global_error_set = module.global_error_set;
3501		const error_index = global_error_set.get(err_name).?;
3502		return MCValue{ .immediate = error_index };
3503		},
3504		else => {
3505		// In this case we are rendering an error union which has a 0 bits payload.
3506		return MCValue{ .immediate = 0 };
3507		},
3508		}
3509		},
3510		.ErrorUnion => {
3511		const error_type = typed_value.ty.errorUnionSet();
3512		const payload_type = typed_value.ty.errorUnionPayload();
3513		const sub_val = typed_value.val.castTag(.eu_payload).?.data;
3514
3515		if (!payload_type.hasCodeGenBits()) {
3516		// We use the error type directly as the type.
3517		return self.genTypedValue(.{ .ty = error_type, .val = sub_val });
3518		}
3519
3520		return self.fail("TODO implement error union const of type '{}'", .{typed_value.ty});
3521		},
3522		else => return self.fail("TODO implement const of type '{}'", .{typed_value.ty}),
3523		}
3524		}
3525
3526		const CallMCValues = struct {
3527		args: []MCValue,
3528		return_value: MCValue,
3529		stack_byte_count: u32,
3530		stack_align: u32,
3531
3532		fn deinit(self: CallMCValues, func: Self) void {
3533		func.gpa.free(self.args);
3534		self.* = undefined;
3535		}
3536		};
3537
3538		/// Caller must call `CallMCValues.deinit`.
3539		fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
3540		const cc = fn_ty.fnCallingConvention();
3541		const param_types = try self.gpa.alloc(Type, fn_ty.fnParamLen());
3542		defer self.gpa.free(param_types);
3543		fn_ty.fnParamTypes(param_types);
3544		var result: CallMCValues = .{
3545		.args = try self.gpa.alloc(MCValue, param_types.len),
3546		// These undefined values must be populated before returning from this function.
3547		.return_value = undefined,
3548		.stack_byte_count = undefined,
3549		.stack_align = undefined,
3550		};
3551		errdefer self.gpa.free(result.args);
3552
3553		const ret_ty = fn_ty.fnReturnType();
3554
3555		switch (arch) {
3556		.arm, .armeb => {
3557		switch (cc) {
3558		.Naked => {
3559		assert(result.args.len == 0);
3560		result.return_value = .{ .unreach = {} };
3561		result.stack_byte_count = 0;
3562		result.stack_align = 1;
3563		return result;
3564		},
3565		.Unspecified, .C => {
3566		// ARM Procedure Call Standard, Chapter 6.5
3567		var ncrn: usize = 0; // Next Core Register Number
3568		var nsaa: u32 = 0; // Next stacked argument address
3569
3570		for (param_types) \|ty, i\| {
3571		if (ty.abiAlignment(self.target.*) == 8)
3572		ncrn = std.mem.alignForwardGeneric(usize, ncrn, 2);
3573
3574		const param_size = @intCast(u32, ty.abiSize(self.target.*));
3575		if (std.math.divCeil(u32, param_size, 4) catch unreachable <= 4 - ncrn) {
3576		if (param_size <= 4) {
3577		result.args[i] = .{ .register = c_abi_int_param_regs[ncrn] };
3578		ncrn += 1;
3579		} else {
3580		return self.fail("TODO MCValues with multiple registers", .{});
3581		}
3582		} else if (ncrn < 4 and nsaa == 0) {
3583		return self.fail("TODO MCValues split between registers and stack", .{});
3584		} else {
3585		ncrn = 4;
3586		if (ty.abiAlignment(self.target.*) == 8)
3587		nsaa = std.mem.alignForwardGeneric(u32, nsaa, 8);
3588
3589		result.args[i] = .{ .stack_offset = nsaa };
3590		nsaa += param_size;
3591		}
3592		}
3593
3594		result.stack_byte_count = nsaa;
3595		result.stack_align = 8;
3596		},
3597		else => return self.fail("TODO implement function parameters for {} on arm", .{cc}),
3598		}
3599		},
3600		else => if (param_types.len != 0)
3601		return self.fail("TODO implement codegen parameters for {}", .{self.target.cpu.arch}),
3602		}
3603
3604		if (ret_ty.zigTypeTag() == .NoReturn) {
3605		result.return_value = .{ .unreach = {} };
3606		} else if (!ret_ty.hasCodeGenBits()) {
3607		result.return_value = .{ .none = {} };
3608		} else switch (arch) {
3609		.arm, .armeb => switch (cc) {
3610		.Naked => unreachable,
3611		.Unspecified, .C => {
3612		const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3613		if (ret_ty_size <= 4) {
3614		result.return_value = .{ .register = c_abi_int_return_regs[0] };
3615		} else {
3616		return self.fail("TODO support more return types for ARM backend", .{});
3617		}
3618		},
3619		else => return self.fail("TODO implement function return values for {}", .{cc}),
3620		},
3621		else => return self.fail("TODO implement codegen return values for {}", .{self.target.cpu.arch}),
3622		}
3623		return result;
3624		}
3625
3626		/// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`.
3627		fn wantSafety(self: *Self) bool {
3628		return switch (self.bin_file.options.optimize_mode) {
3629		.Debug => true,
3630		.ReleaseSafe => true,
3631		.ReleaseFast => false,
3632		.ReleaseSmall => false,
3633		};
3634		}
3635
3636		fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3637		@setCold(true);
3638		assert(self.err_msg == null);
3639		self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3640		return error.CodegenFail;
3641		}
3642
3643		fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3644		@setCold(true);
3645		assert(self.err_msg == null);
3646		self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3647		return error.CodegenFail;
3648		}
3649
3650		const Register = switch (arch) {
3651		.i386 => @import("arch/x86/bits.zig").Register,
3652		.arm, .armeb => @import("arch/arm/bits.zig").Register,
3653		else => enum {
3654		dummy,
3655
3656		pub fn allocIndex(self: Register) ?u4 {
3657		_ = self;
3658		return null;
3659		}
3660		},
3661		};
3662
3663		const Instruction = switch (arch) {
3664		.arm, .armeb => @import("arch/arm/bits.zig").Instruction,
3665		else => void,
3666		};
3667
3668		const Condition = switch (arch) {
3669		.arm, .armeb => @import("arch/arm/bits.zig").Condition,
3670		else => void,
3671		};
3672
3673		const callee_preserved_regs = switch (arch) {
3674		.i386 => @import("arch/x86/bits.zig").callee_preserved_regs,
3675		.arm, .armeb => @import("arch/arm/bits.zig").callee_preserved_regs,
3676		else => [_]Register{},
3677		};
3678
3679		const c_abi_int_param_regs = switch (arch) {
3680		.i386 => @import("arch/x86/bits.zig").c_abi_int_param_regs,
3681		.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_param_regs,
3682		else => [_]Register{},
3683		};
3684
3685		const c_abi_int_return_regs = switch (arch) {
3686		.i386 => @import("arch/x86/bits.zig").c_abi_int_return_regs,
3687		.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_return_regs,
3688		else => [_]Register{},
3689		};
3690
3691		fn parseRegName(name: []const u8) ?Register {
3692		if (@hasDecl(Register, "parseRegName")) {
3693		return Register.parseRegName(name);
3694		}
3695		return std.meta.stringToEnum(Register, name);
3696		}
3697		};
3698		}