Commit 6b5e403e5dcdd55baf318e8734b77ce4bc635fe9 | zig

author	Joachim Schmidt <joachim.schmidt557@outlook.com> 2021-11-07 09:38:04+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2021-11-07 21:20:58+01:00
log	6b5e403e5dcdd55baf318e8734b77ce4bc635fe9
tree	b3272899580f248c901a6de1570c84528e4b096d
parent	0fb26b03690eb327164bbc7a21ba2d69d3b1a5b8

stage2 ARM: move codegen to separate file

This also removes i386 codegen code, which was unused and untested

2 files changed, 3041 insertions(+), 3395 deletions(-)

src
- arch
  - arm
    CodeGen.zig
- codegen.zig

src/arch/arm/CodeGen.zig created+3038

...	@@ -0,0 +1,3038 @@
1	const std = @import("std");
2	const builtin = @import("builtin");
3	const mem = std.mem;
4	const math = std.math;
5	const assert = std.debug.assert;
6	const Air = @import("../../Air.zig");
7	const Zir = @import("../../Zir.zig");
8	const Liveness = @import("../../Liveness.zig");
9	const Type = @import("../../type.zig").Type;
10	const Value = @import("../../value.zig").Value;
11	const TypedValue = @import("../../TypedValue.zig");
12	const link = @import("../../link.zig");
13	const Module = @import("../../Module.zig");
14	const Compilation = @import("../../Compilation.zig");
15	const ErrorMsg = Module.ErrorMsg;
16	const Target = std.Target;
17	const Allocator = mem.Allocator;
18	const trace = @import("../../tracy.zig").trace;
19	const DW = std.dwarf;
20	const leb128 = std.leb;
21	const log = std.log.scoped(.codegen);
22	const build_options = @import("build_options");
23	const RegisterManager = @import("../../register_manager.zig").RegisterManager;
24
25	pub const FnResult = @import("../../codegen.zig").FnResult;
26	pub const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
27	pub const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
28
29	const InnerError = error{
30	OutOfMemory,
31	CodegenFail,
32	};
33
34	gpa: *Allocator,
35	air: Air,
36	liveness: Liveness,
37	bin_file: *link.File,
38	target: *const std.Target,
39	mod_fn: *const Module.Fn,
40	code: *std.ArrayList(u8),
41	debug_output: DebugInfoOutput,
42	err_msg: ?*ErrorMsg,
43	args: []MCValue,
44	ret_mcv: MCValue,
45	fn_type: Type,
46	arg_index: usize,
47	src_loc: Module.SrcLoc,
48	stack_align: u32,
49
50	prev_di_line: u32,
51	prev_di_column: u32,
52	/// Byte offset within the source file of the ending curly.
53	end_di_line: u32,
54	end_di_column: u32,
55	/// Relative to the beginning of `code`.
56	prev_di_pc: usize,
57
58	/// The value is an offset into the `Function` `code` from the beginning.
59	/// To perform the reloc, write 32-bit signed little-endian integer
60	/// which is a relative jump, based on the address following the reloc.
61	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{},
62
63	/// Whenever there is a runtime branch, we push a Branch onto this stack,
64	/// and pop it off when the runtime branch joins. This provides an "overlay"
65	/// of the table of mappings from instructions to `MCValue` from within the branch.
66	/// This way we can modify the `MCValue` for an instruction in different ways
67	/// within different branches. Special consideration is needed when a branch
68	/// joins with its parent, to make sure all instructions have the same MCValue
69	/// across each runtime branch upon joining.
70	branch_stack: *std.ArrayList(Branch),
71
72	// Key is the block instruction
73	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
74
75	register_manager: RegisterManager(Self, Register, &callee_preserved_regs) = .{},
76	/// Maps offset to what is stored there.
77	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
78
79	/// Offset from the stack base, representing the end of the stack frame.
80	max_end_stack: u32 = 0,
81	/// Represents the current end stack offset. If there is no existing slot
82	/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
83	next_stack_offset: u32 = 0,
84
85	/// Debug field, used to find bugs in the compiler.
86	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
87
88	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
89
90	const MCValue = union(enum) {
91	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
92	/// TODO Look into deleting this tag and using `dead` instead, since every use
93	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
94	none,
95	/// Control flow will not allow this value to be observed.
96	unreach,
97	/// No more references to this value remain.
98	dead,
99	/// The value is undefined.
100	undef,
101	/// A pointer-sized integer that fits in a register.
102	/// If the type is a pointer, this is the pointer address in virtual address space.
103	immediate: u64,
104	/// The constant was emitted into the code, at this offset.
105	/// If the type is a pointer, it means the pointer address is embedded in the code.
106	embedded_in_code: usize,
107	/// The value is a pointer to a constant which was emitted into the code, at this offset.
108	ptr_embedded_in_code: usize,
109	/// The value is in a target-specific register.
110	register: Register,
111	/// The value is in memory at a hard-coded address.
112	/// If the type is a pointer, it means the pointer address is at this memory location.
113	memory: u64,
114	/// The value is one of the stack variables.
115	/// If the type is a pointer, it means the pointer address is in the stack at this offset.
116	stack_offset: u32,
117	/// The value is a pointer to one of the stack variables (payload is stack offset).
118	ptr_stack_offset: u32,
119	/// The value is in the compare flags assuming an unsigned operation,
120	/// with this operator applied on top of it.
121	compare_flags_unsigned: math.CompareOperator,
122	/// The value is in the compare flags assuming a signed operation,
123	/// with this operator applied on top of it.
124	compare_flags_signed: math.CompareOperator,
125
126	fn isMemory(mcv: MCValue) bool {
127	return switch (mcv) {
128	.embedded_in_code, .memory, .stack_offset => true,
129	else => false,
130	};
131	}
132
133	fn isImmediate(mcv: MCValue) bool {
134	return switch (mcv) {
135	.immediate => true,
136	else => false,
137	};
138	}
139
140	fn isMutable(mcv: MCValue) bool {
141	return switch (mcv) {
142	.none => unreachable,
143	.unreach => unreachable,
144	.dead => unreachable,
145
146	.immediate,
147	.embedded_in_code,
148	.memory,
149	.compare_flags_unsigned,
150	.compare_flags_signed,
151	.ptr_stack_offset,
152	.ptr_embedded_in_code,
153	.undef,
154	=> false,
155
156	.register,
157	.stack_offset,
158	=> true,
159	};
160	}
161	};
162
163	const Branch = struct {
164	inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .{},
165
166	fn deinit(self: Branch, gpa: Allocator) void {
167	self.inst_table.deinit(gpa);
168	self.* = undefined;
169	}
170	};
171
172	const StackAllocation = struct {
173	inst: Air.Inst.Index,
174	/// TODO do we need size? should be determined by inst.ty.abiSize()
175	size: u32,
176	};
177
178	const BlockData = struct {
179	relocs: std.ArrayListUnmanaged(Reloc),
180	/// The first break instruction encounters `null` here and chooses a
181	/// machine code value for the block result, populating this field.
182	/// Following break instructions encounter that value and use it for
183	/// the location to store their block results.
184	mcv: MCValue,
185	};
186
187	const Reloc = union(enum) {
188	/// The value is an offset into the `Function` `code` from the beginning.
189	/// To perform the reloc, write 32-bit signed little-endian integer
190	/// which is a relative jump, based on the address following the reloc.
191	rel32: usize,
192	/// A branch in the ARM instruction set
193	arm_branch: struct {
194	pos: usize,
195	cond: @import("bits.zig").Condition,
196	},
197	};
198
199	const BigTomb = struct {
200	function: *Self,
201	inst: Air.Inst.Index,
202	tomb_bits: Liveness.Bpi,
203	big_tomb_bits: u32,
204	bit_index: usize,
205
206	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
207	const this_bit_index = bt.bit_index;
208	bt.bit_index += 1;
209
210	const op_int = @enumToInt(op_ref);
211	if (op_int < Air.Inst.Ref.typed_value_map.len) return;
212	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
213
214	if (this_bit_index < Liveness.bpi - 1) {
215	const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
216	if (!dies) return;
217	} else {
218	const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
219	const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
220	if (!dies) return;
221	}
222	bt.function.processDeath(op_index);
223	}
224
225	fn finishAir(bt: *BigTomb, result: MCValue) void {
226	const is_used = !bt.function.liveness.isUnused(bt.inst);
227	if (is_used) {
228	log.debug("%{d} => {}", .{ bt.inst, result });
229	const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
230	branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
231	}
232	bt.function.finishAirBookkeeping();
233	}
234	};
235
236	const Self = @This();
237
238	pub fn generate(
239	bin_file: *link.File,
240	src_loc: Module.SrcLoc,
241	module_fn: *Module.Fn,
242	air: Air,
243	liveness: Liveness,
244	code: *std.ArrayList(u8),
245	debug_output: DebugInfoOutput,
246	) GenerateSymbolError!FnResult {
247	if (build_options.skip_non_native and builtin.cpu.arch != bin_file.options.target.cpu.arch) {
248	@panic("Attempted to compile for architecture that was disabled by build configuration");
249	}
250
251	assert(module_fn.owner_decl.has_tv);
252	const fn_type = module_fn.owner_decl.ty;
253
254	var branch_stack = std.ArrayList(Branch).init(bin_file.allocator);
255	defer {
256	assert(branch_stack.items.len == 1);
257	branch_stack.items[0].deinit(bin_file.allocator);
258	branch_stack.deinit();
259	}
260	try branch_stack.append(.{});
261
262	var function = Self{
263	.gpa = bin_file.allocator,
264	.air = air,
265	.liveness = liveness,
266	.target = &bin_file.options.target,
267	.bin_file = bin_file,
268	.mod_fn = module_fn,
269	.code = code,
270	.debug_output = debug_output,
271	.err_msg = null,
272	.args = undefined, // populated after `resolveCallingConventionValues`
273	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
274	.fn_type = fn_type,
275	.arg_index = 0,
276	.branch_stack = &branch_stack,
277	.src_loc = src_loc,
278	.stack_align = undefined,
279	.prev_di_pc = 0,
280	.prev_di_line = module_fn.lbrace_line,
281	.prev_di_column = module_fn.lbrace_column,
282	.end_di_line = module_fn.rbrace_line,
283	.end_di_column = module_fn.rbrace_column,
284	};
285	defer function.stack.deinit(bin_file.allocator);
286	defer function.blocks.deinit(bin_file.allocator);
287	defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
288
289	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
290	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
291	else => \|e\| return e,
292	};
293	defer call_info.deinit(&function);
294
295	function.args = call_info.args;
296	function.ret_mcv = call_info.return_value;
297	function.stack_align = call_info.stack_align;
298	function.max_end_stack = call_info.stack_byte_count;
299
300	function.gen() catch \|err\| switch (err) {
301	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
302	else => \|e\| return e,
303	};
304
305	if (function.err_msg) \|em\| {
306	return FnResult{ .fail = em };
307	} else {
308	return FnResult{ .appended = {} };
309	}
310	}
311
312	fn gen(self: *Self) !void {
313	const cc = self.fn_type.fnCallingConvention();
314	if (cc != .Naked) {
315	// push {fp, lr}
316	// mov fp, sp
317	// sub sp, sp, #reloc
318	const prologue_reloc = self.code.items.len;
319	try self.code.resize(prologue_reloc + 12);
320	self.writeInt(u32, self.code.items[prologue_reloc + 4 ..][0..4], Instruction.mov(.al, .fp, Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none)).toU32());
321
322	try self.dbgSetPrologueEnd();
323
324	try self.genBody(self.air.getMainBody());
325
326	// Backpatch push callee saved regs
327	var saved_regs = Instruction.RegisterList{
328	.r11 = true, // fp
329	.r14 = true, // lr
330	};
331	inline for (callee_preserved_regs) \|reg\| {
332	if (self.register_manager.isRegAllocated(reg)) {
333	@field(saved_regs, @tagName(reg)) = true;
334	}
335	}
336	self.writeInt(u32, self.code.items[prologue_reloc..][0..4], Instruction.stmdb(.al, .sp, true, saved_regs).toU32());
337
338	// Backpatch stack offset
339	const stack_end = self.max_end_stack;
340	const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
341	if (Instruction.Operand.fromU32(@intCast(u32, aligned_stack_end))) \|op\| {
342	self.writeInt(u32, self.code.items[prologue_reloc + 8 ..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32());
343	} else {
344	return self.failSymbol("TODO ARM: allow larger stacks", .{});
345	}
346
347	try self.dbgSetEpilogueBegin();
348
349	// exitlude jumps
350	if (self.exitlude_jump_relocs.items.len == 1) {
351	// There is only one relocation. Hence,
352	// this relocation must be at the end of
353	// the code. Therefore, we can just delete
354	// the space initially reserved for the
355	// jump
356	self.code.items.len -= 4;
357	} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
358	const amt = @intCast(i32, self.code.items.len) - @intCast(i32, jmp_reloc + 8);
359	if (amt == -4) {
360	// This return is at the end of the
361	// code block. We can't just delete
362	// the space because there may be
363	// other jumps we already relocated to
364	// the address. Instead, insert a nop
365	self.writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.nop().toU32());
366	} else {
367	if (math.cast(i26, amt)) \|offset\| {
368	self.writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(.al, offset).toU32());
369	} else \|_\| {
370	return self.failSymbol("exitlude jump is too large", .{});
371	}
372	}
373	}
374
375	// Epilogue: pop callee saved registers (swap lr with pc in saved_regs)
376	saved_regs.r14 = false; // lr
377	saved_regs.r15 = true; // pc
378
379	// mov sp, fp
380	// pop {fp, pc}
381	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .sp, Instruction.Operand.reg(.fp, Instruction.Operand.Shift.none)).toU32());
382	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldm(.al, .sp, true, saved_regs).toU32());
383	} else {
384	try self.dbgSetPrologueEnd();
385	try self.genBody(self.air.getMainBody());
386	try self.dbgSetEpilogueBegin();
387	}
388
389	// Drop them off at the rbrace.
390	try self.dbgAdvancePCAndLine(self.end_di_line, self.end_di_column);
391	}
392
393	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
394	const air_tags = self.air.instructions.items(.tag);
395
396	for (body) \|inst\| {
397	const old_air_bookkeeping = self.air_bookkeeping;
398	try self.ensureProcessDeathCapacity(Liveness.bpi);
399
400	switch (air_tags[inst]) {
401	// zig fmt: off
402	.add, .ptr_add => try self.airAdd(inst),
403	.addwrap => try self.airAddWrap(inst),
404	.add_sat => try self.airAddSat(inst),
405	.sub, .ptr_sub => try self.airSub(inst),
406	.subwrap => try self.airSubWrap(inst),
407	.sub_sat => try self.airSubSat(inst),
408	.mul => try self.airMul(inst),
409	.mulwrap => try self.airMulWrap(inst),
410	.mul_sat => try self.airMulSat(inst),
411	.rem => try self.airRem(inst),
412	.mod => try self.airMod(inst),
413	.shl, .shl_exact => try self.airShl(inst),
414	.shl_sat => try self.airShlSat(inst),
415	.min => try self.airMin(inst),
416	.max => try self.airMax(inst),
417	.slice => try self.airSlice(inst),
418
419	.div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
420
421	.cmp_lt => try self.airCmp(inst, .lt),
422	.cmp_lte => try self.airCmp(inst, .lte),
423	.cmp_eq => try self.airCmp(inst, .eq),
424	.cmp_gte => try self.airCmp(inst, .gte),
425	.cmp_gt => try self.airCmp(inst, .gt),
426	.cmp_neq => try self.airCmp(inst, .neq),
427
428	.bool_and => try self.airBoolOp(inst),
429	.bool_or => try self.airBoolOp(inst),
430	.bit_and => try self.airBitAnd(inst),
431	.bit_or => try self.airBitOr(inst),
432	.xor => try self.airXor(inst),
433	.shr => try self.airShr(inst),
434
435	.alloc => try self.airAlloc(inst),
436	.ret_ptr => try self.airRetPtr(inst),
437	.arg => try self.airArg(inst),
438	.assembly => try self.airAsm(inst),
439	.bitcast => try self.airBitCast(inst),
440	.block => try self.airBlock(inst),
441	.br => try self.airBr(inst),
442	.breakpoint => try self.airBreakpoint(),
443	.fence => try self.airFence(),
444	.call => try self.airCall(inst),
445	.cond_br => try self.airCondBr(inst),
446	.dbg_stmt => try self.airDbgStmt(inst),
447	.fptrunc => try self.airFptrunc(inst),
448	.fpext => try self.airFpext(inst),
449	.intcast => try self.airIntCast(inst),
450	.trunc => try self.airTrunc(inst),
451	.bool_to_int => try self.airBoolToInt(inst),
452	.is_non_null => try self.airIsNonNull(inst),
453	.is_non_null_ptr => try self.airIsNonNullPtr(inst),
454	.is_null => try self.airIsNull(inst),
455	.is_null_ptr => try self.airIsNullPtr(inst),
456	.is_non_err => try self.airIsNonErr(inst),
457	.is_non_err_ptr => try self.airIsNonErrPtr(inst),
458	.is_err => try self.airIsErr(inst),
459	.is_err_ptr => try self.airIsErrPtr(inst),
460	.load => try self.airLoad(inst),
461	.loop => try self.airLoop(inst),
462	.not => try self.airNot(inst),
463	.ptrtoint => try self.airPtrToInt(inst),
464	.ret => try self.airRet(inst),
465	.ret_load => try self.airRetLoad(inst),
466	.store => try self.airStore(inst),
467	.struct_field_ptr=> try self.airStructFieldPtr(inst),
468	.struct_field_val=> try self.airStructFieldVal(inst),
469	.array_to_slice => try self.airArrayToSlice(inst),
470	.int_to_float => try self.airIntToFloat(inst),
471	.float_to_int => try self.airFloatToInt(inst),
472	.cmpxchg_strong => try self.airCmpxchg(inst),
473	.cmpxchg_weak => try self.airCmpxchg(inst),
474	.atomic_rmw => try self.airAtomicRmw(inst),
475	.atomic_load => try self.airAtomicLoad(inst),
476	.memcpy => try self.airMemcpy(inst),
477	.memset => try self.airMemset(inst),
478	.set_union_tag => try self.airSetUnionTag(inst),
479	.get_union_tag => try self.airGetUnionTag(inst),
480	.clz => try self.airClz(inst),
481	.ctz => try self.airCtz(inst),
482	.popcount => try self.airPopcount(inst),
483
484	.atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
485	.atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
486	.atomic_store_release => try self.airAtomicStore(inst, .Release),
487	.atomic_store_seq_cst => try self.airAtomicStore(inst, .SeqCst),
488
489	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
490	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
491	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
492	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
493
494	.switch_br => try self.airSwitch(inst),
495	.slice_ptr => try self.airSlicePtr(inst),
496	.slice_len => try self.airSliceLen(inst),
497
498	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
499	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
500
501	.array_elem_val => try self.airArrayElemVal(inst),
502	.slice_elem_val => try self.airSliceElemVal(inst),
503	.slice_elem_ptr => try self.airSliceElemPtr(inst),
504	.ptr_elem_val => try self.airPtrElemVal(inst),
505	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
506
507	.constant => unreachable, // excluded from function bodies
508	.const_ty => unreachable, // excluded from function bodies
509	.unreach => self.finishAirBookkeeping(),
510
511	.optional_payload => try self.airOptionalPayload(inst),
512	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
513	.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
514	.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
515	.unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
516	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
517
518	.wrap_optional => try self.airWrapOptional(inst),
519	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
520	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
521	// zig fmt: on
522	}
523	if (std.debug.runtime_safety) {
524	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
525	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] });
526	}
527	}
528	}
529	}
530
531	fn writeInt(self: Self, comptime T: type, buf: [@divExact(@typeInfo(T).Int.bits, 8)]u8, value: T) void {
532	const endian = self.target.cpu.arch.endian();
533	std.mem.writeInt(T, buf, value, endian);
534	}
535
536	fn dbgSetPrologueEnd(self: *Self) InnerError!void {
537	switch (self.debug_output) {
538	.dwarf => \|dbg_out\| {
539	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
540	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
541	},
542	.plan9 => {},
543	.none => {},
544	}
545	}
546
547	fn dbgSetEpilogueBegin(self: *Self) InnerError!void {
548	switch (self.debug_output) {
549	.dwarf => \|dbg_out\| {
550	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
551	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
552	},
553	.plan9 => {},
554	.none => {},
555	}
556	}
557
558	fn dbgAdvancePCAndLine(self: *Self, line: u32, column: u32) InnerError!void {
559	const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
560	const delta_pc: usize = self.code.items.len - self.prev_di_pc;
561	switch (self.debug_output) {
562	.dwarf => \|dbg_out\| {
563	// TODO Look into using the DWARF special opcodes to compress this data.
564	// It lets you emit single-byte opcodes that add different numbers to
565	// both the PC and the line number at the same time.
566	try dbg_out.dbg_line.ensureUnusedCapacity(11);
567	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
568	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
569	if (delta_line != 0) {
570	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
571	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
572	}
573	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
574	self.prev_di_pc = self.code.items.len;
575	self.prev_di_line = line;
576	self.prev_di_column = column;
577	self.prev_di_pc = self.code.items.len;
578	},
579	.plan9 => \|dbg_out\| {
580	if (delta_pc <= 0) return; // only do this when the pc changes
581	// we have already checked the target in the linker to make sure it is compatable
582	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;
583
584	// increasing the line number
585	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
586	// increasing the pc
587	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
588	if (d_pc_p9 > 0) {
589	// minus one because if its the last one, we want to leave space to change the line which is one quanta
590	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
591	if (dbg_out.pcop_change_index.*) \|pci\|
592	dbg_out.dbg_line.items[pci] += 1;
593	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
594	} else if (d_pc_p9 == 0) {
595	// we don't need to do anything, because adding the quant does it for us
596	} else unreachable;
597	if (dbg_out.start_line.* == null)
598	dbg_out.start_line.* = self.prev_di_line;
599	dbg_out.end_line.* = line;
600	// only do this if the pc changed
601	self.prev_di_line = line;
602	self.prev_di_column = column;
603	self.prev_di_pc = self.code.items.len;
604	},
605	.none => {},
606	}
607	}
608
609	/// Asserts there is already capacity to insert into top branch inst_table.
610	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
611	const air_tags = self.air.instructions.items(.tag);
612	if (air_tags[inst] == .constant) return; // Constants are immortal.
613	// When editing this function, note that the logic must synchronize with `reuseOperand`.
614	const prev_value = self.getResolvedInstValue(inst);
615	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
616	branch.inst_table.putAssumeCapacity(inst, .dead);
617	switch (prev_value) {
618	.register => \|reg\| {
619	self.register_manager.freeReg(reg);
620	},
621	else => {}, // TODO process stack allocation death
622	}
623	}
624
625	/// Called when there are no operands, and the instruction is always unreferenced.
626	fn finishAirBookkeeping(self: *Self) void {
627	if (std.debug.runtime_safety) {
628	self.air_bookkeeping += 1;
629	}
630	}
631
632	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
633	var tomb_bits = self.liveness.getTombBits(inst);
634	for (operands) \|op\| {
635	const dies = @truncate(u1, tomb_bits) != 0;
636	tomb_bits >>= 1;
637	if (!dies) continue;
638	const op_int = @enumToInt(op);
639	if (op_int < Air.Inst.Ref.typed_value_map.len) continue;
640	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
641	self.processDeath(op_index);
642	}
643	const is_used = @truncate(u1, tomb_bits) == 0;
644	if (is_used) {
645	log.debug("%{d} => {}", .{ inst, result });
646	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
647	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
648
649	switch (result) {
650	.register => \|reg\| {
651	// In some cases (such as bitcast), an operand
652	// may be the same MCValue as the result. If
653	// that operand died and was a register, it
654	// was freed by processDeath. We have to
655	// "re-allocate" the register.
656	if (self.register_manager.isRegFree(reg)) {
657	self.register_manager.getRegAssumeFree(reg, inst);
658	}
659	},
660	else => {},
661	}
662	}
663	self.finishAirBookkeeping();
664	}
665
666	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
667	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
668	try table.ensureUnusedCapacity(self.gpa, additional_count);
669	}
670
671	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
672	/// after codegen for this symbol is done.
673	fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void {
674	switch (self.debug_output) {
675	.dwarf => \|dbg_out\| {
676	assert(ty.hasCodeGenBits());
677	const index = dbg_out.dbg_info.items.len;
678	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
679
680	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(self.gpa, ty);
681	if (!gop.found_existing) {
682	gop.value_ptr.* = .{
683	.off = undefined,
684	.relocs = .{},
685	};
686	}
687	try gop.value_ptr.relocs.append(self.gpa, @intCast(u32, index));
688	},
689	.plan9 => {},
690	.none => {},
691	}
692	}
693
694	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
695	if (abi_align > self.stack_align)
696	self.stack_align = abi_align;
697	// TODO find a free slot instead of always appending
698	const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align);
699	self.next_stack_offset = offset + abi_size;
700	if (self.next_stack_offset > self.max_end_stack)
701	self.max_end_stack = self.next_stack_offset;
702	try self.stack.putNoClobber(self.gpa, offset, .{
703	.inst = inst,
704	.size = abi_size,
705	});
706	return offset;
707	}
708
709	/// Use a pointer instruction as the basis for allocating stack memory.
710	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
711	const elem_ty = self.air.typeOfIndex(inst).elemType();
712	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
713	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
714	};
715	// TODO swap this for inst.ty.ptrAlign
716	const abi_align = elem_ty.abiAlignment(self.target.*);
717	return self.allocMem(inst, abi_size, abi_align);
718	}
719
720	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
721	const elem_ty = self.air.typeOfIndex(inst);
722	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
723	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
724	};
725	const abi_align = elem_ty.abiAlignment(self.target.*);
726	if (abi_align > self.stack_align)
727	self.stack_align = abi_align;
728
729	if (reg_ok) {
730	// Make sure the type can fit in a register before we try to allocate one.
731	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
732	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
733	if (abi_size <= ptr_bytes) {
734	if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {
735	return MCValue{ .register = reg };
736	}
737	}
738	}
739	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
740	return MCValue{ .stack_offset = stack_offset };
741	}
742
743	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
744	const stack_mcv = try self.allocRegOrMem(inst, false);
745	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
746	const reg_mcv = self.getResolvedInstValue(inst);
747	assert(reg == reg_mcv.register);
748	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
749	try branch.inst_table.put(self.gpa, inst, stack_mcv);
750	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
751	}
752
753	/// Copies a value to a register without tracking the register. The register is not considered
754	/// allocated. A second call to `copyToTmpRegister` may return the same register.
755	/// This can have a side effect of spilling instructions to the stack to free up a register.
756	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
757	const reg = try self.register_manager.allocReg(null, &.{});
758	try self.genSetReg(ty, reg, mcv);
759	return reg;
760	}
761
762	/// Allocates a new register and copies `mcv` into it.
763	/// `reg_owner` is the instruction that gets associated with the register in the register table.
764	/// This can have a side effect of spilling instructions to the stack to free up a register.
765	fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
766	const reg = try self.register_manager.allocReg(reg_owner, &.{});
767	try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
768	return MCValue{ .register = reg };
769	}
770
771	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
772	const stack_offset = try self.allocMemPtr(inst);
773	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
774	}
775
776	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
777	const stack_offset = try self.allocMemPtr(inst);
778	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
779	}
780
781	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
782	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
783	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
784	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
785	}
786
787	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
788	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
789	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
790	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
791	}
792
793	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
794	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
795	if (self.liveness.isUnused(inst))
796	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
797
798	const operand_ty = self.air.typeOf(ty_op.operand);
799	const operand = try self.resolveInst(ty_op.operand);
800	const info_a = operand_ty.intInfo(self.target.*);
801	const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);
802	if (info_a.signedness != info_b.signedness)
803	return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
804
805	if (info_a.bits == info_b.bits)
806	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
807
808	return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch});
809	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
810	}
811
812	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
813	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
814	if (self.liveness.isUnused(inst))
815	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
816
817	const operand = try self.resolveInst(ty_op.operand);
818	_ = operand;
819
820	return self.fail("TODO implement trunc for {}", .{self.target.cpu.arch});
821	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
822	}
823
824	fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
825	const un_op = self.air.instructions.items(.data)[inst].un_op;
826	const operand = try self.resolveInst(un_op);
827	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;
828	return self.finishAir(inst, result, .{ un_op, .none, .none });
829	}
830
831	fn airNot(self: *Self, inst: Air.Inst.Index) !void {
832	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
833	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
834	const operand = try self.resolveInst(ty_op.operand);
835	switch (operand) {
836	.dead => unreachable,
837	.unreach => unreachable,
838	.compare_flags_unsigned => \|op\| {
839	const r = MCValue{
840	.compare_flags_unsigned = switch (op) {
841	.gte => .lt,
842	.gt => .lte,
843	.neq => .eq,
844	.lt => .gte,
845	.lte => .gt,
846	.eq => .neq,
847	},
848	};
849	break :result r;
850	},
851	.compare_flags_signed => \|op\| {
852	const r = MCValue{
853	.compare_flags_signed = switch (op) {
854	.gte => .lt,
855	.gt => .lte,
856	.neq => .eq,
857	.lt => .gte,
858	.lte => .gt,
859	.eq => .neq,
860	},
861	};
862	break :result r;
863	},
864	else => {},
865	}
866
867	break :result try self.genArmBinOp(inst, ty_op.operand, .bool_true, .not);
868	};
869	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
870	}
871
872	fn airMin(self: *Self, inst: Air.Inst.Index) !void {
873	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
874	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement min for {}", .{self.target.cpu.arch});
875	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
876	}
877
878	fn airMax(self: *Self, inst: Air.Inst.Index) !void {
879	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
880	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement max for {}", .{self.target.cpu.arch});
881	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
882	}
883
884	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
885	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
886	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
887	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice for {}", .{self.target.cpu.arch});
888	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
889	}
890
891	fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
892	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
893	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .add);
894	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
895	}
896
897	fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {
898	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
899	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch});
900	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
901	}
902
903	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
904	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
905	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
906	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
907	}
908
909	fn airSub(self: *Self, inst: Air.Inst.Index) !void {
910	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
911	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .sub);
912	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
913	}
914
915	fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
916	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
917	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement subwrap for {}", .{self.target.cpu.arch});
918	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
919	}
920
921	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
922	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
923	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
924	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
925	}
926
927	fn airMul(self: *Self, inst: Air.Inst.Index) !void {
928	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
929	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmMul(inst, bin_op.lhs, bin_op.rhs);
930	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
931	}
932
933	fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {
934	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
935	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch});
936	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
937	}
938
939	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
940	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
941	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
942	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
943	}
944
945	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
946	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
947	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
948	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
949	}
950
951	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
952	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
953	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});
954	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
955	}
956
957	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
958	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
959	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mod for {}", .{self.target.cpu.arch});
960	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
961	}
962
963	fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
964	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
965	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_and);
966	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
967	}
968
969	fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
970	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
971	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_or);
972	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
973	}
974
975	fn airXor(self: *Self, inst: Air.Inst.Index) !void {
976	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
977	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .xor);
978	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
979	}
980
981	fn airShl(self: *Self, inst: Air.Inst.Index) !void {
982	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
983	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shl);
984	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
985	}
986
987	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
988	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
989	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
990	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
991	}
992
993	fn airShr(self: *Self, inst: Air.Inst.Index) !void {
994	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
995	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shr);
996	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
997	}
998
999	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1000	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1001	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});
1002	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1003	}
1004
1005	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1006	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1007	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
1008	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1009	}
1010
1011	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
1012	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1013	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error for {}", .{self.target.cpu.arch});
1014	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1015	}
1016
1017	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
1018	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1019	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload for {}", .{self.target.cpu.arch});
1020	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1021	}
1022
1023	// *(E!T) -> E
1024	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1025	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1026	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
1027	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1028	}
1029
1030	// (E!T) -> T
1031	fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1032	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1033	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
1034	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1035	}
1036
1037	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
1038	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1039	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1040	const optional_ty = self.air.typeOfIndex(inst);
1041
1042	// Optional with a zero-bit payload type is just a boolean true
1043	if (optional_ty.abiSize(self.target.*) == 1)
1044	break :result MCValue{ .immediate = 1 };
1045
1046	return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});
1047	};
1048	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1049	}
1050
1051	/// T to E!T
1052	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
1053	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1054	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});
1055	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1056	}
1057
1058	/// E to E!T
1059	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
1060	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1061	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion error for {}", .{self.target.cpu.arch});
1062	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1063	}
1064
1065	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
1066	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1067	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_ptr for {}", .{self.target.cpu.arch});
1068	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1069	}
1070
1071	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
1072	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1073	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_len for {}", .{self.target.cpu.arch});
1074	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1075	}
1076
1077	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
1078	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1079	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});
1080	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1081	}
1082
1083	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
1084	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1085	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});
1086	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1087	}
1088
1089	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1090	const is_volatile = false; // TODO
1091	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1092	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_val for {}", .{self.target.cpu.arch});
1093	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1094	}
1095
1096	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1097	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1098	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1099	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});
1100	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1101	}
1102
1103	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
1104	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1105	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch});
1106	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1107	}
1108
1109	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
1110	const is_volatile = false; // TODO
1111	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1112	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch});
1113	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1114	}
1115
1116	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1117	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1118	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1119	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});
1120	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1121	}
1122
1123	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1124	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1125	_ = bin_op;
1126	return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch});
1127	// return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1128	}
1129
1130	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1131	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1132	_ = ty_op;
1133	return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
1134	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1135	}
1136
1137	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
1138	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1139	_ = ty_op;
1140	return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
1141	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1142	}
1143
1144	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
1145	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1146	_ = ty_op;
1147	return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
1148	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1149	}
1150
1151	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
1152	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1153	_ = ty_op;
1154	return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
1155	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1156	}
1157
1158	fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {
1159	if (!self.liveness.operandDies(inst, op_index))
1160	return false;
1161
1162	switch (mcv) {
1163	.register => \|reg\| {
1164	// If it's in the registers table, need to associate the register with the
1165	// new instruction.
1166	if (reg.allocIndex()) \|index\| {
1167	if (!self.register_manager.isRegFree(reg)) {
1168	self.register_manager.registers[index] = inst;
1169	}
1170	}
1171	log.debug("%{d} => {} (reused)", .{ inst, reg });
1172	},
1173	.stack_offset => \|off\| {
1174	log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
1175	},
1176	else => return false,
1177	}
1178
1179	// Prevent the operand deaths processing code from deallocating it.
1180	self.liveness.clearOperandDeath(inst, op_index);
1181
1182	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
1183	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1184	branch.inst_table.putAssumeCapacity(Air.refToIndex(operand).?, .dead);
1185
1186	return true;
1187	}
1188
1189	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
1190	const elem_ty = ptr_ty.elemType();
1191	switch (ptr) {
1192	.none => unreachable,
1193	.undef => unreachable,
1194	.unreach => unreachable,
1195	.dead => unreachable,
1196	.compare_flags_unsigned => unreachable,
1197	.compare_flags_signed => unreachable,
1198	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
1199	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
1200	.ptr_embedded_in_code => \|off\| {
1201	try self.setRegOrMem(elem_ty, dst_mcv, .{ .embedded_in_code = off });
1202	},
1203	.embedded_in_code => {
1204	return self.fail("TODO implement loading from MCValue.embedded_in_code", .{});
1205	},
1206	.register => \|reg\| {
1207	switch (dst_mcv) {
1208	.dead => unreachable,
1209	.undef => unreachable,
1210	.compare_flags_signed, .compare_flags_unsigned => unreachable,
1211	.embedded_in_code => unreachable,
1212	.register => \|dst_reg\| {
1213	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, dst_reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
1214	},
1215	else => return self.fail("TODO load from register into {}", .{dst_mcv}),
1216	}
1217	},
1218	.memory => \|addr\| {
1219	const reg = try self.register_manager.allocReg(null, &.{});
1220	try self.genSetReg(ptr_ty, reg, .{ .memory = addr });
1221	try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
1222	},
1223	.stack_offset => {
1224	return self.fail("TODO implement loading from MCValue.stack_offset", .{});
1225	},
1226	}
1227	}
1228
1229	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
1230	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1231	const elem_ty = self.air.typeOfIndex(inst);
1232	const result: MCValue = result: {
1233	if (!elem_ty.hasCodeGenBits())
1234	break :result MCValue.none;
1235
1236	const ptr = try self.resolveInst(ty_op.operand);
1237	const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
1238	if (self.liveness.isUnused(inst) and !is_volatile)
1239	break :result MCValue.dead;
1240
1241	const dst_mcv: MCValue = blk: {
1242	if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
1243	// The MCValue that holds the pointer can be re-used as the value.
1244	break :blk ptr;
1245	} else {
1246	break :blk try self.allocRegOrMem(inst, true);
1247	}
1248	};
1249	try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
1250	break :result dst_mcv;
1251	};
1252	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1253	}
1254
1255	fn airStore(self: *Self, inst: Air.Inst.Index) !void {
1256	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1257	const ptr = try self.resolveInst(bin_op.lhs);
1258	const value = try self.resolveInst(bin_op.rhs);
1259	const elem_ty = self.air.typeOf(bin_op.rhs);
1260	switch (ptr) {
1261	.none => unreachable,
1262	.undef => unreachable,
1263	.unreach => unreachable,
1264	.dead => unreachable,
1265	.compare_flags_unsigned => unreachable,
1266	.compare_flags_signed => unreachable,
1267	.immediate => \|imm\| {
1268	try self.setRegOrMem(elem_ty, .{ .memory = imm }, value);
1269	},
1270	.ptr_stack_offset => \|off\| {
1271	try self.genSetStack(elem_ty, off, value);
1272	},
1273	.ptr_embedded_in_code => \|off\| {
1274	try self.setRegOrMem(elem_ty, .{ .embedded_in_code = off }, value);
1275	},
1276	.embedded_in_code => {
1277	return self.fail("TODO implement storing to MCValue.embedded_in_code", .{});
1278	},
1279	.register => {
1280	return self.fail("TODO implement storing to MCValue.register", .{});
1281	},
1282	.memory => {
1283	return self.fail("TODO implement storing to MCValue.memory", .{});
1284	},
1285	.stack_offset => {
1286	return self.fail("TODO implement storing to MCValue.stack_offset", .{});
1287	},
1288	}
1289	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1290	}
1291
1292	fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
1293	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1294	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1295	return self.structFieldPtr(extra.struct_operand, ty_pl.ty, extra.field_index);
1296	}
1297
1298	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
1299	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1300	return self.structFieldPtr(ty_op.operand, ty_op.ty, index);
1301	}
1302	fn structFieldPtr(self: *Self, operand: Air.Inst.Ref, ty: Air.Inst.Ref, index: u32) !void {
1303	_ = self;
1304	_ = operand;
1305	_ = ty;
1306	_ = index;
1307	return self.fail("TODO implement codegen struct_field_ptr", .{});
1308	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1309	}
1310
1311	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
1312	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1313	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1314	_ = extra;
1315	return self.fail("TODO implement codegen struct_field_val", .{});
1316	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1317	}
1318
1319	fn armOperandShouldBeRegister(self: *Self, mcv: MCValue) !bool {
1320	return switch (mcv) {
1321	.none => unreachable,
1322	.undef => unreachable,
1323	.dead, .unreach => unreachable,
1324	.compare_flags_unsigned => unreachable,
1325	.compare_flags_signed => unreachable,
1326	.ptr_stack_offset => unreachable,
1327	.ptr_embedded_in_code => unreachable,
1328	.immediate => \|imm\| blk: {
1329	if (imm > std.math.maxInt(u32)) return self.fail("TODO ARM binary arithmetic immediate larger than u32", .{});
1330
1331	// Load immediate into register if it doesn't fit
1332	// in an operand
1333	break :blk Instruction.Operand.fromU32(@intCast(u32, imm)) == null;
1334	},
1335	.register => true,
1336	.stack_offset,
1337	.embedded_in_code,
1338	.memory,
1339	=> true,
1340	};
1341	}
1342
1343	fn genArmBinOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref, op: Air.Inst.Tag) !MCValue {
1344	// In the case of bitshifts, the type of rhs is different
1345	// from the resulting type
1346	const ty = self.air.typeOf(op_lhs);
1347
1348	switch (ty.zigTypeTag()) {
1349	.Float => return self.fail("TODO ARM binary operations on floats", .{}),
1350	.Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
1351	.Bool => {
1352	return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, 1, .unsigned);
1353	},
1354	.Int => {
1355	const int_info = ty.intInfo(self.target.*);
1356	return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, int_info.bits, int_info.signedness);
1357	},
1358	else => unreachable,
1359	}
1360	}
1361
1362	fn genArmBinIntOp(
1363	self: *Self,
1364	inst: Air.Inst.Index,
1365	op_lhs: Air.Inst.Ref,
1366	op_rhs: Air.Inst.Ref,
1367	op: Air.Inst.Tag,
1368	bits: u16,
1369	signedness: std.builtin.Signedness,
1370	) !MCValue {
1371	if (bits > 32) {
1372	return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
1373	}
1374
1375	const lhs = try self.resolveInst(op_lhs);
1376	const rhs = try self.resolveInst(op_rhs);
1377
1378	const lhs_is_register = lhs == .register;
1379	const rhs_is_register = rhs == .register;
1380	const lhs_should_be_register = switch (op) {
1381	.shr, .shl => true,
1382	else => try self.armOperandShouldBeRegister(lhs),
1383	};
1384	const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
1385	const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
1386	const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
1387	const can_swap_lhs_and_rhs = switch (op) {
1388	.shr, .shl => false,
1389	else => true,
1390	};
1391
1392	// Destination must be a register
1393	var dst_mcv: MCValue = undefined;
1394	var lhs_mcv = lhs;
1395	var rhs_mcv = rhs;
1396	var swap_lhs_and_rhs = false;
1397
1398	// Allocate registers for operands and/or destination
1399	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1400	if (reuse_lhs) {
1401	// Allocate 0 or 1 registers
1402	if (!rhs_is_register and rhs_should_be_register) {
1403	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
1404	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1405	}
1406	dst_mcv = lhs;
1407	} else if (reuse_rhs and can_swap_lhs_and_rhs) {
1408	// Allocate 0 or 1 registers
1409	if (!lhs_is_register and lhs_should_be_register) {
1410	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
1411	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
1412	}
1413	dst_mcv = rhs;
1414
1415	swap_lhs_and_rhs = true;
1416	} else {
1417	// Allocate 1 or 2 registers
1418	if (lhs_should_be_register and rhs_should_be_register) {
1419	if (lhs_is_register and rhs_is_register) {
1420	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
1421	} else if (lhs_is_register) {
1422	// Move RHS to register
1423	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1424	rhs_mcv = dst_mcv;
1425	} else if (rhs_is_register) {
1426	// Move LHS to register
1427	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1428	lhs_mcv = dst_mcv;
1429	} else {
1430	// Move LHS and RHS to register
1431	const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
1432	lhs_mcv = MCValue{ .register = regs[0] };
1433	rhs_mcv = MCValue{ .register = regs[1] };
1434	dst_mcv = lhs_mcv;
1435
1436	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1437	}
1438	} else if (lhs_should_be_register) {
1439	// RHS is immediate
1440	if (lhs_is_register) {
1441	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1442	} else {
1443	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
1444	lhs_mcv = dst_mcv;
1445	}
1446	} else if (rhs_should_be_register and can_swap_lhs_and_rhs) {
1447	// LHS is immediate
1448	if (rhs_is_register) {
1449	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1450	} else {
1451	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
1452	rhs_mcv = dst_mcv;
1453	}
1454
1455	swap_lhs_and_rhs = true;
1456	} else unreachable; // binary operation on two immediates
1457	}
1458
1459	// Move the operands to the newly allocated registers
1460	if (lhs_mcv == .register and !lhs_is_register) {
1461	try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
1462	}
1463	if (rhs_mcv == .register and !rhs_is_register) {
1464	try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
1465	}
1466
1467	try self.genArmBinOpCode(
1468	dst_mcv.register,
1469	lhs_mcv,
1470	rhs_mcv,
1471	swap_lhs_and_rhs,
1472	op,
1473	signedness,
1474	);
1475	return dst_mcv;
1476	}
1477
1478	fn genArmBinOpCode(
1479	self: *Self,
1480	dst_reg: Register,
1481	lhs_mcv: MCValue,
1482	rhs_mcv: MCValue,
1483	swap_lhs_and_rhs: bool,
1484	op: Air.Inst.Tag,
1485	signedness: std.builtin.Signedness,
1486	) !void {
1487	assert(lhs_mcv == .register or rhs_mcv == .register);
1488
1489	const op1 = if (swap_lhs_and_rhs) rhs_mcv.register else lhs_mcv.register;
1490	const op2 = if (swap_lhs_and_rhs) lhs_mcv else rhs_mcv;
1491
1492	const operand = switch (op2) {
1493	.none => unreachable,
1494	.undef => unreachable,
1495	.dead, .unreach => unreachable,
1496	.compare_flags_unsigned => unreachable,
1497	.compare_flags_signed => unreachable,
1498	.ptr_stack_offset => unreachable,
1499	.ptr_embedded_in_code => unreachable,
1500	.immediate => \|imm\| Instruction.Operand.fromU32(@intCast(u32, imm)).?,
1501	.register => \|reg\| Instruction.Operand.reg(reg, Instruction.Operand.Shift.none),
1502	.stack_offset,
1503	.embedded_in_code,
1504	.memory,
1505	=> unreachable,
1506	};
1507
1508	switch (op) {
1509	.add => {
1510	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.al, dst_reg, op1, operand).toU32());
1511	},
1512	.sub => {
1513	if (swap_lhs_and_rhs) {
1514	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.rsb(.al, dst_reg, op1, operand).toU32());
1515	} else {
1516	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.sub(.al, dst_reg, op1, operand).toU32());
1517	}
1518	},
1519	.bool_and, .bit_and => {
1520	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.@"and"(.al, dst_reg, op1, operand).toU32());
1521	},
1522	.bool_or, .bit_or => {
1523	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, dst_reg, op1, operand).toU32());
1524	},
1525	.not, .xor => {
1526	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.eor(.al, dst_reg, op1, operand).toU32());
1527	},
1528	.cmp_eq => {
1529	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, op1, operand).toU32());
1530	},
1531	.shl => {
1532	assert(!swap_lhs_and_rhs);
1533	const shift_amount = switch (operand) {
1534	.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
1535	.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
1536	};
1537	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.lsl(.al, dst_reg, op1, shift_amount).toU32());
1538	},
1539	.shr => {
1540	assert(!swap_lhs_and_rhs);
1541	const shift_amount = switch (operand) {
1542	.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
1543	.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
1544	};
1545
1546	const shr = switch (signedness) {
1547	.signed => Instruction.asr,
1548	.unsigned => Instruction.lsr,
1549	};
1550	self.writeInt(u32, try self.code.addManyAsArray(4), shr(.al, dst_reg, op1, shift_amount).toU32());
1551	},
1552	else => unreachable, // not a binary instruction
1553	}
1554	}
1555
1556	fn genArmMul(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
1557	const lhs = try self.resolveInst(op_lhs);
1558	const rhs = try self.resolveInst(op_rhs);
1559
1560	const lhs_is_register = lhs == .register;
1561	const rhs_is_register = rhs == .register;
1562	const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
1563	const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
1564
1565	// Destination must be a register
1566	// LHS must be a register
1567	// RHS must be a register
1568	var dst_mcv: MCValue = undefined;
1569	var lhs_mcv: MCValue = lhs;
1570	var rhs_mcv: MCValue = rhs;
1571
1572	// Allocate registers for operands and/or destination
1573	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1574	if (reuse_lhs) {
1575	// Allocate 0 or 1 registers
1576	if (!rhs_is_register) {
1577	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
1578	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1579	}
1580	dst_mcv = lhs;
1581	} else if (reuse_rhs) {
1582	// Allocate 0 or 1 registers
1583	if (!lhs_is_register) {
1584	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
1585	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
1586	}
1587	dst_mcv = rhs;
1588	} else {
1589	// Allocate 1 or 2 registers
1590	if (lhs_is_register and rhs_is_register) {
1591	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
1592	} else if (lhs_is_register) {
1593	// Move RHS to register
1594	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1595	rhs_mcv = dst_mcv;
1596	} else if (rhs_is_register) {
1597	// Move LHS to register
1598	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1599	lhs_mcv = dst_mcv;
1600	} else {
1601	// Move LHS and RHS to register
1602	const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
1603	lhs_mcv = MCValue{ .register = regs[0] };
1604	rhs_mcv = MCValue{ .register = regs[1] };
1605	dst_mcv = lhs_mcv;
1606
1607	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1608	}
1609	}
1610
1611	// Move the operands to the newly allocated registers
1612	if (!lhs_is_register) {
1613	try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
1614	}
1615	if (!rhs_is_register) {
1616	try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
1617	}
1618
1619	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mul(.al, dst_mcv.register, lhs_mcv.register, rhs_mcv.register).toU32());
1620	return dst_mcv;
1621	}
1622
1623	fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
1624	const ty_str = self.air.instructions.items(.data)[inst].ty_str;
1625	const zir = &self.mod_fn.owner_decl.getFileScope().zir;
1626	const name = zir.nullTerminatedString(ty_str.str);
1627	const name_with_null = name.ptr[0 .. name.len + 1];
1628	const ty = self.air.getRefType(ty_str.ty);
1629
1630	switch (mcv) {
1631	.register => \|reg\| {
1632	switch (self.debug_output) {
1633	.dwarf => \|dbg_out\| {
1634	try dbg_out.dbg_info.ensureUnusedCapacity(3);
1635	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
1636	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
1637	1, // ULEB128 dwarf expression length
1638	reg.dwarfLocOp(),
1639	});
1640	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
1641	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
1642	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
1643	},
1644	.plan9 => {},
1645	.none => {},
1646	}
1647	},
1648	.stack_offset => \|offset\| {
1649	switch (self.debug_output) {
1650	.dwarf => \|dbg_out\| {
1651	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
1652	return self.fail("type '{}' too big to fit into stack frame", .{ty});
1653	};
1654	const adjusted_stack_offset = math.negateCast(offset + abi_size) catch {
1655	return self.fail("Stack offset too large for arguments", .{});
1656	};
1657
1658	try dbg_out.dbg_info.append(link.File.Elf.abbrev_parameter);
1659
1660	// Get length of the LEB128 stack offset
1661	var counting_writer = std.io.countingWriter(std.io.null_writer);
1662	leb128.writeILEB128(counting_writer.writer(), adjusted_stack_offset) catch unreachable;
1663
1664	// DW.AT.location, DW.FORM.exprloc
1665	// ULEB128 dwarf expression length
1666	try leb128.writeULEB128(dbg_out.dbg_info.writer(), counting_writer.bytes_written + 1);
1667	try dbg_out.dbg_info.append(DW.OP.breg11);
1668	try leb128.writeILEB128(dbg_out.dbg_info.writer(), adjusted_stack_offset);
1669
1670	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
1671	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
1672	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
1673	},
1674	.plan9 => {},
1675	.none => {},
1676	}
1677	},
1678	else => {},
1679	}
1680	}
1681
1682	fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1683	const arg_index = self.arg_index;
1684	self.arg_index += 1;
1685
1686	const ty = self.air.typeOfIndex(inst);
1687
1688	const result = self.args[arg_index];
1689	const mcv = switch (result) {
1690	// Copy registers to the stack
1691	.register => \|reg\| blk: {
1692	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
1693	return self.fail("type '{}' too big to fit into stack frame", .{ty});
1694	};
1695	const abi_align = ty.abiAlignment(self.target.*);
1696	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
1697	try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
1698
1699	break :blk MCValue{ .stack_offset = stack_offset };
1700	},
1701	else => result,
1702	};
1703	try self.genArgDbgInfo(inst, mcv);
1704
1705	if (self.liveness.isUnused(inst))
1706	return self.finishAirBookkeeping();
1707
1708	switch (mcv) {
1709	.register => \|reg\| {
1710	self.register_manager.getRegAssumeFree(reg, inst);
1711	},
1712	else => {},
1713	}
1714
1715	return self.finishAir(inst, mcv, .{ .none, .none, .none });
1716	}
1717
1718	fn airBreakpoint(self: *Self) !void {
1719	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.bkpt(0).toU32());
1720	return self.finishAirBookkeeping();
1721	}
1722
1723	fn airFence(self: *Self) !void {
1724	return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
1725	//return self.finishAirBookkeeping();
1726	}
1727
1728	fn airCall(self: *Self, inst: Air.Inst.Index) !void {
1729	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
1730	const fn_ty = self.air.typeOf(pl_op.operand);
1731	const callee = pl_op.operand;
1732	const extra = self.air.extraData(Air.Call, pl_op.payload);
1733	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[extra.end..][0..extra.data.args_len]);
1734
1735	var info = try self.resolveCallingConventionValues(fn_ty);
1736	defer info.deinit(self);
1737
1738	// Due to incremental compilation, how function calls are generated depends
1739	// on linking.
1740	if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {
1741	for (info.args) \|mc_arg, arg_i\| {
1742	const arg = args[arg_i];
1743	const arg_ty = self.air.typeOf(arg);
1744	const arg_mcv = try self.resolveInst(args[arg_i]);
1745
1746	switch (mc_arg) {
1747	.none => continue,
1748	.undef => unreachable,
1749	.immediate => unreachable,
1750	.unreach => unreachable,
1751	.dead => unreachable,
1752	.embedded_in_code => unreachable,
1753	.memory => unreachable,
1754	.compare_flags_signed => unreachable,
1755	.compare_flags_unsigned => unreachable,
1756	.register => \|reg\| {
1757	try self.register_manager.getReg(reg, null);
1758	try self.genSetReg(arg_ty, reg, arg_mcv);
1759	},
1760	.stack_offset => {
1761	return self.fail("TODO implement calling with parameters in memory", .{});
1762	},
1763	.ptr_stack_offset => {
1764	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
1765	},
1766	.ptr_embedded_in_code => {
1767	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
1768	},
1769	}
1770	}
1771
1772	if (self.air.value(callee)) \|func_value\| {
1773	if (func_value.castTag(.function)) \|func_payload\| {
1774	const func = func_payload.data;
1775	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1776	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1777	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
1778	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
1779	break :blk @intCast(u32, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * ptr_bytes);
1780	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\|
1781	coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes
1782	else
1783	unreachable;
1784
1785	try self.genSetReg(Type.initTag(.usize), .lr, .{ .memory = got_addr });
1786
1787	// TODO: add Instruction.supportedOn
1788	// function for ARM
1789	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v5t)) {
1790	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.blx(.al, .lr).toU32());
1791	} else {
1792	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .lr, Instruction.Operand.reg(.pc, Instruction.Operand.Shift.none)).toU32());
1793	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.bx(.al, .lr).toU32());
1794	}
1795	} else if (func_value.castTag(.extern_fn)) \|_\| {
1796	return self.fail("TODO implement calling extern functions", .{});
1797	} else {
1798	return self.fail("TODO implement calling bitcasted functions", .{});
1799	}
1800	} else {
1801	return self.fail("TODO implement calling runtime known function pointer", .{});
1802	}
1803	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
1804	unreachable; // unsupported architecture for MachO
1805	} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {
1806	return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});
1807	} else unreachable;
1808
1809	const result: MCValue = result: {
1810	switch (info.return_value) {
1811	.register => \|reg\| {
1812	if (Register.allocIndex(reg) == null) {
1813	// Save function return value in a callee saved register
1814	break :result try self.copyToNewRegister(inst, info.return_value);
1815	}
1816	},
1817	else => {},
1818	}
1819	break :result info.return_value;
1820	};
1821
1822	if (args.len <= Liveness.bpi - 2) {
1823	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
1824	buf[0] = callee;
1825	std.mem.copy(Air.Inst.Ref, buf[1..], args);
1826	return self.finishAir(inst, result, buf);
1827	}
1828	var bt = try self.iterateBigTomb(inst, 1 + args.len);
1829	bt.feed(callee);
1830	for (args) \|arg\| {
1831	bt.feed(arg);
1832	}
1833	return bt.finishAir(result);
1834	}
1835
1836	fn ret(self: *Self, mcv: MCValue) !void {
1837	const ret_ty = self.fn_type.fnReturnType();
1838	try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
1839
1840	// Just add space for an instruction, patch this later
1841	try self.code.resize(self.code.items.len + 4);
1842	try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
1843	}
1844
1845	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
1846	const un_op = self.air.instructions.items(.data)[inst].un_op;
1847	const operand = try self.resolveInst(un_op);
1848	try self.ret(operand);
1849	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
1850	}
1851
1852	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
1853	const un_op = self.air.instructions.items(.data)[inst].un_op;
1854	const ptr = try self.resolveInst(un_op);
1855	_ = ptr;
1856	return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch});
1857	//return self.finishAir(inst, .dead, .{ un_op, .none, .none });
1858	}
1859
1860	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1861	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1862	if (self.liveness.isUnused(inst))
1863	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1864	const ty = self.air.typeOf(bin_op.lhs);
1865	assert(ty.eql(self.air.typeOf(bin_op.rhs)));
1866	if (ty.zigTypeTag() == .ErrorSet)
1867	return self.fail("TODO implement cmp for errors", .{});
1868
1869	const lhs = try self.resolveInst(bin_op.lhs);
1870	const rhs = try self.resolveInst(bin_op.rhs);
1871	const result: MCValue = result: {
1872	const lhs_is_register = lhs == .register;
1873	const rhs_is_register = rhs == .register;
1874	// lhs should always be a register
1875	const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
1876
1877	var lhs_mcv = lhs;
1878	var rhs_mcv = rhs;
1879
1880	// Allocate registers
1881	if (rhs_should_be_register) {
1882	if (!lhs_is_register and !rhs_is_register) {
1883	const regs = try self.register_manager.allocRegs(2, .{
1884	Air.refToIndex(bin_op.rhs).?, Air.refToIndex(bin_op.lhs).?,
1885	}, &.{});
1886	lhs_mcv = MCValue{ .register = regs[0] };
1887	rhs_mcv = MCValue{ .register = regs[1] };
1888	} else if (!rhs_is_register) {
1889	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.rhs).?, &.{}) };
1890	}
1891	}
1892	if (!lhs_is_register) {
1893	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.lhs).?, &.{}) };
1894	}
1895
1896	// Move the operands to the newly allocated registers
1897	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1898	if (lhs_mcv == .register and !lhs_is_register) {
1899	try self.genSetReg(ty, lhs_mcv.register, lhs);
1900	branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.lhs).?, lhs);
1901	}
1902	if (rhs_mcv == .register and !rhs_is_register) {
1903	try self.genSetReg(ty, rhs_mcv.register, rhs);
1904	branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.rhs).?, rhs);
1905	}
1906
1907	// The destination register is not present in the cmp instruction
1908	// The signedness of the integer does not matter for the cmp instruction
1909	try self.genArmBinOpCode(undefined, lhs_mcv, rhs_mcv, false, .cmp_eq, undefined);
1910
1911	break :result switch (ty.isSignedInt()) {
1912	true => MCValue{ .compare_flags_signed = op },
1913	false => MCValue{ .compare_flags_unsigned = op },
1914	};
1915	};
1916	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1917	}
1918
1919	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
1920	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
1921	try self.dbgAdvancePCAndLine(dbg_stmt.line, dbg_stmt.column);
1922	return self.finishAirBookkeeping();
1923	}
1924
1925	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1926	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
1927	const cond = try self.resolveInst(pl_op.operand);
1928	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
1929	const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
1930	const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
1931	const liveness_condbr = self.liveness.getCondBr(inst);
1932
1933	const reloc: Reloc = reloc: {
1934	const condition: Condition = switch (cond) {
1935	.compare_flags_signed => \|cmp_op\| blk: {
1936	// Here we map to the opposite condition because the jump is to the false branch.
1937	const condition = Condition.fromCompareOperatorSigned(cmp_op);
1938	break :blk condition.negate();
1939	},
1940	.compare_flags_unsigned => \|cmp_op\| blk: {
1941	// Here we map to the opposite condition because the jump is to the false branch.
1942	const condition = Condition.fromCompareOperatorUnsigned(cmp_op);
1943	break :blk condition.negate();
1944	},
1945	.register => \|reg\| blk: {
1946	// cmp reg, 1
1947	// bne ...
1948	const op = Instruction.Operand.imm(1, 0);
1949	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, reg, op).toU32());
1950	break :blk .ne;
1951	},
1952	else => return self.fail("TODO implement condbr {} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
1953	};
1954
1955	const reloc = Reloc{
1956	.arm_branch = .{
1957	.pos = self.code.items.len,
1958	.cond = condition,
1959	},
1960	};
1961	try self.code.resize(self.code.items.len + 4);
1962	break :reloc reloc;
1963	};
1964
1965	// Capture the state of register and stack allocation state so that we can revert to it.
1966	const parent_next_stack_offset = self.next_stack_offset;
1967	const parent_free_registers = self.register_manager.free_registers;
1968	var parent_stack = try self.stack.clone(self.gpa);
1969	defer parent_stack.deinit(self.gpa);
1970	const parent_registers = self.register_manager.registers;
1971
1972	try self.branch_stack.append(.{});
1973
1974	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
1975	for (liveness_condbr.then_deaths) \|operand\| {
1976	self.processDeath(operand);
1977	}
1978	try self.genBody(then_body);
1979
1980	// Revert to the previous register and stack allocation state.
1981
1982	var saved_then_branch = self.branch_stack.pop();
1983	defer saved_then_branch.deinit(self.gpa);
1984
1985	self.register_manager.registers = parent_registers;
1986
1987	self.stack.deinit(self.gpa);
1988	self.stack = parent_stack;
1989	parent_stack = .{};
1990
1991	self.next_stack_offset = parent_next_stack_offset;
1992	self.register_manager.free_registers = parent_free_registers;
1993
1994	try self.performReloc(reloc);
1995	const else_branch = self.branch_stack.addOneAssumeCapacity();
1996	else_branch.* = .{};
1997
1998	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
1999	for (liveness_condbr.else_deaths) \|operand\| {
2000	self.processDeath(operand);
2001	}
2002	try self.genBody(else_body);
2003
2004	// At this point, each branch will possibly have conflicting values for where
2005	// each instruction is stored. They agree, however, on which instructions are alive/dead.
2006	// We use the first ("then") branch as canonical, and here emit
2007	// instructions into the second ("else") branch to make it conform.
2008	// We continue respect the data structure semantic guarantees of the else_branch so
2009	// that we can use all the code emitting abstractions. This is why at the bottom we
2010	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
2011	// rather than assigning it.
2012	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
2013	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
2014
2015	const else_slice = else_branch.inst_table.entries.slice();
2016	const else_keys = else_slice.items(.key);
2017	const else_values = else_slice.items(.value);
2018	for (else_keys) \|else_key, else_idx\| {
2019	const else_value = else_values[else_idx];
2020	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
2021	// The instruction's MCValue is overridden in both branches.
2022	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
2023	if (else_value == .dead) {
2024	assert(then_entry.value == .dead);
2025	continue;
2026	}
2027	break :blk then_entry.value;
2028	} else blk: {
2029	if (else_value == .dead)
2030	continue;
2031	// The instruction is only overridden in the else branch.
2032	var i: usize = self.branch_stack.items.len - 2;
2033	while (true) {
2034	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
2035	if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
2036	assert(mcv != .dead);
2037	break :blk mcv;
2038	}
2039	}
2040	};
2041	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
2042	// TODO make sure the destination stack offset / register does not already have something
2043	// going on there.
2044	try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
2045	// TODO track the new register / stack allocation
2046	}
2047	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
2048	const then_slice = saved_then_branch.inst_table.entries.slice();
2049	const then_keys = then_slice.items(.key);
2050	const then_values = then_slice.items(.value);
2051	for (then_keys) \|then_key, then_idx\| {
2052	const then_value = then_values[then_idx];
2053	// We already deleted the items from this table that matched the else_branch.
2054	// So these are all instructions that are only overridden in the then branch.
2055	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
2056	if (then_value == .dead)
2057	continue;
2058	const parent_mcv = blk: {
2059	var i: usize = self.branch_stack.items.len - 2;
2060	while (true) {
2061	i -= 1;
2062	if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
2063	assert(mcv != .dead);
2064	break :blk mcv;
2065	}
2066	}
2067	};
2068	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
2069	// TODO make sure the destination stack offset / register does not already have something
2070	// going on there.
2071	try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
2072	// TODO track the new register / stack allocation
2073	}
2074
2075	self.branch_stack.pop().deinit(self.gpa);
2076
2077	return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
2078	}
2079
2080	fn isNull(self: *Self, operand: MCValue) !MCValue {
2081	_ = operand;
2082	// Here you can specialize this instruction if it makes sense to, otherwise the default
2083	// will call isNonNull and invert the result.
2084	return self.fail("TODO call isNonNull and invert the result", .{});
2085	}
2086
2087	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
2088	_ = operand;
2089	// Here you can specialize this instruction if it makes sense to, otherwise the default
2090	// will call isNull and invert the result.
2091	return self.fail("TODO call isNull and invert the result", .{});
2092	}
2093
2094	fn isErr(self: *Self, operand: MCValue) !MCValue {
2095	_ = operand;
2096	// Here you can specialize this instruction if it makes sense to, otherwise the default
2097	// will call isNonNull and invert the result.
2098	return self.fail("TODO call isNonErr and invert the result", .{});
2099	}
2100
2101	fn isNonErr(self: *Self, operand: MCValue) !MCValue {
2102	_ = operand;
2103	// Here you can specialize this instruction if it makes sense to, otherwise the default
2104	// will call isNull and invert the result.
2105	return self.fail("TODO call isErr and invert the result", .{});
2106	}
2107
2108	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
2109	const un_op = self.air.instructions.items(.data)[inst].un_op;
2110	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2111	const operand = try self.resolveInst(un_op);
2112	break :result try self.isNull(operand);
2113	};
2114	return self.finishAir(inst, result, .{ un_op, .none, .none });
2115	}
2116
2117	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2118	const un_op = self.air.instructions.items(.data)[inst].un_op;
2119	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2120	const operand_ptr = try self.resolveInst(un_op);
2121	const operand: MCValue = blk: {
2122	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2123	// The MCValue that holds the pointer can be re-used as the value.
2124	break :blk operand_ptr;
2125	} else {
2126	break :blk try self.allocRegOrMem(inst, true);
2127	}
2128	};
2129	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2130	break :result try self.isNull(operand);
2131	};
2132	return self.finishAir(inst, result, .{ un_op, .none, .none });
2133	}
2134
2135	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
2136	const un_op = self.air.instructions.items(.data)[inst].un_op;
2137	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2138	const operand = try self.resolveInst(un_op);
2139	break :result try self.isNonNull(operand);
2140	};
2141	return self.finishAir(inst, result, .{ un_op, .none, .none });
2142	}
2143
2144	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2145	const un_op = self.air.instructions.items(.data)[inst].un_op;
2146	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2147	const operand_ptr = try self.resolveInst(un_op);
2148	const operand: MCValue = blk: {
2149	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2150	// The MCValue that holds the pointer can be re-used as the value.
2151	break :blk operand_ptr;
2152	} else {
2153	break :blk try self.allocRegOrMem(inst, true);
2154	}
2155	};
2156	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2157	break :result try self.isNonNull(operand);
2158	};
2159	return self.finishAir(inst, result, .{ un_op, .none, .none });
2160	}
2161
2162	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
2163	const un_op = self.air.instructions.items(.data)[inst].un_op;
2164	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2165	const operand = try self.resolveInst(un_op);
2166	break :result try self.isErr(operand);
2167	};
2168	return self.finishAir(inst, result, .{ un_op, .none, .none });
2169	}
2170
2171	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2172	const un_op = self.air.instructions.items(.data)[inst].un_op;
2173	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2174	const operand_ptr = try self.resolveInst(un_op);
2175	const operand: MCValue = blk: {
2176	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2177	// The MCValue that holds the pointer can be re-used as the value.
2178	break :blk operand_ptr;
2179	} else {
2180	break :blk try self.allocRegOrMem(inst, true);
2181	}
2182	};
2183	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2184	break :result try self.isErr(operand);
2185	};
2186	return self.finishAir(inst, result, .{ un_op, .none, .none });
2187	}
2188
2189	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
2190	const un_op = self.air.instructions.items(.data)[inst].un_op;
2191	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2192	const operand = try self.resolveInst(un_op);
2193	break :result try self.isNonErr(operand);
2194	};
2195	return self.finishAir(inst, result, .{ un_op, .none, .none });
2196	}
2197
2198	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2199	const un_op = self.air.instructions.items(.data)[inst].un_op;
2200	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2201	const operand_ptr = try self.resolveInst(un_op);
2202	const operand: MCValue = blk: {
2203	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2204	// The MCValue that holds the pointer can be re-used as the value.
2205	break :blk operand_ptr;
2206	} else {
2207	break :blk try self.allocRegOrMem(inst, true);
2208	}
2209	};
2210	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2211	break :result try self.isNonErr(operand);
2212	};
2213	return self.finishAir(inst, result, .{ un_op, .none, .none });
2214	}
2215
2216	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
2217	// A loop is a setup to be able to jump back to the beginning.
2218	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2219	const loop = self.air.extraData(Air.Block, ty_pl.payload);
2220	const body = self.air.extra[loop.end..][0..loop.data.body_len];
2221	const start_index = self.code.items.len;
2222	try self.genBody(body);
2223	try self.jump(start_index);
2224	return self.finishAirBookkeeping();
2225	}
2226
2227	/// Send control flow to the `index` of `self.code`.
2228	fn jump(self: *Self, index: usize) !void {
2229	if (math.cast(i26, @intCast(i32, index) - @intCast(i32, self.code.items.len + 8))) \|delta\| {
2230	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.b(.al, delta).toU32());
2231	} else \|_\| {
2232	return self.fail("TODO: enable larger branch offset", .{});
2233	}
2234	}
2235
2236	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
2237	try self.blocks.putNoClobber(self.gpa, inst, .{
2238	// A block is a setup to be able to jump to the end.
2239	.relocs = .{},
2240	// It also acts as a receptacle for break operands.
2241	// Here we use `MCValue.none` to represent a null value so that the first
2242	// break instruction will choose a MCValue for the block result and overwrite
2243	// this field. Following break instructions will use that MCValue to put their
2244	// block results.
2245	.mcv = MCValue{ .none = {} },
2246	});
2247	const block_data = self.blocks.getPtr(inst).?;
2248	defer block_data.relocs.deinit(self.gpa);
2249
2250	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2251	const extra = self.air.extraData(Air.Block, ty_pl.payload);
2252	const body = self.air.extra[extra.end..][0..extra.data.body_len];
2253	try self.genBody(body);
2254
2255	for (block_data.relocs.items) \|reloc\| try self.performReloc(reloc);
2256
2257	const result = @bitCast(MCValue, block_data.mcv);
2258	return self.finishAir(inst, result, .{ .none, .none, .none });
2259	}
2260
2261	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
2262	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2263	const condition = pl_op.operand;
2264	_ = condition;
2265	return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch});
2266	// return self.finishAir(inst, .dead, .{ condition, .none, .none });
2267	}
2268
2269	fn performReloc(self: *Self, reloc: Reloc) !void {
2270	switch (reloc) {
2271	.rel32 => \|pos\| {
2272	const amt = self.code.items.len - (pos + 4);
2273	// Here it would be tempting to implement testing for amt == 0 and then elide the
2274	// jump. However, that will cause a problem because other jumps may assume that they
2275	// can jump to this code. Or maybe I didn't understand something when I was debugging.
2276	// It could be worth another look. Anyway, that's why that isn't done here. Probably the
2277	// best place to elide jumps will be in semantic analysis, by inlining blocks that only
2278	// only have 1 break instruction.
2279	const s32_amt = math.cast(i32, amt) catch
2280	return self.fail("unable to perform relocation: jump too far", .{});
2281	mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt);
2282	},
2283	.arm_branch => \|info\| {
2284	const amt = @intCast(i32, self.code.items.len) - @intCast(i32, info.pos + 8);
2285	if (math.cast(i26, amt)) \|delta\| {
2286	self.writeInt(u32, self.code.items[info.pos..][0..4], Instruction.b(info.cond, delta).toU32());
2287	} else \|_\| {
2288	return self.fail("TODO: enable larger branch offset", .{});
2289	}
2290	},
2291	}
2292	}
2293
2294	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
2295	const branch = self.air.instructions.items(.data)[inst].br;
2296	try self.br(branch.block_inst, branch.operand);
2297	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
2298	}
2299
2300	fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
2301	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2302	const air_tags = self.air.instructions.items(.tag);
2303	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (air_tags[inst]) {
2304	.bool_and => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_and),
2305	.bool_or => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_or),
2306	else => unreachable, // Not a boolean operation
2307	};
2308	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2309	}
2310
2311	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
2312	const block_data = self.blocks.getPtr(block).?;
2313
2314	if (self.air.typeOf(operand).hasCodeGenBits()) {
2315	const operand_mcv = try self.resolveInst(operand);
2316	const block_mcv = block_data.mcv;
2317	if (block_mcv == .none) {
2318	block_data.mcv = operand_mcv;
2319	} else {
2320	try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv);
2321	}
2322	}
2323	return self.brVoid(block);
2324	}
2325
2326	fn brVoid(self: *Self, block: Air.Inst.Index) !void {
2327	const block_data = self.blocks.getPtr(block).?;
2328
2329	// Emit a jump with a relocation. It will be patched up after the block ends.
2330	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
2331
2332	try self.code.resize(self.code.items.len + 4);
2333	block_data.relocs.appendAssumeCapacity(.{
2334	.arm_branch = .{
2335	.pos = self.code.items.len - 4,
2336	.cond = .al,
2337	},
2338	});
2339	}
2340
2341	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
2342	const air_datas = self.air.instructions.items(.data);
2343	const air_extra = self.air.extraData(Air.Asm, air_datas[inst].ty_pl.payload);
2344	const zir = self.mod_fn.owner_decl.getFileScope().zir;
2345	const extended = zir.instructions.items(.data)[air_extra.data.zir_index].extended;
2346	const zir_extra = zir.extraData(Zir.Inst.Asm, extended.operand);
2347	const asm_source = zir.nullTerminatedString(zir_extra.data.asm_source);
2348	const outputs_len = @truncate(u5, extended.small);
2349	const args_len = @truncate(u5, extended.small >> 5);
2350	const clobbers_len = @truncate(u5, extended.small >> 10);
2351	_ = clobbers_len; // TODO honor these
2352	const is_volatile = @truncate(u1, extended.small >> 15) != 0;
2353	const outputs = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end..][0..outputs_len]);
2354	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end + outputs.len ..][0..args_len]);
2355
2356	if (outputs_len > 1) {
2357	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
2358	}
2359	var extra_i: usize = zir_extra.end;
2360	const output_constraint: ?[]const u8 = out: {
2361	var i: usize = 0;
2362	while (i < outputs_len) : (i += 1) {
2363	const output = zir.extraData(Zir.Inst.Asm.Output, extra_i);
2364	extra_i = output.end;
2365	break :out zir.nullTerminatedString(output.data.constraint);
2366	}
2367	break :out null;
2368	};
2369
2370	const dead = !is_volatile and self.liveness.isUnused(inst);
2371	const result: MCValue = if (dead) .dead else result: {
2372	for (args) \|arg\| {
2373	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
2374	extra_i = input.end;
2375	const constraint = zir.nullTerminatedString(input.data.constraint);
2376
2377	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
2378	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
2379	}
2380	const reg_name = constraint[1 .. constraint.len - 1];
2381	const reg = parseRegName(reg_name) orelse
2382	return self.fail("unrecognized register: '{s}'", .{reg_name});
2383
2384	const arg_mcv = try self.resolveInst(arg);
2385	try self.register_manager.getReg(reg, null);
2386	try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
2387	}
2388
2389	if (mem.eql(u8, asm_source, "svc #0")) {
2390	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.svc(.al, 0).toU32());
2391	} else {
2392	return self.fail("TODO implement support for more arm assembly instructions", .{});
2393	}
2394
2395	if (output_constraint) \|output\| {
2396	if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
2397	return self.fail("unrecognized asm output constraint: '{s}'", .{output});
2398	}
2399	const reg_name = output[2 .. output.len - 1];
2400	const reg = parseRegName(reg_name) orelse
2401	return self.fail("unrecognized register: '{s}'", .{reg_name});
2402
2403	break :result MCValue{ .register = reg };
2404	} else {
2405	break :result MCValue{ .none = {} };
2406	}
2407	};
2408	if (outputs.len + args.len <= Liveness.bpi - 1) {
2409	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
2410	std.mem.copy(Air.Inst.Ref, &buf, outputs);
2411	std.mem.copy(Air.Inst.Ref, buf[outputs.len..], args);
2412	return self.finishAir(inst, result, buf);
2413	}
2414	var bt = try self.iterateBigTomb(inst, outputs.len + args.len);
2415	for (outputs) \|output\| {
2416	bt.feed(output);
2417	}
2418	for (args) \|arg\| {
2419	bt.feed(arg);
2420	}
2421	return bt.finishAir(result);
2422	}
2423
2424	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
2425	try self.ensureProcessDeathCapacity(operand_count + 1);
2426	return BigTomb{
2427	.function = self,
2428	.inst = inst,
2429	.tomb_bits = self.liveness.getTombBits(inst),
2430	.big_tomb_bits = self.liveness.special.get(inst) orelse 0,
2431	.bit_index = 0,
2432	};
2433	}
2434
2435	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
2436	fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
2437	switch (loc) {
2438	.none => return,
2439	.register => \|reg\| return self.genSetReg(ty, reg, val),
2440	.stack_offset => \|off\| return self.genSetStack(ty, off, val),
2441	.memory => {
2442	return self.fail("TODO implement setRegOrMem for memory", .{});
2443	},
2444	else => unreachable,
2445	}
2446	}
2447
2448	fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
2449	switch (mcv) {
2450	.dead => unreachable,
2451	.ptr_stack_offset => unreachable,
2452	.ptr_embedded_in_code => unreachable,
2453	.unreach, .none => return, // Nothing to do.
2454	.undef => {
2455	if (!self.wantSafety())
2456	return; // The already existing value will do just fine.
2457	// TODO Upgrade this to a memset call when we have that available.
2458	switch (ty.abiSize(self.target.*)) {
2459	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
2460	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
2461	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
2462	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
2463	else => return self.fail("TODO implement memset", .{}),
2464	}
2465	},
2466	.compare_flags_unsigned,
2467	.compare_flags_signed,
2468	.immediate,
2469	=> {
2470	const reg = try self.copyToTmpRegister(ty, mcv);
2471	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2472	},
2473	.embedded_in_code => \|code_offset\| {
2474	_ = code_offset;
2475	return self.fail("TODO implement set stack variable from embedded_in_code", .{});
2476	},
2477	.register => \|reg\| {
2478	const abi_size = ty.abiSize(self.target.*);
2479	const adj_off = stack_offset + abi_size;
2480
2481	switch (abi_size) {
2482	1, 4 => {
2483	const offset = if (math.cast(u12, adj_off)) \|imm\| blk: {
2484	break :blk Instruction.Offset.imm(imm);
2485	} else \|_\| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
2486	const str = switch (abi_size) {
2487	1 => Instruction.strb,
2488	4 => Instruction.str,
2489	else => unreachable,
2490	};
2491
2492	self.writeInt(u32, try self.code.addManyAsArray(4), str(.al, reg, .fp, .{
2493	.offset = offset,
2494	.positive = false,
2495	}).toU32());
2496	},
2497	2 => {
2498	const offset = if (adj_off <= math.maxInt(u8)) blk: {
2499	break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
2500	} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
2501
2502	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.strh(.al, reg, .fp, .{
2503	.offset = offset,
2504	.positive = false,
2505	}).toU32());
2506	},
2507	else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
2508	}
2509	},
2510	.memory => \|vaddr\| {
2511	_ = vaddr;
2512	return self.fail("TODO implement set stack variable from memory vaddr", .{});
2513	},
2514	.stack_offset => \|off\| {
2515	if (stack_offset == off)
2516	return; // Copy stack variable to itself; nothing to do.
2517
2518	const reg = try self.copyToTmpRegister(ty, mcv);
2519	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2520	},
2521	}
2522	}
2523
2524	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
2525	switch (mcv) {
2526	.dead => unreachable,
2527	.ptr_stack_offset => unreachable,
2528	.ptr_embedded_in_code => unreachable,
2529	.unreach, .none => return, // Nothing to do.
2530	.undef => {
2531	if (!self.wantSafety())
2532	return; // The already existing value will do just fine.
2533	// Write the debug undefined value.
2534	return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa });
2535	},
2536	.compare_flags_unsigned,
2537	.compare_flags_signed,
2538	=> \|op\| {
2539	const condition = switch (mcv) {
2540	.compare_flags_unsigned => Condition.fromCompareOperatorUnsigned(op),
2541	.compare_flags_signed => Condition.fromCompareOperatorSigned(op),
2542	else => unreachable,
2543	};
2544
2545	// mov reg, 0
2546	// moveq reg, 1
2547	const zero = Instruction.Operand.imm(0, 0);
2548	const one = Instruction.Operand.imm(1, 0);
2549	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, zero).toU32());
2550	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(condition, reg, one).toU32());
2551	},
2552	.immediate => \|x\| {
2553	if (x > math.maxInt(u32)) return self.fail("ARM registers are 32-bit wide", .{});
2554
2555	if (Instruction.Operand.fromU32(@intCast(u32, x))) \|op\| {
2556	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, op).toU32());
2557	} else if (Instruction.Operand.fromU32(~@intCast(u32, x))) \|op\| {
2558	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mvn(.al, reg, op).toU32());
2559	} else if (x <= math.maxInt(u16)) {
2560	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
2561	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @intCast(u16, x)).toU32());
2562	} else {
2563	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
2564	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
2565	}
2566	} else {
2567	// TODO write constant to code and load
2568	// relative to pc
2569	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
2570	// immediate: 0xaaaabbbb
2571	// movw reg, #0xbbbb
2572	// movt reg, #0xaaaa
2573	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @truncate(u16, x)).toU32());
2574	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.movt(.al, reg, @truncate(u16, x >> 16)).toU32());
2575	} else {
2576	// immediate: 0xaabbccdd
2577	// mov reg, #0xaa
2578	// orr reg, reg, #0xbb, 24
2579	// orr reg, reg, #0xcc, 16
2580	// orr reg, reg, #0xdd, 8
2581	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
2582	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
2583	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 16), 8)).toU32());
2584	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 24), 4)).toU32());
2585	}
2586	}
2587	},
2588	.register => \|src_reg\| {
2589	// If the registers are the same, nothing to do.
2590	if (src_reg.id() == reg.id())
2591	return;
2592
2593	// mov reg, src_reg
2594	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.reg(src_reg, Instruction.Operand.Shift.none)).toU32());
2595	},
2596	.memory => \|addr\| {
2597	// The value is in memory at a hard-coded address.
2598	// If the type is a pointer, it means the pointer address is at this memory location.
2599	try self.genSetReg(ty, reg, .{ .immediate = addr });
2600	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
2601	},
2602	.stack_offset => \|unadjusted_off\| {
2603	// TODO: maybe addressing from sp instead of fp
2604	const abi_size = ty.abiSize(self.target.*);
2605	const adj_off = unadjusted_off + abi_size;
2606
2607	switch (abi_size) {
2608	1, 4 => {
2609	const offset = if (adj_off <= math.maxInt(u12)) blk: {
2610	break :blk Instruction.Offset.imm(@intCast(u12, adj_off));
2611	} else Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
2612	const ldr = switch (abi_size) {
2613	1 => Instruction.ldrb,
2614	4 => Instruction.ldr,
2615	else => unreachable,
2616	};
2617
2618	self.writeInt(u32, try self.code.addManyAsArray(4), ldr(.al, reg, .fp, .{
2619	.offset = offset,
2620	.positive = false,
2621	}).toU32());
2622	},
2623	2 => {
2624	const offset = if (adj_off <= math.maxInt(u8)) blk: {
2625	break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
2626	} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
2627
2628	self.writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldrh(.al, reg, .fp, .{
2629	.offset = offset,
2630	.positive = false,
2631	}).toU32());
2632	},
2633	else => return self.fail("TODO a type of size {} is not allowed in a register", .{abi_size}),
2634	}
2635	},
2636	else => return self.fail("TODO implement getSetReg for arm {}", .{mcv}),
2637	}
2638	}
2639
2640	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
2641	const un_op = self.air.instructions.items(.data)[inst].un_op;
2642	const result = try self.resolveInst(un_op);
2643	return self.finishAir(inst, result, .{ un_op, .none, .none });
2644	}
2645
2646	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
2647	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2648	const result = try self.resolveInst(ty_op.operand);
2649	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2650	}
2651
2652	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
2653	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2654	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airArrayToSlice for {}", .{
2655	self.target.cpu.arch,
2656	});
2657	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2658	}
2659
2660	fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
2661	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2662	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntToFloat for {}", .{
2663	self.target.cpu.arch,
2664	});
2665	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2666	}
2667
2668	fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
2669	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2670	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatToInt for {}", .{
2671	self.target.cpu.arch,
2672	});
2673	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2674	}
2675
2676	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
2677	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2678	const extra = self.air.extraData(Air.Block, ty_pl.payload);
2679	_ = extra;
2680
2681	return self.fail("TODO implement airCmpxchg for {}", .{
2682	self.target.cpu.arch,
2683	});
2684	}
2685
2686	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
2687	_ = inst;
2688	return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
2689	}
2690
2691	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
2692	_ = inst;
2693	return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
2694	}
2695
2696	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
2697	_ = inst;
2698	_ = order;
2699	return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
2700	}
2701
2702	fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
2703	_ = inst;
2704	return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
2705	}
2706
2707	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
2708	_ = inst;
2709	return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
2710	}
2711
2712	fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
2713	// First section of indexes correspond to a set number of constant values.
2714	const ref_int = @enumToInt(inst);
2715	if (ref_int < Air.Inst.Ref.typed_value_map.len) {
2716	const tv = Air.Inst.Ref.typed_value_map[ref_int];
2717	if (!tv.ty.hasCodeGenBits()) {
2718	return MCValue{ .none = {} };
2719	}
2720	return self.genTypedValue(tv);
2721	}
2722
2723	// If the type has no codegen bits, no need to store it.
2724	const inst_ty = self.air.typeOf(inst);
2725	if (!inst_ty.hasCodeGenBits())
2726	return MCValue{ .none = {} };
2727
2728	const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
2729	switch (self.air.instructions.items(.tag)[inst_index]) {
2730	.constant => {
2731	// Constants have static lifetimes, so they are always memoized in the outer most table.
2732	const branch = &self.branch_stack.items[0];
2733	const gop = try branch.inst_table.getOrPut(self.gpa, inst_index);
2734	if (!gop.found_existing) {
2735	const ty_pl = self.air.instructions.items(.data)[inst_index].ty_pl;
2736	gop.value_ptr.* = try self.genTypedValue(.{
2737	.ty = inst_ty,
2738	.val = self.air.values[ty_pl.payload],
2739	});
2740	}
2741	return gop.value_ptr.*;
2742	},
2743	.const_ty => unreachable,
2744	else => return self.getResolvedInstValue(inst_index),
2745	}
2746	}
2747
2748	fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
2749	// Treat each stack item as a "layer" on top of the previous one.
2750	var i: usize = self.branch_stack.items.len;
2751	while (true) {
2752	i -= 1;
2753	if (self.branch_stack.items[i].inst_table.get(inst)) \|mcv\| {
2754	assert(mcv != .dead);
2755	return mcv;
2756	}
2757	}
2758	}
2759
2760	/// If the MCValue is an immediate, and it does not fit within this type,
2761	/// we put it in a register.
2762	/// A potential opportunity for future optimization here would be keeping track
2763	/// of the fact that the instruction is available both as an immediate
2764	/// and as a register.
2765	fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
2766	const mcv = try self.resolveInst(operand);
2767	const ti = @typeInfo(T).Int;
2768	switch (mcv) {
2769	.immediate => \|imm\| {
2770	// This immediate is unsigned.
2771	const U = std.meta.Int(.unsigned, ti.bits - @boolToInt(ti.signedness == .signed));
2772	if (imm >= math.maxInt(U)) {
2773	return MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.usize), mcv) };
2774	}
2775	},
2776	else => {},
2777	}
2778	return mcv;
2779	}
2780
2781	fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
2782	if (typed_value.val.isUndef())
2783	return MCValue{ .undef = {} };
2784	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2785	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2786	switch (typed_value.ty.zigTypeTag()) {
2787	.Pointer => switch (typed_value.ty.ptrSize()) {
2788	.Slice => {
2789	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
2790	const ptr_type = typed_value.ty.slicePtrFieldType(&buf);
2791	const ptr_mcv = try self.genTypedValue(.{ .ty = ptr_type, .val = typed_value.val });
2792	const slice_len = typed_value.val.sliceLen();
2793	// Codegen can't handle some kinds of indirection. If the wrong union field is accessed here it may mean
2794	// the Sema code needs to use anonymous Decls or alloca instructions to store data.
2795	const ptr_imm = ptr_mcv.memory;
2796	_ = slice_len;
2797	_ = ptr_imm;
2798	// We need more general support for const data being stored in memory to make this work.
2799	return self.fail("TODO codegen for const slices", .{});
2800	},
2801	else => {
2802	if (typed_value.val.castTag(.decl_ref)) \|payload\| {
2803	const decl = payload.data;
2804	decl.alive = true;
2805	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
2806	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
2807	const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes;
2808	return MCValue{ .memory = got_addr };
2809	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
2810	// TODO I'm hacking my way through here by repurposing .memory for storing
2811	// index to the GOT target symbol index.
2812	return MCValue{ .memory = decl.link.macho.local_sym_index };
2813	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
2814	const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes;
2815	return MCValue{ .memory = got_addr };
2816	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
2817	try p9.seeDecl(decl);
2818	const got_addr = p9.bases.data + decl.link.plan9.got_index.? * ptr_bytes;
2819	return MCValue{ .memory = got_addr };
2820	} else {
2821	return self.fail("TODO codegen non-ELF const Decl pointer", .{});
2822	}
2823	}
2824	if (typed_value.val.tag() == .int_u64) {
2825	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
2826	}
2827	return self.fail("TODO codegen more kinds of const pointers", .{});
2828	},
2829	},
2830	.Int => {
2831	const info = typed_value.ty.intInfo(self.target.*);
2832	if (info.bits > ptr_bits or info.signedness == .signed) {
2833	return self.fail("TODO const int bigger than ptr and signed int", .{});
2834	}
2835	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
2836	},
2837	.Bool => {
2838	return MCValue{ .immediate = @boolToInt(typed_value.val.toBool()) };
2839	},
2840	.ComptimeInt => unreachable, // semantic analysis prevents this
2841	.ComptimeFloat => unreachable, // semantic analysis prevents this
2842	.Optional => {
2843	if (typed_value.ty.isPtrLikeOptional()) {
2844	if (typed_value.val.isNull())
2845	return MCValue{ .immediate = 0 };
2846
2847	var buf: Type.Payload.ElemType = undefined;
2848	return self.genTypedValue(.{
2849	.ty = typed_value.ty.optionalChild(&buf),
2850	.val = typed_value.val,
2851	});
2852	} else if (typed_value.ty.abiSize(self.target.*) == 1) {
2853	return MCValue{ .immediate = @boolToInt(typed_value.val.isNull()) };
2854	}
2855	return self.fail("TODO non pointer optionals", .{});
2856	},
2857	.Enum => {
2858	if (typed_value.val.castTag(.enum_field_index)) \|field_index\| {
2859	switch (typed_value.ty.tag()) {
2860	.enum_simple => {
2861	return MCValue{ .immediate = field_index.data };
2862	},
2863	.enum_full, .enum_nonexhaustive => {
2864	const enum_full = typed_value.ty.cast(Type.Payload.EnumFull).?.data;
2865	if (enum_full.values.count() != 0) {
2866	const tag_val = enum_full.values.keys()[field_index.data];
2867	return self.genTypedValue(.{ .ty = enum_full.tag_ty, .val = tag_val });
2868	} else {
2869	return MCValue{ .immediate = field_index.data };
2870	}
2871	},
2872	else => unreachable,
2873	}
2874	} else {
2875	var int_tag_buffer: Type.Payload.Bits = undefined;
2876	const int_tag_ty = typed_value.ty.intTagType(&int_tag_buffer);
2877	return self.genTypedValue(.{ .ty = int_tag_ty, .val = typed_value.val });
2878	}
2879	},
2880	.ErrorSet => {
2881	switch (typed_value.val.tag()) {
2882	.@"error" => {
2883	const err_name = typed_value.val.castTag(.@"error").?.data.name;
2884	const module = self.bin_file.options.module.?;
2885	const global_error_set = module.global_error_set;
2886	const error_index = global_error_set.get(err_name).?;
2887	return MCValue{ .immediate = error_index };
2888	},
2889	else => {
2890	// In this case we are rendering an error union which has a 0 bits payload.
2891	return MCValue{ .immediate = 0 };
2892	},
2893	}
2894	},
2895	.ErrorUnion => {
2896	const error_type = typed_value.ty.errorUnionSet();
2897	const payload_type = typed_value.ty.errorUnionPayload();
2898	const sub_val = typed_value.val.castTag(.eu_payload).?.data;
2899
2900	if (!payload_type.hasCodeGenBits()) {
2901	// We use the error type directly as the type.
2902	return self.genTypedValue(.{ .ty = error_type, .val = sub_val });
2903	}
2904
2905	return self.fail("TODO implement error union const of type '{}'", .{typed_value.ty});
2906	},
2907	else => return self.fail("TODO implement const of type '{}'", .{typed_value.ty}),
2908	}
2909	}
2910
2911	const CallMCValues = struct {
2912	args: []MCValue,
2913	return_value: MCValue,
2914	stack_byte_count: u32,
2915	stack_align: u32,
2916
2917	fn deinit(self: CallMCValues, func: Self) void {
2918	func.gpa.free(self.args);
2919	self.* = undefined;
2920	}
2921	};
2922
2923	/// Caller must call `CallMCValues.deinit`.
2924	fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
2925	const cc = fn_ty.fnCallingConvention();
2926	const param_types = try self.gpa.alloc(Type, fn_ty.fnParamLen());
2927	defer self.gpa.free(param_types);
2928	fn_ty.fnParamTypes(param_types);
2929	var result: CallMCValues = .{
2930	.args = try self.gpa.alloc(MCValue, param_types.len),
2931	// These undefined values must be populated before returning from this function.
2932	.return_value = undefined,
2933	.stack_byte_count = undefined,
2934	.stack_align = undefined,
2935	};
2936	errdefer self.gpa.free(result.args);
2937
2938	const ret_ty = fn_ty.fnReturnType();
2939
2940	switch (cc) {
2941	.Naked => {
2942	assert(result.args.len == 0);
2943	result.return_value = .{ .unreach = {} };
2944	result.stack_byte_count = 0;
2945	result.stack_align = 1;
2946	return result;
2947	},
2948	.Unspecified, .C => {
2949	// ARM Procedure Call Standard, Chapter 6.5
2950	var ncrn: usize = 0; // Next Core Register Number
2951	var nsaa: u32 = 0; // Next stacked argument address
2952
2953	for (param_types) \|ty, i\| {
2954	if (ty.abiAlignment(self.target.*) == 8)
2955	ncrn = std.mem.alignForwardGeneric(usize, ncrn, 2);
2956
2957	const param_size = @intCast(u32, ty.abiSize(self.target.*));
2958	if (std.math.divCeil(u32, param_size, 4) catch unreachable <= 4 - ncrn) {
2959	if (param_size <= 4) {
2960	result.args[i] = .{ .register = c_abi_int_param_regs[ncrn] };
2961	ncrn += 1;
2962	} else {
2963	return self.fail("TODO MCValues with multiple registers", .{});
2964	}
2965	} else if (ncrn < 4 and nsaa == 0) {
2966	return self.fail("TODO MCValues split between registers and stack", .{});
2967	} else {
2968	ncrn = 4;
2969	if (ty.abiAlignment(self.target.*) == 8)
2970	nsaa = std.mem.alignForwardGeneric(u32, nsaa, 8);
2971
2972	result.args[i] = .{ .stack_offset = nsaa };
2973	nsaa += param_size;
2974	}
2975	}
2976
2977	result.stack_byte_count = nsaa;
2978	result.stack_align = 8;
2979	},
2980	else => return self.fail("TODO implement function parameters for {} on arm", .{cc}),
2981	}
2982
2983	if (ret_ty.zigTypeTag() == .NoReturn) {
2984	result.return_value = .{ .unreach = {} };
2985	} else if (!ret_ty.hasCodeGenBits()) {
2986	result.return_value = .{ .none = {} };
2987	} else switch (cc) {
2988	.Naked => unreachable,
2989	.Unspecified, .C => {
2990	const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
2991	if (ret_ty_size <= 4) {
2992	result.return_value = .{ .register = c_abi_int_return_regs[0] };
2993	} else {
2994	return self.fail("TODO support more return types for ARM backend", .{});
2995	}
2996	},
2997	else => return self.fail("TODO implement function return values for {}", .{cc}),
2998	}
2999	return result;
3000	}
3001
3002	/// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`.
3003	fn wantSafety(self: *Self) bool {
3004	return switch (self.bin_file.options.optimize_mode) {
3005	.Debug => true,
3006	.ReleaseSafe => true,
3007	.ReleaseFast => false,
3008	.ReleaseSmall => false,
3009	};
3010	}
3011
3012	fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3013	@setCold(true);
3014	assert(self.err_msg == null);
3015	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3016	return error.CodegenFail;
3017	}
3018
3019	fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3020	@setCold(true);
3021	assert(self.err_msg == null);
3022	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3023	return error.CodegenFail;
3024	}
3025
3026	const Register = @import("bits.zig").Register;
3027	const Instruction = @import("bits.zig").Instruction;
3028	const Condition = @import("bits.zig").Condition;
3029	const callee_preserved_regs = @import("bits.zig").callee_preserved_regs;
3030	const c_abi_int_param_regs = @import("bits.zig").c_abi_int_param_regs;
3031	const c_abi_int_return_regs = @import("bits.zig").c_abi_int_return_regs;
3032
3033	fn parseRegName(name: []const u8) ?Register {
3034	if (@hasDecl(Register, "parseRegName")) {
3035	return Register.parseRegName(name);
3036	}
3037	return std.meta.stringToEnum(Register, name);
3038	}

src/codegen.zig+3-3395

...	@@ -84,8 +84,9 @@ pub fn generateFunction(	...	@@ -84,8 +84,9 @@ pub fn generateFunction(
84	switch (bin_file.options.target.cpu.arch) {	84	switch (bin_file.options.target.cpu.arch) {
85	.wasm32 => unreachable, // has its own code path	85	.wasm32 => unreachable, // has its own code path
86	.wasm64 => unreachable, // has its own code path	86	.wasm64 => unreachable, // has its own code path
87	.arm => return Function(.arm).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	87	.arm,
88	.armeb => return Function(.armeb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	88	.armeb,
		89	=> return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output),
89	.aarch64,	90	.aarch64,
90	.aarch64_be,	91	.aarch64_be,
91	.aarch64_32,	92	.aarch64_32,
...	@@ -303,3396 +304,3 @@ pub fn generateSymbol(	...	@@ -303,3396 +304,3 @@ pub fn generateSymbol(
303	},	304	},
304	}	305	}
305	}	306	}
306
307	const InnerError = error{
308	OutOfMemory,
309	CodegenFail,
310	};
311
312	fn Function(comptime arch: std.Target.Cpu.Arch) type {
313	const writeInt = switch (arch.endian()) {
314	.Little => mem.writeIntLittle,
315	.Big => mem.writeIntBig,
316	};
317
318	return struct {
319	gpa: *Allocator,
320	air: Air,
321	liveness: Liveness,
322	bin_file: *link.File,
323	target: *const std.Target,
324	mod_fn: *const Module.Fn,
325	code: *std.ArrayList(u8),
326	debug_output: DebugInfoOutput,
327	err_msg: ?*ErrorMsg,
328	args: []MCValue,
329	ret_mcv: MCValue,
330	fn_type: Type,
331	arg_index: usize,
332	src_loc: Module.SrcLoc,
333	stack_align: u32,
334
335	prev_di_line: u32,
336	prev_di_column: u32,
337	/// Byte offset within the source file of the ending curly.
338	end_di_line: u32,
339	end_di_column: u32,
340	/// Relative to the beginning of `code`.
341	prev_di_pc: usize,
342
343	/// The value is an offset into the `Function` `code` from the beginning.
344	/// To perform the reloc, write 32-bit signed little-endian integer
345	/// which is a relative jump, based on the address following the reloc.
346	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{},
347
348	/// Whenever there is a runtime branch, we push a Branch onto this stack,
349	/// and pop it off when the runtime branch joins. This provides an "overlay"
350	/// of the table of mappings from instructions to `MCValue` from within the branch.
351	/// This way we can modify the `MCValue` for an instruction in different ways
352	/// within different branches. Special consideration is needed when a branch
353	/// joins with its parent, to make sure all instructions have the same MCValue
354	/// across each runtime branch upon joining.
355	branch_stack: *std.ArrayList(Branch),
356
357	// Key is the block instruction
358	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
359
360	register_manager: RegisterManager(Self, Register, &callee_preserved_regs) = .{},
361	/// Maps offset to what is stored there.
362	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
363
364	/// Offset from the stack base, representing the end of the stack frame.
365	max_end_stack: u32 = 0,
366	/// Represents the current end stack offset. If there is no existing slot
367	/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
368	next_stack_offset: u32 = 0,
369
370	/// Debug field, used to find bugs in the compiler.
371	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
372
373	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
374
375	const MCValue = union(enum) {
376	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
377	/// TODO Look into deleting this tag and using `dead` instead, since every use
378	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
379	none,
380	/// Control flow will not allow this value to be observed.
381	unreach,
382	/// No more references to this value remain.
383	dead,
384	/// The value is undefined.
385	undef,
386	/// A pointer-sized integer that fits in a register.
387	/// If the type is a pointer, this is the pointer address in virtual address space.
388	immediate: u64,
389	/// The constant was emitted into the code, at this offset.
390	/// If the type is a pointer, it means the pointer address is embedded in the code.
391	embedded_in_code: usize,
392	/// The value is a pointer to a constant which was emitted into the code, at this offset.
393	ptr_embedded_in_code: usize,
394	/// The value is in a target-specific register.
395	register: Register,
396	/// The value is in memory at a hard-coded address.
397	/// If the type is a pointer, it means the pointer address is at this memory location.
398	memory: u64,
399	/// The value is one of the stack variables.
400	/// If the type is a pointer, it means the pointer address is in the stack at this offset.
401	stack_offset: u32,
402	/// The value is a pointer to one of the stack variables (payload is stack offset).
403	ptr_stack_offset: u32,
404	/// The value is in the compare flags assuming an unsigned operation,
405	/// with this operator applied on top of it.
406	compare_flags_unsigned: math.CompareOperator,
407	/// The value is in the compare flags assuming a signed operation,
408	/// with this operator applied on top of it.
409	compare_flags_signed: math.CompareOperator,
410
411	fn isMemory(mcv: MCValue) bool {
412	return switch (mcv) {
413	.embedded_in_code, .memory, .stack_offset => true,
414	else => false,
415	};
416	}
417
418	fn isImmediate(mcv: MCValue) bool {
419	return switch (mcv) {
420	.immediate => true,
421	else => false,
422	};
423	}
424
425	fn isMutable(mcv: MCValue) bool {
426	return switch (mcv) {
427	.none => unreachable,
428	.unreach => unreachable,
429	.dead => unreachable,
430
431	.immediate,
432	.embedded_in_code,
433	.memory,
434	.compare_flags_unsigned,
435	.compare_flags_signed,
436	.ptr_stack_offset,
437	.ptr_embedded_in_code,
438	.undef,
439	=> false,
440
441	.register,
442	.stack_offset,
443	=> true,
444	};
445	}
446	};
447
448	const Branch = struct {
449	inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .{},
450
451	fn deinit(self: Branch, gpa: Allocator) void {
452	self.inst_table.deinit(gpa);
453	self.* = undefined;
454	}
455	};
456
457	const StackAllocation = struct {
458	inst: Air.Inst.Index,
459	/// TODO do we need size? should be determined by inst.ty.abiSize()
460	size: u32,
461	};
462
463	const BlockData = struct {
464	relocs: std.ArrayListUnmanaged(Reloc),
465	/// The first break instruction encounters `null` here and chooses a
466	/// machine code value for the block result, populating this field.
467	/// Following break instructions encounter that value and use it for
468	/// the location to store their block results.
469	mcv: MCValue,
470	};
471
472	const Reloc = union(enum) {
473	/// The value is an offset into the `Function` `code` from the beginning.
474	/// To perform the reloc, write 32-bit signed little-endian integer
475	/// which is a relative jump, based on the address following the reloc.
476	rel32: usize,
477	/// A branch in the ARM instruction set
478	arm_branch: struct {
479	pos: usize,
480	cond: @import("arch/arm/bits.zig").Condition,
481	},
482	};
483
484	const BigTomb = struct {
485	function: *Self,
486	inst: Air.Inst.Index,
487	tomb_bits: Liveness.Bpi,
488	big_tomb_bits: u32,
489	bit_index: usize,
490
491	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
492	const this_bit_index = bt.bit_index;
493	bt.bit_index += 1;
494
495	const op_int = @enumToInt(op_ref);
496	if (op_int < Air.Inst.Ref.typed_value_map.len) return;
497	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
498
499	if (this_bit_index < Liveness.bpi - 1) {
500	const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
501	if (!dies) return;
502	} else {
503	const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
504	const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
505	if (!dies) return;
506	}
507	bt.function.processDeath(op_index);
508	}
509
510	fn finishAir(bt: *BigTomb, result: MCValue) void {
511	const is_used = !bt.function.liveness.isUnused(bt.inst);
512	if (is_used) {
513	log.debug("%{d} => {}", .{ bt.inst, result });
514	const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
515	branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
516	}
517	bt.function.finishAirBookkeeping();
518	}
519	};
520
521	const Self = @This();
522
523	fn generate(
524	bin_file: *link.File,
525	src_loc: Module.SrcLoc,
526	module_fn: *Module.Fn,
527	air: Air,
528	liveness: Liveness,
529	code: *std.ArrayList(u8),
530	debug_output: DebugInfoOutput,
531	) GenerateSymbolError!FnResult {
532	if (build_options.skip_non_native and builtin.cpu.arch != arch) {
533	@panic("Attempted to compile for architecture that was disabled by build configuration");
534	}
535
536	assert(module_fn.owner_decl.has_tv);
537	const fn_type = module_fn.owner_decl.ty;
538
539	var branch_stack = std.ArrayList(Branch).init(bin_file.allocator);
540	defer {
541	assert(branch_stack.items.len == 1);
542	branch_stack.items[0].deinit(bin_file.allocator);
543	branch_stack.deinit();
544	}
545	try branch_stack.append(.{});
546
547	var function = Self{
548	.gpa = bin_file.allocator,
549	.air = air,
550	.liveness = liveness,
551	.target = &bin_file.options.target,
552	.bin_file = bin_file,
553	.mod_fn = module_fn,
554	.code = code,
555	.debug_output = debug_output,
556	.err_msg = null,
557	.args = undefined, // populated after `resolveCallingConventionValues`
558	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
559	.fn_type = fn_type,
560	.arg_index = 0,
561	.branch_stack = &branch_stack,
562	.src_loc = src_loc,
563	.stack_align = undefined,
564	.prev_di_pc = 0,
565	.prev_di_line = module_fn.lbrace_line,
566	.prev_di_column = module_fn.lbrace_column,
567	.end_di_line = module_fn.rbrace_line,
568	.end_di_column = module_fn.rbrace_column,
569	};
570	defer function.stack.deinit(bin_file.allocator);
571	defer function.blocks.deinit(bin_file.allocator);
572	defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
573
574	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
575	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
576	else => \|e\| return e,
577	};
578	defer call_info.deinit(&function);
579
580	function.args = call_info.args;
581	function.ret_mcv = call_info.return_value;
582	function.stack_align = call_info.stack_align;
583	function.max_end_stack = call_info.stack_byte_count;
584
585	function.gen() catch \|err\| switch (err) {
586	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
587	else => \|e\| return e,
588	};
589
590	if (function.err_msg) \|em\| {
591	return FnResult{ .fail = em };
592	} else {
593	return FnResult{ .appended = {} };
594	}
595	}
596
597	fn gen(self: *Self) !void {
598	switch (arch) {
599	.arm, .armeb => {
600	const cc = self.fn_type.fnCallingConvention();
601	if (cc != .Naked) {
602	// push {fp, lr}
603	// mov fp, sp
604	// sub sp, sp, #reloc
605	const prologue_reloc = self.code.items.len;
606	try self.code.resize(prologue_reloc + 12);
607	writeInt(u32, self.code.items[prologue_reloc + 4 ..][0..4], Instruction.mov(.al, .fp, Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none)).toU32());
608
609	try self.dbgSetPrologueEnd();
610
611	try self.genBody(self.air.getMainBody());
612
613	// Backpatch push callee saved regs
614	var saved_regs = Instruction.RegisterList{
615	.r11 = true, // fp
616	.r14 = true, // lr
617	};
618	inline for (callee_preserved_regs) \|reg\| {
619	if (self.register_manager.isRegAllocated(reg)) {
620	@field(saved_regs, @tagName(reg)) = true;
621	}
622	}
623	writeInt(u32, self.code.items[prologue_reloc..][0..4], Instruction.stmdb(.al, .sp, true, saved_regs).toU32());
624
625	// Backpatch stack offset
626	const stack_end = self.max_end_stack;
627	const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
628	if (Instruction.Operand.fromU32(@intCast(u32, aligned_stack_end))) \|op\| {
629	writeInt(u32, self.code.items[prologue_reloc + 8 ..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32());
630	} else {
631	return self.failSymbol("TODO ARM: allow larger stacks", .{});
632	}
633
634	try self.dbgSetEpilogueBegin();
635
636	// exitlude jumps
637	if (self.exitlude_jump_relocs.items.len == 1) {
638	// There is only one relocation. Hence,
639	// this relocation must be at the end of
640	// the code. Therefore, we can just delete
641	// the space initially reserved for the
642	// jump
643	self.code.items.len -= 4;
644	} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
645	const amt = @intCast(i32, self.code.items.len) - @intCast(i32, jmp_reloc + 8);
646	if (amt == -4) {
647	// This return is at the end of the
648	// code block. We can't just delete
649	// the space because there may be
650	// other jumps we already relocated to
651	// the address. Instead, insert a nop
652	writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.nop().toU32());
653	} else {
654	if (math.cast(i26, amt)) \|offset\| {
655	writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(.al, offset).toU32());
656	} else \|_\| {
657	return self.failSymbol("exitlude jump is too large", .{});
658	}
659	}
660	}
661
662	// Epilogue: pop callee saved registers (swap lr with pc in saved_regs)
663	saved_regs.r14 = false; // lr
664	saved_regs.r15 = true; // pc
665
666	// mov sp, fp
667	// pop {fp, pc}
668	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .sp, Instruction.Operand.reg(.fp, Instruction.Operand.Shift.none)).toU32());
669	writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldm(.al, .sp, true, saved_regs).toU32());
670	} else {
671	try self.dbgSetPrologueEnd();
672	try self.genBody(self.air.getMainBody());
673	try self.dbgSetEpilogueBegin();
674	}
675	},
676	else => {
677	try self.dbgSetPrologueEnd();
678	try self.genBody(self.air.getMainBody());
679	try self.dbgSetEpilogueBegin();
680	},
681	}
682	// Drop them off at the rbrace.
683	try self.dbgAdvancePCAndLine(self.end_di_line, self.end_di_column);
684	}
685
686	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
687	const air_tags = self.air.instructions.items(.tag);
688
689	for (body) \|inst\| {
690	const old_air_bookkeeping = self.air_bookkeeping;
691	try self.ensureProcessDeathCapacity(Liveness.bpi);
692
693	switch (air_tags[inst]) {
694	// zig fmt: off
695	.add, .ptr_add => try self.airAdd(inst),
696	.addwrap => try self.airAddWrap(inst),
697	.add_sat => try self.airAddSat(inst),
698	.sub, .ptr_sub => try self.airSub(inst),
699	.subwrap => try self.airSubWrap(inst),
700	.sub_sat => try self.airSubSat(inst),
701	.mul => try self.airMul(inst),
702	.mulwrap => try self.airMulWrap(inst),
703	.mul_sat => try self.airMulSat(inst),
704	.rem => try self.airRem(inst),
705	.mod => try self.airMod(inst),
706	.shl, .shl_exact => try self.airShl(inst),
707	.shl_sat => try self.airShlSat(inst),
708	.min => try self.airMin(inst),
709	.max => try self.airMax(inst),
710	.slice => try self.airSlice(inst),
711
712	.div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
713
714	.cmp_lt => try self.airCmp(inst, .lt),
715	.cmp_lte => try self.airCmp(inst, .lte),
716	.cmp_eq => try self.airCmp(inst, .eq),
717	.cmp_gte => try self.airCmp(inst, .gte),
718	.cmp_gt => try self.airCmp(inst, .gt),
719	.cmp_neq => try self.airCmp(inst, .neq),
720
721	.bool_and => try self.airBoolOp(inst),
722	.bool_or => try self.airBoolOp(inst),
723	.bit_and => try self.airBitAnd(inst),
724	.bit_or => try self.airBitOr(inst),
725	.xor => try self.airXor(inst),
726	.shr => try self.airShr(inst),
727
728	.alloc => try self.airAlloc(inst),
729	.ret_ptr => try self.airRetPtr(inst),
730	.arg => try self.airArg(inst),
731	.assembly => try self.airAsm(inst),
732	.bitcast => try self.airBitCast(inst),
733	.block => try self.airBlock(inst),
734	.br => try self.airBr(inst),
735	.breakpoint => try self.airBreakpoint(),
736	.fence => try self.airFence(),
737	.call => try self.airCall(inst),
738	.cond_br => try self.airCondBr(inst),
739	.dbg_stmt => try self.airDbgStmt(inst),
740	.fptrunc => try self.airFptrunc(inst),
741	.fpext => try self.airFpext(inst),
742	.intcast => try self.airIntCast(inst),
743	.trunc => try self.airTrunc(inst),
744	.bool_to_int => try self.airBoolToInt(inst),
745	.is_non_null => try self.airIsNonNull(inst),
746	.is_non_null_ptr => try self.airIsNonNullPtr(inst),
747	.is_null => try self.airIsNull(inst),
748	.is_null_ptr => try self.airIsNullPtr(inst),
749	.is_non_err => try self.airIsNonErr(inst),
750	.is_non_err_ptr => try self.airIsNonErrPtr(inst),
751	.is_err => try self.airIsErr(inst),
752	.is_err_ptr => try self.airIsErrPtr(inst),
753	.load => try self.airLoad(inst),
754	.loop => try self.airLoop(inst),
755	.not => try self.airNot(inst),
756	.ptrtoint => try self.airPtrToInt(inst),
757	.ret => try self.airRet(inst),
758	.ret_load => try self.airRetLoad(inst),
759	.store => try self.airStore(inst),
760	.struct_field_ptr=> try self.airStructFieldPtr(inst),
761	.struct_field_val=> try self.airStructFieldVal(inst),
762	.array_to_slice => try self.airArrayToSlice(inst),
763	.int_to_float => try self.airIntToFloat(inst),
764	.float_to_int => try self.airFloatToInt(inst),
765	.cmpxchg_strong => try self.airCmpxchg(inst),
766	.cmpxchg_weak => try self.airCmpxchg(inst),
767	.atomic_rmw => try self.airAtomicRmw(inst),
768	.atomic_load => try self.airAtomicLoad(inst),
769	.memcpy => try self.airMemcpy(inst),
770	.memset => try self.airMemset(inst),
771	.set_union_tag => try self.airSetUnionTag(inst),
772	.get_union_tag => try self.airGetUnionTag(inst),
773	.clz => try self.airClz(inst),
774	.ctz => try self.airCtz(inst),
775	.popcount => try self.airPopcount(inst),
776
777	.atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
778	.atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
779	.atomic_store_release => try self.airAtomicStore(inst, .Release),
780	.atomic_store_seq_cst => try self.airAtomicStore(inst, .SeqCst),
781
782	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
783	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
784	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
785	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
786
787	.switch_br => try self.airSwitch(inst),
788	.slice_ptr => try self.airSlicePtr(inst),
789	.slice_len => try self.airSliceLen(inst),
790
791	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
792	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
793
794	.array_elem_val => try self.airArrayElemVal(inst),
795	.slice_elem_val => try self.airSliceElemVal(inst),
796	.slice_elem_ptr => try self.airSliceElemPtr(inst),
797	.ptr_elem_val => try self.airPtrElemVal(inst),
798	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
799
800	.constant => unreachable, // excluded from function bodies
801	.const_ty => unreachable, // excluded from function bodies
802	.unreach => self.finishAirBookkeeping(),
803
804	.optional_payload => try self.airOptionalPayload(inst),
805	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
806	.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
807	.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
808	.unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
809	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
810
811	.wrap_optional => try self.airWrapOptional(inst),
812	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
813	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
814	// zig fmt: on
815	}
816	if (std.debug.runtime_safety) {
817	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
818	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] });
819	}
820	}
821	}
822	}
823
824	fn dbgSetPrologueEnd(self: *Self) InnerError!void {
825	switch (self.debug_output) {
826	.dwarf => \|dbg_out\| {
827	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
828	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
829	},
830	.plan9 => {},
831	.none => {},
832	}
833	}
834
835	fn dbgSetEpilogueBegin(self: *Self) InnerError!void {
836	switch (self.debug_output) {
837	.dwarf => \|dbg_out\| {
838	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
839	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
840	},
841	.plan9 => {},
842	.none => {},
843	}
844	}
845
846	fn dbgAdvancePCAndLine(self: *Self, line: u32, column: u32) InnerError!void {
847	const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
848	const delta_pc: usize = self.code.items.len - self.prev_di_pc;
849	switch (self.debug_output) {
850	.dwarf => \|dbg_out\| {
851	// TODO Look into using the DWARF special opcodes to compress this data.
852	// It lets you emit single-byte opcodes that add different numbers to
853	// both the PC and the line number at the same time.
854	try dbg_out.dbg_line.ensureUnusedCapacity(11);
855	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
856	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
857	if (delta_line != 0) {
858	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
859	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
860	}
861	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
862	self.prev_di_pc = self.code.items.len;
863	self.prev_di_line = line;
864	self.prev_di_column = column;
865	self.prev_di_pc = self.code.items.len;
866	},
867	.plan9 => \|dbg_out\| {
868	if (delta_pc <= 0) return; // only do this when the pc changes
869	// we have already checked the target in the linker to make sure it is compatable
870	const quant = @import("link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;
871
872	// increasing the line number
873	try @import("link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
874	// increasing the pc
875	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
876	if (d_pc_p9 > 0) {
877	// minus one because if its the last one, we want to leave space to change the line which is one quanta
878	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
879	if (dbg_out.pcop_change_index.*) \|pci\|
880	dbg_out.dbg_line.items[pci] += 1;
881	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
882	} else if (d_pc_p9 == 0) {
883	// we don't need to do anything, because adding the quant does it for us
884	} else unreachable;
885	if (dbg_out.start_line.* == null)
886	dbg_out.start_line.* = self.prev_di_line;
887	dbg_out.end_line.* = line;
888	// only do this if the pc changed
889	self.prev_di_line = line;
890	self.prev_di_column = column;
891	self.prev_di_pc = self.code.items.len;
892	},
893	.none => {},
894	}
895	}
896
897	/// Asserts there is already capacity to insert into top branch inst_table.
898	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
899	const air_tags = self.air.instructions.items(.tag);
900	if (air_tags[inst] == .constant) return; // Constants are immortal.
901	// When editing this function, note that the logic must synchronize with `reuseOperand`.
902	const prev_value = self.getResolvedInstValue(inst);
903	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
904	branch.inst_table.putAssumeCapacity(inst, .dead);
905	switch (prev_value) {
906	.register => \|reg\| {
907	self.register_manager.freeReg(reg);
908	},
909	else => {}, // TODO process stack allocation death
910	}
911	}
912
913	/// Called when there are no operands, and the instruction is always unreferenced.
914	fn finishAirBookkeeping(self: *Self) void {
915	if (std.debug.runtime_safety) {
916	self.air_bookkeeping += 1;
917	}
918	}
919
920	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
921	var tomb_bits = self.liveness.getTombBits(inst);
922	for (operands) \|op\| {
923	const dies = @truncate(u1, tomb_bits) != 0;
924	tomb_bits >>= 1;
925	if (!dies) continue;
926	const op_int = @enumToInt(op);
927	if (op_int < Air.Inst.Ref.typed_value_map.len) continue;
928	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
929	self.processDeath(op_index);
930	}
931	const is_used = @truncate(u1, tomb_bits) == 0;
932	if (is_used) {
933	log.debug("%{d} => {}", .{ inst, result });
934	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
935	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
936
937	switch (result) {
938	.register => \|reg\| {
939	// In some cases (such as bitcast), an operand
940	// may be the same MCValue as the result. If
941	// that operand died and was a register, it
942	// was freed by processDeath. We have to
943	// "re-allocate" the register.
944	if (self.register_manager.isRegFree(reg)) {
945	self.register_manager.getRegAssumeFree(reg, inst);
946	}
947	},
948	else => {},
949	}
950	}
951	self.finishAirBookkeeping();
952	}
953
954	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
955	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
956	try table.ensureUnusedCapacity(self.gpa, additional_count);
957	}
958
959	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
960	/// after codegen for this symbol is done.
961	fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void {
962	switch (self.debug_output) {
963	.dwarf => \|dbg_out\| {
964	assert(ty.hasCodeGenBits());
965	const index = dbg_out.dbg_info.items.len;
966	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
967
968	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(self.gpa, ty);
969	if (!gop.found_existing) {
970	gop.value_ptr.* = .{
971	.off = undefined,
972	.relocs = .{},
973	};
974	}
975	try gop.value_ptr.relocs.append(self.gpa, @intCast(u32, index));
976	},
977	.plan9 => {},
978	.none => {},
979	}
980	}
981
982	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
983	if (abi_align > self.stack_align)
984	self.stack_align = abi_align;
985	// TODO find a free slot instead of always appending
986	const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align);
987	self.next_stack_offset = offset + abi_size;
988	if (self.next_stack_offset > self.max_end_stack)
989	self.max_end_stack = self.next_stack_offset;
990	try self.stack.putNoClobber(self.gpa, offset, .{
991	.inst = inst,
992	.size = abi_size,
993	});
994	return offset;
995	}
996
997	/// Use a pointer instruction as the basis for allocating stack memory.
998	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
999	const elem_ty = self.air.typeOfIndex(inst).elemType();
1000	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
1001	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
1002	};
1003	// TODO swap this for inst.ty.ptrAlign
1004	const abi_align = elem_ty.abiAlignment(self.target.*);
1005	return self.allocMem(inst, abi_size, abi_align);
1006	}
1007
1008	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
1009	const elem_ty = self.air.typeOfIndex(inst);
1010	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
1011	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
1012	};
1013	const abi_align = elem_ty.abiAlignment(self.target.*);
1014	if (abi_align > self.stack_align)
1015	self.stack_align = abi_align;
1016
1017	if (reg_ok) {
1018	// Make sure the type can fit in a register before we try to allocate one.
1019	const ptr_bits = arch.ptrBitWidth();
1020	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1021	if (abi_size <= ptr_bytes) {
1022	if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {
1023	return MCValue{ .register = reg };
1024	}
1025	}
1026	}
1027	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
1028	return MCValue{ .stack_offset = stack_offset };
1029	}
1030
1031	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1032	const stack_mcv = try self.allocRegOrMem(inst, false);
1033	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
1034	const reg_mcv = self.getResolvedInstValue(inst);
1035	assert(reg == reg_mcv.register);
1036	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1037	try branch.inst_table.put(self.gpa, inst, stack_mcv);
1038	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
1039	}
1040
1041	/// Copies a value to a register without tracking the register. The register is not considered
1042	/// allocated. A second call to `copyToTmpRegister` may return the same register.
1043	/// This can have a side effect of spilling instructions to the stack to free up a register.
1044	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1045	const reg = try self.register_manager.allocReg(null, &.{});
1046	try self.genSetReg(ty, reg, mcv);
1047	return reg;
1048	}
1049
1050	/// Allocates a new register and copies `mcv` into it.
1051	/// `reg_owner` is the instruction that gets associated with the register in the register table.
1052	/// This can have a side effect of spilling instructions to the stack to free up a register.
1053	fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
1054	const reg = try self.register_manager.allocReg(reg_owner, &.{});
1055	try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
1056	return MCValue{ .register = reg };
1057	}
1058
1059	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1060	const stack_offset = try self.allocMemPtr(inst);
1061	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1062	}
1063
1064	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1065	const stack_offset = try self.allocMemPtr(inst);
1066	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1067	}
1068
1069	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
1070	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1071	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1072	else => return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch}),
1073	};
1074	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1075	}
1076
1077	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
1078	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1079	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1080	else => return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch}),
1081	};
1082	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1083	}
1084
1085	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1086	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1087	if (self.liveness.isUnused(inst))
1088	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1089
1090	const operand_ty = self.air.typeOf(ty_op.operand);
1091	const operand = try self.resolveInst(ty_op.operand);
1092	const info_a = operand_ty.intInfo(self.target.*);
1093	const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);
1094	if (info_a.signedness != info_b.signedness)
1095	return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
1096
1097	if (info_a.bits == info_b.bits)
1098	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
1099
1100	const result: MCValue = switch (arch) {
1101	else => return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch}),
1102	};
1103	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1104	}
1105
1106	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1107	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1108	if (self.liveness.isUnused(inst))
1109	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1110
1111	const operand = try self.resolveInst(ty_op.operand);
1112	_ = operand;
1113	const result: MCValue = switch (arch) {
1114	else => return self.fail("TODO implement trunc for {}", .{self.target.cpu.arch}),
1115	};
1116	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1117	}
1118
1119	fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
1120	const un_op = self.air.instructions.items(.data)[inst].un_op;
1121	const operand = try self.resolveInst(un_op);
1122	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;
1123	return self.finishAir(inst, result, .{ un_op, .none, .none });
1124	}
1125
1126	fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1127	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1128	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1129	const operand = try self.resolveInst(ty_op.operand);
1130	switch (operand) {
1131	.dead => unreachable,
1132	.unreach => unreachable,
1133	.compare_flags_unsigned => \|op\| {
1134	const r = MCValue{
1135	.compare_flags_unsigned = switch (op) {
1136	.gte => .lt,
1137	.gt => .lte,
1138	.neq => .eq,
1139	.lt => .gte,
1140	.lte => .gt,
1141	.eq => .neq,
1142	},
1143	};
1144	break :result r;
1145	},
1146	.compare_flags_signed => \|op\| {
1147	const r = MCValue{
1148	.compare_flags_signed = switch (op) {
1149	.gte => .lt,
1150	.gt => .lte,
1151	.neq => .eq,
1152	.lt => .gte,
1153	.lte => .gt,
1154	.eq => .neq,
1155	},
1156	};
1157	break :result r;
1158	},
1159	else => {},
1160	}
1161
1162	switch (arch) {
1163	.arm, .armeb => {
1164	break :result try self.genArmBinOp(inst, ty_op.operand, .bool_true, .not);
1165	},
1166	else => return self.fail("TODO implement NOT for {}", .{self.target.cpu.arch}),
1167	}
1168	};
1169	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1170	}
1171
1172	fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1173	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1174	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1175	else => return self.fail("TODO implement min for {}", .{self.target.cpu.arch}),
1176	};
1177	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1178	}
1179
1180	fn airMax(self: *Self, inst: Air.Inst.Index) !void {
1181	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1182	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1183	else => return self.fail("TODO implement max for {}", .{self.target.cpu.arch}),
1184	};
1185	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1186	}
1187
1188	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1189	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1190	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1191	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1192	else => return self.fail("TODO implement slice for {}", .{self.target.cpu.arch}),
1193	};
1194	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1195	}
1196
1197	fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
1198	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1199	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1200	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .add),
1201	else => return self.fail("TODO implement add for {}", .{self.target.cpu.arch}),
1202	};
1203	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1204	}
1205
1206	fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {
1207	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1208	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1209	else => return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch}),
1210	};
1211	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1212	}
1213
1214	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1215	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1216	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1217	else => return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch}),
1218	};
1219	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1220	}
1221
1222	fn airSub(self: *Self, inst: Air.Inst.Index) !void {
1223	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1224	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1225	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .sub),
1226	else => return self.fail("TODO implement sub for {}", .{self.target.cpu.arch}),
1227	};
1228	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1229	}
1230
1231	fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
1232	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1233	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1234	else => return self.fail("TODO implement subwrap for {}", .{self.target.cpu.arch}),
1235	};
1236	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1237	}
1238
1239	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1240	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1241	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1242	else => return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch}),
1243	};
1244	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1245	}
1246
1247	fn airMul(self: *Self, inst: Air.Inst.Index) !void {
1248	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1249	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1250	.arm, .armeb => try self.genArmMul(inst, bin_op.lhs, bin_op.rhs),
1251	else => return self.fail("TODO implement mul for {}", .{self.target.cpu.arch}),
1252	};
1253	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1254	}
1255
1256	fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {
1257	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1258	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1259	else => return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch}),
1260	};
1261	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1262	}
1263
1264	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1265	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1266	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1267	else => return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch}),
1268	};
1269	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1270	}
1271
1272	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1273	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1274	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1275	else => return self.fail("TODO implement div for {}", .{self.target.cpu.arch}),
1276	};
1277	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1278	}
1279
1280	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
1281	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1282	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1283	else => return self.fail("TODO implement rem for {}", .{self.target.cpu.arch}),
1284	};
1285	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1286	}
1287
1288	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1289	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1290	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1291	else => return self.fail("TODO implement mod for {}", .{self.target.cpu.arch}),
1292	};
1293	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1294	}
1295
1296	fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
1297	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1298	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1299	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_and),
1300	else => return self.fail("TODO implement bitwise and for {}", .{self.target.cpu.arch}),
1301	};
1302	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1303	}
1304
1305	fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
1306	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1307	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1308	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_or),
1309	else => return self.fail("TODO implement bitwise or for {}", .{self.target.cpu.arch}),
1310	};
1311	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1312	}
1313
1314	fn airXor(self: *Self, inst: Air.Inst.Index) !void {
1315	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1316	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1317	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .xor),
1318	else => return self.fail("TODO implement xor for {}", .{self.target.cpu.arch}),
1319	};
1320	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1321	}
1322
1323	fn airShl(self: *Self, inst: Air.Inst.Index) !void {
1324	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1325	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1326	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shl),
1327	else => return self.fail("TODO implement shl for {}", .{self.target.cpu.arch}),
1328	};
1329	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1330	}
1331
1332	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
1333	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1334	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1335	else => return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch}),
1336	};
1337	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1338	}
1339
1340	fn airShr(self: *Self, inst: Air.Inst.Index) !void {
1341	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1342	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1343	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .shr),
1344	else => return self.fail("TODO implement shr for {}", .{self.target.cpu.arch}),
1345	};
1346	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1347	}
1348
1349	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1350	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1351	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1352	else => return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch}),
1353	};
1354	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1355	}
1356
1357	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1358	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1359	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1360	else => return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch}),
1361	};
1362	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1363	}
1364
1365	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
1366	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1367	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1368	else => return self.fail("TODO implement unwrap error union error for {}", .{self.target.cpu.arch}),
1369	};
1370	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1371	}
1372
1373	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
1374	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1375	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1376	else => return self.fail("TODO implement unwrap error union payload for {}", .{self.target.cpu.arch}),
1377	};
1378	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1379	}
1380
1381	// *(E!T) -> E
1382	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1383	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1384	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1385	else => return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch}),
1386	};
1387	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1388	}
1389
1390	// (E!T) -> T
1391	fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1392	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1393	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1394	else => return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch}),
1395	};
1396	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1397	}
1398
1399	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
1400	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1401	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1402	const optional_ty = self.air.typeOfIndex(inst);
1403
1404	// Optional with a zero-bit payload type is just a boolean true
1405	if (optional_ty.abiSize(self.target.*) == 1)
1406	break :result MCValue{ .immediate = 1 };
1407
1408	switch (arch) {
1409	else => return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch}),
1410	}
1411	};
1412	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1413	}
1414
1415	/// T to E!T
1416	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
1417	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1418	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1419	else => return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch}),
1420	};
1421	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1422	}
1423
1424	/// E to E!T
1425	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
1426	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1427	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1428	else => return self.fail("TODO implement wrap errunion error for {}", .{self.target.cpu.arch}),
1429	};
1430	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1431	}
1432
1433	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
1434	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1435	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1436	else => return self.fail("TODO implement slice_ptr for {}", .{self.target.cpu.arch}),
1437	};
1438	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1439	}
1440
1441	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
1442	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1443	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1444	else => return self.fail("TODO implement slice_len for {}", .{self.target.cpu.arch}),
1445	};
1446	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1447	}
1448
1449	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
1450	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1451	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1452	else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch}),
1453	};
1454	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1455	}
1456
1457	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
1458	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1459	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1460	else => return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch}),
1461	};
1462	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1463	}
1464
1465	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1466	const is_volatile = false; // TODO
1467	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1468	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else switch (arch) {
1469	else => return self.fail("TODO implement slice_elem_val for {}", .{self.target.cpu.arch}),
1470	};
1471	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1472	}
1473
1474	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1475	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1476	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1477	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1478	else => return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch}),
1479	};
1480	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1481	}
1482
1483	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
1484	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1485	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1486	else => return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch}),
1487	};
1488	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1489	}
1490
1491	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
1492	const is_volatile = false; // TODO
1493	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1494	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else switch (arch) {
1495	else => return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch}),
1496	};
1497	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1498	}
1499
1500	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1501	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1502	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1503	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1504	else => return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch}),
1505	};
1506	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1507	}
1508
1509	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1510	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1511	const result: MCValue = switch (arch) {
1512	else => return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch}),
1513	};
1514	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1515	}
1516
1517	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1518	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1519	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1520	else => return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch}),
1521	};
1522	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1523	}
1524
1525	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
1526	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1527	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1528	else => return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch}),
1529	};
1530	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1531	}
1532
1533	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
1534	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1535	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1536	else => return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch}),
1537	};
1538	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1539	}
1540
1541	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
1542	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1543	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1544	else => return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch}),
1545	};
1546	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1547	}
1548
1549	fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {
1550	if (!self.liveness.operandDies(inst, op_index))
1551	return false;
1552
1553	switch (mcv) {
1554	.register => \|reg\| {
1555	// If it's in the registers table, need to associate the register with the
1556	// new instruction.
1557	if (reg.allocIndex()) \|index\| {
1558	if (!self.register_manager.isRegFree(reg)) {
1559	self.register_manager.registers[index] = inst;
1560	}
1561	}
1562	log.debug("%{d} => {} (reused)", .{ inst, reg });
1563	},
1564	.stack_offset => \|off\| {
1565	log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
1566	},
1567	else => return false,
1568	}
1569
1570	// Prevent the operand deaths processing code from deallocating it.
1571	self.liveness.clearOperandDeath(inst, op_index);
1572
1573	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
1574	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1575	branch.inst_table.putAssumeCapacity(Air.refToIndex(operand).?, .dead);
1576
1577	return true;
1578	}
1579
1580	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
1581	const elem_ty = ptr_ty.elemType();
1582	switch (ptr) {
1583	.none => unreachable,
1584	.undef => unreachable,
1585	.unreach => unreachable,
1586	.dead => unreachable,
1587	.compare_flags_unsigned => unreachable,
1588	.compare_flags_signed => unreachable,
1589	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
1590	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
1591	.ptr_embedded_in_code => \|off\| {
1592	try self.setRegOrMem(elem_ty, dst_mcv, .{ .embedded_in_code = off });
1593	},
1594	.embedded_in_code => {
1595	return self.fail("TODO implement loading from MCValue.embedded_in_code", .{});
1596	},
1597	.register => \|reg\| {
1598	switch (arch) {
1599	.arm, .armeb => switch (dst_mcv) {
1600	.dead => unreachable,
1601	.undef => unreachable,
1602	.compare_flags_signed, .compare_flags_unsigned => unreachable,
1603	.embedded_in_code => unreachable,
1604	.register => \|dst_reg\| {
1605	writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, dst_reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
1606	},
1607	else => return self.fail("TODO load from register into {}", .{dst_mcv}),
1608	},
1609	else => return self.fail("TODO implement loading from MCValue.register for {}", .{arch}),
1610	}
1611	},
1612	.memory => \|addr\| {
1613	const reg = try self.register_manager.allocReg(null, &.{});
1614	try self.genSetReg(ptr_ty, reg, .{ .memory = addr });
1615	try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
1616	},
1617	.stack_offset => {
1618	return self.fail("TODO implement loading from MCValue.stack_offset", .{});
1619	},
1620	}
1621	}
1622
1623	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
1624	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1625	const elem_ty = self.air.typeOfIndex(inst);
1626	const result: MCValue = result: {
1627	if (!elem_ty.hasCodeGenBits())
1628	break :result MCValue.none;
1629
1630	const ptr = try self.resolveInst(ty_op.operand);
1631	const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
1632	if (self.liveness.isUnused(inst) and !is_volatile)
1633	break :result MCValue.dead;
1634
1635	const dst_mcv: MCValue = blk: {
1636	if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
1637	// The MCValue that holds the pointer can be re-used as the value.
1638	break :blk ptr;
1639	} else {
1640	break :blk try self.allocRegOrMem(inst, true);
1641	}
1642	};
1643	try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
1644	break :result dst_mcv;
1645	};
1646	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1647	}
1648
1649	fn airStore(self: *Self, inst: Air.Inst.Index) !void {
1650	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1651	const ptr = try self.resolveInst(bin_op.lhs);
1652	const value = try self.resolveInst(bin_op.rhs);
1653	const elem_ty = self.air.typeOf(bin_op.rhs);
1654	switch (ptr) {
1655	.none => unreachable,
1656	.undef => unreachable,
1657	.unreach => unreachable,
1658	.dead => unreachable,
1659	.compare_flags_unsigned => unreachable,
1660	.compare_flags_signed => unreachable,
1661	.immediate => \|imm\| {
1662	try self.setRegOrMem(elem_ty, .{ .memory = imm }, value);
1663	},
1664	.ptr_stack_offset => \|off\| {
1665	try self.genSetStack(elem_ty, off, value);
1666	},
1667	.ptr_embedded_in_code => \|off\| {
1668	try self.setRegOrMem(elem_ty, .{ .embedded_in_code = off }, value);
1669	},
1670	.embedded_in_code => {
1671	return self.fail("TODO implement storing to MCValue.embedded_in_code", .{});
1672	},
1673	.register => {
1674	return self.fail("TODO implement storing to MCValue.register", .{});
1675	},
1676	.memory => {
1677	return self.fail("TODO implement storing to MCValue.memory", .{});
1678	},
1679	.stack_offset => {
1680	return self.fail("TODO implement storing to MCValue.stack_offset", .{});
1681	},
1682	}
1683	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1684	}
1685
1686	fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
1687	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1688	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1689	return self.structFieldPtr(extra.struct_operand, ty_pl.ty, extra.field_index);
1690	}
1691
1692	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
1693	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1694	return self.structFieldPtr(ty_op.operand, ty_op.ty, index);
1695	}
1696	fn structFieldPtr(self: *Self, operand: Air.Inst.Ref, ty: Air.Inst.Ref, index: u32) !void {
1697	_ = self;
1698	_ = operand;
1699	_ = ty;
1700	_ = index;
1701	return self.fail("TODO implement codegen struct_field_ptr", .{});
1702	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1703	}
1704
1705	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
1706	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1707	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1708	_ = extra;
1709	return self.fail("TODO implement codegen struct_field_val", .{});
1710	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1711	}
1712
1713	fn armOperandShouldBeRegister(self: *Self, mcv: MCValue) !bool {
1714	return switch (mcv) {
1715	.none => unreachable,
1716	.undef => unreachable,
1717	.dead, .unreach => unreachable,
1718	.compare_flags_unsigned => unreachable,
1719	.compare_flags_signed => unreachable,
1720	.ptr_stack_offset => unreachable,
1721	.ptr_embedded_in_code => unreachable,
1722	.immediate => \|imm\| blk: {
1723	if (imm > std.math.maxInt(u32)) return self.fail("TODO ARM binary arithmetic immediate larger than u32", .{});
1724
1725	// Load immediate into register if it doesn't fit
1726	// in an operand
1727	break :blk Instruction.Operand.fromU32(@intCast(u32, imm)) == null;
1728	},
1729	.register => true,
1730	.stack_offset,
1731	.embedded_in_code,
1732	.memory,
1733	=> true,
1734	};
1735	}
1736
1737	fn genArmBinOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref, op: Air.Inst.Tag) !MCValue {
1738	// In the case of bitshifts, the type of rhs is different
1739	// from the resulting type
1740	const ty = self.air.typeOf(op_lhs);
1741
1742	switch (ty.zigTypeTag()) {
1743	.Float => return self.fail("TODO ARM binary operations on floats", .{}),
1744	.Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
1745	.Bool => {
1746	return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, 1, .unsigned);
1747	},
1748	.Int => {
1749	const int_info = ty.intInfo(self.target.*);
1750	return self.genArmBinIntOp(inst, op_lhs, op_rhs, op, int_info.bits, int_info.signedness);
1751	},
1752	else => unreachable,
1753	}
1754	}
1755
1756	fn genArmBinIntOp(
1757	self: *Self,
1758	inst: Air.Inst.Index,
1759	op_lhs: Air.Inst.Ref,
1760	op_rhs: Air.Inst.Ref,
1761	op: Air.Inst.Tag,
1762	bits: u16,
1763	signedness: std.builtin.Signedness,
1764	) !MCValue {
1765	if (bits > 32) {
1766	return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
1767	}
1768
1769	const lhs = try self.resolveInst(op_lhs);
1770	const rhs = try self.resolveInst(op_rhs);
1771
1772	const lhs_is_register = lhs == .register;
1773	const rhs_is_register = rhs == .register;
1774	const lhs_should_be_register = switch (op) {
1775	.shr, .shl => true,
1776	else => try self.armOperandShouldBeRegister(lhs),
1777	};
1778	const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
1779	const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
1780	const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
1781	const can_swap_lhs_and_rhs = switch (op) {
1782	.shr, .shl => false,
1783	else => true,
1784	};
1785
1786	// Destination must be a register
1787	var dst_mcv: MCValue = undefined;
1788	var lhs_mcv = lhs;
1789	var rhs_mcv = rhs;
1790	var swap_lhs_and_rhs = false;
1791
1792	// Allocate registers for operands and/or destination
1793	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1794	if (reuse_lhs) {
1795	// Allocate 0 or 1 registers
1796	if (!rhs_is_register and rhs_should_be_register) {
1797	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
1798	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1799	}
1800	dst_mcv = lhs;
1801	} else if (reuse_rhs and can_swap_lhs_and_rhs) {
1802	// Allocate 0 or 1 registers
1803	if (!lhs_is_register and lhs_should_be_register) {
1804	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
1805	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
1806	}
1807	dst_mcv = rhs;
1808
1809	swap_lhs_and_rhs = true;
1810	} else {
1811	// Allocate 1 or 2 registers
1812	if (lhs_should_be_register and rhs_should_be_register) {
1813	if (lhs_is_register and rhs_is_register) {
1814	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
1815	} else if (lhs_is_register) {
1816	// Move RHS to register
1817	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1818	rhs_mcv = dst_mcv;
1819	} else if (rhs_is_register) {
1820	// Move LHS to register
1821	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1822	lhs_mcv = dst_mcv;
1823	} else {
1824	// Move LHS and RHS to register
1825	const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
1826	lhs_mcv = MCValue{ .register = regs[0] };
1827	rhs_mcv = MCValue{ .register = regs[1] };
1828	dst_mcv = lhs_mcv;
1829
1830	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1831	}
1832	} else if (lhs_should_be_register) {
1833	// RHS is immediate
1834	if (lhs_is_register) {
1835	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1836	} else {
1837	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
1838	lhs_mcv = dst_mcv;
1839	}
1840	} else if (rhs_should_be_register and can_swap_lhs_and_rhs) {
1841	// LHS is immediate
1842	if (rhs_is_register) {
1843	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1844	} else {
1845	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{}) };
1846	rhs_mcv = dst_mcv;
1847	}
1848
1849	swap_lhs_and_rhs = true;
1850	} else unreachable; // binary operation on two immediates
1851	}
1852
1853	// Move the operands to the newly allocated registers
1854	if (lhs_mcv == .register and !lhs_is_register) {
1855	try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
1856	}
1857	if (rhs_mcv == .register and !rhs_is_register) {
1858	try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
1859	}
1860
1861	try self.genArmBinOpCode(
1862	dst_mcv.register,
1863	lhs_mcv,
1864	rhs_mcv,
1865	swap_lhs_and_rhs,
1866	op,
1867	signedness,
1868	);
1869	return dst_mcv;
1870	}
1871
1872	fn genArmBinOpCode(
1873	self: *Self,
1874	dst_reg: Register,
1875	lhs_mcv: MCValue,
1876	rhs_mcv: MCValue,
1877	swap_lhs_and_rhs: bool,
1878	op: Air.Inst.Tag,
1879	signedness: std.builtin.Signedness,
1880	) !void {
1881	assert(lhs_mcv == .register or rhs_mcv == .register);
1882
1883	const op1 = if (swap_lhs_and_rhs) rhs_mcv.register else lhs_mcv.register;
1884	const op2 = if (swap_lhs_and_rhs) lhs_mcv else rhs_mcv;
1885
1886	const operand = switch (op2) {
1887	.none => unreachable,
1888	.undef => unreachable,
1889	.dead, .unreach => unreachable,
1890	.compare_flags_unsigned => unreachable,
1891	.compare_flags_signed => unreachable,
1892	.ptr_stack_offset => unreachable,
1893	.ptr_embedded_in_code => unreachable,
1894	.immediate => \|imm\| Instruction.Operand.fromU32(@intCast(u32, imm)).?,
1895	.register => \|reg\| Instruction.Operand.reg(reg, Instruction.Operand.Shift.none),
1896	.stack_offset,
1897	.embedded_in_code,
1898	.memory,
1899	=> unreachable,
1900	};
1901
1902	switch (op) {
1903	.add => {
1904	writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.al, dst_reg, op1, operand).toU32());
1905	},
1906	.sub => {
1907	if (swap_lhs_and_rhs) {
1908	writeInt(u32, try self.code.addManyAsArray(4), Instruction.rsb(.al, dst_reg, op1, operand).toU32());
1909	} else {
1910	writeInt(u32, try self.code.addManyAsArray(4), Instruction.sub(.al, dst_reg, op1, operand).toU32());
1911	}
1912	},
1913	.bool_and, .bit_and => {
1914	writeInt(u32, try self.code.addManyAsArray(4), Instruction.@"and"(.al, dst_reg, op1, operand).toU32());
1915	},
1916	.bool_or, .bit_or => {
1917	writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, dst_reg, op1, operand).toU32());
1918	},
1919	.not, .xor => {
1920	writeInt(u32, try self.code.addManyAsArray(4), Instruction.eor(.al, dst_reg, op1, operand).toU32());
1921	},
1922	.cmp_eq => {
1923	writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, op1, operand).toU32());
1924	},
1925	.shl => {
1926	assert(!swap_lhs_and_rhs);
1927	const shift_amount = switch (operand) {
1928	.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
1929	.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
1930	};
1931	writeInt(u32, try self.code.addManyAsArray(4), Instruction.lsl(.al, dst_reg, op1, shift_amount).toU32());
1932	},
1933	.shr => {
1934	assert(!swap_lhs_and_rhs);
1935	const shift_amount = switch (operand) {
1936	.Register => \|reg_op\| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)),
1937	.Immediate => \|imm_op\| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)),
1938	};
1939
1940	const shr = switch (signedness) {
1941	.signed => Instruction.asr,
1942	.unsigned => Instruction.lsr,
1943	};
1944	writeInt(u32, try self.code.addManyAsArray(4), shr(.al, dst_reg, op1, shift_amount).toU32());
1945	},
1946	else => unreachable, // not a binary instruction
1947	}
1948	}
1949
1950	fn genArmMul(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
1951	const lhs = try self.resolveInst(op_lhs);
1952	const rhs = try self.resolveInst(op_rhs);
1953
1954	const lhs_is_register = lhs == .register;
1955	const rhs_is_register = rhs == .register;
1956	const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op_lhs, 0, lhs);
1957	const reuse_rhs = !reuse_lhs and rhs_is_register and self.reuseOperand(inst, op_rhs, 1, rhs);
1958
1959	// Destination must be a register
1960	// LHS must be a register
1961	// RHS must be a register
1962	var dst_mcv: MCValue = undefined;
1963	var lhs_mcv: MCValue = lhs;
1964	var rhs_mcv: MCValue = rhs;
1965
1966	// Allocate registers for operands and/or destination
1967	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1968	if (reuse_lhs) {
1969	// Allocate 0 or 1 registers
1970	if (!rhs_is_register) {
1971	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_rhs).?, &.{lhs.register}) };
1972	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
1973	}
1974	dst_mcv = lhs;
1975	} else if (reuse_rhs) {
1976	// Allocate 0 or 1 registers
1977	if (!lhs_is_register) {
1978	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(op_lhs).?, &.{rhs.register}) };
1979	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_lhs).?, lhs_mcv);
1980	}
1981	dst_mcv = rhs;
1982	} else {
1983	// Allocate 1 or 2 registers
1984	if (lhs_is_register and rhs_is_register) {
1985	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{ lhs.register, rhs.register }) };
1986	} else if (lhs_is_register) {
1987	// Move RHS to register
1988	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{lhs.register}) };
1989	rhs_mcv = dst_mcv;
1990	} else if (rhs_is_register) {
1991	// Move LHS to register
1992	dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, &.{rhs.register}) };
1993	lhs_mcv = dst_mcv;
1994	} else {
1995	// Move LHS and RHS to register
1996	const regs = try self.register_manager.allocRegs(2, .{ inst, Air.refToIndex(op_rhs).? }, &.{});
1997	lhs_mcv = MCValue{ .register = regs[0] };
1998	rhs_mcv = MCValue{ .register = regs[1] };
1999	dst_mcv = lhs_mcv;
2000
2001	branch.inst_table.putAssumeCapacity(Air.refToIndex(op_rhs).?, rhs_mcv);
2002	}
2003	}
2004
2005	// Move the operands to the newly allocated registers
2006	if (!lhs_is_register) {
2007	try self.genSetReg(self.air.typeOf(op_lhs), lhs_mcv.register, lhs);
2008	}
2009	if (!rhs_is_register) {
2010	try self.genSetReg(self.air.typeOf(op_rhs), rhs_mcv.register, rhs);
2011	}
2012
2013	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mul(.al, dst_mcv.register, lhs_mcv.register, rhs_mcv.register).toU32());
2014	return dst_mcv;
2015	}
2016
2017	fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
2018	const ty_str = self.air.instructions.items(.data)[inst].ty_str;
2019	const zir = &self.mod_fn.owner_decl.getFileScope().zir;
2020	const name = zir.nullTerminatedString(ty_str.str);
2021	const name_with_null = name.ptr[0 .. name.len + 1];
2022	const ty = self.air.getRefType(ty_str.ty);
2023
2024	switch (mcv) {
2025	.register => \|reg\| {
2026	switch (self.debug_output) {
2027	.dwarf => \|dbg_out\| {
2028	try dbg_out.dbg_info.ensureUnusedCapacity(3);
2029	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
2030	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
2031	1, // ULEB128 dwarf expression length
2032	reg.dwarfLocOp(),
2033	});
2034	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
2035	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
2036	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
2037	},
2038	.plan9 => {},
2039	.none => {},
2040	}
2041	},
2042	.stack_offset => \|offset\| {
2043	switch (self.debug_output) {
2044	.dwarf => \|dbg_out\| {
2045	switch (arch) {
2046	.arm, .armeb => {
2047	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
2048	return self.fail("type '{}' too big to fit into stack frame", .{ty});
2049	};
2050	const adjusted_stack_offset = math.negateCast(offset + abi_size) catch {
2051	return self.fail("Stack offset too large for arguments", .{});
2052	};
2053
2054	try dbg_out.dbg_info.append(link.File.Elf.abbrev_parameter);
2055
2056	// Get length of the LEB128 stack offset
2057	var counting_writer = std.io.countingWriter(std.io.null_writer);
2058	leb128.writeILEB128(counting_writer.writer(), adjusted_stack_offset) catch unreachable;
2059
2060	// DW.AT.location, DW.FORM.exprloc
2061	// ULEB128 dwarf expression length
2062	try leb128.writeULEB128(dbg_out.dbg_info.writer(), counting_writer.bytes_written + 1);
2063	try dbg_out.dbg_info.append(DW.OP.breg11);
2064	try leb128.writeILEB128(dbg_out.dbg_info.writer(), adjusted_stack_offset);
2065
2066	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
2067	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
2068	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
2069	},
2070	else => {},
2071	}
2072	},
2073	.plan9 => {},
2074	.none => {},
2075	}
2076	},
2077	else => {},
2078	}
2079	}
2080
2081	fn airArg(self: *Self, inst: Air.Inst.Index) !void {
2082	const arg_index = self.arg_index;
2083	self.arg_index += 1;
2084
2085	const ty = self.air.typeOfIndex(inst);
2086
2087	const result = self.args[arg_index];
2088	const mcv = switch (arch) {
2089	// TODO support stack-only arguments on all target architectures
2090	.arm, .armeb => switch (result) {
2091	// Copy registers to the stack
2092	.register => \|reg\| blk: {
2093	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch {
2094	return self.fail("type '{}' too big to fit into stack frame", .{ty});
2095	};
2096	const abi_align = ty.abiAlignment(self.target.*);
2097	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
2098	try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2099
2100	break :blk MCValue{ .stack_offset = stack_offset };
2101	},
2102	else => result,
2103	},
2104	else => result,
2105	};
2106	try self.genArgDbgInfo(inst, mcv);
2107
2108	if (self.liveness.isUnused(inst))
2109	return self.finishAirBookkeeping();
2110
2111	switch (mcv) {
2112	.register => \|reg\| {
2113	self.register_manager.getRegAssumeFree(reg, inst);
2114	},
2115	else => {},
2116	}
2117
2118	return self.finishAir(inst, mcv, .{ .none, .none, .none });
2119	}
2120
2121	fn airBreakpoint(self: *Self) !void {
2122	switch (arch) {
2123	.i386 => {
2124	try self.code.append(0xcc); // int3
2125	},
2126	.arm, .armeb => {
2127	writeInt(u32, try self.code.addManyAsArray(4), Instruction.bkpt(0).toU32());
2128	},
2129	else => return self.fail("TODO implement @breakpoint() for {}", .{self.target.cpu.arch}),
2130	}
2131	return self.finishAirBookkeeping();
2132	}
2133
2134	fn airFence(self: *Self) !void {
2135	return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
2136	//return self.finishAirBookkeeping();
2137	}
2138
2139	fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2140	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2141	const fn_ty = self.air.typeOf(pl_op.operand);
2142	const callee = pl_op.operand;
2143	const extra = self.air.extraData(Air.Call, pl_op.payload);
2144	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[extra.end..][0..extra.data.args_len]);
2145
2146	var info = try self.resolveCallingConventionValues(fn_ty);
2147	defer info.deinit(self);
2148
2149	// Due to incremental compilation, how function calls are generated depends
2150	// on linking.
2151	if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {
2152	switch (arch) {
2153	.arm, .armeb => {
2154	for (info.args) \|mc_arg, arg_i\| {
2155	const arg = args[arg_i];
2156	const arg_ty = self.air.typeOf(arg);
2157	const arg_mcv = try self.resolveInst(args[arg_i]);
2158
2159	switch (mc_arg) {
2160	.none => continue,
2161	.undef => unreachable,
2162	.immediate => unreachable,
2163	.unreach => unreachable,
2164	.dead => unreachable,
2165	.embedded_in_code => unreachable,
2166	.memory => unreachable,
2167	.compare_flags_signed => unreachable,
2168	.compare_flags_unsigned => unreachable,
2169	.register => \|reg\| {
2170	try self.register_manager.getReg(reg, null);
2171	try self.genSetReg(arg_ty, reg, arg_mcv);
2172	},
2173	.stack_offset => {
2174	return self.fail("TODO implement calling with parameters in memory", .{});
2175	},
2176	.ptr_stack_offset => {
2177	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2178	},
2179	.ptr_embedded_in_code => {
2180	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2181	},
2182	}
2183	}
2184
2185	if (self.air.value(callee)) \|func_value\| {
2186	if (func_value.castTag(.function)) \|func_payload\| {
2187	const func = func_payload.data;
2188	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2189	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2190	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
2191	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
2192	break :blk @intCast(u32, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * ptr_bytes);
2193	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\|
2194	coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes
2195	else
2196	unreachable;
2197
2198	try self.genSetReg(Type.initTag(.usize), .lr, .{ .memory = got_addr });
2199
2200	// TODO: add Instruction.supportedOn
2201	// function for ARM
2202	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v5t)) {
2203	writeInt(u32, try self.code.addManyAsArray(4), Instruction.blx(.al, .lr).toU32());
2204	} else {
2205	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .lr, Instruction.Operand.reg(.pc, Instruction.Operand.Shift.none)).toU32());
2206	writeInt(u32, try self.code.addManyAsArray(4), Instruction.bx(.al, .lr).toU32());
2207	}
2208	} else if (func_value.castTag(.extern_fn)) \|_\| {
2209	return self.fail("TODO implement calling extern functions", .{});
2210	} else {
2211	return self.fail("TODO implement calling bitcasted functions", .{});
2212	}
2213	} else {
2214	return self.fail("TODO implement calling runtime known function pointer", .{});
2215	}
2216	},
2217	else => return self.fail("TODO implement call for {}", .{self.target.cpu.arch}),
2218	}
2219	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
2220	unreachable; // unsupported architecture for MachO
2221	} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {
2222	return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});
2223	} else unreachable;
2224
2225	const result: MCValue = result: {
2226	switch (info.return_value) {
2227	.register => \|reg\| {
2228	if (Register.allocIndex(reg) == null) {
2229	// Save function return value in a callee saved register
2230	break :result try self.copyToNewRegister(inst, info.return_value);
2231	}
2232	},
2233	else => {},
2234	}
2235	break :result info.return_value;
2236	};
2237
2238	if (args.len <= Liveness.bpi - 2) {
2239	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
2240	buf[0] = callee;
2241	std.mem.copy(Air.Inst.Ref, buf[1..], args);
2242	return self.finishAir(inst, result, buf);
2243	}
2244	var bt = try self.iterateBigTomb(inst, 1 + args.len);
2245	bt.feed(callee);
2246	for (args) \|arg\| {
2247	bt.feed(arg);
2248	}
2249	return bt.finishAir(result);
2250	}
2251
2252	fn ret(self: *Self, mcv: MCValue) !void {
2253	const ret_ty = self.fn_type.fnReturnType();
2254	try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
2255	switch (arch) {
2256	.i386 => {
2257	try self.code.append(0xc3); // ret
2258	},
2259	.arm, .armeb => {
2260	// Just add space for an instruction, patch this later
2261	try self.code.resize(self.code.items.len + 4);
2262	try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
2263	},
2264	else => return self.fail("TODO implement return for {}", .{self.target.cpu.arch}),
2265	}
2266	}
2267
2268	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
2269	const un_op = self.air.instructions.items(.data)[inst].un_op;
2270	const operand = try self.resolveInst(un_op);
2271	try self.ret(operand);
2272	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2273	}
2274
2275	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
2276	const un_op = self.air.instructions.items(.data)[inst].un_op;
2277	const ptr = try self.resolveInst(un_op);
2278	_ = ptr;
2279	return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch});
2280	//return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2281	}
2282
2283	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2284	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2285	if (self.liveness.isUnused(inst))
2286	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2287	const ty = self.air.typeOf(bin_op.lhs);
2288	assert(ty.eql(self.air.typeOf(bin_op.rhs)));
2289	if (ty.zigTypeTag() == .ErrorSet)
2290	return self.fail("TODO implement cmp for errors", .{});
2291
2292	const lhs = try self.resolveInst(bin_op.lhs);
2293	const rhs = try self.resolveInst(bin_op.rhs);
2294	const result: MCValue = switch (arch) {
2295	.arm, .armeb => result: {
2296	const lhs_is_register = lhs == .register;
2297	const rhs_is_register = rhs == .register;
2298	// lhs should always be a register
2299	const rhs_should_be_register = try self.armOperandShouldBeRegister(rhs);
2300
2301	var lhs_mcv = lhs;
2302	var rhs_mcv = rhs;
2303
2304	// Allocate registers
2305	if (rhs_should_be_register) {
2306	if (!lhs_is_register and !rhs_is_register) {
2307	const regs = try self.register_manager.allocRegs(2, .{
2308	Air.refToIndex(bin_op.rhs).?, Air.refToIndex(bin_op.lhs).?,
2309	}, &.{});
2310	lhs_mcv = MCValue{ .register = regs[0] };
2311	rhs_mcv = MCValue{ .register = regs[1] };
2312	} else if (!rhs_is_register) {
2313	rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.rhs).?, &.{}) };
2314	}
2315	}
2316	if (!lhs_is_register) {
2317	lhs_mcv = MCValue{ .register = try self.register_manager.allocReg(Air.refToIndex(bin_op.lhs).?, &.{}) };
2318	}
2319
2320	// Move the operands to the newly allocated registers
2321	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
2322	if (lhs_mcv == .register and !lhs_is_register) {
2323	try self.genSetReg(ty, lhs_mcv.register, lhs);
2324	branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.lhs).?, lhs);
2325	}
2326	if (rhs_mcv == .register and !rhs_is_register) {
2327	try self.genSetReg(ty, rhs_mcv.register, rhs);
2328	branch.inst_table.putAssumeCapacity(Air.refToIndex(bin_op.rhs).?, rhs);
2329	}
2330
2331	// The destination register is not present in the cmp instruction
2332	// The signedness of the integer does not matter for the cmp instruction
2333	try self.genArmBinOpCode(undefined, lhs_mcv, rhs_mcv, false, .cmp_eq, undefined);
2334
2335	break :result switch (ty.isSignedInt()) {
2336	true => MCValue{ .compare_flags_signed = op },
2337	false => MCValue{ .compare_flags_unsigned = op },
2338	};
2339	},
2340	else => return self.fail("TODO implement cmp for {}", .{self.target.cpu.arch}),
2341	};
2342	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2343	}
2344
2345	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
2346	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
2347	try self.dbgAdvancePCAndLine(dbg_stmt.line, dbg_stmt.column);
2348	return self.finishAirBookkeeping();
2349	}
2350
2351	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2352	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2353	const cond = try self.resolveInst(pl_op.operand);
2354	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
2355	const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
2356	const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
2357	const liveness_condbr = self.liveness.getCondBr(inst);
2358
2359	const reloc: Reloc = switch (arch) {
2360	.i386 => reloc: {
2361	try self.code.ensureUnusedCapacity(6);
2362
2363	const opcode: u8 = switch (cond) {
2364	.compare_flags_signed => \|cmp_op\| blk: {
2365	// Here we map to the opposite opcode because the jump is to the false branch.
2366	const opcode: u8 = switch (cmp_op) {
2367	.gte => 0x8c,
2368	.gt => 0x8e,
2369	.neq => 0x84,
2370	.lt => 0x8d,
2371	.lte => 0x8f,
2372	.eq => 0x85,
2373	};
2374	break :blk opcode;
2375	},
2376	.compare_flags_unsigned => \|cmp_op\| blk: {
2377	// Here we map to the opposite opcode because the jump is to the false branch.
2378	const opcode: u8 = switch (cmp_op) {
2379	.gte => 0x82,
2380	.gt => 0x86,
2381	.neq => 0x84,
2382	.lt => 0x83,
2383	.lte => 0x87,
2384	.eq => 0x85,
2385	};
2386	break :blk opcode;
2387	},
2388	.register => \|reg\| blk: {
2389	// test reg, 1
2390	// TODO detect al, ax, eax
2391	const Encoder = @import("arch/x86_64/bits.zig").Encoder;
2392	const encoder = try Encoder.init(self.code, 4);
2393	encoder.rex(.{
2394	// TODO audit this codegen: we force w = true here to make
2395	// the value affect the big register
2396	.w = true,
2397	.b = reg.isExtended(),
2398	});
2399	encoder.opcode_1byte(0xf6);
2400	encoder.modRm_direct(
2401	0,
2402	reg.low_id(),
2403	);
2404	encoder.disp8(1);
2405	break :blk 0x84;
2406	},
2407	else => return self.fail("TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
2408	};
2409	self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode });
2410	const reloc = Reloc{ .rel32 = self.code.items.len };
2411	self.code.items.len += 4;
2412	break :reloc reloc;
2413	},
2414	.arm, .armeb => reloc: {
2415	const condition: Condition = switch (cond) {
2416	.compare_flags_signed => \|cmp_op\| blk: {
2417	// Here we map to the opposite condition because the jump is to the false branch.
2418	const condition = Condition.fromCompareOperatorSigned(cmp_op);
2419	break :blk condition.negate();
2420	},
2421	.compare_flags_unsigned => \|cmp_op\| blk: {
2422	// Here we map to the opposite condition because the jump is to the false branch.
2423	const condition = Condition.fromCompareOperatorUnsigned(cmp_op);
2424	break :blk condition.negate();
2425	},
2426	.register => \|reg\| blk: {
2427	// cmp reg, 1
2428	// bne ...
2429	const op = Instruction.Operand.imm(1, 0);
2430	writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, reg, op).toU32());
2431	break :blk .ne;
2432	},
2433	else => return self.fail("TODO implement condbr {} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
2434	};
2435
2436	const reloc = Reloc{
2437	.arm_branch = .{
2438	.pos = self.code.items.len,
2439	.cond = condition,
2440	},
2441	};
2442	try self.code.resize(self.code.items.len + 4);
2443	break :reloc reloc;
2444	},
2445	else => return self.fail("TODO implement condbr {}", .{self.target.cpu.arch}),
2446	};
2447
2448	// Capture the state of register and stack allocation state so that we can revert to it.
2449	const parent_next_stack_offset = self.next_stack_offset;
2450	const parent_free_registers = self.register_manager.free_registers;
2451	var parent_stack = try self.stack.clone(self.gpa);
2452	defer parent_stack.deinit(self.gpa);
2453	const parent_registers = self.register_manager.registers;
2454
2455	try self.branch_stack.append(.{});
2456
2457	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
2458	for (liveness_condbr.then_deaths) \|operand\| {
2459	self.processDeath(operand);
2460	}
2461	try self.genBody(then_body);
2462
2463	// Revert to the previous register and stack allocation state.
2464
2465	var saved_then_branch = self.branch_stack.pop();
2466	defer saved_then_branch.deinit(self.gpa);
2467
2468	self.register_manager.registers = parent_registers;
2469
2470	self.stack.deinit(self.gpa);
2471	self.stack = parent_stack;
2472	parent_stack = .{};
2473
2474	self.next_stack_offset = parent_next_stack_offset;
2475	self.register_manager.free_registers = parent_free_registers;
2476
2477	try self.performReloc(reloc);
2478	const else_branch = self.branch_stack.addOneAssumeCapacity();
2479	else_branch.* = .{};
2480
2481	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
2482	for (liveness_condbr.else_deaths) \|operand\| {
2483	self.processDeath(operand);
2484	}
2485	try self.genBody(else_body);
2486
2487	// At this point, each branch will possibly have conflicting values for where
2488	// each instruction is stored. They agree, however, on which instructions are alive/dead.
2489	// We use the first ("then") branch as canonical, and here emit
2490	// instructions into the second ("else") branch to make it conform.
2491	// We continue respect the data structure semantic guarantees of the else_branch so
2492	// that we can use all the code emitting abstractions. This is why at the bottom we
2493	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
2494	// rather than assigning it.
2495	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
2496	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
2497
2498	const else_slice = else_branch.inst_table.entries.slice();
2499	const else_keys = else_slice.items(.key);
2500	const else_values = else_slice.items(.value);
2501	for (else_keys) \|else_key, else_idx\| {
2502	const else_value = else_values[else_idx];
2503	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
2504	// The instruction's MCValue is overridden in both branches.
2505	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
2506	if (else_value == .dead) {
2507	assert(then_entry.value == .dead);
2508	continue;
2509	}
2510	break :blk then_entry.value;
2511	} else blk: {
2512	if (else_value == .dead)
2513	continue;
2514	// The instruction is only overridden in the else branch.
2515	var i: usize = self.branch_stack.items.len - 2;
2516	while (true) {
2517	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
2518	if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
2519	assert(mcv != .dead);
2520	break :blk mcv;
2521	}
2522	}
2523	};
2524	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
2525	// TODO make sure the destination stack offset / register does not already have something
2526	// going on there.
2527	try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
2528	// TODO track the new register / stack allocation
2529	}
2530	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
2531	const then_slice = saved_then_branch.inst_table.entries.slice();
2532	const then_keys = then_slice.items(.key);
2533	const then_values = then_slice.items(.value);
2534	for (then_keys) \|then_key, then_idx\| {
2535	const then_value = then_values[then_idx];
2536	// We already deleted the items from this table that matched the else_branch.
2537	// So these are all instructions that are only overridden in the then branch.
2538	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
2539	if (then_value == .dead)
2540	continue;
2541	const parent_mcv = blk: {
2542	var i: usize = self.branch_stack.items.len - 2;
2543	while (true) {
2544	i -= 1;
2545	if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
2546	assert(mcv != .dead);
2547	break :blk mcv;
2548	}
2549	}
2550	};
2551	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
2552	// TODO make sure the destination stack offset / register does not already have something
2553	// going on there.
2554	try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
2555	// TODO track the new register / stack allocation
2556	}
2557
2558	self.branch_stack.pop().deinit(self.gpa);
2559
2560	return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
2561	}
2562
2563	fn isNull(self: *Self, operand: MCValue) !MCValue {
2564	_ = operand;
2565	// Here you can specialize this instruction if it makes sense to, otherwise the default
2566	// will call isNonNull and invert the result.
2567	switch (arch) {
2568	else => return self.fail("TODO call isNonNull and invert the result", .{}),
2569	}
2570	}
2571
2572	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
2573	_ = operand;
2574	// Here you can specialize this instruction if it makes sense to, otherwise the default
2575	// will call isNull and invert the result.
2576	switch (arch) {
2577	else => return self.fail("TODO call isNull and invert the result", .{}),
2578	}
2579	}
2580
2581	fn isErr(self: *Self, operand: MCValue) !MCValue {
2582	_ = operand;
2583	// Here you can specialize this instruction if it makes sense to, otherwise the default
2584	// will call isNonNull and invert the result.
2585	switch (arch) {
2586	else => return self.fail("TODO call isNonErr and invert the result", .{}),
2587	}
2588	}
2589
2590	fn isNonErr(self: *Self, operand: MCValue) !MCValue {
2591	_ = operand;
2592	// Here you can specialize this instruction if it makes sense to, otherwise the default
2593	// will call isNull and invert the result.
2594	switch (arch) {
2595	else => return self.fail("TODO call isErr and invert the result", .{}),
2596	}
2597	}
2598
2599	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
2600	const un_op = self.air.instructions.items(.data)[inst].un_op;
2601	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2602	const operand = try self.resolveInst(un_op);
2603	break :result try self.isNull(operand);
2604	};
2605	return self.finishAir(inst, result, .{ un_op, .none, .none });
2606	}
2607
2608	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2609	const un_op = self.air.instructions.items(.data)[inst].un_op;
2610	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2611	const operand_ptr = try self.resolveInst(un_op);
2612	const operand: MCValue = blk: {
2613	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2614	// The MCValue that holds the pointer can be re-used as the value.
2615	break :blk operand_ptr;
2616	} else {
2617	break :blk try self.allocRegOrMem(inst, true);
2618	}
2619	};
2620	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2621	break :result try self.isNull(operand);
2622	};
2623	return self.finishAir(inst, result, .{ un_op, .none, .none });
2624	}
2625
2626	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
2627	const un_op = self.air.instructions.items(.data)[inst].un_op;
2628	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2629	const operand = try self.resolveInst(un_op);
2630	break :result try self.isNonNull(operand);
2631	};
2632	return self.finishAir(inst, result, .{ un_op, .none, .none });
2633	}
2634
2635	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2636	const un_op = self.air.instructions.items(.data)[inst].un_op;
2637	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2638	const operand_ptr = try self.resolveInst(un_op);
2639	const operand: MCValue = blk: {
2640	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2641	// The MCValue that holds the pointer can be re-used as the value.
2642	break :blk operand_ptr;
2643	} else {
2644	break :blk try self.allocRegOrMem(inst, true);
2645	}
2646	};
2647	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2648	break :result try self.isNonNull(operand);
2649	};
2650	return self.finishAir(inst, result, .{ un_op, .none, .none });
2651	}
2652
2653	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
2654	const un_op = self.air.instructions.items(.data)[inst].un_op;
2655	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2656	const operand = try self.resolveInst(un_op);
2657	break :result try self.isErr(operand);
2658	};
2659	return self.finishAir(inst, result, .{ un_op, .none, .none });
2660	}
2661
2662	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2663	const un_op = self.air.instructions.items(.data)[inst].un_op;
2664	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2665	const operand_ptr = try self.resolveInst(un_op);
2666	const operand: MCValue = blk: {
2667	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2668	// The MCValue that holds the pointer can be re-used as the value.
2669	break :blk operand_ptr;
2670	} else {
2671	break :blk try self.allocRegOrMem(inst, true);
2672	}
2673	};
2674	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2675	break :result try self.isErr(operand);
2676	};
2677	return self.finishAir(inst, result, .{ un_op, .none, .none });
2678	}
2679
2680	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
2681	const un_op = self.air.instructions.items(.data)[inst].un_op;
2682	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2683	const operand = try self.resolveInst(un_op);
2684	break :result try self.isNonErr(operand);
2685	};
2686	return self.finishAir(inst, result, .{ un_op, .none, .none });
2687	}
2688
2689	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2690	const un_op = self.air.instructions.items(.data)[inst].un_op;
2691	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2692	const operand_ptr = try self.resolveInst(un_op);
2693	const operand: MCValue = blk: {
2694	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2695	// The MCValue that holds the pointer can be re-used as the value.
2696	break :blk operand_ptr;
2697	} else {
2698	break :blk try self.allocRegOrMem(inst, true);
2699	}
2700	};
2701	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2702	break :result try self.isNonErr(operand);
2703	};
2704	return self.finishAir(inst, result, .{ un_op, .none, .none });
2705	}
2706
2707	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
2708	// A loop is a setup to be able to jump back to the beginning.
2709	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2710	const loop = self.air.extraData(Air.Block, ty_pl.payload);
2711	const body = self.air.extra[loop.end..][0..loop.data.body_len];
2712	const start_index = self.code.items.len;
2713	try self.genBody(body);
2714	try self.jump(start_index);
2715	return self.finishAirBookkeeping();
2716	}
2717
2718	/// Send control flow to the `index` of `self.code`.
2719	fn jump(self: *Self, index: usize) !void {
2720	switch (arch) {
2721	.i386 => {
2722	try self.code.ensureUnusedCapacity(5);
2723	if (math.cast(i8, @intCast(i32, index) - (@intCast(i32, self.code.items.len + 2)))) \|delta\| {
2724	self.code.appendAssumeCapacity(0xeb); // jmp rel8
2725	self.code.appendAssumeCapacity(@bitCast(u8, delta));
2726	} else \|_\| {
2727	const delta = @intCast(i32, index) - (@intCast(i32, self.code.items.len + 5));
2728	self.code.appendAssumeCapacity(0xe9); // jmp rel32
2729	mem.writeIntLittle(i32, self.code.addManyAsArrayAssumeCapacity(4), delta);
2730	}
2731	},
2732	.arm, .armeb => {
2733	if (math.cast(i26, @intCast(i32, index) - @intCast(i32, self.code.items.len + 8))) \|delta\| {
2734	writeInt(u32, try self.code.addManyAsArray(4), Instruction.b(.al, delta).toU32());
2735	} else \|_\| {
2736	return self.fail("TODO: enable larger branch offset", .{});
2737	}
2738	},
2739	else => return self.fail("TODO implement jump for {}", .{self.target.cpu.arch}),
2740	}
2741	}
2742
2743	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
2744	try self.blocks.putNoClobber(self.gpa, inst, .{
2745	// A block is a setup to be able to jump to the end.
2746	.relocs = .{},
2747	// It also acts as a receptacle for break operands.
2748	// Here we use `MCValue.none` to represent a null value so that the first
2749	// break instruction will choose a MCValue for the block result and overwrite
2750	// this field. Following break instructions will use that MCValue to put their
2751	// block results.
2752	.mcv = MCValue{ .none = {} },
2753	});
2754	const block_data = self.blocks.getPtr(inst).?;
2755	defer block_data.relocs.deinit(self.gpa);
2756
2757	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2758	const extra = self.air.extraData(Air.Block, ty_pl.payload);
2759	const body = self.air.extra[extra.end..][0..extra.data.body_len];
2760	try self.genBody(body);
2761
2762	for (block_data.relocs.items) \|reloc\| try self.performReloc(reloc);
2763
2764	const result = @bitCast(MCValue, block_data.mcv);
2765	return self.finishAir(inst, result, .{ .none, .none, .none });
2766	}
2767
2768	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
2769	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2770	const condition = pl_op.operand;
2771	switch (arch) {
2772	else => return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch}),
2773	}
2774	return self.finishAir(inst, .dead, .{ condition, .none, .none });
2775	}
2776
2777	fn performReloc(self: *Self, reloc: Reloc) !void {
2778	switch (reloc) {
2779	.rel32 => \|pos\| {
2780	const amt = self.code.items.len - (pos + 4);
2781	// Here it would be tempting to implement testing for amt == 0 and then elide the
2782	// jump. However, that will cause a problem because other jumps may assume that they
2783	// can jump to this code. Or maybe I didn't understand something when I was debugging.
2784	// It could be worth another look. Anyway, that's why that isn't done here. Probably the
2785	// best place to elide jumps will be in semantic analysis, by inlining blocks that only
2786	// only have 1 break instruction.
2787	const s32_amt = math.cast(i32, amt) catch
2788	return self.fail("unable to perform relocation: jump too far", .{});
2789	mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt);
2790	},
2791	.arm_branch => \|info\| {
2792	switch (arch) {
2793	.arm, .armeb => {
2794	const amt = @intCast(i32, self.code.items.len) - @intCast(i32, info.pos + 8);
2795	if (math.cast(i26, amt)) \|delta\| {
2796	writeInt(u32, self.code.items[info.pos..][0..4], Instruction.b(info.cond, delta).toU32());
2797	} else \|_\| {
2798	return self.fail("TODO: enable larger branch offset", .{});
2799	}
2800	},
2801	else => unreachable, // attempting to perform an ARM relocation on a non-ARM target arch
2802	}
2803	},
2804	}
2805	}
2806
2807	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
2808	const branch = self.air.instructions.items(.data)[inst].br;
2809	try self.br(branch.block_inst, branch.operand);
2810	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
2811	}
2812
2813	fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
2814	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2815	const air_tags = self.air.instructions.items(.tag);
2816	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
2817	.arm, .armeb => switch (air_tags[inst]) {
2818	.bool_and => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_and),
2819	.bool_or => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_or),
2820	else => unreachable, // Not a boolean operation
2821	},
2822	else => return self.fail("TODO implement boolean operations for {}", .{self.target.cpu.arch}),
2823	};
2824	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2825	}
2826
2827	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
2828	const block_data = self.blocks.getPtr(block).?;
2829
2830	if (self.air.typeOf(operand).hasCodeGenBits()) {
2831	const operand_mcv = try self.resolveInst(operand);
2832	const block_mcv = block_data.mcv;
2833	if (block_mcv == .none) {
2834	block_data.mcv = operand_mcv;
2835	} else {
2836	try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv);
2837	}
2838	}
2839	return self.brVoid(block);
2840	}
2841
2842	fn brVoid(self: *Self, block: Air.Inst.Index) !void {
2843	const block_data = self.blocks.getPtr(block).?;
2844
2845	// Emit a jump with a relocation. It will be patched up after the block ends.
2846	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
2847
2848	switch (arch) {
2849	.i386 => {
2850	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
2851	// which is available if the jump is 127 bytes or less forward.
2852	try self.code.resize(self.code.items.len + 5);
2853	self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
2854	// Leave the jump offset undefined
2855	block_data.relocs.appendAssumeCapacity(.{ .rel32 = self.code.items.len - 4 });
2856	},
2857	.arm, .armeb => {
2858	try self.code.resize(self.code.items.len + 4);
2859	block_data.relocs.appendAssumeCapacity(.{
2860	.arm_branch = .{
2861	.pos = self.code.items.len - 4,
2862	.cond = .al,
2863	},
2864	});
2865	},
2866	else => return self.fail("TODO implement brvoid for {}", .{self.target.cpu.arch}),
2867	}
2868	}
2869
2870	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
2871	const air_datas = self.air.instructions.items(.data);
2872	const air_extra = self.air.extraData(Air.Asm, air_datas[inst].ty_pl.payload);
2873	const zir = self.mod_fn.owner_decl.getFileScope().zir;
2874	const extended = zir.instructions.items(.data)[air_extra.data.zir_index].extended;
2875	const zir_extra = zir.extraData(Zir.Inst.Asm, extended.operand);
2876	const asm_source = zir.nullTerminatedString(zir_extra.data.asm_source);
2877	const outputs_len = @truncate(u5, extended.small);
2878	const args_len = @truncate(u5, extended.small >> 5);
2879	const clobbers_len = @truncate(u5, extended.small >> 10);
2880	_ = clobbers_len; // TODO honor these
2881	const is_volatile = @truncate(u1, extended.small >> 15) != 0;
2882	const outputs = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end..][0..outputs_len]);
2883	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end + outputs.len ..][0..args_len]);
2884
2885	if (outputs_len > 1) {
2886	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
2887	}
2888	var extra_i: usize = zir_extra.end;
2889	const output_constraint: ?[]const u8 = out: {
2890	var i: usize = 0;
2891	while (i < outputs_len) : (i += 1) {
2892	const output = zir.extraData(Zir.Inst.Asm.Output, extra_i);
2893	extra_i = output.end;
2894	break :out zir.nullTerminatedString(output.data.constraint);
2895	}
2896	break :out null;
2897	};
2898
2899	const dead = !is_volatile and self.liveness.isUnused(inst);
2900	const result: MCValue = if (dead) .dead else switch (arch) {
2901	.arm, .armeb => result: {
2902	for (args) \|arg\| {
2903	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
2904	extra_i = input.end;
2905	const constraint = zir.nullTerminatedString(input.data.constraint);
2906
2907	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
2908	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
2909	}
2910	const reg_name = constraint[1 .. constraint.len - 1];
2911	const reg = parseRegName(reg_name) orelse
2912	return self.fail("unrecognized register: '{s}'", .{reg_name});
2913
2914	const arg_mcv = try self.resolveInst(arg);
2915	try self.register_manager.getReg(reg, null);
2916	try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
2917	}
2918
2919	if (mem.eql(u8, asm_source, "svc #0")) {
2920	writeInt(u32, try self.code.addManyAsArray(4), Instruction.svc(.al, 0).toU32());
2921	} else {
2922	return self.fail("TODO implement support for more arm assembly instructions", .{});
2923	}
2924
2925	if (output_constraint) \|output\| {
2926	if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
2927	return self.fail("unrecognized asm output constraint: '{s}'", .{output});
2928	}
2929	const reg_name = output[2 .. output.len - 1];
2930	const reg = parseRegName(reg_name) orelse
2931	return self.fail("unrecognized register: '{s}'", .{reg_name});
2932
2933	break :result MCValue{ .register = reg };
2934	} else {
2935	break :result MCValue{ .none = {} };
2936	}
2937	},
2938	.i386 => result: {
2939	for (args) \|arg\| {
2940	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
2941	extra_i = input.end;
2942	const constraint = zir.nullTerminatedString(input.data.constraint);
2943
2944	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
2945	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
2946	}
2947	const reg_name = constraint[1 .. constraint.len - 1];
2948	const reg = parseRegName(reg_name) orelse
2949	return self.fail("unrecognized register: '{s}'", .{reg_name});
2950
2951	const arg_mcv = try self.resolveInst(arg);
2952	try self.register_manager.getReg(reg, null);
2953	try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
2954	}
2955
2956	{
2957	var iter = std.mem.tokenize(u8, asm_source, "\n\r");
2958	while (iter.next()) \|ins\| {
2959	if (mem.eql(u8, ins, "syscall")) {
2960	try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });
2961	} else if (mem.indexOf(u8, ins, "push")) \|_\| {
2962	const arg = ins[4..];
2963	if (mem.indexOf(u8, arg, "$")) \|l\| {
2964	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch return self.fail("TODO implement more inline asm int parsing", .{});
2965	try self.code.appendSlice(&.{ 0x6a, n });
2966	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
2967	const reg_name = ins[4 + l + 2 ..];
2968	const reg = parseRegName(reg_name) orelse
2969	return self.fail("unrecognized register: '{s}'", .{reg_name});
2970	const low_id: u8 = reg.low_id();
2971	if (reg.isExtended()) {
2972	try self.code.appendSlice(&.{ 0x41, 0b1010000 \| low_id });
2973	} else {
2974	try self.code.append(0b1010000 \| low_id);
2975	}
2976	} else return self.fail("TODO more push operands", .{});
2977	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
2978	const arg = ins[3..];
2979	if (mem.indexOf(u8, arg, "%%")) \|l\| {
2980	const reg_name = ins[3 + l + 2 ..];
2981	const reg = parseRegName(reg_name) orelse
2982	return self.fail("unrecognized register: '{s}'", .{reg_name});
2983	const low_id: u8 = reg.low_id();
2984	if (reg.isExtended()) {
2985	try self.code.appendSlice(&.{ 0x41, 0b1011000 \| low_id });
2986	} else {
2987	try self.code.append(0b1011000 \| low_id);
2988	}
2989	} else return self.fail("TODO more pop operands", .{});
2990	} else {
2991	return self.fail("TODO implement support for more x86 assembly instructions", .{});
2992	}
2993	}
2994	}
2995
2996	if (output_constraint) \|output\| {
2997	if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
2998	return self.fail("unrecognized asm output constraint: '{s}'", .{output});
2999	}
3000	const reg_name = output[2 .. output.len - 1];
3001	const reg = parseRegName(reg_name) orelse
3002	return self.fail("unrecognized register: '{s}'", .{reg_name});
3003	break :result MCValue{ .register = reg };
3004	} else {
3005	break :result MCValue{ .none = {} };
3006	}
3007	},
3008	else => return self.fail("TODO implement inline asm support for more architectures", .{}),
3009	};
3010	if (outputs.len + args.len <= Liveness.bpi - 1) {
3011	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
3012	std.mem.copy(Air.Inst.Ref, &buf, outputs);
3013	std.mem.copy(Air.Inst.Ref, buf[outputs.len..], args);
3014	return self.finishAir(inst, result, buf);
3015	}
3016	var bt = try self.iterateBigTomb(inst, outputs.len + args.len);
3017	for (outputs) \|output\| {
3018	bt.feed(output);
3019	}
3020	for (args) \|arg\| {
3021	bt.feed(arg);
3022	}
3023	return bt.finishAir(result);
3024	}
3025
3026	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
3027	try self.ensureProcessDeathCapacity(operand_count + 1);
3028	return BigTomb{
3029	.function = self,
3030	.inst = inst,
3031	.tomb_bits = self.liveness.getTombBits(inst),
3032	.big_tomb_bits = self.liveness.special.get(inst) orelse 0,
3033	.bit_index = 0,
3034	};
3035	}
3036
3037	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
3038	fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
3039	switch (loc) {
3040	.none => return,
3041	.register => \|reg\| return self.genSetReg(ty, reg, val),
3042	.stack_offset => \|off\| return self.genSetStack(ty, off, val),
3043	.memory => {
3044	return self.fail("TODO implement setRegOrMem for memory", .{});
3045	},
3046	else => unreachable,
3047	}
3048	}
3049
3050	fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
3051	switch (arch) {
3052	.arm, .armeb => switch (mcv) {
3053	.dead => unreachable,
3054	.ptr_stack_offset => unreachable,
3055	.ptr_embedded_in_code => unreachable,
3056	.unreach, .none => return, // Nothing to do.
3057	.undef => {
3058	if (!self.wantSafety())
3059	return; // The already existing value will do just fine.
3060	// TODO Upgrade this to a memset call when we have that available.
3061	switch (ty.abiSize(self.target.*)) {
3062	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
3063	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
3064	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
3065	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
3066	else => return self.fail("TODO implement memset", .{}),
3067	}
3068	},
3069	.compare_flags_unsigned,
3070	.compare_flags_signed,
3071	.immediate,
3072	=> {
3073	const reg = try self.copyToTmpRegister(ty, mcv);
3074	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3075	},
3076	.embedded_in_code => \|code_offset\| {
3077	_ = code_offset;
3078	return self.fail("TODO implement set stack variable from embedded_in_code", .{});
3079	},
3080	.register => \|reg\| {
3081	const abi_size = ty.abiSize(self.target.*);
3082	const adj_off = stack_offset + abi_size;
3083
3084	switch (abi_size) {
3085	1, 4 => {
3086	const offset = if (math.cast(u12, adj_off)) \|imm\| blk: {
3087	break :blk Instruction.Offset.imm(imm);
3088	} else \|_\| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
3089	const str = switch (abi_size) {
3090	1 => Instruction.strb,
3091	4 => Instruction.str,
3092	else => unreachable,
3093	};
3094
3095	writeInt(u32, try self.code.addManyAsArray(4), str(.al, reg, .fp, .{
3096	.offset = offset,
3097	.positive = false,
3098	}).toU32());
3099	},
3100	2 => {
3101	const offset = if (adj_off <= math.maxInt(u8)) blk: {
3102	break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
3103	} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
3104
3105	writeInt(u32, try self.code.addManyAsArray(4), Instruction.strh(.al, reg, .fp, .{
3106	.offset = offset,
3107	.positive = false,
3108	}).toU32());
3109	},
3110	else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
3111	}
3112	},
3113	.memory => \|vaddr\| {
3114	_ = vaddr;
3115	return self.fail("TODO implement set stack variable from memory vaddr", .{});
3116	},
3117	.stack_offset => \|off\| {
3118	if (stack_offset == off)
3119	return; // Copy stack variable to itself; nothing to do.
3120
3121	const reg = try self.copyToTmpRegister(ty, mcv);
3122	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3123	},
3124	},
3125	else => return self.fail("TODO implement getSetStack for {}", .{self.target.cpu.arch}),
3126	}
3127	}
3128
3129	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
3130	switch (arch) {
3131	.arm, .armeb => switch (mcv) {
3132	.dead => unreachable,
3133	.ptr_stack_offset => unreachable,
3134	.ptr_embedded_in_code => unreachable,
3135	.unreach, .none => return, // Nothing to do.
3136	.undef => {
3137	if (!self.wantSafety())
3138	return; // The already existing value will do just fine.
3139	// Write the debug undefined value.
3140	return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa });
3141	},
3142	.compare_flags_unsigned,
3143	.compare_flags_signed,
3144	=> \|op\| {
3145	const condition = switch (mcv) {
3146	.compare_flags_unsigned => Condition.fromCompareOperatorUnsigned(op),
3147	.compare_flags_signed => Condition.fromCompareOperatorSigned(op),
3148	else => unreachable,
3149	};
3150
3151	// mov reg, 0
3152	// moveq reg, 1
3153	const zero = Instruction.Operand.imm(0, 0);
3154	const one = Instruction.Operand.imm(1, 0);
3155	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, zero).toU32());
3156	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(condition, reg, one).toU32());
3157	},
3158	.immediate => \|x\| {
3159	if (x > math.maxInt(u32)) return self.fail("ARM registers are 32-bit wide", .{});
3160
3161	if (Instruction.Operand.fromU32(@intCast(u32, x))) \|op\| {
3162	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, op).toU32());
3163	} else if (Instruction.Operand.fromU32(~@intCast(u32, x))) \|op\| {
3164	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mvn(.al, reg, op).toU32());
3165	} else if (x <= math.maxInt(u16)) {
3166	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
3167	writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @intCast(u16, x)).toU32());
3168	} else {
3169	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
3170	writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
3171	}
3172	} else {
3173	// TODO write constant to code and load
3174	// relative to pc
3175	if (Target.arm.featureSetHas(self.target.cpu.features, .has_v7)) {
3176	// immediate: 0xaaaabbbb
3177	// movw reg, #0xbbbb
3178	// movt reg, #0xaaaa
3179	writeInt(u32, try self.code.addManyAsArray(4), Instruction.movw(.al, reg, @truncate(u16, x)).toU32());
3180	writeInt(u32, try self.code.addManyAsArray(4), Instruction.movt(.al, reg, @truncate(u16, x >> 16)).toU32());
3181	} else {
3182	// immediate: 0xaabbccdd
3183	// mov reg, #0xaa
3184	// orr reg, reg, #0xbb, 24
3185	// orr reg, reg, #0xcc, 16
3186	// orr reg, reg, #0xdd, 8
3187	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.imm(@truncate(u8, x), 0)).toU32());
3188	writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 8), 12)).toU32());
3189	writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 16), 8)).toU32());
3190	writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, reg, reg, Instruction.Operand.imm(@truncate(u8, x >> 24), 4)).toU32());
3191	}
3192	}
3193	},
3194	.register => \|src_reg\| {
3195	// If the registers are the same, nothing to do.
3196	if (src_reg.id() == reg.id())
3197	return;
3198
3199	// mov reg, src_reg
3200	writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, Instruction.Operand.reg(src_reg, Instruction.Operand.Shift.none)).toU32());
3201	},
3202	.memory => \|addr\| {
3203	// The value is in memory at a hard-coded address.
3204	// If the type is a pointer, it means the pointer address is at this memory location.
3205	try self.genSetReg(ty, reg, .{ .immediate = addr });
3206	writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, reg, reg, .{ .offset = Instruction.Offset.none }).toU32());
3207	},
3208	.stack_offset => \|unadjusted_off\| {
3209	// TODO: maybe addressing from sp instead of fp
3210	const abi_size = ty.abiSize(self.target.*);
3211	const adj_off = unadjusted_off + abi_size;
3212
3213	switch (abi_size) {
3214	1, 4 => {
3215	const offset = if (adj_off <= math.maxInt(u12)) blk: {
3216	break :blk Instruction.Offset.imm(@intCast(u12, adj_off));
3217	} else Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0);
3218	const ldr = switch (abi_size) {
3219	1 => Instruction.ldrb,
3220	4 => Instruction.ldr,
3221	else => unreachable,
3222	};
3223
3224	writeInt(u32, try self.code.addManyAsArray(4), ldr(.al, reg, .fp, .{
3225	.offset = offset,
3226	.positive = false,
3227	}).toU32());
3228	},
3229	2 => {
3230	const offset = if (adj_off <= math.maxInt(u8)) blk: {
3231	break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off));
3232	} else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }));
3233
3234	writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldrh(.al, reg, .fp, .{
3235	.offset = offset,
3236	.positive = false,
3237	}).toU32());
3238	},
3239	else => return self.fail("TODO a type of size {} is not allowed in a register", .{abi_size}),
3240	}
3241	},
3242	else => return self.fail("TODO implement getSetReg for arm {}", .{mcv}),
3243	},
3244	else => return self.fail("TODO implement getSetReg for {}", .{self.target.cpu.arch}),
3245	}
3246	}
3247
3248	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
3249	const un_op = self.air.instructions.items(.data)[inst].un_op;
3250	const result = try self.resolveInst(un_op);
3251	return self.finishAir(inst, result, .{ un_op, .none, .none });
3252	}
3253
3254	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
3255	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3256	const result = try self.resolveInst(ty_op.operand);
3257	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3258	}
3259
3260	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
3261	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3262	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3263	else => return self.fail("TODO implement airArrayToSlice for {}", .{
3264	self.target.cpu.arch,
3265	}),
3266	};
3267	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3268	}
3269
3270	fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
3271	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3272	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3273	else => return self.fail("TODO implement airIntToFloat for {}", .{
3274	self.target.cpu.arch,
3275	}),
3276	};
3277	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3278	}
3279
3280	fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
3281	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3282	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3283	else => return self.fail("TODO implement airFloatToInt for {}", .{
3284	self.target.cpu.arch,
3285	}),
3286	};
3287	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3288	}
3289
3290	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
3291	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
3292	const extra = self.air.extraData(Air.Block, ty_pl.payload);
3293	const result: MCValue = switch (arch) {
3294	else => return self.fail("TODO implement airCmpxchg for {}", .{
3295	self.target.cpu.arch,
3296	}),
3297	};
3298	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
3299	}
3300
3301	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
3302	_ = inst;
3303	return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
3304	}
3305
3306	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
3307	_ = inst;
3308	return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
3309	}
3310
3311	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
3312	_ = inst;
3313	_ = order;
3314	return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
3315	}
3316
3317	fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
3318	_ = inst;
3319	return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
3320	}
3321
3322	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
3323	_ = inst;
3324	return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
3325	}
3326
3327	fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3328	// First section of indexes correspond to a set number of constant values.
3329	const ref_int = @enumToInt(inst);
3330	if (ref_int < Air.Inst.Ref.typed_value_map.len) {
3331	const tv = Air.Inst.Ref.typed_value_map[ref_int];
3332	if (!tv.ty.hasCodeGenBits()) {
3333	return MCValue{ .none = {} };
3334	}
3335	return self.genTypedValue(tv);
3336	}
3337
3338	// If the type has no codegen bits, no need to store it.
3339	const inst_ty = self.air.typeOf(inst);
3340	if (!inst_ty.hasCodeGenBits())
3341	return MCValue{ .none = {} };
3342
3343	const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
3344	switch (self.air.instructions.items(.tag)[inst_index]) {
3345	.constant => {
3346	// Constants have static lifetimes, so they are always memoized in the outer most table.
3347	const branch = &self.branch_stack.items[0];
3348	const gop = try branch.inst_table.getOrPut(self.gpa, inst_index);
3349	if (!gop.found_existing) {
3350	const ty_pl = self.air.instructions.items(.data)[inst_index].ty_pl;
3351	gop.value_ptr.* = try self.genTypedValue(.{
3352	.ty = inst_ty,
3353	.val = self.air.values[ty_pl.payload],
3354	});
3355	}
3356	return gop.value_ptr.*;
3357	},
3358	.const_ty => unreachable,
3359	else => return self.getResolvedInstValue(inst_index),
3360	}
3361	}
3362
3363	fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
3364	// Treat each stack item as a "layer" on top of the previous one.
3365	var i: usize = self.branch_stack.items.len;
3366	while (true) {
3367	i -= 1;
3368	if (self.branch_stack.items[i].inst_table.get(inst)) \|mcv\| {
3369	assert(mcv != .dead);
3370	return mcv;
3371	}
3372	}
3373	}
3374
3375	/// If the MCValue is an immediate, and it does not fit within this type,
3376	/// we put it in a register.
3377	/// A potential opportunity for future optimization here would be keeping track
3378	/// of the fact that the instruction is available both as an immediate
3379	/// and as a register.
3380	fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
3381	const mcv = try self.resolveInst(operand);
3382	const ti = @typeInfo(T).Int;
3383	switch (mcv) {
3384	.immediate => \|imm\| {
3385	// This immediate is unsigned.
3386	const U = std.meta.Int(.unsigned, ti.bits - @boolToInt(ti.signedness == .signed));
3387	if (imm >= math.maxInt(U)) {
3388	return MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.usize), mcv) };
3389	}
3390	},
3391	else => {},
3392	}
3393	return mcv;
3394	}
3395
3396	fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
3397	if (typed_value.val.isUndef())
3398	return MCValue{ .undef = {} };
3399	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
3400	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
3401	switch (typed_value.ty.zigTypeTag()) {
3402	.Pointer => switch (typed_value.ty.ptrSize()) {
3403	.Slice => {
3404	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
3405	const ptr_type = typed_value.ty.slicePtrFieldType(&buf);
3406	const ptr_mcv = try self.genTypedValue(.{ .ty = ptr_type, .val = typed_value.val });
3407	const slice_len = typed_value.val.sliceLen();
3408	// Codegen can't handle some kinds of indirection. If the wrong union field is accessed here it may mean
3409	// the Sema code needs to use anonymous Decls or alloca instructions to store data.
3410	const ptr_imm = ptr_mcv.memory;
3411	_ = slice_len;
3412	_ = ptr_imm;
3413	// We need more general support for const data being stored in memory to make this work.
3414	return self.fail("TODO codegen for const slices", .{});
3415	},
3416	else => {
3417	if (typed_value.val.castTag(.decl_ref)) \|payload\| {
3418	const decl = payload.data;
3419	decl.alive = true;
3420	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
3421	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
3422	const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes;
3423	return MCValue{ .memory = got_addr };
3424	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
3425	// TODO I'm hacking my way through here by repurposing .memory for storing
3426	// index to the GOT target symbol index.
3427	return MCValue{ .memory = decl.link.macho.local_sym_index };
3428	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
3429	const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes;
3430	return MCValue{ .memory = got_addr };
3431	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
3432	try p9.seeDecl(decl);
3433	const got_addr = p9.bases.data + decl.link.plan9.got_index.? * ptr_bytes;
3434	return MCValue{ .memory = got_addr };
3435	} else {
3436	return self.fail("TODO codegen non-ELF const Decl pointer", .{});
3437	}
3438	}
3439	if (typed_value.val.tag() == .int_u64) {
3440	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
3441	}
3442	return self.fail("TODO codegen more kinds of const pointers", .{});
3443	},
3444	},
3445	.Int => {
3446	const info = typed_value.ty.intInfo(self.target.*);
3447	if (info.bits > ptr_bits or info.signedness == .signed) {
3448	return self.fail("TODO const int bigger than ptr and signed int", .{});
3449	}
3450	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
3451	},
3452	.Bool => {
3453	return MCValue{ .immediate = @boolToInt(typed_value.val.toBool()) };
3454	},
3455	.ComptimeInt => unreachable, // semantic analysis prevents this
3456	.ComptimeFloat => unreachable, // semantic analysis prevents this
3457	.Optional => {
3458	if (typed_value.ty.isPtrLikeOptional()) {
3459	if (typed_value.val.isNull())
3460	return MCValue{ .immediate = 0 };
3461
3462	var buf: Type.Payload.ElemType = undefined;
3463	return self.genTypedValue(.{
3464	.ty = typed_value.ty.optionalChild(&buf),
3465	.val = typed_value.val,
3466	});
3467	} else if (typed_value.ty.abiSize(self.target.*) == 1) {
3468	return MCValue{ .immediate = @boolToInt(typed_value.val.isNull()) };
3469	}
3470	return self.fail("TODO non pointer optionals", .{});
3471	},
3472	.Enum => {
3473	if (typed_value.val.castTag(.enum_field_index)) \|field_index\| {
3474	switch (typed_value.ty.tag()) {
3475	.enum_simple => {
3476	return MCValue{ .immediate = field_index.data };
3477	},
3478	.enum_full, .enum_nonexhaustive => {
3479	const enum_full = typed_value.ty.cast(Type.Payload.EnumFull).?.data;
3480	if (enum_full.values.count() != 0) {
3481	const tag_val = enum_full.values.keys()[field_index.data];
3482	return self.genTypedValue(.{ .ty = enum_full.tag_ty, .val = tag_val });
3483	} else {
3484	return MCValue{ .immediate = field_index.data };
3485	}
3486	},
3487	else => unreachable,
3488	}
3489	} else {
3490	var int_tag_buffer: Type.Payload.Bits = undefined;
3491	const int_tag_ty = typed_value.ty.intTagType(&int_tag_buffer);
3492	return self.genTypedValue(.{ .ty = int_tag_ty, .val = typed_value.val });
3493	}
3494	},
3495	.ErrorSet => {
3496	switch (typed_value.val.tag()) {
3497	.@"error" => {
3498	const err_name = typed_value.val.castTag(.@"error").?.data.name;
3499	const module = self.bin_file.options.module.?;
3500	const global_error_set = module.global_error_set;
3501	const error_index = global_error_set.get(err_name).?;
3502	return MCValue{ .immediate = error_index };
3503	},
3504	else => {
3505	// In this case we are rendering an error union which has a 0 bits payload.
3506	return MCValue{ .immediate = 0 };
3507	},
3508	}
3509	},
3510	.ErrorUnion => {
3511	const error_type = typed_value.ty.errorUnionSet();
3512	const payload_type = typed_value.ty.errorUnionPayload();
3513	const sub_val = typed_value.val.castTag(.eu_payload).?.data;
3514
3515	if (!payload_type.hasCodeGenBits()) {
3516	// We use the error type directly as the type.
3517	return self.genTypedValue(.{ .ty = error_type, .val = sub_val });
3518	}
3519
3520	return self.fail("TODO implement error union const of type '{}'", .{typed_value.ty});
3521	},
3522	else => return self.fail("TODO implement const of type '{}'", .{typed_value.ty}),
3523	}
3524	}
3525
3526	const CallMCValues = struct {
3527	args: []MCValue,
3528	return_value: MCValue,
3529	stack_byte_count: u32,
3530	stack_align: u32,
3531
3532	fn deinit(self: CallMCValues, func: Self) void {
3533	func.gpa.free(self.args);
3534	self.* = undefined;
3535	}
3536	};
3537
3538	/// Caller must call `CallMCValues.deinit`.
3539	fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
3540	const cc = fn_ty.fnCallingConvention();
3541	const param_types = try self.gpa.alloc(Type, fn_ty.fnParamLen());
3542	defer self.gpa.free(param_types);
3543	fn_ty.fnParamTypes(param_types);
3544	var result: CallMCValues = .{
3545	.args = try self.gpa.alloc(MCValue, param_types.len),
3546	// These undefined values must be populated before returning from this function.
3547	.return_value = undefined,
3548	.stack_byte_count = undefined,
3549	.stack_align = undefined,
3550	};
3551	errdefer self.gpa.free(result.args);
3552
3553	const ret_ty = fn_ty.fnReturnType();
3554
3555	switch (arch) {
3556	.arm, .armeb => {
3557	switch (cc) {
3558	.Naked => {
3559	assert(result.args.len == 0);
3560	result.return_value = .{ .unreach = {} };
3561	result.stack_byte_count = 0;
3562	result.stack_align = 1;
3563	return result;
3564	},
3565	.Unspecified, .C => {
3566	// ARM Procedure Call Standard, Chapter 6.5
3567	var ncrn: usize = 0; // Next Core Register Number
3568	var nsaa: u32 = 0; // Next stacked argument address
3569
3570	for (param_types) \|ty, i\| {
3571	if (ty.abiAlignment(self.target.*) == 8)
3572	ncrn = std.mem.alignForwardGeneric(usize, ncrn, 2);
3573
3574	const param_size = @intCast(u32, ty.abiSize(self.target.*));
3575	if (std.math.divCeil(u32, param_size, 4) catch unreachable <= 4 - ncrn) {
3576	if (param_size <= 4) {
3577	result.args[i] = .{ .register = c_abi_int_param_regs[ncrn] };
3578	ncrn += 1;
3579	} else {
3580	return self.fail("TODO MCValues with multiple registers", .{});
3581	}
3582	} else if (ncrn < 4 and nsaa == 0) {
3583	return self.fail("TODO MCValues split between registers and stack", .{});
3584	} else {
3585	ncrn = 4;
3586	if (ty.abiAlignment(self.target.*) == 8)
3587	nsaa = std.mem.alignForwardGeneric(u32, nsaa, 8);
3588
3589	result.args[i] = .{ .stack_offset = nsaa };
3590	nsaa += param_size;
3591	}
3592	}
3593
3594	result.stack_byte_count = nsaa;
3595	result.stack_align = 8;
3596	},
3597	else => return self.fail("TODO implement function parameters for {} on arm", .{cc}),
3598	}
3599	},
3600	else => if (param_types.len != 0)
3601	return self.fail("TODO implement codegen parameters for {}", .{self.target.cpu.arch}),
3602	}
3603
3604	if (ret_ty.zigTypeTag() == .NoReturn) {
3605	result.return_value = .{ .unreach = {} };
3606	} else if (!ret_ty.hasCodeGenBits()) {
3607	result.return_value = .{ .none = {} };
3608	} else switch (arch) {
3609	.arm, .armeb => switch (cc) {
3610	.Naked => unreachable,
3611	.Unspecified, .C => {
3612	const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3613	if (ret_ty_size <= 4) {
3614	result.return_value = .{ .register = c_abi_int_return_regs[0] };
3615	} else {
3616	return self.fail("TODO support more return types for ARM backend", .{});
3617	}
3618	},
3619	else => return self.fail("TODO implement function return values for {}", .{cc}),
3620	},
3621	else => return self.fail("TODO implement codegen return values for {}", .{self.target.cpu.arch}),
3622	}
3623	return result;
3624	}
3625
3626	/// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`.
3627	fn wantSafety(self: *Self) bool {
3628	return switch (self.bin_file.options.optimize_mode) {
3629	.Debug => true,
3630	.ReleaseSafe => true,
3631	.ReleaseFast => false,
3632	.ReleaseSmall => false,
3633	};
3634	}
3635
3636	fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3637	@setCold(true);
3638	assert(self.err_msg == null);
3639	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3640	return error.CodegenFail;
3641	}
3642
3643	fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3644	@setCold(true);
3645	assert(self.err_msg == null);
3646	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3647	return error.CodegenFail;
3648	}
3649
3650	const Register = switch (arch) {
3651	.i386 => @import("arch/x86/bits.zig").Register,
3652	.arm, .armeb => @import("arch/arm/bits.zig").Register,
3653	else => enum {
3654	dummy,
3655
3656	pub fn allocIndex(self: Register) ?u4 {
3657	_ = self;
3658	return null;
3659	}
3660	},
3661	};
3662
3663	const Instruction = switch (arch) {
3664	.arm, .armeb => @import("arch/arm/bits.zig").Instruction,
3665	else => void,
3666	};
3667
3668	const Condition = switch (arch) {
3669	.arm, .armeb => @import("arch/arm/bits.zig").Condition,
3670	else => void,
3671	};
3672
3673	const callee_preserved_regs = switch (arch) {
3674	.i386 => @import("arch/x86/bits.zig").callee_preserved_regs,
3675	.arm, .armeb => @import("arch/arm/bits.zig").callee_preserved_regs,
3676	else => [_]Register{},
3677	};
3678
3679	const c_abi_int_param_regs = switch (arch) {
3680	.i386 => @import("arch/x86/bits.zig").c_abi_int_param_regs,
3681	.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_param_regs,
3682	else => [_]Register{},
3683	};
3684
3685	const c_abi_int_return_regs = switch (arch) {
3686	.i386 => @import("arch/x86/bits.zig").c_abi_int_return_regs,
3687	.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_return_regs,
3688	else => [_]Register{},
3689	};
3690
3691	fn parseRegName(name: []const u8) ?Register {
3692	if (@hasDecl(Register, "parseRegName")) {
3693	return Register.parseRegName(name);
3694	}
3695	return std.meta.stringToEnum(Register, name);
3696	}
3697	};
3698	}