Meg's Mirrors

zig

1	const Register = @import("bits.zig").Register;
2	const encoding = @import("encoding.zig");
3	const Instruction = encoding.Instruction;
4	const Mir = @import("Mir.zig");
5	const Assemble = @import("Assemble.zig");
6	const Disassemble = @import("Disassemble.zig");
7
8	const verify_target_features = false;
9	const assume_memmove_no_overlap = true;
10	/// https://github.com/ziglang/zig/issues/11307
11	/// Enabling this flag generates "break 0xAA" for unimplemented things.
12	const debug_trap_unimplemented_code = false;
13	/// Saves AIR index to $r21 for debugging.
14	const debug_r21_as_air = false;
15
16	pt: Zcu.PerThread,
17	target: *const std.Target,
18	opt_mode: std.builtin.OptimizeMode,
19	air: Air,
20	nav_index: InternPool.Nav.Index,
21
22	// WIP MIR
23	saved_registers: RegisterSet = .empty,
24	instructions: std.ArrayList(Instruction) = .empty,
25	nav_relocs: std.ArrayList(Mir.Reloc.Nav) = .empty,
26	uav_relocs: std.ArrayList(Mir.Reloc.Uav) = .empty,
27	lazy_relocs: std.ArrayList(Mir.Reloc.Lazy) = .empty,
28	global_relocs: std.ArrayList(Mir.Reloc.Global) = .empty,
29	internal_relocs: std.ArrayList(Mir.Reloc.Internal) = .empty,
30
31	// Stack Frame
32	returns: bool = false,
33	stack_size: u24 = 0,
34	stack_align: InternPool.Alignment = .@"16",
35	/// Relocations for reading incoming registers.
36	///
37	/// The instruction must be `ori rd, rj, 0`.
38	/// These relocations are applied in `Select.layout`,
39	/// and the instruction may be replaced with `ld.[w/d] rd, sp, ?`
40	/// if `rj` is spilled to stack.
41	///
42	/// See `Select.ldIncoming`.
43	layout_relocs: std.ArrayList(usize) = .empty,
44
45	// Value Tracking
46	live_registers: LiveRegisters = .initFill(.free),
47	live_values: std.AutoHashMapUnmanaged(Air.Inst.Index, Value.Index) = .empty,
48	values: std.ArrayList(Value) = .empty,
49	value_types: std.ArrayList(ZigType) = .empty,
50
51	// Calling Convention
52	arg_layouts: []const Value.Index = &.{},
53
54	// Analysis
55	/// Definition order of AIR instructions.
56	def_order: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, void) = .empty,
57	/// Stack of active blocks. Value is undefined during analysis.
58	active_blocks: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, Block) = .empty,
59	/// Loops. The last entry is Loop.invalid, which is added in `finishAnalysis`.
60	loops: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, Loop) = .empty,
61	/// Stack of active loops.
62	active_loops: std.ArrayList(Loop.Index) = .empty,
63	/// Loop liveness
64	loop_outer_live: struct {
65	/// Pairs of loops and AIRs that is used in the loop body but is defined
66	/// earlier than the loop entry.
67	/// Populated during analysis phase, in analyseUse.
68	///
69	/// Includes only references where the loop and the AIR are in the same upper loop.
70	/// For example, in the following structure:
71	/// %1 arg
72	/// %2 arg
73	/// %3 arg
74	/// %4 loop (loop 0)
75	/// %5 add %1 %2
76	/// %6 loop (loop 1)
77	/// %7 add %3 %5
78	/// %8 add %2 %5
79	/// Only (loop 0, %1), (loop 0, %2), (loop 0, %3), (loop 1, %5) will be recorded, because,
80	/// although %2 and %3 are used in loop 1, they are in the outer layer of loop 0, not loop 1.
81	set: std.AutoArrayHashMapUnmanaged(struct { Loop.Index, Air.Inst.Index }, void) = .empty,
82	/// List representation of `loop_live.set`, for faster indexing.
83	list: std.ArrayList(Air.Inst.Index) = .empty,
84	} = .{},
85
86	pub const RegisterSet = std.enums.EnumSet(Register);
87	pub const LiveRegisters = std.enums.EnumArray(Register, Value.Index);
88
89	pub const Block = struct {
90	snapshot: LocationSnapshot = .empty,
91	target_label: u32,
92
93	pub const main: Air.Inst.Index = @fromBackingInt(
94	std.math.maxInt(@typeInfo(Air.Inst.Index).@"enum".tag_type),
95	);
96
97	pub fn deinit(target_block: *Block, isel: *Select) void {
98	target_block.snapshot.deinit(isel);
99	}
100
101	fn branch(target_block: *Block, isel: *Select) !void {
102	if (isel.instructions.items.len > target_block.target_label) {
103	try isel.internal_relocs.append(isel.pt.zcu.gpa, .{
104	.target = target_block.target_label,
105	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B26 },
106	});
107	try isel.emit(.b(0, 0));
108	}
109	try target_block.snapshot.merge(isel);
110	}
111	};
112
113	pub const Loop = struct {
114	def_order: u32,
115	outer_live: u32,
116	repeat_list: u32,
117	/// Used during code selection. Location snapshot before entering loop bodyies.
118	/// Cleared after leaving the loop body.
119	snapshot: LocationSnapshot = .empty,
120	/// Used during code selection. Registers that are written during a loop body.
121	/// See Select.markRegWritten.
122	/// After leaving a loop, written register set is copied to the outer loop.
123	written_regs: RegisterSet = .empty,
124
125	pub const invalid: Air.Inst.Index = @fromBackingInt(
126	std.math.maxInt(@typeInfo(Air.Inst.Index).@"enum".tag_type),
127	);
128
129	pub const Index = enum(u32) {
130	_,
131
132	fn inst(li: Loop.Index, isel: *Select) Air.Inst.Index {
133	return isel.loops.keys()[@backingInt(li)];
134	}
135
136	fn get(li: Loop.Index, isel: *Select) *Loop {
137	return &isel.loops.values()[@backingInt(li)];
138	}
139	};
140
141	pub const empty_list: u32 = std.math.maxInt(u32);
142
143	fn branch(target_loop: *Loop, isel: *Select) !void {
144	try isel.instructions.ensureUnusedCapacity(isel.pt.zcu.gpa, 1);
145	const repeat_list_tail = target_loop.repeat_list;
146	target_loop.repeat_list = @intCast(isel.instructions.items.len);
147	isel.instructions.appendAssumeCapacity(@bitCast(repeat_list_tail));
148	try target_loop.snapshot.merge(isel);
149	}
150	};
151
152	pub fn deinit(isel: *Select) void {
153	const gpa = isel.pt.zcu.gpa;
154
155	isel.instructions.deinit(gpa);
156	isel.nav_relocs.deinit(gpa);
157	isel.uav_relocs.deinit(gpa);
158	isel.lazy_relocs.deinit(gpa);
159	isel.global_relocs.deinit(gpa);
160	isel.internal_relocs.deinit(gpa);
161
162	isel.layout_relocs.deinit(gpa);
163
164	isel.live_values.deinit(gpa);
165	isel.values.deinit(gpa);
166	isel.value_types.deinit(gpa);
167
168	if (isel.arg_layouts.len != 0) gpa.free(isel.arg_layouts);
169
170	isel.def_order.deinit(gpa);
171	isel.active_blocks.deinit(gpa);
172	isel.loops.deinit(gpa);
173	isel.active_loops.deinit(gpa);
174	isel.loop_outer_live.set.deinit(gpa);
175	isel.loop_outer_live.list.deinit(gpa);
176
177	isel.* = undefined;
178	}
179
180	/// A node in the value tree.
181	pub const Value = struct {
182	refs: u32,
183	flags: Flags,
184	offset_from_parent: u64,
185	parent_payload: Parent.Payload,
186	location_payload: LocationInfo.Payload,
187	parts: Value.Index,
188
189	/// Must be at least 16 to compute call ABI.
190	/// Must be at least 16, the largest hardware alignment.
191	pub const max_parts = 16;
192	pub const PartsLen = std.math.IntFittingRange(0, Value.max_parts);
193
194	comptime {
195	if (!std.debug.runtime_safety) assert(@sizeOf(Value) == 32);
196	}
197
198	pub const Flags = packed struct(u32) {
199	alignment: InternPool.Alignment,
200	parent_tag: Parent.Tag,
201	location_tag: LocationInfo.Tag,
202	parts_len_minus_one: std.math.IntFittingRange(0, Value.max_parts - 1),
203	splitted: bool,
204	unused: u17 = 0,
205	};
206
207	pub const Parent = union(enum(u2)) {
208	none: void,
209	value: Value.Index,
210	constant: Constant,
211	/// Dereferencing. Only used for layout values at ABI boundaries.
212	address: Value.Index,
213
214	pub const Tag = @typeInfo(Parent).@"union".tag_type.?;
215	pub const Payload = Payload: {
216	const info = @typeInfo(Parent).@"union";
217	break :Payload @Union(.auto, null, info.field_names, info.field_types[0..], &@splat(.{}));
218	};
219	};
220
221	pub const LocationInfo = union(enum(u2)) {
222	/// Small values that fit into a register
223	small: struct {
224	flags: packed struct {
225	/// Byte-size of the part
226	size: u6,
227	/// Way in which the unused bits are filled
228	/// For subtrees whose root has Parent.address, immutable after initialization
229	extension: Extension,
230	/// Register access modifier
231	hint_modifier: Register.Modifier,
232	/// Preferred register, maybe ignore, $zero = unset
233	hint_register: Register,
234	/// The current expected location
235	location_tag: Location.Tag,
236	},
237	location_payload: Location.Payload,
238	},
239	/// Large values that can only be stored in stack slots
240	large: struct {
241	/// Byte-size of the part
242	size: u32,
243	/// The current expected location
244	/// Well-shaped values are always in pcs extended, ill-shaped are garbage extended
245	stack_slot: Indirect,
246	},
247	/// Extreme values that are too large to be materialized in stack slots
248	extreme: struct {
249	size: u64,
250	},
251
252	pub const Tag = @typeInfo(LocationInfo).@"union".tag_type.?;
253	pub const Payload = Payload: {
254	const info = @typeInfo(LocationInfo).@"union";
255	break :Payload @Union(.auto, null, info.field_names, info.field_types[0..], &@splat(.{}));
256	};
257	};
258
259	pub const Location = union(enum(u1)) {
260	register: Register.Alias,
261	stack_slot: Indirect,
262
263	pub const unallocated: Location = .{ .register = .zero };
264
265	pub inline fn isUnallocated(loc: Location) bool {
266	return switch (loc) {
267	.register => |ra| ra.reg == Register.zero,
268	else => false,
269	};
270	}
271
272	fn tryLock(loc: Location, isel: *Select) RegLock {
273	return if (loc.asRegister()) |reg| isel.tryLockReg(reg) else .empty;
274	}
275
276	pub fn asRegisterAlias(loc: Location) ?Register.Alias {
277	return switch (loc) {
278	.register => |ra| if (ra.reg == Register.zero) null else ra,
279	else => null,
280	};
281	}
282
283	pub fn asRegister(loc: Location) ?Register {
284	return if (loc.asRegisterAlias()) |ra| ra.reg else null;
285	}
286
287	pub fn asStackSlot(loc: Location) ?Indirect {
288	return switch (loc) {
289	.stack_slot => |stack_slot| stack_slot,
290	else => null,
291	};
292	}
293
294	pub fn format(loc: Location, w: *std.Io.Writer) std.Io.Writer.Error!void {
295	if (loc.isUnallocated()) return w.writeAll("unallocated");
296	switch (loc) {
297	inline else => |loc_pl| try loc_pl.format(w),
298	}
299	}
300
301	pub fn markRegWritten(loc: Location, isel: *Select) void {
302	if (loc.asRegister()) |loc_reg| isel.markRegWritten(loc_reg);
303	}
304
305	pub const Tag = @typeInfo(Location).@"union".tag_type.?;
306	pub const Payload = Payload: {
307	const info = @typeInfo(Location).@"union";
308	break :Payload @Union(.auto, null, info.field_names, info.field_types[0..], &@splat(.{}));
309	};
310	};
311
312	// TODO far indirect
313	pub const Indirect = packed struct(u32) {
314	base: Register,
315	offset: i25,
316
317	pub const unallocated: Indirect = .{ .base = .zero, .offset = 0 };
318
319	pub fn withOffset(ind: Indirect, offset: i25) Indirect {
320	return .{
321	.base = ind.base,
322	.offset = ind.offset + offset,
323	};
324	}
325
326	pub fn format(self: Indirect, w: *std.Io.Writer) std.Io.Writer.Error!void {
327	try w.print("[${t}, #{s}0x{x}]", .{
328	self.base,
329	if (self.offset < 0) "-" else "",
330	@abs(self.offset),
331	});
332	}
333	};
334
335	pub const Extension = enum(u2) {
336	garbage,
337	sign_ext,
338	zero_ext,
339
340	pub fn fromSignedness(signedness: std.builtin.Signedness) Extension {
341	return switch (signedness) {
342	.signed => .sign_ext,
343	.unsigned => .zero_ext,
344	};
345	}
346
347	fn signednessForLoad(fill_mode: Extension) std.builtin.Signedness {
348	return switch (fill_mode) {
349	.garbage, .zero_ext => .unsigned,
350	.sign_ext => .signed,
351	};
352	}
353
354	pub fn mix(a: Extension, b: Extension) Extension {
355	if (a == b) return a;
356	return .garbage;
357	}
358
359	fn pcsMode(isel: *Select, ty: ZigType) Extension {
360	const zcu = isel.pt.zcu;
361	const int_info = switch (ty.zigTypeTag(zcu)) {
362	.bool => ZigType.u1.intInfo(zcu),
363	.int, .@"enum", .error_set => ty.intInfo(zcu),
364	else => return .garbage,
365	};
366	return switch (int_info.bits) {
367	32 => .sign_ext,
368	else => .fromSignedness(int_info.signedness),
369	};
370	}
371	};
372
373	pub const Index = enum(u32) {
374	allocating = std.math.maxInt(u32) - 1,
375	free = std.math.maxInt(u32) - 0,
376	_,
377
378	fn get(vi: Value.Index, isel: *Select) *Value {
379	return &isel.values.items[@backingInt(vi)];
380	}
381
382	fn typeOf(vi: Value.Index, isel: *Select) ?ZigType {
383	const ty = isel.value_types.items[@backingInt(vi)];
384	if (ty.ip_index == .none) return null;
385	return ty;
386	}
387
388	pub fn format(vi: Value.Index, w: *std.Io.Writer) std.Io.Writer.Error!void {
389	return switch (vi) {
390	_ => w.print("${d}", .{@backingInt(vi)}),
391	.allocating => w.writeAll("(allocating)"),
392	.free => w.writeAll("(free)"),
393	};
394	}
395
396	fn setAlignment(vi: Value.Index, isel: *Select, new_alignment: InternPool.Alignment) void {
397	vi.get(isel).flags.alignment = new_alignment;
398	}
399
400	pub fn alignment(vi: Value.Index, isel: *Select) InternPool.Alignment {
401	return vi.get(isel).flags.alignment;
402	}
403
404	pub fn setParent(vi: Value.Index, isel: *Select, new_parent: Parent) void {
405	const value = vi.get(isel);
406	if (value.refs > 0) {
407	switch (value.flags.parent_tag) {
408	.none, .constant => {},
409	inline .address, .value => |tag| @field(value.parent_payload, @tagName(tag)).deref(isel),
410	}
411	switch (new_parent) {
412	.none => unreachable,
413	.constant => {},
414	.address, .value => |parent_vi| _ = parent_vi.ref(isel),
415	}
416	}
417	value.flags.parent_tag = new_parent;
418	value.parent_payload = switch (new_parent) {
419	.none => unreachable,
420	inline else => |payload, tag| @unionInit(Parent.Payload, @tagName(tag), payload),
421	};
422	}
423
424	pub fn parent(vi: Value.Index, isel: *Select) Parent {
425	const value = vi.get(isel);
426	return switch (value.flags.parent_tag) {
427	inline else => |tag| @unionInit(
428	Parent,
429	@tagName(tag),
430	@field(value.parent_payload, @tagName(tag)),
431	),
432	};
433	}
434
435	pub fn parentValue(vi: Value.Index, isel: *Select) ?Value.Index {
436	const value = vi.get(isel);
437	return switch (value.flags.parent_tag) {
438	.value => value.parent_payload.value,
439	else => null,
440	};
441	}
442
443	pub fn valueRoot(initial_vi: Value.Index, isel: *Select) struct { u64, Value.Index } {
444	var offset: u64 = 0;
445	var vi = initial_vi;
446	parent: switch (vi.parent(isel)) {
447	else => return .{ offset, vi },
448	.value => |parent_vi| {
449	offset += vi.get(isel).offset_from_parent;
450	vi = parent_vi;
451	continue :parent parent_vi.parent(isel);
452	},
453	}
454	}
455
456	pub fn locationInfo(vi: Value.Index, isel: *Select) LocationInfo {
457	const value = vi.get(isel);
458	return switch (value.flags.location_tag) {
459	inline else => |tag| @unionInit(
460	LocationInfo,
461	@tagName(tag),
462	@field(value.location_payload, @tagName(tag)),
463	),
464	};
465	}
466
467	pub fn isSmall(vi: Value.Index, isel: *Select) bool {
468	return vi.get(isel).flags.location_tag == .small;
469	}
470
471	pub fn setSmallLocation(vi: Value.Index, isel: *Select, new_location: Location) void {
472	const value = vi.get(isel);
473	value.location_payload.small.flags.location_tag = new_location;
474	value.location_payload.small.location_payload = switch (new_location) {
475	inline else => |payload, tag| @unionInit(Location.Payload, @tagName(tag), payload),
476	};
477	}
478
479	pub fn smallLocation(vi: Value.Index, isel: *Select) Location {
480	const value = vi.get(isel);
481	return switch (value.location_payload.small.flags.location_tag) {
482	inline else => |tag| @unionInit(
483	Location,
484	@tagName(tag),
485	@field(value.location_payload.small.location_payload, @tagName(tag)),
486	),
487	};
488	}
489
490	pub fn positionInParent(vi: Value.Index, isel: *Select) struct { u64, u64 } {
491	return .{ vi.get(isel).offset_from_parent, vi.size(isel) };
492	}
493
494	pub fn offsetIn(initial_vi: Value.Index, isel: *Select, ancestor_vi: Value.Index) u64 {
495	if (initial_vi == ancestor_vi) return 0;
496	var offset: u64 = 0;
497	var vi = initial_vi;
498	parent: switch (vi.parent(isel)) {
499	else => unreachable, // ancestor_vi is not an ancestor of initial_vi
500	.value => |parent_vi| {
501	offset += vi.get(isel).offset_from_parent;
502	if (parent_vi != ancestor_vi) {
503	vi = parent_vi;
504	continue :parent parent_vi.parent(isel);
505	} else return offset;
506	},
507	}
508	}
509
510	pub fn size(vi: Value.Index, isel: *Select) u64 {
511	return switch (vi.locationInfo(isel)) {
512	.small => |loc| loc.flags.size,
513	inline else => |loc| loc.size,
514	};
515	}
516
517	pub fn bitSize(vi: Value.Index, isel: *Select) u64 {
518	if (vi.typeOf(isel)) |init_ty| bit_size: {
519	const zcu = isel.pt.zcu;
520	var ty = init_ty;
521	check_ty: while (true) {
522	switch (ty.zigTypeTag(zcu)) {
523	else => {},
524	.error_union => break :bit_size,
525	.@"struct", .@"union" => if (ty.containerLayout(zcu) != .@"packed") break :bit_size,
526	.pointer, .optional => if (!ty.isPtrAtRuntime(zcu)) break :bit_size,
527	.array, .vector => {
528	ty = ty.childType(zcu);
529	continue :check_ty;
530	},
531	}
532	break :check_ty;
533	}
534	return init_ty.bitSize(zcu);
535	}
536	return vi.size(isel) * 8;
537	}
538
539	fn setExtension(vi: Value.Index, isel: *Select, new_mode: Extension) void {
540	const value = vi.get(isel);
541	if (value.flags.location_tag == .small)
542	value.location_payload.small.flags.extension = new_mode;
543	}
544
545	/// For values on stack, unused bits are the highest ((size * 8) - bit_size) bits.
546	/// For values on registers, unused bits are the highest (ra_width - bit_size) bits.
547	/// That is, for a u3 (3b, 1B) stored in LA64 GPR (64b, 8B), the unused bits to be filled
548	/// are reg[3..63] instead of reg[3..7].
549	pub fn extension(vi: Value.Index, isel: *Select) Extension {
550	const value = vi.get(isel);
551	return switch (value.flags.location_tag) {
552	.small => value.location_payload.small.flags.extension,
553	.large, .extreme => if (vi.typeOf(isel)) |ty| .pcsMode(isel, ty) else .garbage,
554	};
555	}
556
557	fn setHintModifier(vi: Value.Index, isel: *Select, new_modifier: Register.Modifier) void {
558	vi.get(isel).location_payload.small.flags.hint_modifier = new_modifier;
559	}
560
561	pub fn hintModifier(vi: Value.Index, isel: *Select) Register.Modifier {
562	return switch (vi.locationInfo(isel)) {
563	.small => |loc| loc.flags.hint_modifier,
564	.large, .extreme => .undef,
565	};
566	}
567
568	fn setHintRegister(vi: Value.Index, isel: *Select, new_hint: Register) void {
569	vi.get(isel).location_payload.small.flags.hint_register = new_hint;
570	}
571
572	pub fn hintRegister(vi: Value.Index, isel: *Select) ?Register {
573	return switch (vi.locationInfo(isel)) {
574	.small => |loc| switch (loc.flags.hint_register) {
575	Register.zero => null,
576	else => |hint_reg| hint_reg,
577	},
578	.large, .extreme => null,
579	};
580	}
581
582	pub fn hintRegisterAlias(vi: Value.Index, isel: *Select) ?Register.Alias {
583	return switch (vi.locationInfo(isel)) {
584	.small => |loc| switch (loc.flags.hint_register) {
585	Register.zero => null,
586	else => |hint_reg| .{ .mod = vi.hintModifier(isel), .reg = hint_reg },
587	},
588	.large, .extreme => null,
589	};
590	}
591
592	pub fn location(vi: Value.Index, isel: *Select) ?Location {
593	return switch (vi.locationInfo(isel)) {
594	.small => |loc| if (loc.flags.location_tag == .register and loc.location_payload.register.reg == Register.zero)
595	null
596	else switch (loc.flags.location_tag) {
597	inline else => |tag| @unionInit(
598	Location,
599	@tagName(tag),
600	@field(loc.location_payload, @tagName(tag)),
601	),
602	},
603	.large => |loc| if (loc.stack_slot == Indirect.unallocated)
604	null
605	else
606	.{ .stack_slot = loc.stack_slot },
607	.extreme => null,
608	};
609	}
610
611	pub fn register(vi: Value.Index, isel: *Select) ?Register.Alias {
612	return switch (vi.location(isel) orelse return null) {
613	.register => |ra| ra,
614	.stack_slot => null,
615	};
616	}
617
618	pub fn stackSlot(vi: Value.Index, isel: *Select) ?Indirect {
619	return switch (vi.location(isel) orelse return null) {
620	.register => null,
621	.stack_slot => |slot| slot,
622	};
623	}
624
625	/// Takes the expected location. Registers are free.
626	fn takeLocation(vi: Value.Index, isel: *Select) ?Location {
627	const value = vi.get(isel);
628	return switch (value.flags.location_tag) {
629	.small => loc: {
630	const loc = vi.smallLocation(isel);
631	if (loc.isUnallocated()) break :loc null;
632	if (loc.asRegister()) |reg| {
633	const live_vi = isel.live_registers.getPtr(reg);
634	assert(live_vi.* == vi);
635	live_vi.* = .free;
636	}
637	vi.setSmallLocation(isel, .unallocated);
638	break :loc loc;
639	},
640	.large => loc: {
641	const stack_slot = value.location_payload.large.stack_slot;
642	if (stack_slot == Indirect.unallocated) break :loc null;
643	value.location_payload.large.stack_slot = .unallocated;
644	break :loc .{ .stack_slot = stack_slot };
645	},
646	.extreme => null,
647	};
648	}
649
650	/// Takes the expected location. Registers are free and marked written.
651	fn takeLocationMarkWritten(vi: Value.Index, isel: *Select) ?Location {
652	const maybe_loc = vi.takeLocation(isel);
653	if (maybe_loc) |loc| loc.markRegWritten(isel);
654	return maybe_loc;
655	}
656
657	fn setStackSlot(vi: Value.Index, isel: *Select, new_slot: Indirect) void {
658	const value = vi.get(isel);
659	return switch (value.flags.location_tag) {
660	.small => vi.setSmallLocation(isel, .{ .stack_slot = new_slot }),
661	.large => value.location_payload.large.stack_slot = new_slot,
662	.extreme => unreachable,
663	};
664	}
665
666	pub fn isUsed(vi: Value.Index, isel: *Select) bool {
667	return vi.valueRoot(isel)[1].parent(isel) != .none or vi.hasLocationRecursive(isel);
668	}
669
670	fn hasLocationRecursive(vi: Value.Index, isel: *Select) bool {
671	if (vi.location(isel) != null) return true;
672	var part_it = vi.parts(isel);
673	if (part_it.only() == null)
674	while (part_it.next()) |part_vi|
675	if (part_vi.hasLocationRecursive(isel)) return true;
676	return false;
677	}
678
679	fn setParts(vi: Value.Index, isel: *Select, parts_len: Value.PartsLen) void {
680	assert(parts_len > 1);
681	const value = vi.get(isel);
682	assert(value.flags.parts_len_minus_one == 0);
683	value.parts = @fromBackingInt(@intCast(isel.values.items.len));
684	value.flags.parts_len_minus_one = @intCast(parts_len - 1);
685	}
686
687	fn addPart(vi: Value.Index, isel: *Select, part_offset: u64, part_size: u64, maybe_ty: ?ZigType) Value.Index {
688	const part_vi = isel.initValueAdvanced(
689	vi.alignment(isel),
690	part_offset,
691	part_size,
692	maybe_ty,
693	);
694	if (maybe_ty) |ty|
695	tracking_log.debug("{f} <- {f}[{d}] ({d}B, {f})", .{ part_vi, vi, part_offset, part_size, isel.fmtType(ty) })
696	else
697	tracking_log.debug("{f} <- {f}[{d}] ({d}B, untyped)", .{ part_vi, vi, part_offset, part_size });
698	part_vi.setParent(isel, .{ .value = vi });
699	return part_vi;
700	}
701
702	fn addIntPart(vi: Value.Index, isel: *Select, part_offset: u64, part_size: u64, part_bit_size: u9) !Value.Index {
703	const part_vi = isel.initValueAdvanced(vi.alignment(isel), part_offset, part_size, try isel.pt.intType(.unsigned, part_bit_size));
704	tracking_log.debug("{f} <- {f}[{d}] ({d}B, {d}b)", .{ part_vi, vi, part_offset, part_size, part_bit_size });
705	part_vi.setParent(isel, .{ .value = vi });
706	return part_vi;
707	}
708
709	pub fn parts(vi: Value.Index, isel: *Select) Value.PartIterator {
710	const value = vi.get(isel);
711	return switch (value.flags.parts_len_minus_one) {
712	0 => .initOne(vi),
713	else => |parts_len_minus_one| .{
714	.vi = value.parts,
715	.remaining = @as(Value.PartsLen, parts_len_minus_one) + 1,
716	},
717	};
718	}
719
720	pub fn hasParts(vi: Value.Index, isel: *Select) bool {
721	return vi.get(isel).flags.parts_len_minus_one != 0;
722	}
723
724	fn partAtOffset(vi: Value.Index, isel: *Select, offset: u64) Value.Index {
725	const SearchPartIndex = std.math.IntFittingRange(0, Value.max_parts * 2 - 1);
726	const value = vi.get(isel);
727	var last: SearchPartIndex = value.flags.parts_len_minus_one;
728	if (last == 0) return vi;
729	var first: SearchPartIndex = 0;
730	last += 1;
731	while (true) {
732	const mid = (first + last) / 2;
733	const mid_vi: Value.Index = @fromBackingInt(@backingInt(value.parts) + mid);
734	if (mid == first) return mid_vi;
735	if (offset < mid_vi.get(isel).offset_from_parent) last = mid else first = mid;
736	}
737	}
738
739	fn partExact(vi: Value.Index, isel: *Select, offset: u64, part_size: u64) !Value.Index {
740	try vi.split(isel, false);
741	const part_vi = vi.partAtOffset(isel, offset);
742	if (part_vi.offsetIn(isel, vi) != offset or part_vi.size(isel) != part_size) {
743	isel.dumpValues(.all);
744	tracking_log.debug("{f}.partExact({}, {}) selected {f}", .{ vi, offset, part_size, part_vi });
745	unreachable;
746	}
747	return part_vi;
748	}
749
750	fn partExactRecursive(vi: Value.Index, isel: *Select, init_offset: u64, part_size: u64) !Value.Index {
751	if (init_offset == 0 and vi.size(isel) == part_size) return vi;
752	var part_vi = vi;
753	var offset = init_offset;
754	while (true) {
755	try part_vi.split(isel, false);
756	const subpart_vi = part_vi.partAtOffset(isel, offset);
757	if (subpart_vi == part_vi) {
758	isel.dumpValues(.all);
759	tracking_log.debug("{f}.partExactRecursive({}, {}) selected {f}", .{ vi, init_offset, part_size, part_vi });
760	unreachable;
761	}
762	const subpart_offset = subpart_vi.get(isel).offset_from_parent;
763	offset -= subpart_offset;
764	if (offset == 0 and subpart_vi.size(isel) == part_size) return subpart_vi;
765	part_vi = subpart_vi;
766	}
767	}
768
769	fn partAtLargerThan(vi: Value.Index, isel: *Select, offset: u64, part_size: u64) !Value.Index {
770	try vi.split(isel, false);
771	const part_vi = vi.partAtOffset(isel, offset);
772	if (part_vi.offsetIn(isel, vi) != offset or part_vi.size(isel) < part_size) {
773	isel.dumpValues(.all);
774	tracking_log.debug("{f}.partAtLargerThan({}, {}) selected {f}", .{ vi, offset, part_size, part_vi });
775	unreachable;
776	}
777	return part_vi;
778	}
779
780	fn walk(vi: Value.Index, isel: *Select, opts: Walk.Options) Walk {
781	return .{ .isel = isel, .root_vi = vi, .next_vi = vi, .opts = opts };
782	}
783
784	fn ref(initial_vi: Value.Index, isel: *Select) Value.Index {
785	var vi = initial_vi;
786	while (true) {
787	const refs = &vi.get(isel).refs;
788	refs.* += 1;
789	if (refs.* > 1) return initial_vi;
790	switch (vi.parent(isel)) {
791	.none, .constant => {},
792	.address, .value => |parent_vi| {
793	vi = parent_vi;
794	continue;
795	},
796	}
797	return initial_vi;
798	}
799	}
800
801	pub fn deref(initial_vi: Value.Index, isel: *Select) void {
802	var vi = initial_vi;
803	while (true) {
804	const refs = &vi.get(isel).refs;
805	refs.* -= 1;
806	if (refs.* > 0) return;
807	switch (vi.parent(isel)) {
808	.none, .constant => {},
809	.address, .value => |parent_vi| {
810	vi = parent_vi;
811	continue;
812	},
813	}
814	return;
815	}
816	}
817
818	/// Allocates a stack slot for this value, not updating the value location.
819	fn allocStackSlot(vi: Value.Index, isel: *Select) Indirect {
820	const offset = vi.alignment(isel).forward(isel.stack_size);
821	isel.stack_size = @intCast(offset + vi.size(isel));
822	tracking_log.debug("[sp, #0x{x}] -> allocated for {f}", .{ @abs(offset), vi });
823	return .{
824	.base = .sp,
825	.offset = @intCast(offset),
826	};
827	}
828
829	/// Allocates a register for this value, not updating the value location.
830	fn allocRegister(vi: Value.Index, isel: *Select) !?Register.Alias {
831	// Try to allocate hint register
832	if (vi.hintRegister(isel)) |hint_reg| {
833	const live_vi = isel.live_registers.getPtr(hint_reg);
834	if (live_vi.* == .free) {
835	live_vi.* = .allocating;
836	isel.saved_registers.insert(hint_reg);
837	return .{ .reg = hint_reg, .mod = vi.hintModifier(isel) };
838	}
839	}
840	// Try to allocate a register
841	const value = vi.get(isel);
842	switch (value.flags.location_tag) {
843	.small => {
844	const reg_mod = vi.hintModifier(isel);
845	const reg = try isel.allocReg(reg_mod.class());
846	return .{ .reg = reg, .mod = reg_mod };
847	},
848	.large, .extreme => return null,
849	}
850	}
851
852	fn reextend(vi: Value.Index, isel: *Select, new_ext: Extension) !void {
853	if (!vi.isSmall(isel)) return;
854	return vi.reextendAdvanced(isel, vi.bitSize(isel), null, new_ext);
855	}
856
857	fn reextendToGarbage(vi: Value.Index, isel: *Select) !void {
858	if (!vi.isSmall(isel)) return;
859	return vi.reextendAdvanced(isel, vi.bitSize(isel), null, .garbage);
860	}
861
862	fn reextendToPcs(vi: Value.Index, isel: *Select) !void {
863	if (!vi.isSmall(isel)) return;
864	const ty = vi.typeOf(isel) orelse unreachable; // cannot reextend ill-shaped values to PCS mode
865	return vi.reextendAdvanced(isel, vi.bitSize(isel), null, .pcsMode(isel, ty));
866	}
867
868	fn reextendAdvanced(
869	vi: Value.Index,
870	isel: *Select,
871	old_bits: u64,
872	override_old_ext: ?Extension,
873	new_ext: Extension,
874	) !void {
875	if (vi.location(isel) == null) return;
876	const value = vi.get(isel);
877	const old_ext = override_old_ext orelse vi.extension(isel);
878	const bit_size = vi.bitSize(isel);
879	if (bit_size == 0) return;
880	const vi_bits = vi.size(isel) * 8;
881	const old_unused_bits = vi_bits - @min(old_bits, vi_bits);
882	const new_unused_bits = vi_bits - bit_size;
883	const dst_ext, const src_ext = if (bit_size == old_bits)
884	.{ old_ext, new_ext }
885	else if (bit_size < old_bits)
886	.{ .garbage, new_ext }
887	else ext_config: {
888	// To cast an ABI int to a wider one, signedness of the int must be specified
889	// in new_ext, so bits that are previously unused but now used can be properly
890	// re-filled.
891	if (old_ext != .garbage)
892	break :ext_config .{ old_ext, new_ext }
893	else
894	break :ext_config .{ .zero_ext, new_ext };
895	};
896	const unused_bits = @max(new_unused_bits, old_unused_bits);
897	if (dst_ext == src_ext and bit_size <= old_bits) return;
898	tracking_log.debug("{f}: {t} ({t}) -> {t} ({t}), {d}b -> {d}b", .{ vi, src_ext, new_ext, dst_ext, old_ext, old_bits, bit_size });
899
900	// avoid setting extension to .garbage to reduce MIR for sequences like
901	// zero_ext -> garbage -> zero_ext
902	if (dst_ext == .garbage) return;
903	if (value.flags.location_tag == .small)
904	value.location_payload.small.flags.extension = new_ext;
905	if (vi_bits <= isel.gprBits()) {
906	const vi_mat = try vi.mat(isel, .{ .pref = .only_reg });
907	try isel.fillUnusedBits(
908	vi_mat.reg(),
909	vi_mat.reg(),
910	dst_ext,
911	src_ext,
912	@intCast(vi_mat.ra().mod.bitSize(isel.target) - vi_bits + unused_bits),
913	);
914	try vi_mat.finish(isel);
915	} else {
916	const unused_bytes = std.math.divCeil(u64, unused_bits, 8) catch unreachable;
917	assert(unused_bytes <= isel.gprSize()); // TODO larger extending
918	const used_bytes = vi.size(isel) - unused_bytes;
919
920	var hit = false;
921	var walker = vi.walk(isel, .{});
922	while (walker.next()) |part_vi| {
923	const part_offset = part_vi.offsetIn(isel, vi);
924	const part_size = part_vi.size(isel);
925	const part_end = part_offset + part_size;
926	if (part_end <= used_bytes) continue;
927	if (part_size > isel.gprSize()) continue;
928
929	walker.skipChildren(part_vi);
930
931	const part_mat = try part_vi.mat(isel, .{ .pref = .only_reg });
932	try isel.fillUnusedBits(
933	part_mat.reg(),
934	part_mat.reg(),
935	dst_ext,
936	src_ext,
937	@intCast(unused_bits - ((vi.size(isel) - part_end) * 8)),
938	);
939	try part_mat.finish(isel);
940	if (hit) unreachable; // TODO
941	hit = true;
942	}
943	}
944	}
945
946	/// Defines ancestors by combining their children
947	fn defChildren(def_vi: Value.Index, isel: *Select) !void {
948	if (def_vi.parentValue(isel)) |parent_vi|
949	try parent_vi.defChildren(isel);
950	assert(def_vi.hasParts(isel));
951	if (def_vi.location(isel) == null) return;
952	wip_mir_log.debug(" | # merge children -> {f}", .{def_vi});
953	const def_bit_size = def_vi.bitSize(isel);
954
955	// If def_vi fits into a register, reextend def_vi
956	var reextend_parts = true;
957	if (def_vi.isSmall(isel)) {
958	const maybe_mixed_ext = mix_ext: {
959	var maybe_mixed_ext: ?Extension = null;
960	var part_it = def_vi.parts(isel);
961	while (part_it.next()) |part_vi| {
962	const part_offset, const part_size = part_vi.positionInParent(isel);
963	if ((part_offset + part_size) * 8 > def_bit_size) {
964	if (maybe_mixed_ext) |mixed_ext|
965	maybe_mixed_ext = mixed_ext.mix(part_vi.extension(isel))
966	else
967	maybe_mixed_ext = part_vi.extension(isel);
968	}
969	}
970	break :mix_ext maybe_mixed_ext;
971	};
972	if (maybe_mixed_ext) |mixed_ext| {
973	try def_vi.reextend(isel, mixed_ext);
974	reextend_parts = false;
975	}
976	}
977
978	const def_loc = def_vi.takeLocationMarkWritten(isel).?;
979	const def_reg_lock = def_loc.tryLock(isel);
980	defer def_reg_lock.unlock(isel);
981	const def_ext = def_vi.extension(isel);
982	var part_it = def_vi.parts(isel);
983	while (part_it.next()) |part_vi| {
984	const part_offset, const part_size = part_vi.positionInParent(isel);
985	const part_mat = try part_vi.mat(isel, .{});
986	try isel.moveLoc(def_loc, part_offset, part_mat.loc(), 0, part_size, .preserved);
987	try part_mat.finish(isel);
988	if (reextend_parts)
989	try part_vi.reextend(isel, def_ext);
990	}
991	}
992
993	/// Defines descendants by deriving from their parents
994	fn defParent(def_vi: Value.Index, isel: *Select) !void {
995	if (def_vi.hasParts(isel)) {
996	// DFS descendants
997	var part_it = def_vi.parts(isel);
998	while (part_it.next()) |part_vi| try part_vi.defParent(isel);
999	}
1000	wip_mir_log.debug(" | # derive parent -> {f}", .{def_vi});
1001	const parent_vi = def_vi.parentValue(isel).?;
1002	try def_vi.reextendAdvanced(isel, parent_vi.bitSize(isel), null, parent_vi.extension(isel));
1003	const def_loc = def_vi.takeLocationMarkWritten(isel) orelse return;
1004	const def_offset, const def_size = def_vi.positionInParent(isel);
1005	const parent_mat = try parent_vi.mat(isel, .{});
1006	try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .none);
1007	try parent_mat.finish(isel);
1008	}
1009
1010	/// Defines ancestors and descendants
1011	fn collectDefs(vi: Value.Index, isel: *Select) !void {
1012	if (vi.parentValue(isel)) |parent_vi|
1013	try parent_vi.defChildren(isel);
1014	if (vi.hasParts(isel)) {
1015	var part_it = vi.parts(isel);
1016	while (part_it.next()) |part_vi| try part_vi.defParent(isel);
1017	}
1018	}
1019
1020	/// Defines a value with a location.
1021	/// Returned location must be free-ed by caller.
1022	/// Extension unchanged.
1023	fn def(vi: Value.Index, isel: *Select) codegen.Error!?Location {
1024	try vi.collectDefs(isel);
1025	return vi.takeLocationMarkWritten(isel);
1026	}
1027
1028	/// Defines a value with a register.
1029	/// Returned registers are free-ed and marked as written.
1030	/// Extension unchanged.
1031	fn defReg(vi: Value.Index, isel: *Select) !?Register.Alias {
1032	const value = vi.get(isel);
1033	assert(value.flags.location_tag == .small); // must fit into a register
1034	try vi.collectDefs(isel);
1035
1036	const loc = vi.takeLocationMarkWritten(isel) orelse return null;
1037	switch (loc) {
1038	.register => |ra| return ra,
1039	.stack_slot => |stack| {
1040	const reg_mod = vi.hintModifier(isel);
1041	const reg = try isel.allocRegForWrite(reg_mod.class());
1042	defer isel.freeReg(reg);
1043	const ra: Register.Alias = .{ .mod = reg_mod, .reg = reg };
1044	try isel.storeReg(reg, vi.size(isel), stack.base, stack.offset);
1045	return ra;
1046	},
1047	}
1048	}
1049
1050	/// Defines a value with a register.
1051	/// Returned registers are free-ed.
1052	/// Extension unchanged.
1053	fn defRegMod(vi: Value.Index, isel: *Select, mod: Register.Modifier) !?Register {
1054	assert(mod != .undef);
1055	const loc = try vi.defReg(isel) orelse return null;
1056	if (loc.mod == mod) return loc.reg;
1057	const new_reg = try isel.allocRegForWrite(mod.class());
1058	try isel.moveReg(
1059	loc,
1060	0,
1061	.{ .reg = new_reg, .mod = mod },
1062	0,
1063	@min(loc.mod.bitSize(isel.target), mod.bitSize(isel.target)),
1064	.none,
1065	);
1066	return new_reg;
1067	}
1068
1069	/// Defines a value with a stack slot.
1070	/// Reextended in PCS mode.
1071	fn defStack(vi: Value.Index, isel: *Select) !?Indirect {
1072	try vi.reextendToPcs(isel);
1073	try vi.collectDefs(isel);
1074	const loc = vi.takeLocationMarkWritten(isel) orelse return null;
1075	switch (loc) {
1076	.register => |ra| {
1077	const stack_slot = vi.allocStackSlot(isel);
1078	try isel.loadReg(ra.reg, vi.size(isel), vi.extension(isel).signednessForLoad(), stack_slot.base, stack_slot.offset);
1079	return stack_slot;
1080	},
1081	.stack_slot => |stack| return stack,
1082	}
1083	}
1084
1085	/// Defines a value with undefined bytes.
1086	fn defUndef(vi: Value.Index, isel: *Select) !void {
1087	try vi.reextendToGarbage(isel);
1088	try vi.collectDefs(isel);
1089	const loc = vi.takeLocationMarkWritten(isel) orelse return;
1090	wip_mir_log.debug(" | # undef -> {f}", .{vi});
1091	try isel.moveUndef(loc, vi.size(isel));
1092	}
1093
1094	/// Defines a value by loading from memory.
1095	/// Reextended to PCS mode.
1096	///
1097	/// Returns true if vi has a location.
1098	fn defLoad(
1099	vi: Value.Index,
1100	isel: *Select,
1101	base_reg: Register,
1102	offset: u64,
1103	opts: MemoryAccessOptions,
1104	) !bool {
1105	try vi.reextendToPcs(isel);
1106	try vi.collectDefs(isel);
1107	const loc = vi.takeLocationMarkWritten(isel) orelse return false;
1108	wip_mir_log.debug(" | # load {f} <- [${t}, #{d}] ({d}B)", .{ vi, base_reg, offset, vi.size(isel) });
1109	_ = opts;
1110
1111	try isel.moveLoc(
1112	loc,
1113	0,
1114	.{ .stack_slot = .{ .base = base_reg, .offset = 0 } },
1115	offset,
1116	vi.size(isel),
1117	.none,
1118	);
1119	return true;
1120	}
1121
1122	/// Defines a value by copying another value.
1123	/// PCS aware.
1124	fn defMove(dst_vi: Value.Index, isel: *Select, src_ref: Air.Inst.Ref) !void {
1125	try dst_vi.defCopy(isel, try isel.use(src_ref));
1126	}
1127
1128	/// Defines a value by copying another value.
1129	/// PCS aware.
1130	fn defCopy(dst_vi: Value.Index, isel: *Select, src_vi: Value.Index) !void {
1131	try dst_vi.collectDefs(isel);
1132	wip_mir_log.debug(" | # copy {f} <- {f}", .{ dst_vi, src_vi });
1133	const copy_size = @min(dst_vi.size(isel), src_vi.size(isel));
1134
1135	// select reextension strategy
1136	const ext_strat: enum { dst_to_src, src_to_dst } = ext_strat: {
1137	const dst_has_loc = dst_vi.location(isel) != null;
1138	const src_has_loc = src_vi.location(isel) != null;
1139	if (dst_has_loc and !src_has_loc and src_vi.isSmall(isel)) break :ext_strat .src_to_dst;
1140	if (src_has_loc and !dst_has_loc) break :ext_strat .dst_to_src;
1141	break :ext_strat .dst_to_src; // random choice
1142	};
1143
1144	// reextend dst
1145	if (ext_strat == .dst_to_src) {
1146	try dst_vi.reextendAdvanced(
1147	isel,
1148	dst_vi.bitSize(isel),
1149	null,
1150	src_vi.extension(isel),
1151	);
1152	}
1153
1154	// do copy
1155	{
1156	const loc = dst_vi.takeLocation(isel) orelse return;
1157	const src_mat = try src_vi.mat(isel, .{
1158	.size = @intCast(copy_size),
1159	.pref = switch (loc) {
1160	.register => .prefer_reg,
1161	.stack_slot => .prefer_stack,
1162	},
1163	.hint_ra = loc.asRegisterAlias() orelse .zero,
1164	.hint_stack = loc.asStackSlot() orelse .unallocated,
1165	});
1166	const src_loc = src_mat.loc();
1167	if (!std.meta.eql(loc, src_loc)) {
1168	loc.markRegWritten(isel);
1169	try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .none);
1170	}
1171	try src_mat.finish(isel);
1172	}
1173
1174	// reextend src
1175	if (ext_strat == .src_to_dst) {
1176	try src_vi.reextend(isel, dst_vi.extension(isel));
1177	}
1178	}
1179
1180	/// Defines a value in a certain layout, commonly used near basic block boundaries.
1181	/// Reextends to PCS mode.
1182	pub fn defLiveIn(def_vi: Value.Index, isel: *Select, layout_vi: Value.Index, opts: struct {
1183	/// Whether registers should be freed.
1184	fill_regs: bool = true,
1185	}) !void {
1186	wip_mir_log.debug(" | # live in {f}, layout={f}", .{ def_vi, layout_vi });
1187	assert(def_vi.size(isel) == layout_vi.size(isel));
1188	const gpa = isel.pt.zcu.gpa;
1189
1190	var maybe_def_addr_mat: ?Value.Mat = null;
1191	switch (def_vi.parent(isel)) {
1192	.none => {},
1193	.value => |parent_vi| try parent_vi.defChildren(isel),
1194	.address => |def_addr_vi| {
1195	switch (layout_vi.parent(isel)) {
1196	.address => |layout_addr_vi| {
1197	try def_addr_vi.defLiveIn(isel, layout_addr_vi, opts);
1198	},
1199	.none, .value => {
1200	maybe_def_addr_mat = try def_vi.parent(isel).address.matIntRegZeroExt(isel);
1201	},
1202	.constant => unreachable,
1203	}
1204	},
1205	.constant => unreachable,
1206	}
1207
1208	// TODO optimize this O(n^2)
1209	var def_walk = def_vi.walk(isel, .{});
1210	while (def_walk.next()) |def_part_vi| {
1211	const part_offset = def_part_vi.offsetIn(isel, def_vi);
1212	const part_size = def_part_vi.size(isel);
1213	const part_end_plus1 = part_offset + part_size;
1214
1215	var layout_walk = layout_vi.walk(isel, .{});
1216	var layout_parts: std.ArrayList(struct {
1217	vi: Value.Index,
1218	offset: u64,
1219	end_plus1: u64,
1220	}) = .empty;
1221	defer layout_parts.deinit(gpa);
1222	var maybe_mixed_layout_ext: ?Extension = null;
1223	while (layout_walk.next()) |layout_part_vi| {
1224	if (layout_part_vi.location(isel) == null and layout_part_vi.hintRegister(isel) == null) continue;
1225	const layout_part_offset = layout_part_vi.offsetIn(isel, layout_vi);
1226	const layout_part_size = layout_part_vi.size(isel);
1227	const layout_part_end_plus1 = layout_part_offset + layout_part_size;
1228	if (layout_part_end_plus1 <= part_offset or
1229	layout_part_offset >= part_end_plus1) continue;
1230
1231	try layout_parts.append(gpa, .{
1232	.vi = layout_part_vi,
1233	.offset = layout_part_offset,
1234	.end_plus1 = layout_part_end_plus1,
1235	});
1236
1237	const layout_part_ext = layout_part_vi.extension(isel);
1238	if (maybe_mixed_layout_ext) |mixed_layout_ext| {
1239	maybe_mixed_layout_ext = mixed_layout_ext.mix(layout_part_ext);
1240	} else {
1241	maybe_mixed_layout_ext = layout_part_ext;
1242	}
1243	}
1244	if (maybe_mixed_layout_ext) |mixed_layout_ext| {
1245	try def_part_vi.reextend(isel, mixed_layout_ext);
1246	} else unreachable;
1247
1248	const def_part_loc = if (maybe_def_addr_mat == null or def_part_vi != def_vi) def_part_loc: {
1249	break :def_part_loc def_part_vi.takeLocationMarkWritten(isel) orelse continue;
1250	} else def_part_loc: {
1251	break :def_part_loc maybe_def_addr_mat.?.loc();
1252	};
1253	const def_part_lock = def_part_loc.tryLock(isel);
1254	defer def_part_lock.unlock(isel);
1255
1256	for (layout_parts.items) |layout_part| {
1257	const dst_offset = layout_part.offset -| part_offset;
1258	const src_offset = part_offset -| layout_part.offset;
1259
1260	const mat_size = @min(part_end_plus1, layout_part.end_plus1) - @max(part_offset, layout_part.offset);
1261	assert(mat_size != 0);
1262	const src_loc: Location = if (layout_part.vi.location(isel)) |loc|
1263	loc
1264	else if (layout_part.vi.hintRegisterAlias(isel)) |hint_ra|
1265	.{ .register = hint_ra }
1266	else
1267	unreachable;
1268	if (opts.fill_regs) {
1269	if (src_loc.asRegister()) |src_reg|
1270	_ = try isel.fillReg(src_reg);
1271	}
1272	// TODO: replace reextending def_part_vi to .zero_ext with moveLoc .wipe when applicable
1273	try isel.moveLoc(def_part_loc, dst_offset, src_loc, src_offset, mat_size, .preserved);
1274	}
1275	}
1276	if (maybe_def_addr_mat) |def_addr_mat| try def_addr_mat.finish(isel);
1277	}
1278
1279	const MemoryAccessOptions = struct {
1280	// TODO unimplemented, remove?
1281	@"volatile": bool = false,
1282	};
1283
1284	const MatOptions = struct {
1285	/// Offset of materialized part
1286	offset: u64 = 0,
1287	/// Size, coerced to [0, part size - offset]
1288	size: u32 = std.math.maxInt(u32),
1289	/// Location preference
1290	pref: LocPreference = .none,
1291	reg_mod: Register.Modifier = .undef,
1292	/// Expected extension mode
1293	extension: Extension = .garbage,
1294	hint_ra: Register.Alias = .zero,
1295	hint_stack: Indirect = .unallocated,
1296
1297	const LocPreference = enum {
1298	none,
1299	/// Loads value to a register if possible, otherwise returns a stack slot
1300	prefer_reg,
1301	/// Loads value to a register, asserts the value fitting into a register
1302	only_reg,
1303	/// If there isn't an exisiting location, allocate a stack slot
1304	prefer_stack,
1305	/// Stores value to a stack slot
1306	only_stack,
1307	};
1308	};
1309
1310	/// Materializes a value
1311	fn mat(vi: Value.Index, isel: *Select, opts: MatOptions) Mat.Error!Mat {
1312	// try vi.split(isel, true);
1313	const mat_size = @min(opts.size, @as(u32, @intCast(vi.size(isel) - opts.offset)));
1314	const loc_pref = if (opts.extension == .garbage)
1315	opts.pref
1316	else switch (opts.pref) {
1317	.none, .prefer_reg, .prefer_stack => .prefer_reg,
1318	.only_reg, .only_stack => |loc_pref| loc_pref,
1319	};
1320	var maybe_prev_loc: ?Location = null;
1321	const loc: Location, var full = loc: {
1322	// Try to reuse existing location
1323	if (vi.location(isel)) |loc| {
1324	maybe_prev_loc = loc;
1325	switch (loc) {
1326	.register => |loc_ra| if (opts.offset == 0 and (opts.reg_mod == .undef or opts.reg_mod == loc_ra.mod)) {
1327	switch (loc_pref) {
1328	.none, .prefer_reg, .only_reg, .prefer_stack => break :loc .{ loc, false },
1329	.only_stack => {},
1330	}
1331	},
1332	.stack_slot => switch (loc_pref) {
1333	.none, .prefer_stack, .only_stack => break :loc .{ loc, true },
1334	.prefer_reg, .only_reg => {},
1335	},
1336	}
1337	}
1338	if (loc_pref != .only_stack and loc_pref != .prefer_stack) {
1339	// Try to allocate hint RA
1340	if (opts.hint_ra.reg != Register.zero) {
1341	if (isel.live_registers.get(opts.hint_ra.reg) == .free) {
1342	isel.saved_registers.insert(opts.hint_ra.reg);
1343	break :loc .{ .{ .register = opts.hint_ra }, false };
1344	}
1345	}
1346	// Try to allocate a register
1347	if (opts.reg_mod == .undef or opts.reg_mod == vi.hintModifier(isel)) {
1348	if (try vi.allocRegister(isel)) |ra|
1349	break :loc .{ .{ .register = ra }, false };
1350	} else try_alloc: {
1351	const reg = isel.allocReg(opts.reg_mod.class()) catch break :try_alloc;
1352	break :loc .{ .{ .register = .{ .reg = reg, .mod = opts.reg_mod } }, false };
1353	}
1354	}
1355	// Use existing stack slot if cannot mat into regs
1356	switch (loc_pref) {
1357	.none, .prefer_stack, .only_stack => {},
1358	.prefer_reg => if (maybe_prev_loc) |loc| break :loc .{ loc, true },
1359	.only_reg => unreachable, // too large to fit in registers
1360	}
1361	// Use hint stack slot
1362	if (false) {
1363	// TODO needs stack slot tracking
1364	if (opts.hint_stack != .unallocated) {
1365	break :loc .{ .{ .stack_slot = opts.hint_stack }, false };
1366	}
1367	}
1368	// Allocate on stack
1369	break :loc .{ .{ .stack_slot = vi.allocStackSlot(isel) }, true };
1370	};
1371	if (maybe_prev_loc) |prev_loc| {
1372	if (std.meta.eql(loc, prev_loc)) {
1373	if (opts.extension != .garbage) {
1374	try vi.reextendAdvanced(isel, vi.bitSize(isel), null, opts.extension);
1375	}
1376	_ = vi.takeLocation(isel);
1377	}
1378	}
1379	if (loc.asRegister()) |reg| {
1380	const live_vi = isel.live_registers.getPtr(reg);
1381	switch (live_vi.*) {
1382	_ => unreachable,
1383	.allocating => {},
1384	.free => live_vi.* = .allocating,
1385	}
1386	full = opts.offset == 0 and mat_size == vi.size(isel);
1387	}
1388	if (full) {
1389	tracking_log.debug("{f}[{d}..{d}] -> {f}[...] (mat, {t})", .{ vi, opts.offset, opts.offset + mat_size - 1, loc, opts.extension });
1390	} else {
1391	tracking_log.debug("{f}[{d}..{d}] -> {f} (mat, {t})", .{ vi, opts.offset, opts.offset + mat_size - 1, loc, opts.extension });
1392	}
1393	return .{
1394	.vi = vi,
1395	.location = loc,
1396	.offset = opts.offset,
1397	.size = mat_size,
1398	.extension = opts.extension,
1399	.full = full,
1400	};
1401	}
1402
1403	fn matReg(vi: Value.Index, isel: *Select) !Mat {
1404	return vi.mat(isel, .{ .pref = .only_reg });
1405	}
1406
1407	fn matRegMod(vi: Value.Index, isel: *Select, mod: Register.Modifier) !Mat {
1408	return vi.mat(isel, .{ .pref = .only_reg, .reg_mod = mod });
1409	}
1410
1411	fn matIntRegZeroExt(vi: Value.Index, isel: *Select) !Mat {
1412	return vi.mat(isel, .{
1413	.pref = .only_reg,
1414	.reg_mod = .integer,
1415	.extension = .zero_ext,
1416	});
1417	}
1418
1419	/// Moves the address of vi, plus offset, to ptr_reg
1420	fn matAddress(vi: Value.Index, isel: *Select, ptr_reg: Register, offset: u64) !void {
1421	wip_mir_log.debug(" | # address ${t} <- (&{f} + {d})", .{ ptr_reg, vi, offset });
1422	const offset_from_root, const root_vi = vi.valueRoot(isel);
1423	const total_root_offset = offset_from_root + offset;
1424	switch (root_vi.parent(isel)) {
1425	.none => {
1426	const value_mat = try vi.mat(isel, .{ .pref = .only_stack });
1427	const value_stack = value_mat.loc().stack_slot;
1428	try isel.addImm(ptr_reg, value_stack.base, @as(i65, value_stack.offset) + offset);
1429	try value_mat.finish(isel);
1430	},
1431	.address => |addr_vi| {
1432	const addr_mat = try addr_vi.mat(isel, .{
1433	.pref = .only_reg,
1434	.hint_ra = .{ .mod = .integer, .reg = ptr_reg },
1435	});
1436	try isel.addImm(ptr_reg, addr_mat.reg(), total_root_offset);
1437	try addr_mat.finish(isel);
1438	},
1439	.value => unreachable,
1440	.constant => |constant| {
1441	const pt = isel.pt;
1442	const zcu = pt.zcu;
1443
1444	try isel.uav_relocs.append(zcu.gpa, .{
1445	.uav = .{
1446	.val = constant.toIntern(),
1447	.orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
1448	},
1449	.reloc = .{
1450	.label = @intCast(isel.instructions.items.len),
1451	.addend = @intCast(total_root_offset),
1452	.type = .PCALA_LO12,
1453	},
1454	});
1455	try isel.emit(.@"addi.d"(ptr_reg, ptr_reg, 0));
1456	try isel.uav_relocs.append(zcu.gpa, .{
1457	.uav = .{
1458	.val = constant.toIntern(),
1459	.orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
1460	},
1461	.reloc = .{
1462	.label = @intCast(isel.instructions.items.len),
1463	.addend = @intCast(total_root_offset),
1464	.type = .PCALA_HI20,
1465	},
1466	});
1467	try isel.emit(.pcalau12i(ptr_reg, 0));
1468	},
1469	}
1470	}
1471
1472	/// Stores a value to memory.
1473	fn matStore(
1474	vi: Value.Index,
1475	isel: *Select,
1476	base_reg: Register,
1477	offset: u64,
1478	opts: MemoryAccessOptions,
1479	) !void {
1480	wip_mir_log.debug(" | # store {f} -> [${t}, #{d}]", .{ vi, base_reg, offset });
1481	_ = opts;
1482
1483	const hint_stack: Indirect = if (std.math.cast(@FieldType(Indirect, "offset"), offset)) |stack_off|
1484	.{ .base = base_reg, .offset = stack_off }
1485	else
1486	.unallocated;
1487	const value_mat = try vi.mat(isel, .{ .hint_stack = hint_stack });
1488	try isel.moveLoc(
1489	.{ .stack_slot = .{ .base = base_reg, .offset = 0 } },
1490	offset,
1491	value_mat.loc(),
1492	0,
1493	vi.size(isel),
1494	.none,
1495	);
1496	try value_mat.finish(isel);
1497	}
1498
1499	/// Stores a value in a certain layout, commonly used near basic block boundaries.
1500	/// Reextends to PCS mode.
1501	fn matLiveOut(
1502	vi: Value.Index,
1503	isel: *Select,
1504	layout_vi: Value.Index,
1505	opts: struct {
1506	mode: enum { param, ret },
1507	},
1508	) !void {
1509	wip_mir_log.debug(" | # live out {f}, layout={f}, opts: regs={t}", .{ vi, layout_vi, opts.mode });
1510
1511	wip_mir_log.debug(" | # live out {f}: fill registers", .{vi});
1512	switch (opts.mode) {
1513	.param => {
1514	var layout_walk = layout_vi.walk(isel, .{});
1515	while (layout_walk.next()) |part_vi| {
1516	if (part_vi.hintRegister(isel)) |part_reg| {
1517	_ = try isel.fillReg(part_reg);
1518	}
1519	}
1520	},
1521	.ret => {
1522	var layout_walk = layout_vi.walk(isel, .{});
1523	while (layout_walk.next()) |part_vi| {
1524	if (part_vi.hintRegister(isel)) |part_reg| {
1525	assert(try isel.forgetReg(part_reg));
1526	_ = isel.lockReg(part_reg);
1527	}
1528	}
1529	},
1530	}
1531
1532	wip_mir_log.debug(" | # live out {f}: move values", .{vi});
1533	var layout_walk = layout_vi.walk(isel, .{});
1534	while (layout_walk.next()) |part_vi| {
1535	if (part_vi.hintRegisterAlias(isel)) |part_ra| {
1536	const part_offset = part_vi.offsetIn(isel, layout_vi);
1537	const part_size = part_vi.size(isel);
1538
1539	if (opts.mode == .ret) isel.freeReg(part_ra.reg);
1540	const value_mat = try vi.mat(isel, .{
1541	.hint_ra = part_ra,
1542	.offset = part_offset,
1543	.size = @intCast(part_size),
1544	.extension = part_vi.extension(isel),
1545	});
1546	try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .none);
1547	try value_mat.finish(isel);
1548	}
1549
1550	if (part_vi.location(isel)) |layout_part_loc| {
1551	const layout_part_stack = layout_part_loc.asStackSlot().?;
1552	const part_offset = part_vi.offsetIn(isel, layout_vi);
1553	const part_size = part_vi.size(isel);
1554
1555	const value_mat = try vi.mat(isel, .{
1556	.hint_stack = layout_part_stack,
1557	.offset = part_offset,
1558	.size = @intCast(part_size),
1559	.extension = part_vi.extension(isel),
1560	});
1561	try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .none);
1562	try value_mat.finish(isel);
1563	}
1564	}
1565	}
1566
1567	/// Moves the expected location to another location.
1568	fn moveTo(vi: Value.Index, isel: *Select, src_loc: Location) !void {
1569	if (src_loc.asRegister()) |src_reg| _ = try isel.fillReg(src_reg);
1570	tracking_log.debug("{f} -> {f} (move to)", .{ vi, src_loc });
1571	if (vi.takeLocationMarkWritten(isel)) |dst_loc|
1572	try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .none);
1573	if (vi.isSmall(isel)) {
1574	vi.setSmallLocation(isel, src_loc);
1575	if (src_loc.asRegister()) |src_reg| {
1576	const src_live_vi = isel.live_registers.getPtr(src_reg);
1577	assert(src_live_vi.* == .free);
1578	src_live_vi.* = vi;
1579	}
1580	} else {
1581	switch (src_loc) {
1582	.register => unreachable, // large values cannot be moved into a register
1583	.stack_slot => |src_stack| vi.setStackSlot(isel, src_stack),
1584	}
1585	}
1586	}
1587
1588	pub fn isSplitted(vi: Value.Index, isel: *Select) bool {
1589	const value = vi.get(isel);
1590	return value.flags.parts_len_minus_one != 0 or value.flags.splitted;
1591	}
1592
1593	pub fn split(vi: Value.Index, isel: *Select, force: bool) !void {
1594	const zcu = isel.pt.zcu;
1595	const ip = &zcu.intern_pool;
1596
1597	const value1 = vi.get(isel);
1598	if (value1.flags.splitted and !force) return;
1599	value1.flags.splitted = true;
1600	if (value1.flags.parts_len_minus_one != 0) return;
1601	var ty = vi.typeOf(isel) orelse {
1602	if (force)
1603	return vi.splitBlindly(isel)
1604	else
1605	return;
1606	};
1607
1608	try isel.values.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
1609	try isel.value_types.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
1610	const value = vi.get(isel);
1611	type_key: switch (ip.indexToKey(ty.toIntern())) {
1612	else => return isel.fail("unimplemented Value.split({f})", .{isel.fmtType(ty)}),
1613	.int_type => |int_type| {
1614	const gpr_size = isel.gprSize();
1615	const gpr_bits = isel.gprBits();
1616	const parts_len = std.math.divCeil(u16, int_type.bits, gpr_bits) catch unreachable;
1617	if (parts_len == 1) break :type_key;
1618	vi.setParts(isel, @intCast(parts_len));
1619	for (0..parts_len) |part_index|
1620	_ = try vi.addIntPart(
1621	isel,
1622	part_index * gpr_size,
1623	gpr_size,
1624	@intCast(@min(int_type.bits - (part_index * gpr_bits), gpr_bits)),
1625	);
1626	},
1627	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
1628	.one, .many, .c => break :type_key,
1629	.slice => {
1630	const ptr_size = isel.gprSize();
1631	vi.setParts(isel, 2);
1632	_ = vi.addPart(isel, 0, ptr_size, ty.slicePtrFieldType(zcu));
1633	_ = vi.addPart(isel, ptr_size, ptr_size, .usize);
1634	},
1635	},
1636	.opt_type => |child_type| if (ty.optionalReprIsPayload(zcu)) {
1637	ty = .fromInterned(child_type);
1638	continue :type_key ip.indexToKey(child_type);
1639	} else {
1640	const child_ty: ZigType = .fromInterned(child_type);
1641	const child_size = child_ty.abiSize(zcu);
1642	vi.setParts(isel, 2);
1643	_ = vi.addPart(isel, 0, child_size, child_ty);
1644	_ = vi.addPart(isel, child_size, 1, .bool);
1645	},
1646	.array_type => |array_type| {
1647	const full_len = array_type.lenIncludingSentinel();
1648	const child_ty: ZigType = .fromInterned(array_type.child);
1649	const child_size = child_ty.abiSize(zcu);
1650	const aligned_size = child_ty.abiAlignment(zcu).forward(child_size);
1651	if (full_len == 1) {
1652	continue :type_key ip.indexToKey(child_ty.ip_index);
1653	} else if (full_len <= Value.max_parts) {
1654	vi.setParts(isel, @intCast(full_len));
1655	for (0..@intCast(full_len)) |part_i| {
1656	_ = vi.addPart(
1657	isel,
1658	@intCast(part_i * aligned_size),
1659	child_size,
1660	child_ty,
1661	);
1662	}
1663	} else {
1664	// Construct a tree with minimum nodes and depth
1665	// Minimum number of direct/indirect intermediate nodes to contain full_len leaf nodes
1666	const min_intermediate_nodes = (std.math.divCeil(u64, full_len - 1, Value.max_parts - 1) catch unreachable) - 1;
1667	assert(min_intermediate_nodes >= 1);
1668	// Number of direct intermediate children
1669	const intermediate_children = @min(Value.max_parts, min_intermediate_nodes);
1670	// Number of direct leaf children
1671	const leaf_children = @as(u64, Value.max_parts) - intermediate_children;
1672	// Number of indirect leaf children
1673	const indirect_leaf_children = full_len - leaf_children;
1674	// Length of each intermediate children
1675	const group_len = indirect_leaf_children / intermediate_children;
1676	const group_tail = indirect_leaf_children % intermediate_children;
1677	const tail_group_len = group_len + group_tail;
1678	const group_size = group_len * child_size;
1679	const tail_group_size = tail_group_len * child_size;
1680	const group_aligned_size = group_len * aligned_size;
1681	const group_ty: ZigType = if (intermediate_children == 1) undefined else try isel.pt.arrayType(.{
1682	.child = child_ty.ip_index,
1683	.len = group_len,
1684	});
1685	const tail_group_ty = if (array_type.sentinel == .none) try isel.pt.arrayType(.{
1686	.child = child_ty.ip_index,
1687	.len = tail_group_len,
1688	}) else try isel.pt.arrayType(.{
1689	.child = child_ty.ip_index,
1690	.len = tail_group_len - 1,
1691	.sentinel = array_type.sentinel,
1692	});
1693
1694	vi.setParts(isel, Value.max_parts);
1695	for (0..@intCast(leaf_children)) |part_i| {
1696	_ = vi.addPart(
1697	isel,
1698	@intCast(part_i * aligned_size),
1699	child_size,
1700	child_ty,
1701	);
1702	}
1703	const leaf_offset = leaf_children * aligned_size;
1704	for (0..@intCast(intermediate_children - 1)) |part_i| {
1705	_ = vi.addPart(
1706	isel,
1707	@intCast(leaf_offset + (part_i * group_aligned_size)),
1708	group_size,
1709	group_ty,
1710	);
1711	}
1712	_ = vi.addPart(
1713	isel,
1714	@intCast(leaf_offset + ((intermediate_children - 1) * group_aligned_size)),
1715	tail_group_size,
1716	tail_group_ty,
1717	);
1718	}
1719	},
1720	.anyframe_type => unreachable,
1721	.error_union_type => |error_union_type| {
1722	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
1723	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1724	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
1725
1726	var fields: [2]SplitStructField = undefined;
1727	var part_len: usize = 0;
1728	for (0..2) |field_index| {
1729	const field_name: enum { error_set, payload } = switch (field_index) {
1730	0 => if (error_set_offset < payload_offset) .error_set else .payload,
1731	1 => if (error_set_offset < payload_offset) .payload else .error_set,
1732	else => unreachable,
1733	};
1734	const field_ty: ZigType, const field_begin = switch (field_name) {
1735	.error_set => .{ .fromInterned(error_union_type.error_set_type), error_set_offset },
1736	.payload => .{ payload_ty, payload_offset },
1737	};
1738	const field_size = field_ty.abiSize(zcu);
1739	if (field_size == 0) continue;
1740
1741	fields[part_len] = .{ .offset = field_begin, .size = field_size };
1742	part_len += 1;
1743	}
1744
1745	try vi.splitStruct(isel, fields[0..part_len], .{
1746	.ty_size = vi.size(isel),
1747	.ty_alignment = vi.alignment(isel),
1748	.combine = false,
1749	});
1750	},
1751	.simple_type => |simple_type| switch (simple_type) {
1752	.f16, .f32, .f64, .f128, .c_longdouble => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
1753	.f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
1754	.usize,
1755	.isize,
1756	.c_char,
1757	.c_short,
1758	.c_ushort,
1759	.c_int,
1760	.c_uint,
1761	.c_long,
1762	.c_ulong,
1763	.c_longlong,
1764	.c_ulonglong,
1765	=> continue :type_key .{ .int_type = ty.intInfo(zcu) },
1766	.anyopaque,
1767	.void,
1768	.type,
1769	.comptime_int,
1770	.comptime_float,
1771	.noreturn,
1772	.null,
1773	.undefined,
1774	.enum_literal,
1775	.adhoc_inferred_error_set,
1776	.generic_poison,
1777	=> unreachable,
1778	.bool => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 1 } },
1779	.anyerror => continue :type_key .{ .int_type = .{
1780	.signedness = .unsigned,
1781	.bits = zcu.errorSetBits(),
1782	} },
1783	},
1784	.struct_type => {
1785	const loaded_struct = ip.loadStructType(ty.toIntern());
1786	switch (loaded_struct.layout) {
1787	.auto, .@"extern" => {},
1788	.@"packed" => {
1789	ty = .fromInterned(loaded_struct.packed_backing_int_type);
1790	continue :type_key ip.indexToKey(loaded_struct.packed_backing_int_type);
1791	},
1792	}
1793
1794	var field_end: u64 = 0;
1795	var field_it = loaded_struct.iterateRuntimeOrder(ip);
1796	var fields: []SplitStructField = try zcu.gpa.alloc(SplitStructField, loaded_struct.field_types.len);
1797	defer zcu.gpa.free(fields);
1798	var part_len: usize = 0;
1799	while (field_it.next()) |field_index| {
1800	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
1801	const field_begin = switch (loaded_struct.field_aligns.getOrNone(ip, field_index)) {
1802	.none => field_ty.abiAlignment(zcu),
1803	else => |field_align| field_align,
1804	}.forward(field_end);
1805	const field_size = field_ty.abiSize(zcu);
1806	if (field_size == 0) continue;
1807	field_end = field_begin + field_size;
1808
1809	fields[part_len] = .{ .offset = field_begin, .size = field_size, .ty = field_ty };
1810	part_len += 1;
1811	}
1812
1813	try vi.splitStruct(isel, fields[0..part_len], .{
1814	.ty_size = vi.size(isel),
1815	.ty_alignment = vi.alignment(isel),
1816	.combine = true,
1817	});
1818	},
1819	.tuple_type => |tuple_type| {
1820	var field_end: u64 = 0;
1821	var fields: []SplitStructField = try zcu.gpa.alloc(SplitStructField, tuple_type.types.len);
1822	defer zcu.gpa.free(fields);
1823	var part_len: usize = 0;
1824
1825	for (tuple_type.types.get(ip), tuple_type.values.get(ip)) |field_type, field_value| {
1826	if (field_value != .none) continue;
1827	const field_ty: ZigType = .fromInterned(field_type);
1828	const field_begin = field_ty.abiAlignment(zcu).forward(field_end);
1829	const field_size = field_ty.abiSize(zcu);
1830	if (field_size == 0) continue;
1831	field_end = field_begin + field_size;
1832
1833	fields[part_len] = .{ .offset = field_begin, .size = field_size, .ty = field_ty };
1834	part_len += 1;
1835	}
1836
1837	try vi.splitStruct(isel, fields[0..part_len], .{
1838	.ty_size = vi.size(isel),
1839	.ty_alignment = vi.alignment(isel),
1840	.combine = true,
1841	});
1842	},
1843	.union_type => {
1844	const loaded_union = ip.loadUnionType(ty.toIntern());
1845	switch (loaded_union.layout) {
1846	.auto, .@"extern" => {},
1847	.@"packed" => continue :type_key .{ .int_type = .{
1848	.signedness = .unsigned,
1849	.bits = @intCast(ty.bitSize(zcu)),
1850	} },
1851	}
1852
1853	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
1854	const tag_offset = union_layout.tagOffset();
1855	const payload_offset = union_layout.payloadOffset();
1856
1857	var field_end: u64 = 0;
1858	var fields: [2]SplitStructField = undefined;
1859	var part_len: usize = 0;
1860
1861	for (0..2) |field_index| {
1862	const field_name: enum { tag, payload } = switch (field_index) {
1863	0 => if (tag_offset < payload_offset) .tag else .payload,
1864	1 => if (tag_offset < payload_offset) .payload else .tag,
1865	else => unreachable,
1866	};
1867	const field_size, const field_begin = switch (field_name) {
1868	.tag => .{ union_layout.tag_size, tag_offset },
1869	.payload => .{ union_layout.payload_size, payload_offset },
1870	};
1871	if (field_size == 0) continue;
1872	field_end = field_begin + field_size;
1873
1874	fields[part_len] = .{ .offset = field_begin, .size = field_size };
1875	part_len += 1;
1876	}
1877
1878	try vi.splitStruct(isel, fields[0..part_len], .{
1879	.ty_size = vi.size(isel),
1880	.ty_alignment = vi.alignment(isel),
1881	.combine = false,
1882	});
1883	},
1884	.opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
1885	.enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).int_tag_type),
1886	.error_set_type,
1887	.inferred_error_set_type,
1888	=> continue :type_key .{ .simple_type = .anyerror },
1889	}
1890
1891	if (force and value.flags.parts_len_minus_one == 0) try vi.splitBlindly(isel);
1892	}
1893
1894	pub fn splitBlindly(vi: Value.Index, isel: *Select) !void {
1895	const value = vi.get(isel);
1896	value.flags.splitted = true;
1897	if (value.flags.parts_len_minus_one != 0) return;
1898
1899	return isel.fail("splitBlindly unimplemented", .{});
1900	}
1901
1902	const SplitStructField = struct {
1903	offset: u64,
1904	size: u64,
1905	ty: ZigType = .void,
1906	};
1907
1908	const SplitStructOpts = struct {
1909	ty_size: u64,
1910	ty_alignment: InternPool.Alignment,
1911	combine: bool,
1912	};
1913
1914	fn splitStruct(vi: Value.Index, isel: *Select, fields: []SplitStructField, opts: SplitStructOpts) !void {
1915	const min_part_log2_stride: u5 = switch (opts.ty_size) {
1916	0...4 => 0,
1917	5...8 => 2,
1918	9...16 => 3,
1919	else => 4,
1920	};
1921	if (fields.len > Value.max_parts and
1922	(std.math.divCeil(u64, opts.ty_size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts)
1923	{
1924	// fast path for structs with too many parts
1925	return;
1926	}
1927
1928	// split parts with combination
1929	const Part = struct {
1930	offset: u64,
1931	size: u64,
1932	ty: ZigType,
1933	vi: Value.Index,
1934	subparts: Value.PartsLen,
1935	};
1936	var new_parts: [Value.max_parts]Part = undefined;
1937	var parts_len: Value.PartsLen = 0;
1938	var field_end: u64 = 0;
1939	for (fields) |*struct_field| {
1940	const field_ty = struct_field.ty;
1941	const field_begin = struct_field.offset;
1942	const field_size = struct_field.size;
1943	field_end = field_begin + field_size;
1944	if (opts.combine and parts_len > 0) combine: {
1945	const prev_part = &new_parts[parts_len - 1];
1946	const combined_size = field_end - prev_part.offset;
1947	if (combined_size > @as(u64, 1) << @min(
1948	min_part_log2_stride,
1949	opts.ty_alignment.toLog2Units(),
1950	@ctz(prev_part.offset),
1951	)) break :combine;
1952	prev_part.size = combined_size;
1953	prev_part.ty = undefined;
1954	prev_part.subparts += 1;
1955	continue;
1956	}
1957	if (parts_len == Value.max_parts) return;
1958	new_parts[parts_len] = .{
1959	.offset = field_begin,
1960	.size = field_size,
1961	.ty = field_ty,
1962	.vi = undefined,
1963	.subparts = 1,
1964	};
1965	parts_len += 1;
1966	}
1967	if (parts_len <= 1) return;
1968	vi.setParts(isel, parts_len);
1969	for (new_parts[0..parts_len]) |*part| {
1970	part.vi = vi.addPart(
1971	isel,
1972	part.offset,
1973	part.size,
1974	if (part.subparts == 1 and part.ty.ip_index != .void_type) part.ty else null,
1975	);
1976	}
1977	const last_part = new_parts[parts_len - 1];
1978	const remaining_size = opts.ty_size - last_part.offset - last_part.size;
1979	if (remaining_size != 0)
1980	_ = vi.addPart(isel, last_part.offset, remaining_size, null);
1981
1982	// split combined parts
1983	var part_index: Value.PartsLen = 0;
1984	for (fields) |*struct_field| {
1985	const field_ty = struct_field.ty;
1986	const field_begin = struct_field.offset;
1987	const field_size = struct_field.size;
1988
1989	var new_part = &new_parts[part_index];
1990	while (new_part.offset + new_part.size <= field_begin) {
1991	part_index += 1;
1992	new_part = &new_parts[part_index];
1993	}
1994	if (new_part.subparts == 1) continue;
1995	if (!new_part.vi.hasParts(isel))
1996	new_part.vi.setParts(isel, new_part.subparts);
1997	_ = new_part.vi.addPart(
1998	isel,
1999	field_begin - new_part.offset,
2000	field_size,
2001	if (field_ty.ip_index != .void_type) field_ty else null,
2002	);
2003	}
2004	}
2005	};
2006
2007	pub const PartIterator = struct {
2008	vi: Value.Index,
2009	remaining: Value.PartsLen,
2010
2011	fn initOne(vi: Value.Index) PartIterator {
2012	return .{ .vi = vi, .remaining = 1 };
2013	}
2014
2015	pub fn next(it: *PartIterator) ?Value.Index {
2016	if (it.remaining == 0) return null;
2017	it.remaining -= 1;
2018	defer it.vi = @fromBackingInt(@backingInt(it.vi) + 1);
2019	return it.vi;
2020	}
2021
2022	pub fn peek(it: PartIterator) ?Value.Index {
2023	var it_mut = it;
2024	return it_mut.next();
2025	}
2026
2027	pub fn only(it: PartIterator) ?Value.Index {
2028	return if (it.remaining == 1) it.vi else null;
2029	}
2030	};
2031
2032	const Mat = struct {
2033	vi: Value.Index,
2034	/// Position of the materialized part
2035	offset: u64,
2036	/// Size of the materialized part
2037	size: u32,
2038	/// Expected live-in extension mode
2039	extension: Extension,
2040	/// Register are locked.
2041	location: Location,
2042	/// Whether the location stores the whole value or the materialized part
2043	full: bool,
2044
2045	comptime {
2046	if (!std.debug.runtime_safety) assert(@sizeOf(Mat) <= 32);
2047	}
2048
2049	const Error = codegen.Error;
2050
2051	pub fn ra(mat: Value.Mat) Register.Alias {
2052	return mat.location.register;
2053	}
2054
2055	pub fn reg(mat: Value.Mat) Register {
2056	return mat.location.register.reg;
2057	}
2058
2059	pub fn loc(mat: Value.Mat) Location {
2060	return switch (mat.location) {
2061	.register => |loc_ra| .{ .register = loc_ra },
2062	.stack_slot => |stack_slot| if (mat.full)
2063	.{ .stack_slot = stack_slot.withOffset(@intCast(mat.offset)) }
2064	else
2065	.{ .stack_slot = stack_slot },
2066	};
2067	}
2068
2069	fn finish(mat: Value.Mat, isel: *Select) Mat.Error!void {
2070	const vi = mat.vi;
2071	const value = vi.get(isel);
2072	tracking_log.debug("{f}[{d}..{d}] <- {f} (mat finish)", .{ vi, mat.offset, mat.offset + mat.size - 1, mat.loc() });
2073
2074	if (mat.location.asRegister()) |mat_reg|
2075	isel.freeReg(mat_reg);
2076
2077	const offset_from_root, const root_vi = vi.valueRoot(isel);
2078	switch (root_vi.parent(isel)) {
2079	.none => {
2080	// Try to set the location as expected
2081	if (mat.full and vi.location(isel) == null) {
2082	switch (value.flags.location_tag) {
2083	.extreme => unreachable,
2084	.small => {
2085	vi.setSmallLocation(isel, mat.location);
2086	vi.setExtension(isel, mat.extension);
2087	if (mat.location.asRegister()) |loc_reg|
2088	isel.live_registers.set(loc_reg, vi);
2089	return;
2090	},
2091	.large => switch (mat.location) {
2092	.stack_slot => |stack_slot| {
2093	value.location_payload.large.stack_slot = stack_slot;
2094	try vi.reextendAdvanced(isel, vi.bitSize(isel), mat.extension, vi.extension(isel));
2095	return;
2096	},
2097	else => {},
2098	},
2099	}
2100	}
2101
2102	// Initialize a location and copy
2103	if (vi.location(isel) == null) {
2104	switch (value.flags.location_tag) {
2105	.extreme => unreachable,
2106	.small => {
2107	const new_ra = (try vi.allocRegister(isel)).?;
2108	vi.setSmallLocation(isel, .{ .register = new_ra });
2109	isel.live_registers.set(new_ra.reg, vi);
2110	},
2111	.large => value.location_payload.large.stack_slot = vi.allocStackSlot(isel),
2112	}
2113	}
2114	switch (value.flags.location_tag) {
2115	.extreme => unreachable,
2116	.small => {},
2117	.large => {
2118	try vi.reextendAdvanced(isel, vi.bitSize(isel), mat.extension, vi.extension(isel));
2119	},
2120	}
2121	const vi_loc = vi.location(isel).?;
2122	const maybe_loc_reg = vi_loc.asRegister();
2123	if (maybe_loc_reg) |loc_reg| {
2124	const loc_live = isel.live_registers.getPtr(loc_reg);
2125	assert(loc_live.* == vi);
2126	loc_live.* = .allocating;
2127	}
2128	vi_loc.markRegWritten(isel);
2129	try isel.moveLoc(
2130	mat.location,
2131	if (mat.full) mat.offset else 0,
2132	vi_loc,
2133	mat.offset,
2134	mat.size,
2135	.preserved,
2136	);
2137	if (maybe_loc_reg) |loc_reg| {
2138	const loc_live = isel.live_registers.getPtr(loc_reg);
2139	assert(loc_live.* == .allocating);
2140	loc_live.* = vi;
2141	}
2142	},
2143	.value => unreachable,
2144	.address => |addr_vi| {
2145	try vi.reextendAdvanced(isel, vi.bitSize(isel), mat.extension, vi.extension(isel));
2146
2147	// reextend
2148	reextend: {
2149	const dst_ext = vi.extension(isel);
2150	const src_ext = mat.extension;
2151	if (dst_ext == src_ext or dst_ext == .garbage) break :reextend;
2152
2153	const bit_size = vi.bitSize(isel);
2154	if (bit_size == 0) break :reextend;
2155
2156	switch (mat.location) {
2157	.register => |loc_ra| {
2158	const offset_fixup = if (mat.full) 0 else mat.offset;
2159	const reg_bits = loc_ra.mod.bitSize(isel.target);
2160	const unused_bits = reg_bits - @min(bit_size - (offset_fixup * 8), reg_bits);
2161	try isel.fillUnusedBits(loc_ra.reg, loc_ra.reg, dst_ext, src_ext, @intCast(unused_bits));
2162	},
2163	.stack_slot => |stack| {
2164	const total_size = vi.size(isel);
2165	const unused_bits = (total_size * 8) - bit_size;
2166	const reg_mod: Register.Modifier = if (vi.isSmall(isel)) vi.hintModifier(isel) else .integer;
2167	const reg_class = reg_mod.class();
2168	const reg_size = reg_mod.byteSize(isel.target);
2169	const reg_alignment: InternPool.Alignment = .fromByteUnits(reg_size);
2170	const base_offset = @as(i65, stack.offset) - (if (mat.full) 0 else mat.offset);
2171
2172	var offset = reg_alignment.backward(bit_size / 8);
2173	const tmp_reg = try isel.allocRegForWrite(reg_class);
2174	defer isel.freeReg(tmp_reg);
2175	while (offset < total_size) {
2176	const part_size = @min(reg_size, total_size - offset);
2177	defer offset += part_size;
2178
2179	try isel.storeReg(tmp_reg, part_size, stack.base, base_offset + offset);
2180	try isel.fillUnusedBits(tmp_reg, tmp_reg, dst_ext, src_ext, @intCast(unused_bits));
2181	try isel.loadReg(tmp_reg, part_size, vi.extension(isel).signednessForLoad(), stack.base, base_offset + offset);
2182	}
2183	},
2184	}
2185	}
2186
2187	const addr_mat = try addr_vi.matIntRegZeroExt(isel);
2188	assert(addr_mat.ra().mod == .integer);
2189	try isel.moveLoc(
2190	mat.location,
2191	if (mat.full) mat.offset else 0,
2192	.{ .stack_slot = .{ .base = addr_mat.reg(), .offset = 0 } },
2193	offset_from_root + mat.offset,
2194	mat.size,
2195	.none,
2196	);
2197	try addr_mat.finish(isel);
2198	},
2199	.constant => |constant| {
2200	const mat_loc = mat.loc();
2201	mat_loc.markRegWritten(isel);
2202	try isel.moveConstant(mat_loc, constant, offset_from_root + mat.offset, mat.size);
2203	},
2204	}
2205	}
2206	};
2207
2208	/// DFS iterator over a sub-tree.
2209	const Walk = struct {
2210	isel: *Select,
2211	root_vi: Value.Index,
2212	next_vi: Value.Index,
2213	opts: Options,
2214
2215	const Options = packed struct {
2216	/// Reversed order
2217	reverse: bool = true,
2218	/// Whether to include root nodes
2219	root: bool = true,
2220	/// Whether to include intermdiate nodes
2221	/// (i.e. nodes that are not leaf vertexes)
2222	intermdiate: bool = true,
2223	/// Whether to include leaf vertexes
2224	leaves: bool = true,
2225	};
2226
2227	pub fn next(it: *Walk) ?Value.Index {
2228	const isel = it.isel;
2229	const opts = it.opts;
2230	while (it.next_vi != .free) {
2231	const node_vi = it.next_vi;
2232
2233	// find next node
2234	next_node: {
2235	// go to the first child
2236	if (node_vi.hasParts(isel)) {
2237	it.next_vi = if (!opts.reverse)
2238	node_vi.get(isel).parts
2239	else last_child: {
2240	const node_value = node_vi.get(isel);
2241	break :last_child @fromBackingInt(@backingInt(node_value.parts) + node_value.flags.parts_len_minus_one);
2242	};
2243	break :next_node;
2244	}
2245	if (node_vi.parentValue(isel) != null) {
2246	var iter_vi = node_vi;
2247	while (true) {
2248	// go to the next sibling
2249	const parent_vi = iter_vi.get(isel).parent_payload.value;
2250	const parent_value = parent_vi.get(isel);
2251	if (!opts.reverse) {
2252	const last_sibling = @backingInt(parent_value.parts) + parent_value.flags.parts_len_minus_one;
2253	if (@backingInt(iter_vi) < last_sibling) {
2254	it.next_vi = @fromBackingInt(@backingInt(iter_vi) + 1);
2255	break :next_node;
2256	}
2257	} else {
2258	if (@backingInt(iter_vi) > @backingInt(parent_value.parts)) {
2259	it.next_vi = @fromBackingInt(@backingInt(iter_vi) - 1);
2260	break :next_node;
2261	}
2262	}
2263	// return to ancestor's sibling
2264	if (parent_value.flags.parent_tag == .value)
2265	iter_vi = parent_vi
2266	else
2267	break;
2268	}
2269	}
2270	it.next_vi = .free;
2271	}
2272
2273	// filter nodes
2274	if (!it.opts.root and node_vi == it.root_vi) continue;
2275	if (!it.opts.intermdiate and node_vi.hasParts(isel)) continue;
2276	if (!it.opts.leaves and !node_vi.hasParts(isel)) continue;
2277	return node_vi;
2278	}
2279	return null;
2280	}
2281
2282	pub fn skipChildren(it: *Walk, current_vi: Value.Index) void {
2283	const isel = it.isel;
2284	const current_value = current_vi.get(isel);
2285	if (current_value.flags.parts_len_minus_one != 0) {
2286	const last_part = @backingInt(current_value.parts) + current_value.flags.parts_len_minus_one;
2287	it.next_vi = @fromBackingInt(last_part);
2288	_ = it.next();
2289	}
2290	}
2291
2292	pub fn peek(it: Walk) ?Value.Index {
2293	var it_mut = it;
2294	return it_mut.next();
2295	}
2296	};
2297	};
2298
2299	fn fail(isel: *Select, comptime format: []const u8, args: anytype) codegen.Error {
2300	@branchHint(.cold);
2301	wip_mir_log.debug("codegen error: " ++ format, args);
2302	return isel.pt.zcu.codegenFail(isel.nav_index, format, args);
2303	}
2304
2305	fn failUnimplemented(isel: *Select, comptime format: []const u8, args: anytype) codegen.Error!void {
2306	@branchHint(.cold);
2307	if (debug_trap_unimplemented_code) {
2308	const gpa = isel.pt.zcu.gpa;
2309
2310	const msg = try std.fmt.allocPrintSentinel(gpa, format, args, 0);
2311	defer gpa.free(msg);
2312	wip_mir_log.err("{s}", .{msg});
2313	try isel.emit(.@"break"(0xaa));
2314	try isel.moveDebugString(.r22, msg);
2315	} else return isel.fail(format, args);
2316	}
2317
2318	fn moveDebugString(isel: *Select, reg: Register, msg: [:0]const u8) error{ OutOfMemory, AlreadyReported }!void {
2319	@branchHint(.cold);
2320	assert(debug_trap_unimplemented_code);
2321
2322	const pt = isel.pt;
2323	const zcu = pt.zcu;
2324	const ip = &zcu.intern_pool;
2325	const gpa = zcu.gpa;
2326
2327	const msg_ty = try pt.arrayType(.{
2328	.len = msg.len,
2329	.child = .u8_type,
2330	.sentinel = .zero_u8,
2331	});
2332	const msg_str = try ip.getOrPutString(gpa, zcu.comp.io, pt.tid, msg, .maybe_embedded_nulls);
2333	const msg_val = try pt.intern(.{ .aggregate = .{
2334	.ty = msg_ty.ip_index,
2335	.storage = .{ .bytes = msg_str },
2336	} });
2337	const msg_ptr = try pt.intern(.{ .ptr = .{
2338	.ty = .manyptr_const_u8_sentinel_0_type,
2339	.base_addr = .{ .uav = .{
2340	.val = msg_val,
2341	.orig_ty = .manyptr_const_u8_sentinel_0_type,
2342	} },
2343	.byte_offset = 0,
2344	} });
2345	try isel.moveConstant(
2346	.{ .register = .{ .reg = reg, .mod = .integer } },
2347	.fromInterned(msg_ptr),
2348	0,
2349	isel.gprSize(),
2350	);
2351	}
2352
2353	pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
2354	const zcu = isel.pt.zcu;
2355	const ip = &zcu.intern_pool;
2356	const gpa = zcu.gpa;
2357	const air_tags = isel.air.instructions.items(.tag);
2358	const air_data = isel.air.instructions.items(.data);
2359	const initial_def_order_len = isel.def_order.count();
2360
2361	for (air_body) |air_inst_index| {
2362	switch (air_tags[@backingInt(air_inst_index)]) {
2363	else => |air_tag| return isel.fail("unimplemented analyze for {t}", .{air_tag}),
2364	.arg,
2365	.ret_addr,
2366	.frame_addr,
2367	.err_return_trace,
2368	.save_err_return_trace_index,
2369	.runtime_nav_ptr,
2370	.c_va_start,
2371	=> {
2372	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2373	},
2374	.add,
2375	.add_safe,
2376	.add_optimized,
2377	.add_wrap,
2378	.add_sat,
2379	.sub,
2380	.sub_safe,
2381	.sub_optimized,
2382	.sub_wrap,
2383	.sub_sat,
2384	.mul,
2385	.mul_safe,
2386	.mul_optimized,
2387	.mul_wrap,
2388	.mul_sat,
2389	.div_float,
2390	.div_float_optimized,
2391	.div_trunc,
2392	.div_trunc_optimized,
2393	.div_floor,
2394	.div_floor_optimized,
2395	.div_exact,
2396	.div_exact_optimized,
2397	.rem,
2398	.rem_optimized,
2399	.mod,
2400	.mod_optimized,
2401	.max,
2402	.min,
2403	.bit_and,
2404	.bit_or,
2405	.shr,
2406	.shr_exact,
2407	.shl,
2408	.shl_exact,
2409	.shl_sat,
2410	.xor,
2411	.cmp_lt,
2412	.cmp_lt_optimized,
2413	.cmp_lte,
2414	.cmp_lte_optimized,
2415	.cmp_eq,
2416	.cmp_eq_optimized,
2417	.cmp_gte,
2418	.cmp_gte_optimized,
2419	.cmp_gt,
2420	.cmp_gt_optimized,
2421	.cmp_neq,
2422	.cmp_neq_optimized,
2423	.array_elem_val,
2424	.slice_elem_val,
2425	.ptr_elem_val,
2426	=> {
2427	const bin_op = air_data[@backingInt(air_inst_index)].bin_op;
2428
2429	try isel.analyzeUse(bin_op.lhs);
2430	try isel.analyzeUse(bin_op.rhs);
2431	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2432	},
2433	.ptr_add,
2434	.ptr_sub,
2435	.add_with_overflow,
2436	.sub_with_overflow,
2437	.mul_with_overflow,
2438	.shl_with_overflow,
2439	.slice,
2440	.slice_elem_ptr,
2441	.ptr_elem_ptr,
2442	=> {
2443	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2444	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
2445
2446	try isel.analyzeUse(bin_op.lhs);
2447	try isel.analyzeUse(bin_op.rhs);
2448	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2449	},
2450	.alloc => {
2451	const ty = air_data[@backingInt(air_inst_index)].ty;
2452
2453	isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu));
2454	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2455	},
2456	.inferred_alloc,
2457	.inferred_alloc_comptime,
2458	.wasm_memory_size,
2459	.wasm_memory_grow,
2460	.work_item_id,
2461	.work_group_size,
2462	.work_group_id,
2463	=> unreachable,
2464	.ret, .ret_safe, .ret_load => {
2465	const un_op = air_data[@backingInt(air_inst_index)].un_op;
2466	isel.returns = true;
2467
2468	assert(isel.active_blocks.keys()[0] == Block.main);
2469
2470	try isel.analyzeUse(un_op);
2471	},
2472	.ret_ptr => {
2473	const ty = air_data[@backingInt(air_inst_index)].ty;
2474
2475	if (isel.live_values.get(Block.main)) |ret_vi| {
2476	switch (ret_vi.parent(isel)) {
2477	.none => isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu)),
2478	.value, .constant => unreachable,
2479	.address => |address_vi| try isel.live_values.putNoClobber(gpa, air_inst_index, address_vi.ref(isel)),
2480	}
2481	if (ret_vi.stackSlot(isel) != null)
2482	isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu));
2483	}
2484	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2485	},
2486	.assembly => {
2487	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2488	const extra = isel.air.extraData(Air.Asm, ty_pl.payload);
2489	const operands: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0 .. extra.data.flags.outputs_len + extra.data.inputs_len]);
2490
2491	for (operands) |operand| if (operand != .none) try isel.analyzeUse(operand);
2492	if (ty_pl.ty.ip_index != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2493	},
2494	.not,
2495	.clz,
2496	.ctz,
2497	.popcount,
2498	.byte_swap,
2499	.bit_reverse,
2500	.abs,
2501	.load,
2502	.fptrunc,
2503	.fpext,
2504	.int_cast,
2505	.int_cast_safe,
2506	.trunc,
2507	.optional_payload,
2508	.optional_payload_ptr,
2509	.optional_payload_ptr_set,
2510	.wrap_optional,
2511	.unwrap_errunion_payload,
2512	.unwrap_errunion_err,
2513	.unwrap_errunion_payload_ptr,
2514	.unwrap_errunion_err_ptr,
2515	.errunion_payload_ptr_set,
2516	.wrap_errunion_payload,
2517	.wrap_errunion_err,
2518	.struct_field_ptr_index_0,
2519	.struct_field_ptr_index_1,
2520	.struct_field_ptr_index_2,
2521	.struct_field_ptr_index_3,
2522	.get_union_tag,
2523	.ptr_slice_len_ptr,
2524	.ptr_slice_ptr_ptr,
2525	.array_to_slice,
2526	.int_from_float,
2527	.int_from_float_optimized,
2528	.int_from_float_safe,
2529	.int_from_float_optimized_safe,
2530	.float_from_int,
2531	.splat,
2532	.error_set_has_value,
2533	.addrspace_cast,
2534	.c_va_arg,
2535	.c_va_copy,
2536	.bit_cast,
2537	.ptr_cast,
2538	.ptr_from_int,
2539	.int_from_ptr,
2540	.error_cast,
2541	.error_from_int,
2542	.int_from_error,
2543	.union_from_enum,
2544	=> {
2545	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
2546
2547	try isel.analyzeUse(ty_op.operand);
2548	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2549	},
2550	.loop => {
2551	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2552	const extra = isel.air.extraData(Air.Block, ty_pl.payload);
2553
2554	try isel.active_loops.append(gpa, @fromBackingInt(@intCast(isel.loops.count())));
2555	try isel.loops.putNoClobber(gpa, air_inst_index, .{
2556	.def_order = @intCast(isel.def_order.count()),
2557	.outer_live = 0,
2558	.repeat_list = undefined,
2559	});
2560	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
2561	assert(isel.active_loops.pop().?.inst(isel) == air_inst_index);
2562	},
2563	.repeat, .trap, .unreach => {},
2564	.br => {
2565	const br = air_data[@backingInt(air_inst_index)].br;
2566	try isel.analyzeUse(br.operand);
2567	},
2568	.breakpoint, .dbg_stmt, .dbg_empty_stmt, .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline, .c_va_end => {},
2569	.sqrt,
2570	.sin,
2571	.cos,
2572	.tan,
2573	.exp,
2574	.exp2,
2575	.log,
2576	.log2,
2577	.log10,
2578	.floor,
2579	.ceil,
2580	.round,
2581	.trunc_float,
2582	.neg,
2583	.neg_optimized,
2584	.is_null,
2585	.is_non_null,
2586	.is_null_ptr,
2587	.is_non_null_ptr,
2588	.is_err,
2589	.is_non_err,
2590	.is_err_ptr,
2591	.is_non_err_ptr,
2592	.is_named_enum_value,
2593	.tag_name,
2594	.error_name,
2595	.cmp_lte_errors_len,
2596	=> {
2597	const un_op = air_data[@backingInt(air_inst_index)].un_op;
2598
2599	try isel.analyzeUse(un_op);
2600	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2601	},
2602	.cmp_vector, .cmp_vector_optimized => {
2603	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2604	const extra = isel.air.extraData(Air.VectorCmp, ty_pl.payload).data;
2605
2606	try isel.analyzeUse(extra.lhs);
2607	try isel.analyzeUse(extra.rhs);
2608	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2609	},
2610	.store,
2611	.store_safe,
2612	.set_union_tag,
2613	.memset,
2614	.memset_safe,
2615	.memcpy,
2616	.memmove,
2617	.atomic_store_unordered,
2618	.atomic_store_monotonic,
2619	.atomic_store_release,
2620	.atomic_store_seq_cst,
2621	=> {
2622	const bin_op = air_data[@backingInt(air_inst_index)].bin_op;
2623
2624	try isel.analyzeUse(bin_op.lhs);
2625	try isel.analyzeUse(bin_op.rhs);
2626	},
2627	.struct_field_ptr, .agg_field_val => {
2628	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2629	const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
2630
2631	try isel.analyzeUse(extra.struct_operand);
2632	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2633	},
2634	.aggregate_init => {
2635	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2636	const elements: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[ty_pl.payload..][0..@intCast(ty_pl.ty.arrayLen(zcu))]);
2637
2638	for (elements) |element| try isel.analyzeUse(element);
2639	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2640	},
2641	.union_init => {
2642	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2643	const extra = isel.air.extraData(Air.UnionInit, ty_pl.payload).data;
2644
2645	try isel.analyzeUse(extra.init);
2646	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2647	},
2648	.prefetch => {
2649	const prefetch = air_data[@backingInt(air_inst_index)].prefetch;
2650	try isel.analyzeUse(prefetch.ptr);
2651	},
2652	.field_parent_ptr => {
2653	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2654	const extra = isel.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
2655
2656	try isel.analyzeUse(extra.field_ptr);
2657	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2658	},
2659	.set_err_return_trace => {
2660	const un_op = air_data[@backingInt(air_inst_index)].un_op;
2661	try isel.analyzeUse(un_op);
2662	},
2663	inline .block, .dbg_inline_block => |air_tag| {
2664	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2665	const extra = isel.air.extraData(switch (air_tag) {
2666	else => comptime unreachable,
2667	.block => Air.Block,
2668	.dbg_inline_block => Air.DbgInlineBlock,
2669	}, ty_pl.payload);
2670	const result_ty = ty_pl.ty;
2671
2672	if (result_ty.ip_index == .noreturn_type) {
2673	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
2674	break;
2675	}
2676
2677	assert(!(try isel.active_blocks.getOrPut(gpa, air_inst_index)).found_existing);
2678	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
2679	const block_entry = isel.active_blocks.pop().?;
2680	assert(block_entry.key == air_inst_index);
2681
2682	if (result_ty.ip_index != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2683	},
2684	.call,
2685	.call_always_tail,
2686	.call_never_tail,
2687	.call_never_inline,
2688	=> {
2689	const pl_op = air_data[@backingInt(air_inst_index)].pl_op;
2690	const extra = isel.air.extraData(Air.Call, pl_op.payload);
2691	const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
2692	isel.saved_registers.insert(.ra);
2693	const callee_ty = isel.air.typeOf(pl_op.operand, ip);
2694	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
2695	else => unreachable,
2696	.func_type => |func_type| func_type,
2697	.ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type,
2698	};
2699
2700	try isel.analyzeUse(pl_op.operand);
2701	var cc_it: CallAbiIterator = .{ .isel = isel, .cc = &func_info.cc };
2702
2703	const ret_ty = isel.air.typeOfIndex(air_inst_index, ip);
2704	if (try cc_it.resolve(ret_ty, true)) |ret_vi| {
2705	tracking_log.debug("{f} <- %{d} (call return)", .{ ret_vi, @backingInt(air_inst_index) });
2706	switch (ret_vi.parent(isel)) {
2707	.none => {},
2708	.value, .constant => unreachable,
2709	.address => |address_vi| {
2710	defer address_vi.deref(isel);
2711	const ret_value = ret_vi.get(isel);
2712	ret_value.flags.parent_tag = .none;
2713	ret_value.parent_payload = .{ .none = {} };
2714	},
2715	}
2716	try isel.live_values.putNoClobber(gpa, air_inst_index, ret_vi);
2717
2718	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2719	}
2720
2721	for (args) |arg| {
2722	{
2723	const restore_values_len = isel.values.items.len;
2724	defer isel.values.shrinkRetainingCapacity(restore_values_len);
2725	defer isel.value_types.shrinkRetainingCapacity(restore_values_len);
2726
2727	const param_ty = isel.air.typeOf(arg, ip);
2728	const param_vi = try cc_it.resolve(param_ty, false) orelse continue;
2729	defer param_vi.deref(isel);
2730
2731	const passed_vi = switch (param_vi.parent(isel)) {
2732	.none => param_vi,
2733	.value, .constant => unreachable,
2734	.address => |address_vi| address_vi,
2735	};
2736	if (passed_vi.stackSlot(isel)) |stack_slot| {
2737	assert(stack_slot.base == Register.sp);
2738	isel.stack_size = @max(
2739	isel.stack_size,
2740	stack_slot.offset + @as(u24, @intCast(passed_vi.size(isel))),
2741	);
2742	}
2743	}
2744
2745	try isel.analyzeUse(arg);
2746	}
2747	},
2748	.cond_br => {
2749	const pl_op = air_data[@backingInt(air_inst_index)].pl_op;
2750	const extra = isel.air.extraData(Air.CondBr, pl_op.payload);
2751
2752	try isel.analyzeUse(pl_op.operand);
2753
2754	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.then_body_len]));
2755	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
2756	},
2757	.switch_br => {
2758	const switch_br = isel.air.unwrapSwitch(air_inst_index);
2759
2760	try isel.analyzeUse(switch_br.operand);
2761
2762	var cases_it = switch_br.iterateCases();
2763	while (cases_it.next()) |case| try isel.analyze(case.body);
2764	if (switch_br.else_body_len > 0) try isel.analyze(cases_it.elseBody());
2765	},
2766	.loop_switch_br => {
2767	const switch_br = isel.air.unwrapSwitch(air_inst_index);
2768
2769	try isel.active_loops.append(gpa, @fromBackingInt(@intCast(isel.loops.count())));
2770	try isel.loops.putNoClobber(gpa, air_inst_index, .{
2771	.def_order = @intCast(isel.def_order.count()),
2772	.outer_live = 0,
2773	.repeat_list = undefined,
2774	});
2775
2776	var cases_it = switch_br.iterateCases();
2777	while (cases_it.next()) |case| try isel.analyze(case.body);
2778	if (switch_br.else_body_len > 0) try isel.analyze(cases_it.elseBody());
2779
2780	assert(isel.active_loops.pop().?.inst(isel) == air_inst_index);
2781	},
2782	.switch_dispatch => {
2783	const br = air_data[@backingInt(air_inst_index)].br;
2784	try isel.analyzeUse(br.operand);
2785	},
2786	.slice_ptr => {
2787	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
2788
2789	try isel.analyzeUse(ty_op.operand);
2790	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2791
2792	const slice_vi = try isel.use(ty_op.operand);
2793	const ptr_part_vi = try slice_vi.partExact(isel, 0, 8);
2794	try isel.live_values.putNoClobber(gpa, air_inst_index, ptr_part_vi.ref(isel));
2795	},
2796	.slice_len => {
2797	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
2798
2799	try isel.analyzeUse(ty_op.operand);
2800	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2801
2802	const slice_vi = try isel.use(ty_op.operand);
2803	const len_part_vi = try slice_vi.partExact(isel, 8, 8);
2804	try isel.live_values.putNoClobber(gpa, air_inst_index, len_part_vi.ref(isel));
2805	},
2806	.reduce, .reduce_optimized => {
2807	const reduce = air_data[@backingInt(air_inst_index)].reduce;
2808
2809	try isel.analyzeUse(reduce.operand);
2810	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2811	},
2812	.shuffle_one => {
2813	const extra = isel.air.unwrapShuffleOne(zcu, air_inst_index);
2814
2815	try isel.analyzeUse(extra.operand);
2816	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2817	},
2818	.shuffle_two => {
2819	const extra = isel.air.unwrapShuffleTwo(zcu, air_inst_index);
2820
2821	try isel.analyzeUse(extra.operand_a);
2822	try isel.analyzeUse(extra.operand_b);
2823	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2824	},
2825	.@"try", .try_cold => {
2826	const pl_op = air_data[@backingInt(air_inst_index)].pl_op;
2827	const extra = isel.air.extraData(Air.Try, pl_op.payload);
2828
2829	try isel.analyzeUse(pl_op.operand);
2830	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
2831	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2832	},
2833	.try_ptr, .try_ptr_cold => {
2834	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2835	const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload);
2836
2837	try isel.analyzeUse(extra.data.ptr);
2838	try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
2839	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2840	},
2841	.cmpxchg_weak, .cmpxchg_strong => {
2842	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
2843	const extra = isel.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
2844
2845	try isel.analyzeUse(extra.ptr);
2846	try isel.analyzeUse(extra.expected_value);
2847	try isel.analyzeUse(extra.new_value);
2848	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2849	},
2850	.atomic_load => {
2851	const atomic_load = air_data[@backingInt(air_inst_index)].atomic_load;
2852
2853	try isel.analyzeUse(atomic_load.ptr);
2854	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2855	},
2856	.atomic_rmw => {
2857	const pl_op = air_data[@backingInt(air_inst_index)].pl_op;
2858	const extra = isel.air.extraData(Air.AtomicRmw, pl_op.payload).data;
2859
2860	try isel.analyzeUse(extra.operand);
2861	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
2862	},
2863	}
2864	}
2865	isel.def_order.shrinkRetainingCapacity(initial_def_order_len);
2866	}
2867
2868	fn analyzeUse(isel: *Select, air_ref: Air.Inst.Ref) !void {
2869	const air_inst_index = air_ref.toIndex() orelse return;
2870	const def_order_index = isel.def_order.getIndex(air_inst_index).?;
2871
2872	// Loop liveness
2873	var active_loop_index = isel.active_loops.items.len;
2874	while (active_loop_index > 0) {
2875	const prev_active_loop_index = active_loop_index - 1;
2876	const active_loop = isel.active_loops.items[prev_active_loop_index];
2877	if (def_order_index >= active_loop.get(isel).def_order) break;
2878	active_loop_index = prev_active_loop_index;
2879	}
2880	if (active_loop_index < isel.active_loops.items.len) {
2881	const active_loop = isel.active_loops.items[active_loop_index];
2882	const loop_live_gop =
2883	try isel.loop_outer_live.set.getOrPut(isel.pt.zcu.gpa, .{ active_loop, air_inst_index });
2884	if (!loop_live_gop.found_existing) active_loop.get(isel).outer_live += 1;
2885	}
2886	}
2887
2888	pub fn finishAnalysis(isel: *Select) !void {
2889	const gpa = isel.pt.zcu.gpa;
2890
2891	// Loop liveness
2892	if (isel.loops.count() > 0) {
2893	try isel.loops.ensureUnusedCapacity(gpa, 1);
2894
2895	const loop_live_len: u32 = @intCast(isel.loop_outer_live.set.count());
2896	if (loop_live_len > 0) {
2897	try isel.loop_outer_live.list.resize(gpa, loop_live_len);
2898
2899	// prefix sum
2900	const loops = isel.loops.values();
2901	for (loops[1..], loops[0 .. loops.len - 1]) |*loop, prev_loop| loop.outer_live += prev_loop.outer_live;
2902	assert(loops[loops.len - 1].outer_live == loop_live_len);
2903
2904	for (isel.loop_outer_live.set.keys()) |entry| {
2905	const loop, const inst = entry;
2906	const loop_live = &loop.get(isel).outer_live;
2907	loop_live.* -= 1;
2908	isel.loop_outer_live.list.items[loop_live.*] = inst;
2909	}
2910	assert(loops[0].outer_live == 0);
2911	}
2912
2913	const invalid_gop = isel.loops.getOrPutAssumeCapacity(Loop.invalid);
2914	assert(!invalid_gop.found_existing);
2915	invalid_gop.value_ptr.* = .{
2916	.def_order = undefined,
2917	.outer_live = loop_live_len,
2918	.repeat_list = undefined,
2919	};
2920	}
2921
2922	assert(isel.active_blocks.count() == 1 and isel.active_blocks.keys()[0] == Select.Block.main);
2923	assert(isel.active_loops.items.len == 0);
2924	}
2925
2926	pub fn verify(isel: *Select, check_values: bool) void {
2927	if (!std.debug.runtime_safety) return;
2928	assert(isel.active_blocks.count() == 1 and isel.active_blocks.keys()[0] == Select.Block.main);
2929	assert(isel.active_loops.items.len == 0);
2930	assert(isel.values.items.len == isel.value_types.items.len);
2931
2932	// Verify register state
2933	var live_reg_it = isel.live_registers.iterator();
2934	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
2935	_ => {
2936	tracking_log.err("{f}: still using ${t}", .{ live_reg_entry.value.*, live_reg_entry.key });
2937	isel.dumpValues(.all);
2938	unreachable;
2939	},
2940	.allocating, .free => {},
2941	};
2942
2943	// Check values state
2944	if (!check_values) return;
2945	for (isel.values.items, 0..) |value, vi_i| {
2946	const vi: Value.Index = @fromBackingInt(@as(@typeInfo(Value.Index).@"enum".tag_type, @intCast(vi_i)));
2947	if (value.refs != 0) {
2948	tracking_log.err("{f}: still referenced", .{vi});
2949	isel.dumpValues(.all);
2950	unreachable;
2951	}
2952	if (value.flags.parent_tag == .none and value.offset_from_parent != 0) {
2953	tracking_log.err("{f}: values without none cannot have offset from parent", .{vi});
2954	isel.dumpValues(.all);
2955	unreachable;
2956	}
2957	// Stack slot locations are allowed because layout values use them
2958	if (vi.register(isel) != null) {
2959	tracking_log.err("{f}: still has a location", .{vi});
2960	isel.dumpValues(.all);
2961	unreachable;
2962	}
2963	}
2964	}
2965
2966	pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void {
2967	const zcu = isel.pt.zcu;
2968	const ip = &zcu.intern_pool;
2969	const gpa = zcu.gpa;
2970
2971	{
2972	var live_reg_it = isel.live_registers.iterator();
2973	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
2974	.allocating => {
2975	tracking_log.err("${t} is allocated", .{live_reg_entry.key});
2976	isel.dumpValues(.all);
2977	unreachable;
2978	},
2979	_, .free => {},
2980	};
2981	}
2982
2983	var air: struct {
2984	isel: *Select,
2985	tag_items: []const Air.Inst.Tag,
2986	data_items: []const Air.Inst.Data,
2987	body: []const Air.Inst.Index,
2988	body_index: u32,
2989	inst_index: Air.Inst.Index,
2990
2991	fn tag(it: *@This(), inst_index: Air.Inst.Index) Air.Inst.Tag {
2992	return it.tag_items[@backingInt(inst_index)];
2993	}
2994
2995	fn data(it: *@This(), inst_index: Air.Inst.Index) Air.Inst.Data {
2996	return it.data_items[@backingInt(inst_index)];
2997	}
2998
2999	fn next(it: *@This()) ?Air.Inst.Tag {
3000	if (it.body_index == 0) {
3001	@branchHint(.unlikely);
3002	return null;
3003	}
3004	it.body_index -= 1;
3005	it.inst_index = it.body[it.body_index];
3006	wip_mir_log.debug("{f}", .{it.fmtAir(it.inst_index)});
3007	if (@import("builtin").mode == .debug) {
3008	if (it.isel.live_values.get(it.inst_index)) |def_vi| {
3009	wip_mir_log.debug(" <- {f}", .{it.isel.fmtValue(def_vi)});
3010	}
3011	}
3012	return it.tag(it.inst_index);
3013	}
3014
3015	fn fmtAir(it: @This(), inst: Air.Inst.Index) struct {
3016	isel: *Select,
3017	inst: Air.Inst.Index,
3018	pub fn format(fmt_air: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
3019	fmt_air.isel.air.writeInst(writer, fmt_air.inst, fmt_air.isel.pt, null);
3020	}
3021	} {
3022	return .{ .isel = it.isel, .inst = inst };
3023	}
3024	} = .{
3025	.isel = isel,
3026	.tag_items = isel.air.instructions.items(.tag),
3027	.data_items = isel.air.instructions.items(.data),
3028	.body = air_body,
3029	.body_index = @intCast(air_body.len),
3030	.inst_index = undefined,
3031	};
3032	while (air.next()) |air_tag| {
3033	switch (air_tag) {
3034	else => if (debug_trap_unimplemented_code) {
3035	if (isel.live_values.fetchRemove(air.inst_index)) |vi| {
3036	vi.value.deref(isel);
3037	isel.wipeLocationDfs(vi.value);
3038	}
3039	try isel.failUnimplemented("unimplemented select for {s}", .{@tagName(air_tag)});
3040	} else return isel.fail("unimplemented select for {s}", .{@tagName(air_tag)}),
3041
3042	// Misc
3043	.unreach => {},
3044	.trap, .breakpoint => try isel.emit(.@"break"(0)),
3045
3046	// Arguments & return
3047	.arg => {
3048	const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
3049	defer arg_vi.deref(isel);
3050	const layout_vi = isel.arg_layouts[@backingInt(air.inst_index)];
3051	layout_vi.deref(isel);
3052	switch (layout_vi.parent(isel)) {
3053	.none => try arg_vi.defLiveIn(isel, layout_vi, .{}),
3054	.value, .constant => unreachable,
3055	.address => |layout_addr_vi| {
3056	switch (arg_vi.parent(isel)) {
3057	else => unreachable,
3058	.address => |arg_addr_vi| {
3059	try arg_addr_vi.defLiveIn(isel, layout_addr_vi, .{});
3060	},
3061	}
3062	},
3063	}
3064	},
3065	.ret, .ret_safe => {
3066	assert(isel.active_blocks.keys()[0] == Block.main);
3067	try isel.active_blocks.values()[0].branch(isel);
3068	if (isel.live_values.get(Block.main)) |ret_vi| {
3069	const un_op = air.data(air.inst_index).un_op;
3070	const src_vi = try isel.use(un_op);
3071	switch (ret_vi.parent(isel)) {
3072	.none => try src_vi.matLiveOut(isel, ret_vi, .{ .mode = .ret }),
3073	.value, .constant => unreachable,
3074	.address => |addr_vi| {
3075	const addr_mat = try addr_vi.matIntRegZeroExt(isel);
3076	try src_vi.matStore(isel, addr_mat.reg(), 0, .{});
3077	try addr_mat.finish(isel);
3078	},
3079	}
3080	}
3081	},
3082	.ret_load => {
3083	const un_op = air.data(air.inst_index).un_op;
3084	const ptr_ty = isel.air.typeOf(un_op, ip);
3085	const ptr_info = ptr_ty.ptrInfo(zcu);
3086	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load ret_load", .{});
3087
3088	assert(isel.active_blocks.keys()[0] == Block.main);
3089	try isel.active_blocks.values()[0].branch(isel);
3090	if (isel.live_values.get(Block.main)) |layout_vi| switch (layout_vi.parent(isel)) {
3091	.none => {
3092	const ptr_vi = try isel.use(un_op);
3093	const ret_ty = ptr_ty.childType(zcu);
3094	const ret_vi = try isel.initValue(ret_ty);
3095	ret_vi.setParent(isel, .{ .address = ptr_vi });
3096	try ret_vi.matLiveOut(isel, layout_vi, .{ .mode = .ret });
3097	},
3098	.value, .constant => unreachable,
3099	.address => {},
3100	};
3101	},
3102
3103	// Frame addresses
3104	.ret_addr => if (isel.live_values.fetchRemove(air.inst_index)) |addr_vi| unused: {
3105	defer addr_vi.value.deref(isel);
3106	const addr_reg = try addr_vi.value.defRegMod(isel, .integer) orelse break :unused;
3107	try isel.ldIncoming(addr_reg, .ra);
3108	},
3109	.frame_addr => if (isel.live_values.fetchRemove(air.inst_index)) |addr_vi| unused: {
3110	defer addr_vi.value.deref(isel);
3111	const addr_reg = try addr_vi.value.defRegMod(isel, .integer) orelse break :unused;
3112	isel.saved_registers.insert(.fp);
3113	try isel.emit(.ori(addr_reg, .fp, 0));
3114	},
3115
3116	// Debugging
3117	.dbg_stmt, .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => {},
3118	.dbg_empty_stmt => try isel.emit(.andi(.r0, .r0, 0)),
3119
3120	// Control-flows
3121	.dbg_inline_block => {
3122	const ty_pl = air.data(air.inst_index).ty_pl;
3123	const extra = isel.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
3124	try isel.block(air.inst_index, ty_pl.ty, @ptrCast(
3125	isel.air.extra.items[extra.end..][0..extra.data.body_len],
3126	));
3127	},
3128	.block => {
3129	const ty_pl = air.data(air.inst_index).ty_pl;
3130	const extra = isel.air.extraData(Air.Block, ty_pl.payload);
3131	try isel.block(air.inst_index, ty_pl.ty, @ptrCast(
3132	isel.air.extra.items[extra.end..][0..extra.data.body_len],
3133	));
3134	},
3135	.loop => {
3136	const ty_pl = air.data(air.inst_index).ty_pl;
3137	const extra = isel.air.extraData(Air.Block, ty_pl.payload);
3138	const loops = isel.loops.values();
3139	const loop_index = isel.loops.getIndex(air.inst_index).?;
3140	const loop = &loops[loop_index];
3141
3142	tracking_log.debug("{f}", .{isel.fmtLoopLive(air.inst_index)});
3143	loop.snapshot = try isel.takeLocationSnapshot();
3144	tracking_log.debug("loop snapshot taken:\n{f}", .{loop.snapshot});
3145	loop.repeat_list = Loop.empty_list;
3146
3147	try isel.active_loops.append(gpa, @fromBackingInt(@intCast(loop_index)));
3148	try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
3149	assert(isel.active_loops.pop().?.inst(isel) == air.inst_index);
3150
3151	tracking_log.debug("loop %{d}: merge snapshot after loop body", .{@backingInt(air.inst_index)});
3152	try loop.snapshot.merge(isel);
3153	loop.snapshot.deinit(isel);
3154	loop.snapshot = .empty;
3155
3156	tracking_log.debug("loop %{d}: kill registers written in loop body", .{@backingInt(air.inst_index)});
3157	try isel.fillRegsBatch(loop.written_regs, false);
3158	// copy written registers to outer loops
3159	isel.markRegsWritten(loop.written_regs);
3160
3161	// relocate branches
3162	var repeat_label = loop.repeat_list;
3163	assert(repeat_label != Loop.empty_list);
3164	while (repeat_label != Loop.empty_list) {
3165	const instruction = &isel.instructions.items[repeat_label];
3166	const next_repeat_label = instruction.*;
3167	instruction.* = .b(0, 0);
3168	try isel.internal_relocs.append(gpa, .{
3169	.target = isel.instructions.items.len,
3170	.reloc = .{ .label = repeat_label, .type = .B26 },
3171	});
3172	repeat_label = @bitCast(next_repeat_label);
3173	}
3174	},
3175	.repeat => {
3176	const repeat = air.data(air.inst_index).repeat;
3177	try isel.loops.getPtr(repeat.loop_inst).?.branch(isel);
3178	},
3179	.br => {
3180	const br = air.data(air.inst_index).br;
3181	try isel.active_blocks.getPtr(br.block_inst).?.branch(isel);
3182	if (isel.live_values.get(br.block_inst)) |dst_vi| try dst_vi.defMove(isel, br.operand);
3183	},
3184	.cond_br => {
3185	const pl_op = air.data(air.inst_index).pl_op;
3186	const extra = isel.air.extraData(Air.CondBr, pl_op.payload);
3187
3188	try isel.body(@ptrCast(isel.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
3189	const else_label = isel.instructions.items.len;
3190	var else_snapshot = try isel.takeLocationSnapshot();
3191	defer else_snapshot.deinit(isel);
3192	tracking_log.debug("if-body snapshot taken:\n{f}", .{else_snapshot});
3193	try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.then_body_len]));
3194	try else_snapshot.merge(isel);
3195
3196	const cond_vi = try isel.use(pl_op.operand);
3197	const cond_mat = try cond_vi.mat(isel, .{
3198	.pref = .only_reg,
3199	.extension = .zero_ext,
3200	});
3201	try isel.internal_relocs.append(gpa, .{
3202	.target = else_label,
3203	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B21 },
3204	});
3205	try isel.emit(.beqz(cond_mat.reg(), 0, 0));
3206	try cond_mat.finish(isel);
3207	},
3208	.switch_br, .loop_switch_br => {
3209	// TODO loop switch br and switch dispatch
3210	if (air_tag == .loop_switch_br) try isel.failUnimplemented("TODO loop_switch_br", .{});
3211	const switch_br = isel.air.unwrapSwitch(air.inst_index);
3212
3213	var final_case = true;
3214	if (switch_br.else_body_len > 0) {
3215	var cases_it = switch_br.iterateCases();
3216	while (cases_it.next()) |_| {}
3217	try isel.body(cases_it.elseBody());
3218	assert(final_case);
3219	final_case = false;
3220	}
3221	var cases_it = switch_br.iterateCases();
3222	while (cases_it.next()) |case| {
3223	wip_mir_log.debug(" case {d}:", .{case.idx});
3224
3225	const next_label = isel.instructions.items.len;
3226	var next_snapshot = try isel.takeLocationSnapshot();
3227	defer next_snapshot.deinit(isel);
3228	tracking_log.debug("switch case snapshot taken:\n{f}", .{next_snapshot});
3229	try isel.body(case.body);
3230	try next_snapshot.merge(isel);
3231	if (final_case) {
3232	final_case = false;
3233	continue;
3234	}
3235
3236	const case_label = isel.instructions.items.len;
3237
3238	var cond_vi = try isel.use(switch_br.operand);
3239	const cond_mat = try cond_vi.mat(isel, .{
3240	.pref = .only_reg,
3241	.reg_mod = .integer,
3242	.extension = .zero_ext,
3243	});
3244
3245	try isel.internal_relocs.append(gpa, .{
3246	.target = next_label,
3247	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B26 },
3248	});
3249	try isel.emit(.b(0, 0));
3250
3251	var case_range_index = case.ranges.len;
3252	while (case_range_index > 0) {
3253	case_range_index -= 1;
3254	try isel.failUnimplemented("TODO switch_br range", .{});
3255	}
3256	var case_item_index = case.items.len;
3257	while (case_item_index > 0) {
3258	case_item_index -= 1;
3259
3260	const item_val: Constant = .fromInterned(case.items[case_item_index].toInterned().?);
3261	var item_bigint_space: Constant.BigIntSpace = undefined;
3262	const item_bigint = item_val.toBigInt(&item_bigint_space, zcu);
3263	const item_int: i64 = if (item_bigint.positive) @bitCast(
3264	item_bigint.toInt(u64) catch
3265	return isel.fail("too big case item: {f}", .{isel.fmtConstant(item_val)}),
3266	) else item_bigint.toInt(i64) catch
3267	return isel.fail("too big case item: {f}", .{isel.fmtConstant(item_val)});
3268
3269	const item_reg = try isel.allocRegForWrite(.int);
3270	defer isel.freeReg(item_reg);
3271
3272	try isel.internal_relocs.append(gpa, .{
3273	.target = case_label,
3274	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B16 },
3275	});
3276	try isel.emit(.beq(cond_mat.reg(), item_reg, 0));
3277	try isel.moveIntImm(item_reg, @bitCast(item_int));
3278	}
3279
3280	try cond_mat.finish(isel);
3281	}
3282	},
3283
3284	// Procedure call
3285	.call => {
3286	const pl_op = air.data(air.inst_index).pl_op;
3287	const extra = isel.air.extraData(Air.Call, pl_op.payload);
3288	const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
3289	const callee_ty = isel.air.typeOf(pl_op.operand, ip);
3290	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
3291	else => unreachable,
3292	.func_type => |func_type| func_type,
3293	.ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type,
3294	};
3295
3296	var cc_it: CallAbiIterator = .{ .isel = isel, .cc = &func_info.cc };
3297
3298	// return
3299	try call.prepareReturn(isel);
3300	const ret_ty = isel.air.typeOfIndex(air.inst_index, ip);
3301	const maybe_def_ret_vi = isel.live_values.fetchRemove(air.inst_index);
3302	const ret_vi = try cc_it.resolve(ret_ty, true) orelse .free;
3303	defer if (ret_vi != .free) ret_vi.deref(isel);
3304
3305	var def_ret_stack: Value.Indirect = .unallocated;
3306	if (maybe_def_ret_vi) |def_ret_vi| {
3307	defer def_ret_vi.value.deref(isel);
3308	assert(ret_vi != .free);
3309	switch (ret_vi.parent(isel)) {
3310	else => {
3311	try def_ret_vi.value.defLiveIn(isel, ret_vi, .{});
3312	},
3313	.address => {
3314	def_ret_stack = try def_ret_vi.value.defStack(isel) orelse ret_vi.allocStackSlot(isel);
3315	},
3316	}
3317	}
3318	try call.finishReturn(isel);
3319
3320	// call
3321	try call.prepareCallee(isel);
3322	if (pl_op.operand.toInterned()) |ct_callee| {
3323	try isel.emit(.jirl(.ra, .ra, 0));
3324	try isel.nav_relocs.append(gpa, switch (ip.indexToKey(ct_callee)) {
3325	else => unreachable,
3326	inline .@"extern", .func => |func| .{
3327	.nav = func.owner_nav,
3328	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .CALL36 },
3329	},
3330	.ptr => |ptr| .{
3331	.nav = ptr.base_addr.nav,
3332	.reloc = .{
3333	.label = @intCast(isel.instructions.items.len),
3334	.addend = @intCast(ptr.byte_offset),
3335	.type = .CALL36,
3336	},
3337	},
3338	});
3339	try isel.emit(.pcaddu18i(.ra, 0));
3340	} else {
3341	const callee_vi = try isel.use(pl_op.operand);
3342	const callee_mat = try callee_vi.matIntRegZeroExt(isel);
3343	try isel.emit(.jirl(.ra, callee_mat.reg(), 0));
3344	try callee_mat.finish(isel);
3345	}
3346	try call.finishCallee(isel);
3347
3348	// params
3349	try call.prepareParams(isel);
3350	if (ret_vi != .free) switch (ret_vi.parent(isel)) {
3351	else => {},
3352	.address => |addr_vi| try call.paramAddress(isel, def_ret_stack, addr_vi),
3353	};
3354	for (args) |arg| {
3355	const param_ty = isel.air.typeOf(arg, ip);
3356	const param_vi = try cc_it.resolve(param_ty, false) orelse continue;
3357	defer param_vi.deref(isel);
3358	const arg_vi = try isel.use(arg);
3359	try call.paramLiveOut(isel, arg_vi, param_vi);
3360	}
3361	try call.finishParams(isel);
3362	},
3363
3364	// Stack allocation
3365	.alloc, .ret_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| unused: {
3366	defer ptr_vi.value.deref(isel);
3367	switch (air_tag) {
3368	else => unreachable,
3369	.alloc => {},
3370	.ret_ptr => if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
3371	.none => {},
3372	.value, .constant => unreachable,
3373	.address => break :unused,
3374	},
3375	}
3376	const ptr_reg = try ptr_vi.value.defRegMod(isel, .integer) orelse break :unused;
3377
3378	const ty = air.data(air.inst_index).ty;
3379	const slot_size = ty.childType(zcu).abiSize(zcu);
3380	const slot_align = ty.ptrAlignment(zcu);
3381	const slot_offset = slot_align.forward(isel.stack_size);
3382	isel.stack_size = @intCast(slot_offset + slot_size);
3383
3384	try isel.addImm(ptr_reg, .sp, slot_offset);
3385	},
3386	.inferred_alloc, .inferred_alloc_comptime => unreachable,
3387
3388	// Assembly
3389	.assembly => {
3390	const unwrapped_asm = isel.air.unwrapAsm(air.inst_index);
3391	const inputs = unwrapped_asm.inputs;
3392
3393	var as: Assemble = .{ .source = unwrapped_asm.source };
3394	defer as.deinit(gpa);
3395
3396	var it = unwrapped_asm.iterateOutputs();
3397	while (it.next()) |output| {
3398	const constraint = output.constraint;
3399	const name = output.name;
3400
3401	switch (output.operand) {
3402	else => return isel.fail("invalid constraint: '{s}'", .{constraint}),
3403	.none => {
3404	const output_reg = output_reg: {
3405	if (std.mem.startsWith(u8, constraint, "={") and std.mem.endsWith(u8, constraint, "}")) {
3406	const output_reg = Register.parse(constraint["={".len .. constraint.len - "}".len]) orelse
3407	return isel.fail("invalid constraint: '{s}'", .{constraint});
3408	assert(try isel.fillReg(output_reg));
3409	isel.markRegWritten(output_reg);
3410	if (isel.live_values.fetchRemove(air.inst_index)) |output_vi| {
3411	defer output_vi.value.deref(isel);
3412	try output_vi.value.reextendToPcs(isel);
3413	if (try output_vi.value.def(isel)) |output_loc|
3414	try isel.moveLoc(
3415	.{ .register = .{ .mod = .integer, .reg = output_reg } },
3416	0,
3417	output_loc,
3418	0,
3419	output_vi.value.size(isel),
3420	.none,
3421	);
3422	}
3423	break :output_reg output_reg;
3424	} else if (std.mem.eql(u8, constraint, "=r")) {
3425	if (isel.live_values.fetchRemove(air.inst_index)) |output_vi| {
3426	defer output_vi.value.deref(isel);
3427	try output_vi.value.reextendToPcs(isel);
3428	break :output_reg try output_vi.value.defRegMod(isel, .integer) orelse try isel.allocRegForWrite(.int);
3429	} else break :output_reg try isel.allocRegForWrite(.int);
3430	} else return isel.fail("invalid constraint: '{s}'", .{constraint});
3431	};
3432	if (!std.mem.eql(u8, name, "_")) {
3433	const arg_gop = try as.args.getOrPut(gpa, name);
3434	if (arg_gop.found_existing) return isel.fail("duplicate output name: '{s}'", .{name});
3435	arg_gop.value_ptr.* = .{ .register = output_reg };
3436	}
3437	},
3438	}
3439	}
3440
3441	const clobbers_val: Constant = .fromInterned(unwrapped_asm.clobbers);
3442	const clobbers_ty = clobbers_val.typeOf(zcu);
3443	var clobbers_bigint_buf: Constant.BigIntSpace = undefined;
3444	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
3445	var clobbered_regs: RegisterSet = .empty;
3446	for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
3447	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
3448	const limb_bits = @bitSizeOf(std.math.big.Limb);
3449	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
3450	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
3451	0 => continue, // field is false
3452	1 => {}, // field is true
3453	}
3454
3455	const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
3456	if (std.mem.eql(u8, clobber_name, "memory")) continue;
3457	if (std.mem.startsWith(u8, clobber_name, "fcsr")) continue;
3458	const clobber_reg = Register.parse(clobber_name) orelse
3459	return isel.fail("unable to parse clobber: '{s}'", .{clobber_name});
3460	if (clobbered_regs.contains(clobber_reg))
3461	return isel.fail("clobbered twice: '{t}'", .{clobber_reg});
3462	clobbered_regs.insert(clobber_reg);
3463	}
3464	try isel.fillRegsBatch(clobbered_regs, true);
3465	isel.markRegsWritten(clobbered_regs);
3466
3467	const InputMat = union(enum(u1)) {
3468	reg: Register.Alias,
3469	mat: Value.Mat,
3470	};
3471	const input_mats = try gpa.alloc(InputMat, inputs.len);
3472	defer gpa.free(input_mats);
3473	var index: u32 = 0;
3474	it = unwrapped_asm.iterateInputs();
3475	while (it.next()) |input| : (index += 1) {
3476	const constraint = input.constraint;
3477	const name = input.name;
3478	const input_mat = &input_mats[index];
3479
3480	const input_vi = try isel.use(input.operand);
3481	try input_vi.reextendToPcs(isel);
3482
3483	// TODO support X constraint
3484	if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
3485	const input_reg = Register.parse(constraint["{".len .. constraint.len - "}".len]) orelse
3486	return isel.fail("invalid constraint: '{s}'", .{constraint});
3487	input_mat.* = .{ .reg = .{ .mod = .integer, .reg = input_reg } };
3488	} else if (std.mem.eql(u8, constraint, "r")) {
3489	const input_value_mat = try input_vi.mat(isel, .{
3490	.pref = .only_reg,
3491	.reg_mod = .integer,
3492	.extension = if (input_vi.typeOf(isel)) |input_ty|
3493	.pcsMode(isel, input_ty)
3494	else
3495	.zero_ext,
3496	});
3497	input_mat.* = .{ .mat = input_value_mat };
3498	} else if (std.mem.eql(u8, name, "_")) {
3499	input_mat.* = .{ .reg = .zero };
3500	} else return isel.fail("invalid constraint: '{s}'", .{constraint});
3501
3502	if (!std.mem.eql(u8, name, "_")) {
3503	const arg_gop = try as.args.getOrPut(gpa, name);
3504	if (arg_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
3505	arg_gop.value_ptr.* = .{ .register = switch (input_mat.*) {
3506	.reg => |input_ra| input_ra.reg,
3507	.mat => |input_val_mat| input_val_mat.reg(),
3508	} };
3509	}
3510	}
3511
3512	const asm_start = isel.instructions.items.len;
3513	while (instruction: {
3514	const line = as.nextLine();
3515	break :instruction as.parseLine(line) catch |err| switch (err) {
3516	error.InvalidSyntax => {
3517	if (debug_trap_unimplemented_code) {
3518	wip_mir_log.err("unable to assemble: '{s}'", .{std.mem.trim(
3519	u8,
3520	line,
3521	&std.ascii.whitespace,
3522	)});
3523	break :instruction Instruction.@"break"(0xaa);
3524	} else return isel.fail("unable to assemble: '{s}'", .{std.mem.trim(
3525	u8,
3526	line,
3527	&std.ascii.whitespace,
3528	)});
3529	},
3530	};
3531	}) |instruction| try isel.emit(instruction);
3532	std.mem.reverse(Instruction, isel.instructions.items[asm_start..]);
3533
3534	it = unwrapped_asm.iterateInputs();
3535	index = 0;
3536	while (it.next()) |input| : (index += 1) {
3537	const input_mat = &input_mats[index];
3538	const input_vi = try isel.use(input.operand);
3539	switch (input_mat.*) {
3540	.reg => |input_ra| {
3541	const input_val_mat = try input_vi.mat(isel, .{
3542	.pref = .prefer_reg,
3543	.hint_ra = input_ra,
3544	});
3545	const input_val_loc = input_val_mat.loc();
3546	const dst_loc: Value.Location = .{ .register = input_ra };
3547	if (!std.meta.eql(input_val_loc, dst_loc)) {
3548	dst_loc.markRegWritten(isel);
3549	try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .none);
3550	}
3551	try input_val_mat.finish(isel);
3552	},
3553	.mat => |input_val_mat| try input_val_mat.finish(isel),
3554	}
3555	}
3556
3557	var clobber_regs_it = clobbered_regs.iterator();
3558	while (clobber_regs_it.next()) |clobber_reg| isel.freeReg(clobber_reg);
3559	},
3560
3561	// Arithmetic
3562	.add, .add_safe, .add_optimized, .add_wrap, .sub, .sub_safe, .sub_optimized, .sub_wrap => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
3563	defer res_vi.value.deref(isel);
3564
3565	const bin_op = air.data(air.inst_index).bin_op;
3566	const ty = isel.air.typeOf(bin_op.lhs, ip);
3567	if (!ty.isRuntimeFloat()) try isel.addOrSubtract(ty, res_vi.value, switch (air_tag) {
3568	else => unreachable,
3569	.add, .add_safe, .add_wrap => .add,
3570	.sub, .sub_safe, .sub_wrap => .sub,
3571	}, try isel.use(bin_op.lhs), try isel.use(bin_op.rhs), .{
3572	.overflow = switch (air_tag) {
3573	else => unreachable,
3574	.add, .sub => .@"unreachable",
3575	.add_safe, .sub_safe => .{ .panic = .integer_overflow },
3576	.add_wrap, .sub_wrap => .wrap,
3577	},
3578	}) else return isel.fail("unimplemented float", .{});
3579	},
3580	.add_with_overflow, .sub_with_overflow => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
3581	defer res_vi.value.deref(isel);
3582
3583	const ty_pl = air.data(air.inst_index).ty_pl;
3584	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
3585	const ty = isel.air.typeOf(bin_op.lhs, ip);
3586	const lhs_vi = try isel.use(bin_op.lhs);
3587	const rhs_vi = try isel.use(bin_op.rhs);
3588	const ty_size = lhs_vi.size(isel);
3589
3590	const wrapped_vi = try res_vi.value.partExact(isel, 0, ty_size);
3591	const overflow_vi = try res_vi.value.partExact(isel, ty_size, 1);
3592	try isel.addOrSubtract(ty, wrapped_vi, switch (air_tag) {
3593	else => unreachable,
3594	.add_with_overflow => .add,
3595	.sub_with_overflow => .sub,
3596	}, lhs_vi, rhs_vi, .{
3597	.overflow = if (try overflow_vi.defReg(isel)) |overflow_ra| .{ .overflow_ra = overflow_ra } else .wrap,
3598	});
3599	},
3600	.not => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3601	defer res_vi.value.deref(isel);
3602
3603	const ty_op = air.data(air.inst_index).ty_op;
3604	const src_vi = try isel.use(ty_op.operand);
3605	const ty = ty_op.ty;
3606	switch (ty.zigTypeTag(zcu)) {
3607	.bool => {
3608	// boolean not
3609	try res_vi.value.reextend(isel, .zero_ext);
3610	const res_reg = try res_vi.value.defRegMod(isel, .integer) orelse break :unused;
3611	// TODO optimize fcc path
3612	const src_mat = try src_vi.matIntRegZeroExt(isel);
3613	const src_reg = src_mat.reg();
3614	try isel.emit(.xori(res_reg, src_reg, 1));
3615	try src_mat.finish(isel);
3616	},
3617	.int => {
3618	// bitwise not
3619	var res_walk = res_vi.value.walk(isel, .{});
3620	const gpr_size = isel.gprSize();
3621	while (res_walk.next()) |res_part_vi| {
3622	if (res_part_vi.size(isel) > gpr_size) continue;
3623	res_walk.skipChildren(res_part_vi);
3624	const res_part_ra = try res_part_vi.defReg(isel) orelse continue;
3625	const src_part_mat = try src_vi.mat(isel, .{
3626	.offset = res_part_vi.offsetIn(isel, res_vi.value),
3627	.size = @intCast(res_part_vi.size(isel)),
3628	.pref = .only_reg,
3629	.reg_mod = res_part_ra.mod,
3630	});
3631	const src_part_reg = src_part_mat.reg();
3632	switch (res_part_ra.mod) {
3633	.undef => unreachable,
3634	.integer => try isel.emit(.nor(res_part_ra.reg, src_part_reg, .zero)),
3635	else => return isel.fail("unimplemented not {t}", .{res_part_ra.mod}),
3636	}
3637	try src_part_mat.finish(isel);
3638	}
3639	},
3640	else => |ty_tag| return isel.fail("unimplemented not on {t}", .{ty_tag}),
3641	}
3642	},
3643	.trunc => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
3644	defer res_vi.value.deref(isel);
3645
3646	const ty_op = air.data(air.inst_index).ty_op;
3647	const src_vi = try isel.use(ty_op.operand);
3648	const src_ty = ty_op.ty;
3649	const src_bits = src_ty.bitSize(zcu);
3650	try res_vi.value.reextendAdvanced(
3651	isel,
3652	src_bits,
3653	src_vi.extension(isel),
3654	res_vi.value.extension(isel),
3655	);
3656	try res_vi.value.defCopy(isel, src_vi);
3657	},
3658	.div_trunc, .div_trunc_optimized, .div_floor, .div_floor_optimized, .div_exact, .div_exact_optimized => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3659	defer res_vi.value.deref(isel);
3660
3661	const bin_op = air.data(air.inst_index).bin_op;
3662	const ty = isel.air.typeOf(bin_op.lhs, ip);
3663	if (!ty.isRuntimeFloat()) {
3664	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3665	const int_info = ty.intInfo(zcu);
3666	switch (int_info.bits) {
3667	0 => unreachable,
3668	1...64 => |bits| {
3669	const res_reg = try res_vi.value.defRegMod(isel, .integer) orelse break :unused;
3670	const lhs_vi = try isel.use(bin_op.lhs);
3671	const rhs_vi = try isel.use(bin_op.rhs);
3672	const mat_opts: Value.Index.MatOptions = .{
3673	.pref = .only_reg,
3674	.reg_mod = .integer,
3675	.extension = ext_mode: {
3676	if (bits == 32 and isel.hasCpuFeature(.@"64bit") and isel.hasCpuFeature(.div32)) {
3677	break :ext_mode .garbage;
3678	}
3679	break :ext_mode .fromSignedness(int_info.signedness);
3680	},
3681	};
3682	const lhs_mat = try lhs_vi.mat(isel, mat_opts);
3683	const rhs_mat = try rhs_vi.mat(isel, mat_opts);
3684	const lhs_reg = lhs_mat.reg();
3685	const rhs_reg = rhs_mat.reg();
3686
3687	switch (bits) {
3688	else => unreachable,
3689	1...32 => try isel.emit(switch (int_info.signedness) {
3690	.signed => .@"div.w"(res_reg, lhs_reg, rhs_reg),
3691	.unsigned => .@"div.wu"(res_reg, lhs_reg, rhs_reg),
3692	}),
3693	33...64 => if (isel.hasCpuFeature(.@"64bit")) {
3694	try isel.emit(switch (int_info.signedness) {
3695	.signed => .@"div.d"(res_reg, lhs_reg, rhs_reg),
3696	.unsigned => .@"div.du"(res_reg, lhs_reg, rhs_reg),
3697	});
3698	} else return isel.fail("unimplemented 64bit division on LA32", .{}),
3699	}
3700	try rhs_mat.finish(isel);
3701	try lhs_mat.finish(isel);
3702	},
3703	else => {
3704	_ = try res_vi.value.def(isel);
3705	try isel.failUnimplemented("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
3706	},
3707	}
3708	} else try isel.failUnimplemented("unimplemented float div", .{});
3709	},
3710	.bit_cast,
3711	.ptr_cast,
3712	.ptr_from_int,
3713	.int_from_ptr,
3714	.error_cast,
3715	.error_from_int,
3716	.int_from_error,
3717	.union_from_enum,
3718	=> if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
3719	defer dst_vi.value.deref(isel);
3720	const ty_op = air.data(air.inst_index).ty_op;
3721	const dst_ty = ty_op.ty;
3722	const dst_tag = dst_ty.zigTypeTag(zcu);
3723	const src_ty = isel.air.typeOf(ty_op.operand, ip);
3724	const src_tag = src_ty.zigTypeTag(zcu);
3725
3726	if ((dst_tag == .bool or dst_ty.isAbiInt(zcu)) and (src_tag == .bool or src_ty.isAbiInt(zcu))) {
3727	const dst_int_info: std.builtin.Type.Int = if (dst_tag == .bool) .{ .signedness = .unsigned, .bits = 1 } else dst_ty.intInfo(zcu);
3728	const src_int_info: std.builtin.Type.Int = if (src_tag == .bool) .{ .signedness = .unsigned, .bits = 1 } else src_ty.intInfo(zcu);
3729	assert(dst_int_info.bits == src_int_info.bits);
3730	if (dst_tag != .@"struct" and src_tag != .@"struct") {
3731	try dst_vi.value.defMove(isel, ty_op.operand);
3732	} else switch (dst_int_info.bits) {
3733	0 => unreachable,
3734	1...31, 33...63 => |bits| {
3735	try dst_vi.value.reextendToGarbage(isel);
3736	const dst_reg = try dst_vi.value.defRegMod(isel, .integer) orelse break :unused;
3737	const src_vi = try isel.use(ty_op.operand);
3738	const src_mat = try src_vi.matReg(isel);
3739	try isel.fillUnusedBits(
3740	dst_reg,
3741	src_mat.reg(),
3742	.fromSignedness(dst_int_info.signedness),
3743	.fromSignedness(src_int_info.signedness),
3744	@intCast(bits),
3745	);
3746	try src_mat.finish(isel);
3747	},
3748	32, 64 => try dst_vi.value.defMove(isel, ty_op.operand),
3749	else => return isel.fail("unimplemented {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
3750	}
3751	} else if ((dst_ty.isPtrAtRuntime(zcu) or dst_ty.isAbiInt(zcu)) and (src_ty.isPtrAtRuntime(zcu) or src_ty.isAbiInt(zcu))) {
3752	try dst_vi.value.defMove(isel, ty_op.operand);
3753	} else if (dst_ty.isSliceAtRuntime(zcu) and src_ty.isSliceAtRuntime(zcu)) {
3754	try dst_vi.value.defMove(isel, ty_op.operand);
3755	} else if (dst_tag == .error_union and src_tag == .error_union) {
3756	assert(dst_ty.errorUnionSet(zcu).hasRuntimeBits(zcu) ==
3757	src_ty.errorUnionSet(zcu).hasRuntimeBits(zcu));
3758	if (dst_ty.errorUnionPayload(zcu).toIntern() == src_ty.errorUnionPayload(zcu).toIntern()) {
3759	try dst_vi.value.defMove(isel, ty_op.operand);
3760	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3761	} else if (dst_tag == .float and src_tag == .float) {
3762	assert(dst_ty.floatBits(isel.target) == src_ty.floatBits(isel.target));
3763	try dst_vi.value.defMove(isel, ty_op.operand);
3764	} else if (dst_ty.isAbiInt(zcu) and src_tag == .float and isel.canUseFprForFloat(src_ty.floatBits(isel.target))) {
3765	const dst_int_info = dst_ty.intInfo(zcu);
3766	assert(dst_int_info.bits == src_ty.floatBits(isel.target));
3767
3768	try dst_vi.value.reextendToGarbage(isel);
3769	const dst_reg = try dst_vi.value.defRegMod(isel, .fromFloatBits(dst_int_info.bits)) orelse break :unused;
3770	const src_vi = try isel.use(ty_op.operand);
3771	const src_mat = try src_vi.matReg(isel);
3772	const src_reg = src_mat.reg();
3773	try isel.emit(switch (dst_int_info.bits) {
3774	else => unreachable,
3775	32 => .@"movfr2gr.s"(dst_reg, src_reg),
3776	64 => .@"movfr2gr.d"(dst_reg, src_reg),
3777	});
3778	try src_mat.finish(isel);
3779	} else if (dst_tag == .float and src_ty.isAbiInt(zcu) and isel.canUseFprForFloat(dst_ty.floatBits(isel.target))) {
3780	const src_int_info = src_ty.intInfo(zcu);
3781	assert(dst_ty.floatBits(isel.target) == src_int_info.bits);
3782
3783	try dst_vi.value.reextendToGarbage(isel);
3784	const dst_reg = try dst_vi.value.defRegMod(isel, .fromFloatBits(src_int_info.bits)) orelse break :unused;
3785	const src_vi = try isel.use(ty_op.operand);
3786	const src_mat = try src_vi.matReg(isel);
3787	const src_reg = src_mat.reg();
3788	try isel.emit(switch (src_int_info.bits) {
3789	else => unreachable,
3790	32 => .@"movgr2fr.w"(dst_reg, src_reg),
3791	64 => .@"movfr2gr.d"(dst_reg, src_reg),
3792	});
3793	try src_mat.finish(isel);
3794	} else if (dst_ty.isAbiInt(zcu) and src_tag == .array and src_ty.childType(zcu).isAbiInt(zcu)) {
3795	const dst_int_info = dst_ty.intInfo(zcu);
3796	const src_child_int_info = src_ty.childType(zcu).intInfo(zcu);
3797	const src_len = src_ty.arrayLenIncludingSentinel(zcu);
3798	assert(dst_int_info.bits == src_child_int_info.bits * src_len);
3799	const src_child_size = src_ty.childType(zcu).abiSize(zcu);
3800	if (8 * src_child_size == src_child_int_info.bits) {
3801	const src_vi = try isel.use(ty_op.operand);
3802	try dst_vi.value.defCopy(isel, src_vi);
3803	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3804	} else if (dst_tag == .array and dst_ty.childType(zcu).isAbiInt(zcu) and src_ty.isAbiInt(zcu)) {
3805	const dst_child_int_info = dst_ty.childType(zcu).intInfo(zcu);
3806	const src_int_info = src_ty.intInfo(zcu);
3807	const dst_len = dst_ty.arrayLenIncludingSentinel(zcu);
3808	assert(dst_child_int_info.bits * dst_len == src_int_info.bits);
3809	const dst_child_size = dst_ty.childType(zcu).abiSize(zcu);
3810	if (8 * dst_child_size == dst_child_int_info.bits) {
3811	const src_vi = try isel.use(ty_op.operand);
3812	try dst_vi.value.defCopy(isel, src_vi);
3813	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3814	} else if (dst_tag == .array and dst_ty.childType(zcu).isAbiInt(zcu) and
3815	src_tag == .array and src_ty.childType(zcu).isAbiInt(zcu))
3816	{
3817	const dst_child_int_info = dst_ty.childType(zcu).intInfo(zcu);
3818	const dst_len = dst_ty.arrayLenIncludingSentinel(zcu);
3819	const src_child_int_info = src_ty.childType(zcu).intInfo(zcu);
3820	const src_len = src_ty.arrayLenIncludingSentinel(zcu);
3821	assert(dst_child_int_info.bits * dst_len == src_child_int_info.bits * src_len);
3822	const dst_child_size = dst_ty.childType(zcu).abiSize(zcu);
3823	const src_child_size = src_ty.childType(zcu).abiSize(zcu);
3824	if (8 * dst_child_size == dst_child_int_info.bits and 8 * src_child_size == src_child_int_info.bits) {
3825	const src_vi = try isel.use(ty_op.operand);
3826	try dst_vi.value.defCopy(isel, src_vi);
3827	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3828	} else return isel.fail("unimplemented {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3829	},
3830	.bit_and, .bit_or, .xor => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
3831	defer res_vi.value.deref(isel);
3832
3833	const bin_op = air.data(air.inst_index).bin_op;
3834
3835	const lhs_vi = try isel.use(bin_op.lhs);
3836	const rhs_vi = try isel.use(bin_op.rhs);
3837
3838	const lhs_ext_mode = lhs_vi.extension(isel);
3839	const rhs_ext_mode = rhs_vi.extension(isel);
3840	try res_vi.value.reextend(isel, res_ext_mode: switch (air_tag) {
3841	else => unreachable,
3842	.bit_and => {
3843	if (lhs_ext_mode == rhs_ext_mode) break :res_ext_mode lhs_ext_mode;
3844	if (lhs_ext_mode == .zero_ext or rhs_ext_mode == .zero_ext) break :res_ext_mode .zero_ext;
3845	break :res_ext_mode .garbage;
3846	},
3847	.bit_or => if (lhs_ext_mode == rhs_ext_mode) lhs_ext_mode else .garbage,
3848	.xor => .garbage,
3849	});
3850
3851	var res_walk = res_vi.value.walk(isel, .{});
3852	const gpr_size = isel.gprSize();
3853	while (res_walk.next()) |res_part_vi| {
3854	if (res_part_vi.size(isel) > gpr_size) continue;
3855	res_walk.skipChildren(res_part_vi);
3856	const part_offset = res_part_vi.offsetIn(isel, res_vi.value);
3857	const part_size = res_part_vi.size(isel);
3858	// TODO implement vectors
3859	const res_part_ra = try res_part_vi.defReg(isel) orelse continue;
3860	const res_part_reg = res_part_ra.reg;
3861	const lhs_part_mat = try lhs_vi.mat(isel, .{
3862	.offset = part_offset,
3863	.size = @intCast(part_size),
3864	.pref = .only_reg,
3865	.reg_mod = res_part_ra.mod,
3866	});
3867	const lhs_part_reg = lhs_part_mat.reg();
3868	const rhs_part_mat = try lhs_vi.mat(isel, .{
3869	.offset = part_offset,
3870	.size = @intCast(part_size),
3871	.pref = .only_reg,
3872	.reg_mod = res_part_ra.mod,
3873	});
3874	const rhs_part_reg = rhs_part_mat.reg();
3875
3876	try isel.emit(switch (air_tag) {
3877	else => unreachable,
3878	.bit_and => .@"and"(res_part_reg, lhs_part_reg, rhs_part_reg),
3879	.bit_or => .@"or"(res_part_reg, lhs_part_reg, rhs_part_reg),
3880	.xor => .xor(res_part_reg, lhs_part_reg, rhs_part_reg),
3881	});
3882	try rhs_part_mat.finish(isel);
3883	try lhs_part_mat.finish(isel);
3884	}
3885	},
3886	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3887	defer res_vi.value.deref(isel);
3888
3889	const bin_op = air.data(air.inst_index).bin_op;
3890	const ty = isel.air.typeOf(bin_op.lhs, ip);
3891	const lhs_vi = try isel.use(bin_op.lhs);
3892	const rhs_vi = try isel.use(bin_op.rhs);
3893
3894	switch (ip.indexToKey(ty.toIntern())) {
3895	else => {},
3896	.opt_type => |payload_ty| switch (air_tag) {
3897	else => unreachable,
3898	.cmp_eq, .cmp_neq => if (!ty.optionalReprIsPayload(zcu)) {
3899	const payload_size = ZigType.abiSize(.fromInterned(payload_ty), zcu);
3900	try res_vi.value.reextendToGarbage(isel);
3901	const res_reg = try res_vi.value.defRegMod(isel, .integer) orelse break :unused;
3902
3903	const cmp_label = isel.instructions.items.len;
3904	try isel.cmp(
3905	res_reg,
3906	.fromInterned(payload_ty),
3907	try lhs_vi.partExact(isel, 0, payload_size),
3908	air_tag.toCmpOp().?,
3909	try rhs_vi.partExact(isel, 0, payload_size),
3910	);
3911	const lhs_tag_mat = try lhs_vi.mat(isel, .{
3912	.offset = payload_size,
3913	.size = 1,
3914	.pref = .only_reg,
3915	.reg_mod = .integer,
3916	.extension = .zero_ext,
3917	});
3918	const rhs_tag_mat = try rhs_vi.mat(isel, .{
3919	.offset = payload_size,
3920	.size = 1,
3921	.pref = .only_reg,
3922	.reg_mod = .integer,
3923	.extension = .zero_ext,
3924	});
3925	try isel.internal_relocs.append(gpa, .{
3926	.target = cmp_label,
3927	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B21 },
3928	});
3929	try isel.emit(.beqz(lhs_tag_mat.reg(), 0, 0));
3930	try isel.internal_relocs.append(gpa, .{
3931	.target = cmp_label,
3932	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B21 },
3933	});
3934	try isel.emit(.beqz(res_reg, 0, 0));
3935
3936	try isel.emit(.xori(res_reg, res_reg, 1));
3937	try isel.emit(.xor(res_reg, lhs_tag_mat.reg(), rhs_tag_mat.reg()));
3938	try rhs_tag_mat.finish(isel);
3939	try lhs_tag_mat.finish(isel);
3940	break :unused;
3941	},
3942	},
3943	}
3944
3945	// TODO optimize fcc path
3946	try res_vi.value.reextendToPcs(isel);
3947	try isel.cmp(
3948	try res_vi.value.defRegMod(isel, .integer) orelse break :unused,
3949	ty,
3950	lhs_vi,
3951	air_tag.toCmpOp().?,
3952	rhs_vi,
3953	);
3954	},
3955	.store, .store_safe, .atomic_store_unordered => unused: {
3956	const bin_op = air.data(air.inst_index).bin_op;
3957	const ptr_ty = isel.air.typeOf(bin_op.lhs, ip);
3958	const ptr_info = ptr_ty.ptrInfo(zcu);
3959	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed store", .{});
3960	if (bin_op.rhs.toInterned()) |rhs_val| if (ip.isUndef(rhs_val)) break :unused;
3961
3962	const src_vi = try isel.use(bin_op.rhs);
3963	const ptr_vi = try isel.use(bin_op.lhs);
3964	const ptr_mat = try ptr_vi.matReg(isel);
3965	try src_vi.matStore(isel, ptr_mat.reg(), 0, .{
3966	.@"volatile" = ptr_info.flags.is_volatile,
3967	});
3968	try ptr_mat.finish(isel);
3969	},
3970	.load => {
3971	const ty_op = air.data(air.inst_index).ty_op;
3972	const ptr_ty = isel.air.typeOf(ty_op.operand, ip);
3973	const ptr_info = ptr_ty.ptrInfo(zcu);
3974	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load", .{});
3975
3976	if (ptr_info.flags.is_volatile) _ = try isel.use(air.inst_index.toRef());
3977	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
3978	defer dst_vi.value.deref(isel);
3979
3980	// TODO unaligned loads
3981	assert(isel.target.cpu.has(.loongarch, .ual));
3982	const ptr_vi = try isel.use(ty_op.operand);
3983	const ptr_mat = try ptr_vi.matIntRegZeroExt(isel);
3984	_ = try dst_vi.value.defLoad(isel, ptr_mat.reg(), 0, .{
3985	.@"volatile" = ptr_info.flags.is_volatile,
3986	});
3987	try ptr_mat.finish(isel);
3988	}
3989	},
3990	.int_cast => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
3991	defer dst_vi.value.deref(isel);
3992
3993	const ty_op = air.data(air.inst_index).ty_op;
3994	const dst_ty = ty_op.ty;
3995	const dst_int_info = dst_ty.intInfo(zcu);
3996	const src_ty = isel.air.typeOf(ty_op.operand, ip);
3997	const src_int_info = src_ty.intInfo(zcu);
3998
3999	if (dst_int_info.bits == src_int_info.bits) {
4000	try dst_vi.value.defMove(isel, ty_op.operand);
4001	} else {
4002	const src_vi = try isel.use(ty_op.operand);
4003	try dst_vi.value.reextendAdvanced(isel, src_int_info.bits, null, src_vi.extension(isel));
4004	try dst_vi.value.defCopy(isel, src_vi);
4005	}
4006	},
4007	.is_null, .is_non_null => if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
4008	defer is_vi.value.deref(isel);
4009	const is_reg = try is_vi.value.defRegMod(isel, .integer) orelse break :unused;
4010
4011	const un_op = air.data(air.inst_index).un_op;
4012	const opt_ty = isel.air.typeOf(un_op, ip);
4013	const payload_ty = opt_ty.optionalChild(zcu);
4014	const payload_size = payload_ty.abiSize(zcu);
4015	const has_value_offset, const has_value_size = if (!opt_ty.optionalReprIsPayload(zcu))
4016	.{ payload_size, 1 }
4017	else if (payload_ty.isSlice(zcu))
4018	.{ 0, 8 }
4019	else
4020	.{ 0, @as(u32, @intCast(payload_size)) };
4021
4022	const opt_vi = try isel.use(un_op);
4023	const has_value_mat = try opt_vi.mat(isel, .{
4024	.offset = has_value_offset,
4025	.size = has_value_size,
4026	.pref = .only_reg,
4027	.reg_mod = .integer,
4028	.extension = .zero_ext,
4029	.hint_ra = .{ .reg = is_reg, .mod = .integer },
4030	});
4031	const has_value_reg = has_value_mat.reg();
4032	try isel.emit(switch (air_tag) {
4033	else => unreachable,
4034	.is_null => .sltui(is_reg, has_value_reg, 1),
4035	.is_non_null => .sltu(is_reg, .zero, has_value_reg),
4036	});
4037	try has_value_mat.finish(isel);
4038	},
4039	.is_err, .is_non_err => if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
4040	defer is_vi.value.deref(isel);
4041	const is_reg = try is_vi.value.defRegMod(isel, .integer) orelse break :unused;
4042
4043	const un_op = air.data(air.inst_index).un_op;
4044	const error_union_ty = isel.air.typeOf(un_op, ip);
4045	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4046	const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
4047	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4048	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
4049	const error_set_size = error_set_ty.abiSize(zcu);
4050
4051	const error_union_vi = try isel.use(un_op);
4052	const error_set_mat = try error_union_vi.mat(isel, .{
4053	.offset = error_set_offset,
4054	.size = @intCast(error_set_size),
4055	.pref = .only_reg,
4056	.reg_mod = .integer,
4057	.hint_ra = .{ .reg = is_reg, .mod = .integer },
4058	});
4059	try isel.emit(switch (air_tag) {
4060	else => unreachable,
4061	.is_err => .sltu(is_reg, .zero, is_reg),
4062	.is_non_err => .sltui(is_reg, is_reg, 1),
4063	});
4064	try error_set_mat.finish(isel);
4065	},
4066	.max, .min => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
4067	defer res_vi.value.deref(isel);
4068
4069	const bin_op = air.data(air.inst_index).bin_op;
4070	const ty = isel.air.typeOf(bin_op.lhs, ip);
4071	if (!ty.isRuntimeFloat()) {
4072	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
4073	const int_info = ty.intInfo(zcu);
4074	if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
4075
4076	try res_vi.value.reextendToGarbage(isel);
4077	const res_reg = try res_vi.value.defRegMod(isel, .integer) orelse break :unused;
4078	const lhs_vi = try isel.use(bin_op.lhs);
4079	// TODO: relax LHS and RHS requirements to "not garbage filled"
4080	const lhs_mat = try lhs_vi.matIntRegZeroExt(isel);
4081	const lhs_reg = lhs_mat.reg();
4082	const rhs_vi = try isel.use(bin_op.rhs);
4083	const rhs_mat = try rhs_vi.matIntRegZeroExt(isel);
4084	const rhs_reg = rhs_mat.reg();
4085
4086	const tmp_reg = try isel.allocRegForWrite(.int);
4087	defer isel.freeReg(tmp_reg);
4088	const cond_reg = try isel.allocRegForWrite(.int);
4089	defer isel.freeReg(cond_reg);
4090
4091	try isel.emit(.@"or"(res_reg, res_reg, tmp_reg));
4092	try isel.emit(.maskeqz(res_reg, lhs_reg, cond_reg));
4093	try isel.emit(.masknez(tmp_reg, rhs_reg, cond_reg));
4094	switch (air_tag) {
4095	else => unreachable,
4096	.min => try isel.emit(.sltu(cond_reg, lhs_reg, rhs_reg)),
4097	.max => try isel.emit(.sltu(cond_reg, rhs_reg, lhs_reg)),
4098	}
4099
4100	try rhs_mat.finish(isel);
4101	try lhs_mat.finish(isel);
4102	} else switch (ty.floatBits(isel.target)) {
4103	else => unreachable,
4104	32, 64 => return isel.fail("TODO float min/max", .{}),
4105	}
4106	},
4107	.slice => if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| {
4108	defer slice_vi.value.deref(isel);
4109	const ty_pl = air.data(air.inst_index).ty_pl;
4110	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
4111	const gpr_size = isel.gprSize();
4112	const ptr_part_vi = try slice_vi.value.partExact(isel, 0, gpr_size);
4113	try ptr_part_vi.defMove(isel, bin_op.lhs);
4114	const len_part_vi = try slice_vi.value.partExact(isel, gpr_size, gpr_size);
4115	try len_part_vi.defMove(isel, bin_op.rhs);
4116	},
4117	.slice_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| {
4118	defer ptr_vi.value.deref(isel);
4119	const ty_op = air.data(air.inst_index).ty_op;
4120	const gpr_size = isel.gprSize();
4121	const slice_vi = try isel.use(ty_op.operand);
4122	const ptr_part_vi = try slice_vi.partExact(isel, 0, gpr_size);
4123	try ptr_vi.value.defCopy(isel, ptr_part_vi);
4124	},
4125	.slice_len => if (isel.live_values.fetchRemove(air.inst_index)) |len_vi| {
4126	defer len_vi.value.deref(isel);
4127	const ty_op = air.data(air.inst_index).ty_op;
4128	const gpr_size = isel.gprSize();
4129	const slice_vi = try isel.use(ty_op.operand);
4130	const len_part_vi = try slice_vi.partExact(isel, gpr_size, gpr_size);
4131	try len_vi.value.defCopy(isel, len_part_vi);
4132	},
4133	.ptr_slice_ptr_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
4134	defer dst_vi.value.deref(isel);
4135	const ty_op = air.data(air.inst_index).ty_op;
4136	try dst_vi.value.defMove(isel, ty_op.operand);
4137	},
4138	.ptr_slice_len_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
4139	defer dst_vi.value.deref(isel);
4140	const ty_op = air.data(air.inst_index).ty_op;
4141	const dst_reg = try dst_vi.value.defRegMod(isel, .integer) orelse break :unused;
4142	const src_vi = try isel.use(ty_op.operand);
4143	const src_mat = try src_vi.matIntRegZeroExt(isel);
4144	const src_reg = src_mat.reg();
4145	switch (isel.gprSize()) {
4146	else => unreachable,
4147	4 => try isel.emit(.@"addi.w"(dst_reg, src_reg, 4)),
4148	8 => try isel.emit(.@"addi.d"(dst_reg, src_reg, 8)),
4149	}
4150	try src_mat.finish(isel);
4151	},
4152	.slice_elem_val => if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
4153	defer elem_vi.value.deref(isel);
4154
4155	const bin_op = air.data(air.inst_index).bin_op;
4156	const slice_ty = isel.air.typeOf(bin_op.lhs, ip);
4157	const ptr_info = slice_ty.ptrInfo(zcu);
4158	const elem_size = elem_vi.value.size(isel);
4159
4160	const elem_ptr_reg = try isel.allocRegForWrite(.int);
4161	defer isel.freeReg(elem_ptr_reg);
4162
4163	if (!try elem_vi.value.defLoad(isel, elem_ptr_reg, 0, .{
4164	.@"volatile" = ptr_info.flags.is_volatile,
4165	})) break :unused;
4166
4167	const slice_vi = try isel.use(bin_op.lhs);
4168	const base_ptr_mat = try slice_vi.mat(isel, .{
4169	.offset = 0,
4170	.size = isel.gprSize(),
4171	.pref = .only_reg,
4172	.reg_mod = .integer,
4173	});
4174	const index_vi = try isel.use(bin_op.rhs);
4175	try isel.elemPtr(elem_ptr_reg, base_ptr_mat.reg(), .add, elem_size, index_vi);
4176	try base_ptr_mat.finish(isel);
4177	},
4178	.slice_elem_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |elem_ptr_vi| unused: {
4179	defer elem_ptr_vi.value.deref(isel);
4180	const elem_ptr_reg = try elem_ptr_vi.value.defRegMod(isel, .integer) orelse break :unused;
4181
4182	const ty_pl = air.data(air.inst_index).ty_pl;
4183	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
4184	const elem_size = ty_pl.ty.childType(zcu).abiSize(zcu);
4185
4186	const slice_vi = try isel.use(bin_op.lhs);
4187	const base_ptr_mat = try slice_vi.mat(isel, .{
4188	.offset = 0,
4189	.size = isel.gprSize(),
4190	.pref = .only_reg,
4191	.reg_mod = .integer,
4192	});
4193	const index_vi = try isel.use(bin_op.rhs);
4194	try isel.elemPtr(elem_ptr_reg, base_ptr_mat.reg(), .add, elem_size, index_vi);
4195	try base_ptr_mat.finish(isel);
4196	},
4197	.ptr_add, .ptr_sub => if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
4198	defer res_vi.value.deref(isel);
4199	const res_reg = try res_vi.value.defRegMod(isel, .integer) orelse break :unused;
4200
4201	const ty_pl = air.data(air.inst_index).ty_pl;
4202	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
4203	const elem_size = ty_pl.ty.childType(zcu).abiSize(zcu);
4204
4205	const base_vi = try isel.use(bin_op.lhs);
4206	const base_ptr_mat = try base_vi.mat(isel, .{
4207	.offset = 0,
4208	.size = isel.gprSize(),
4209	.pref = .only_reg,
4210	.reg_mod = .integer,
4211	});
4212	const index_vi = try isel.use(bin_op.rhs);
4213	try isel.elemPtr(res_reg, base_ptr_mat.reg(), switch (air_tag) {
4214	else => unreachable,
4215	.ptr_add => .add,
4216	.ptr_sub => .sub,
4217	}, elem_size, index_vi);
4218	try base_ptr_mat.finish(isel);
4219	},
4220	.ptr_elem_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |elem_ptr_vi| unused: {
4221	defer elem_ptr_vi.value.deref(isel);
4222	const elem_ptr_reg = try elem_ptr_vi.value.defRegMod(isel, .integer) orelse break :unused;
4223
4224	const ty_pl = air.data(air.inst_index).ty_pl;
4225	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
4226	const elem_size = ty_pl.ty.childType(zcu).abiSize(zcu);
4227
4228	const base_vi = try isel.use(bin_op.lhs);
4229	const base_mat = try base_vi.matIntRegZeroExt(isel);
4230	const index_vi = try isel.use(bin_op.rhs);
4231	try isel.elemPtr(elem_ptr_reg, base_mat.reg(), .add, elem_size, index_vi);
4232	try base_mat.finish(isel);
4233	},
4234	.array_to_slice => if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| {
4235	defer slice_vi.value.deref(isel);
4236	const ty_op = air.data(air.inst_index).ty_op;
4237	const gpr_size = isel.gprSize();
4238	const array_len = isel.air.typeOf(ty_op.operand, ip).childType(zcu).arrayLen(zcu);
4239
4240	const len_part_vi = try slice_vi.value.partExact(isel, gpr_size, gpr_size);
4241	if (try len_part_vi.defRegMod(isel, .integer)) |len_reg|
4242	try isel.moveIntImm(len_reg, @bitCast(array_len));
4243
4244	const ptr_part_vi = try slice_vi.value.partExact(isel, 0, gpr_size);
4245	try ptr_part_vi.defMove(isel, ty_op.operand);
4246	},
4247	.@"try", .try_cold => {
4248	const pl_op = air.data(air.inst_index).pl_op;
4249	const extra = isel.air.extraData(Air.Try, pl_op.payload);
4250	const error_union_ty = isel.air.typeOf(pl_op.operand, ip);
4251	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4252	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4253
4254	const error_union_vi = try isel.use(pl_op.operand);
4255	if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| {
4256	defer payload_vi.value.deref(isel);
4257
4258	const payload_part_vi = try error_union_vi.partExact(
4259	isel,
4260	codegen.errUnionPayloadOffset(payload_ty, zcu),
4261	payload_vi.value.size(isel),
4262	);
4263	try payload_vi.value.defCopy(isel, payload_part_vi);
4264	}
4265
4266	const cont_label = isel.instructions.items.len;
4267	var cont_snapshot = try isel.takeLocationSnapshot();
4268	defer cont_snapshot.deinit(isel);
4269	tracking_log.debug("try-continue snapshot taken:\n{f}", .{cont_snapshot});
4270	try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
4271	try cont_snapshot.merge(isel);
4272
4273	const error_set_part_vi = try error_union_vi.partExact(
4274	isel,
4275	codegen.errUnionErrorOffset(payload_ty, zcu),
4276	ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu),
4277	);
4278	const error_set_part_mat = try error_set_part_vi.matIntRegZeroExt(isel);
4279	try isel.internal_relocs.append(gpa, .{
4280	.target = cont_label,
4281	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B26 },
4282	});
4283	try isel.emit(.beqz(error_set_part_mat.reg(), 0, 0));
4284	try error_set_part_mat.finish(isel);
4285	},
4286	.try_ptr, .try_ptr_cold => {
4287	const unwrapped_try = isel.air.unwrapTryPtr(air.inst_index);
4288	const error_union_ty = isel.air.typeOf(unwrapped_try.error_union_ptr, ip).childType(zcu);
4289	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4290	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4291
4292	const error_union_ptr_vi = try isel.use(unwrapped_try.error_union_ptr);
4293	if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
4294	defer payload_ptr_vi.value.deref(isel);
4295
4296	const payload_offset = codegen.errUnionPayloadOffset(unwrapped_try.error_union_payload_ptr_ty.childType(zcu), zcu);
4297	if (payload_offset == 0) {
4298	try payload_ptr_vi.value.defMove(isel, unwrapped_try.error_union_ptr);
4299	} else {
4300	const payload_ptr_reg = try payload_ptr_vi.value.defRegMod(isel, .integer) orelse break :unused;
4301	const error_union_ptr_mat = try error_union_ptr_vi.matIntRegZeroExt(isel);
4302	try isel.addImm(payload_ptr_reg, error_union_ptr_mat.reg(), payload_offset);
4303	try error_union_ptr_mat.finish(isel);
4304	}
4305	}
4306
4307	const cont_label = isel.instructions.items.len;
4308	var cont_snapshot = try isel.takeLocationSnapshot();
4309	defer cont_snapshot.deinit(isel);
4310	tracking_log.debug("try_ptr-continue snapshot taken:\n{f}", .{cont_snapshot});
4311	try isel.body(unwrapped_try.else_body);
4312	try cont_snapshot.merge(isel);
4313
4314	const tmp_reg = try isel.allocRegForWrite(.int);
4315	defer isel.freeReg(tmp_reg);
4316
4317	try isel.internal_relocs.append(gpa, .{
4318	.target = cont_label,
4319	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .B26 },
4320	});
4321	try isel.emit(.beqz(tmp_reg, 0, 0));
4322
4323	const error_union_ptr_mat = try error_union_ptr_vi.matIntRegZeroExt(isel);
4324	try isel.loadReg(
4325	tmp_reg,
4326	ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu),
4327	.unsigned,
4328	error_union_ptr_mat.reg(),
4329	codegen.errUnionErrorOffset(payload_ty, zcu),
4330	);
4331	try error_union_ptr_mat.finish(isel);
4332	},
4333	.aggregate_init => if (isel.live_values.fetchRemove(air.inst_index)) |agg_vi| {
4334	defer agg_vi.value.deref(isel);
4335
4336	const ty_pl = air.data(air.inst_index).ty_pl;
4337	const agg_ty = ty_pl.ty;
4338	switch (ip.indexToKey(agg_ty.toIntern())) {
4339	.array_type => |array_type| {
4340	const elem_ty = ZigType.fromInterned(array_type.child);
4341	const elem_size = elem_ty.abiSize(zcu);
4342	const elems: []const Air.Inst.Ref =
4343	@ptrCast(isel.air.extra.items[ty_pl.payload..][0..@intCast(array_type.len)]);
4344	var elem_offset: u64 = 0;
4345
4346	try agg_vi.value.split(isel, false);
4347	for (elems) |elem| {
4348	const agg_part_vi = try agg_vi.value.partExactRecursive(isel, elem_offset, elem_size);
4349	try agg_part_vi.defMove(isel, elem);
4350	elem_offset += elem_size;
4351	}
4352	switch (array_type.sentinel) {
4353	.none => {},
4354	else => |sentinel| {
4355	const agg_part_vi = try agg_vi.value.partExactRecursive(isel, elem_offset, elem_size);
4356	try agg_part_vi.defMove(isel, .fromIntern(sentinel));
4357	},
4358	}
4359	},
4360	.struct_type => {
4361	const loaded_struct = ip.loadStructType(agg_ty.toIntern());
4362	const elems: []const Air.Inst.Ref =
4363	@ptrCast(isel.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]);
4364	var field_offset: u64 = 0;
4365	var field_it = loaded_struct.iterateRuntimeOrder(ip);
4366	while (field_it.next()) |field_index| {
4367	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
4368	field_offset = loaded_struct.field_offsets.get(ip)[field_index];
4369	const field_size = field_ty.abiSize(zcu);
4370	if (field_size == 0) continue;
4371	const agg_part_vi = try agg_vi.value.partExactRecursive(isel, field_offset, field_size);
4372	try agg_part_vi.defMove(isel, elems[field_index]);
4373	field_offset += field_size;
4374	}
4375	assert(loaded_struct.alignment.forward(field_offset) == agg_vi.value.size(isel));
4376	},
4377	.tuple_type => |tuple_type| {
4378	const elems: []const Air.Inst.Ref =
4379	@ptrCast(isel.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]);
4380	var tuple_align: InternPool.Alignment = .@"1";
4381	var field_offset: u64 = 0;
4382	for (
4383	tuple_type.types.get(ip),
4384	tuple_type.values.get(ip),
4385	elems,
4386	) |field_ty_index, field_val, elem| {
4387	if (field_val != .none) continue;
4388	const field_ty: ZigType = .fromInterned(field_ty_index);
4389	const field_align = field_ty.abiAlignment(zcu);
4390	tuple_align = tuple_align.maxStrict(field_align);
4391	field_offset = field_align.forward(field_offset);
4392	const field_size = field_ty.abiSize(zcu);
4393	if (field_size == 0) continue;
4394	const agg_part_vi = try agg_vi.value.partExactRecursive(isel, field_offset, field_size);
4395	try agg_part_vi.defMove(isel, elem);
4396	field_offset += field_size;
4397	}
4398	assert(tuple_align.forward(field_offset) == agg_vi.value.size(isel));
4399	},
4400	.vector_type => try isel.failUnimplemented("agg init vector", .{}),
4401	else => unreachable,
4402	}
4403	},
4404	.struct_field_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
4405	defer dst_vi.value.deref(isel);
4406	const ty_pl = air.data(air.inst_index).ty_pl;
4407	const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
4408	switch (codegen.fieldOffset(
4409	isel.air.typeOf(extra.struct_operand, ip),
4410	ty_pl.ty,
4411	extra.field_index,
4412	zcu,
4413	)) {
4414	0 => try dst_vi.value.defMove(isel, extra.struct_operand),
4415	else => |field_offset| {
4416	const dst_reg = try dst_vi.value.defRegMod(isel, .integer) orelse break :unused;
4417	const src_vi = try isel.use(extra.struct_operand);
4418	const src_mat = try src_vi.matIntRegZeroExt(isel);
4419	try isel.addImm(dst_reg, src_mat.reg(), field_offset);
4420	try src_mat.finish(isel);
4421	},
4422	}
4423	},
4424	.struct_field_ptr_index_0,
4425	.struct_field_ptr_index_1,
4426	.struct_field_ptr_index_2,
4427	.struct_field_ptr_index_3,
4428	=> if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
4429	defer dst_vi.value.deref(isel);
4430	const ty_op = air.data(air.inst_index).ty_op;
4431	switch (codegen.fieldOffset(
4432	isel.air.typeOf(ty_op.operand, ip),
4433	ty_op.ty,
4434	switch (air_tag) {
4435	else => unreachable,
4436	.struct_field_ptr_index_0 => 0,
4437	.struct_field_ptr_index_1 => 1,
4438	.struct_field_ptr_index_2 => 2,
4439	.struct_field_ptr_index_3 => 3,
4440	},
4441	zcu,
4442	)) {
4443	0 => try dst_vi.value.defMove(isel, ty_op.operand),
4444	else => |field_offset| {
4445	const dst_reg = try dst_vi.value.defRegMod(isel, .integer) orelse break :unused;
4446	const src_vi = try isel.use(ty_op.operand);
4447	const src_mat = try src_vi.matIntRegZeroExt(isel);
4448	try isel.addImm(dst_reg, src_mat.reg(), field_offset);
4449	try src_mat.finish(isel);
4450	},
4451	}
4452	},
4453	.agg_field_val => if (isel.live_values.fetchRemove(air.inst_index)) |field_vi| {
4454	defer field_vi.value.deref(isel);
4455
4456	const ty_pl = air.data(air.inst_index).ty_pl;
4457	const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
4458	const agg_ty = isel.air.typeOf(extra.struct_operand, ip);
4459	const field_ty = ty_pl.ty;
4460
4461	const field_bit_offset, const field_bit_size, const is_packed = switch (agg_ty.containerLayout(zcu)) {
4462	.auto, .@"extern" => .{
4463	8 * agg_ty.structFieldOffset(extra.field_index, zcu),
4464	8 * field_ty.abiSize(zcu),
4465	false,
4466	},
4467	.@"packed" => .{
4468	if (zcu.typeToPackedStruct(agg_ty)) |loaded_struct|
4469	zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index)
4470	else
4471	0,
4472	field_ty.bitSize(zcu),
4473	true,
4474	},
4475	};
4476	if (is_packed) return isel.fail("packed field of {f}", .{
4477	isel.fmtType(agg_ty),
4478	});
4479
4480	const agg_vi = try isel.use(extra.struct_operand);
4481	switch (agg_ty.zigTypeTag(zcu)) {
4482	else => unreachable,
4483	.@"struct" => {
4484	const agg_part_vi = try agg_vi.partExactRecursive(
4485	isel,
4486	@divExact(field_bit_offset, 8),
4487	@divExact(field_bit_size, 8),
4488	);
4489	try field_vi.value.defCopy(isel, agg_part_vi);
4490	},
4491	.@"union" => {
4492	const agg_part_vi = try agg_vi.partAtLargerThan(
4493	isel,
4494	@divExact(field_bit_offset, 8),
4495	@divExact(field_bit_size, 8),
4496	);
4497	try field_vi.value.defCopy(isel, agg_part_vi);
4498	},
4499	}
4500	},
4501	.union_init => if (isel.live_values.fetchRemove(air.inst_index)) |union_vi| {
4502	defer union_vi.value.deref(isel);
4503
4504	const ty_pl = air.data(air.inst_index).ty_pl;
4505	const extra = isel.air.extraData(Air.UnionInit, ty_pl.payload).data;
4506	const union_ty = ty_pl.ty;
4507	const loaded_union = ip.loadUnionType(union_ty.toIntern());
4508	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
4509
4510	if (union_layout.tag_size > 0) unused_tag: {
4511	const loaded_tag = ip.loadEnumType(loaded_union.enum_tag_type);
4512	const tag_vi = try union_vi.value.partExact(
4513	isel,
4514	union_layout.tagOffset(),
4515	union_layout.tag_size,
4516	);
4517	if (tag_vi.extension(isel) == .sign_ext)
4518	try tag_vi.reextendToGarbage(isel);
4519	const tag_reg = try tag_vi.defRegMod(isel, .integer) orelse break :unused_tag;
4520	const tag_val: i64 = switch (loaded_tag.field_values.len) {
4521	0 => extra.field_index,
4522	else => switch (ip.indexToKey(loaded_tag.field_values.get(ip)[extra.field_index]).int.storage) {
4523	.u64 => |imm| @bitCast(imm),
4524	.i64 => |imm| imm,
4525	else => unreachable,
4526	},
4527	};
4528	try isel.moveIntImm(tag_reg, tag_val);
4529	}
4530	const payload_vi = try union_vi.value.partExact(
4531	isel,
4532	union_layout.payloadOffset(),
4533	union_layout.payload_size,
4534	);
4535	try payload_vi.defMove(isel, extra.init);
4536	},
4537	.set_union_tag => {
4538	const bin_op = air.data(air.inst_index).bin_op;
4539	const union_ty = isel.air.typeOf(bin_op.lhs, ip).childType(zcu);
4540	const union_layout = union_ty.unionGetLayout(zcu);
4541	const tag_vi = try isel.use(bin_op.rhs);
4542	const union_ptr_vi = try isel.use(bin_op.lhs);
4543	const union_ptr_mat = try union_ptr_vi.matIntRegZeroExt(isel);
4544	try tag_vi.matStore(isel, union_ptr_mat.reg(), union_layout.tagOffset(), .{});
4545	try union_ptr_mat.finish(isel);
4546	},
4547	.get_union_tag => if (isel.live_values.fetchRemove(air.inst_index)) |tag_vi| {
4548	defer tag_vi.value.deref(isel);
4549	const ty_op = air.data(air.inst_index).ty_op;
4550	const union_ty = isel.air.typeOf(ty_op.operand, ip);
4551	const union_layout = union_ty.unionGetLayout(zcu);
4552	const union_vi = try isel.use(ty_op.operand);
4553	const tag_part_vi = try union_vi.partExact(isel, union_layout.tagOffset(), union_layout.tag_size);
4554	try tag_vi.value.defCopy(isel, tag_part_vi);
4555	},
4556	.optional_payload => if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| unused: {
4557	defer payload_vi.value.deref(isel);
4558
4559	const ty_op = air.data(air.inst_index).ty_op;
4560	const opt_ty = isel.air.typeOf(ty_op.operand, ip);
4561	if (opt_ty.optionalReprIsPayload(zcu)) {
4562	try payload_vi.value.defMove(isel, ty_op.operand);
4563	break :unused;
4564	}
4565
4566	const opt_vi = try isel.use(ty_op.operand);
4567	const payload_part_vi = try opt_vi.partExact(isel, 0, payload_vi.value.size(isel));
4568	try payload_vi.value.defCopy(isel, payload_part_vi);
4569	},
4570	.optional_payload_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| {
4571	defer payload_ptr_vi.value.deref(isel);
4572	const ty_op = air.data(air.inst_index).ty_op;
4573	try payload_ptr_vi.value.defMove(isel, ty_op.operand);
4574	},
4575	.wrap_optional => if (isel.live_values.fetchRemove(air.inst_index)) |opt_vi| unused: {
4576	defer opt_vi.value.deref(isel);
4577
4578	const ty_op = air.data(air.inst_index).ty_op;
4579	if (ty_op.ty.optionalReprIsPayload(zcu)) {
4580	try opt_vi.value.defMove(isel, ty_op.operand);
4581	break :unused;
4582	}
4583
4584	const payload_size = isel.air.typeOf(ty_op.operand, ip).abiSize(zcu);
4585
4586	const payload_part_vi = try opt_vi.value.partExact(isel, 0, payload_size);
4587	const has_value_part_vi = try opt_vi.value.partExact(isel, payload_size, 1);
4588	try payload_part_vi.defMove(isel, ty_op.operand);
4589	const maybe_has_value_part_reg = try has_value_part_vi.defRegMod(isel, .integer);
4590	if (maybe_has_value_part_reg) |has_value_part_reg|
4591	try isel.emit(.ori(has_value_part_reg, .zero, 0));
4592	},
4593	.field_parent_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
4594	defer dst_vi.value.deref(isel);
4595	const ty_pl = air.data(air.inst_index).ty_pl;
4596	const extra = isel.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
4597	switch (codegen.fieldOffset(
4598	ty_pl.ty,
4599	isel.air.typeOf(extra.field_ptr, ip),
4600	extra.field_index,
4601	zcu,
4602	)) {
4603	0 => try dst_vi.value.defMove(isel, extra.field_ptr),
4604	else => |field_offset| {
4605	const dst_reg = try dst_vi.value.defRegMod(isel, .integer) orelse break :unused;
4606	const src_vi = try isel.use(extra.field_ptr);
4607	const src_mat = try src_vi.matIntRegZeroExt(isel);
4608	try isel.addImm(dst_reg, src_mat.reg(), -@as(i65, field_offset));
4609	try src_mat.finish(isel);
4610	},
4611	}
4612	},
4613	.unwrap_errunion_payload => if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| {
4614	defer payload_vi.value.deref(isel);
4615
4616	const ty_op = air.data(air.inst_index).ty_op;
4617	const error_union_vi = try isel.use(ty_op.operand);
4618	try payload_vi.value.defCopy(
4619	isel,
4620	try error_union_vi.partExact(
4621	isel,
4622	codegen.errUnionPayloadOffset(ty_op.ty, zcu),
4623	payload_vi.value.size(isel),
4624	),
4625	);
4626	},
4627	.unwrap_errunion_err => if (isel.live_values.fetchRemove(air.inst_index)) |error_set_vi| {
4628	defer error_set_vi.value.deref(isel);
4629
4630	const ty_op = air.data(air.inst_index).ty_op;
4631	const error_union_ty = isel.air.typeOf(ty_op.operand, ip);
4632	const error_union_vi = try isel.use(ty_op.operand);
4633	try error_set_vi.value.defCopy(
4634	isel,
4635	try error_union_vi.partExact(
4636	isel,
4637	codegen.errUnionErrorOffset(error_union_ty.errorUnionPayload(zcu), zcu),
4638	error_set_vi.value.size(isel),
4639	),
4640	);
4641	},
4642	.wrap_errunion_payload => if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| {
4643	defer error_union_vi.value.deref(isel);
4644
4645	const ty_op = air.data(air.inst_index).ty_op;
4646	const error_union_ty = ty_op.ty;
4647	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4648	const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
4649	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4650	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
4651	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
4652	const error_set_size = error_set_ty.abiSize(zcu);
4653	const payload_size = payload_ty.abiSize(zcu);
4654
4655	try error_union_vi.value.collectDefs(isel);
4656
4657	if (payload_size > 0) {
4658	const payload_part_vi = try error_union_vi.value.partExact(isel, payload_offset, payload_size);
4659	try payload_part_vi.defMove(isel, ty_op.operand);
4660	}
4661	const error_set_part_vi = try error_union_vi.value.partExact(isel, error_set_offset, error_set_size);
4662	if (try error_set_part_vi.defRegMod(isel, .integer)) |error_set_part_reg|
4663	try isel.emit(.ori(error_set_part_reg, .zero, 0));
4664	},
4665	.wrap_errunion_err => if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| {
4666	defer error_union_vi.value.deref(isel);
4667
4668	const ty_op = air.data(air.inst_index).ty_op;
4669	const error_union_ty = ty_op.ty;
4670	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4671	const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
4672	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4673	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
4674	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
4675	const error_set_size = error_set_ty.abiSize(zcu);
4676	const payload_size = payload_ty.abiSize(zcu);
4677
4678	const error_set_part_vi = try error_union_vi.value.partExact(isel, error_set_offset, error_set_size);
4679	try error_set_part_vi.defMove(isel, ty_op.operand);
4680	if (payload_size > 0) {
4681	const payload_part_vi = try error_union_vi.value.partExact(isel, payload_offset, payload_size);
4682	try payload_part_vi.defUndef(isel);
4683	}
4684	},
4685	.errunion_payload_ptr_set => if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
4686	defer payload_ptr_vi.value.deref(isel);
4687	const ty_op = air.data(air.inst_index).ty_op;
4688	const payload_ty = ty_op.ty.childType(zcu);
4689	const eu_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu);
4690	const error_set_size = eu_ty.errorUnionSet(zcu).abiSize(zcu);
4691
4692	const eu_ptr_vi = try isel.use(ty_op.operand);
4693	const error_union_ptr_mat = try eu_ptr_vi.matIntRegZeroExt(isel);
4694	if (error_set_size != 0) {
4695	try isel.storeReg(
4696	.zero,
4697	error_set_size,
4698	error_union_ptr_mat.reg(),
4699	codegen.errUnionErrorOffset(payload_ty, zcu),
4700	);
4701	}
4702	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
4703	if (payload_offset == 0) {
4704	try error_union_ptr_mat.finish(isel);
4705	try payload_ptr_vi.value.defMove(isel, ty_op.operand);
4706	} else {
4707	const payload_ptr_reg = try payload_ptr_vi.value.defRegMod(isel, .integer) orelse break :unused;
4708	try isel.addImm(payload_ptr_reg, error_union_ptr_mat.reg(), payload_offset);
4709	try error_union_ptr_mat.finish(isel);
4710	}
4711	},
4712	}
4713	if (air_tag != .arg) {
4714	var live_reg_it = isel.live_registers.iterator();
4715	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
4716	.allocating => {
4717	tracking_log.err("${t} is still allocated", .{live_reg_entry.key});
4718	isel.dumpValues(.all);
4719	unreachable;
4720	},
4721	_, .free => {},
4722	};
4723	}
4724	if (debug_r21_as_air) {
4725	try isel.moveIntImm(.r21, @backingInt(air.inst_index));
4726	}
4727	}
4728	assert(air.body_index == 0);
4729	}
4730
4731	/// Generates prologue and epilogue. Returns the length of epilogue.
4732	///
4733	/// Stack Frame Layout
4734	/// +-+-----------------------------------+
4735	/// |R| caller frame |
4736	/// +-+-----------------------------------+
4737	/// |S| incoming stack arguments | +---------------+
4738	/// +-+-----------------------------------+ <-| align(16) |
4739	/// |L| callee saved FP | | entry/exit SP |
4740	/// +-+-----------------------------------+ | FP |
4741	/// |L| callee saved GPR area | +---------------+
4742	/// +-+-----------------------------------+
4743	/// |L| callee saved FPR area | +-----------------+
4744	/// +-+-----------------------------------+ <-| FP - saves_size |
4745	/// |L| realignment gap | +-----------------+
4746	/// +-+-----------------------------------+ <-| align(16) |
4747	/// |L| locals | +-----------------+
4748	/// +-+-----------------------------------+
4749	/// |S| outgoing stack arguments | +----+
4750	/// +-+-----------------------------------+ <-| SP |
4751	/// +----+
4752	/// [S] Size computed by `analyze`, can be used by the body.
4753	/// [L] Size computed by `layout`, can be used by the prologue/epilogue.
4754	/// [R] Size unknown until runtime, can vary from one call to the next.
4755	///
4756	/// FP saving/restoring is not yet implemented.
4757	pub fn layout(isel: *Select, cc_it: CallAbiIterator, mod: *const Module) !usize {
4758	_ = cc_it;
4759	_ = mod;
4760	const zcu = isel.pt.zcu;
4761	const ip = &zcu.intern_pool;
4762	const nav = ip.getNav(isel.nav_index);
4763	wip_mir_log.debug("{f}<body>:\n", .{nav.fqn.fmt(ip)});
4764
4765	const gpr_size = isel.gprSize();
4766
4767	var saves_buf: [10 + 2 + 8]struct {
4768	register: Register,
4769	needs_restore: bool,
4770	offset: u11,
4771	size: u5,
4772	} = undefined;
4773	var saved_offset: std.EnumArray(Register, u11) = .initUndefined();
4774	const saves, const saves_size = saves: {
4775	var saves_len: usize = 0;
4776	var saves_size: u11 = 0;
4777	var save_reg: Register = undefined;
4778
4779	// callee saved GPR area
4780	save_reg = .r23;
4781	while (true) : (save_reg = @fromBackingInt(@backingInt(save_reg) + 1)) {
4782	if (isel.saved_registers.contains(save_reg)) {
4783	saves_size = std.mem.alignForward(u11, saves_size, gpr_size);
4784	saves_buf[saves_len] = .{
4785	.register = save_reg,
4786	.needs_restore = true,
4787	.offset = saves_size,
4788	.size = gpr_size,
4789	};
4790	saved_offset.set(save_reg, saves_size);
4791	saves_len += 1;
4792	saves_size += gpr_size;
4793	}
4794	if (save_reg == .r31) break;
4795	}
4796	inline for (.{ Register.ra, Register.fp }) |reg| {
4797	if (isel.saved_registers.contains(reg)) {
4798	saves_size = std.mem.alignForward(u11, saves_size, gpr_size);
4799	saves_buf[saves_len] = .{
4800	.register = reg,
4801	.needs_restore = true,
4802	.offset = saves_size,
4803	.size = gpr_size,
4804	};
4805	saved_offset.set(reg, saves_size);
4806	saves_len += 1;
4807	saves_size += gpr_size;
4808	}
4809	}
4810
4811	// callee saved FPR area
4812	save_reg = .f24;
4813	while (true) : (save_reg = @fromBackingInt(@backingInt(save_reg) + 1)) {
4814	if (isel.saved_registers.contains(save_reg)) {
4815	saves_size = std.mem.alignForward(u11, saves_size, 8);
4816	saves_buf[saves_len] = .{
4817	.register = save_reg,
4818	.needs_restore = true,
4819	.offset = saves_size,
4820	.size = 8,
4821	};
4822	saved_offset.set(save_reg, saves_size);
4823	saves_len += 1;
4824	saves_size += 8;
4825	}
4826	if (save_reg == .f31) break;
4827	}
4828	break :saves .{ saves_buf[0..saves_len], std.mem.Alignment.@"16".forward(saves_size) };
4829	};
4830
4831	const stack_frame_size = isel.stack_align.forward(saves_size + isel.stack_size);
4832
4833	// apply layout relocs
4834	for (isel.layout_relocs.items) |label| {
4835	const instruction = isel.instructions.items[label];
4836	const rj: Register = .decode(.int, instruction.DJUk12.rj);
4837	if (isel.saved_registers.contains(rj)) {
4838	const rd: Register = .decode(.int, instruction.DJUk12.rd);
4839	const offset = saved_offset.get(rj);
4840	isel.instructions.items[label] = switch (gpr_size) {
4841	else => unreachable,
4842	4 => .@"ld.w"(rd, .sp, @intCast(stack_frame_size - 8 - offset)),
4843	8 => .@"ld.d"(rd, .sp, @intCast(stack_frame_size - 8 - offset)),
4844	};
4845	}
4846	}
4847
4848	// prologue
4849	{
4850	// move SP
4851	if (stack_frame_size == 0) {} else if (std.math.cast(i12, stack_frame_size)) |stack_size12| {
4852	switch (gpr_size) {
4853	4 => try isel.emit(.@"addi.w"(.sp, .sp, -stack_size12)),
4854	8 => try isel.emit(.@"addi.d"(.sp, .sp, -stack_size12)),
4855	else => unreachable,
4856	}
4857	} else {
4858	switch (gpr_size) {
4859	4 => try isel.emit(.@"sub.w"(.sp, .sp, .t0)),
4860	8 => try isel.emit(.@"sub.d"(.sp, .sp, .t0)),
4861	else => unreachable,
4862	}
4863	try isel.moveIntImm(.t0, @intCast(stack_frame_size));
4864	}
4865
4866	// set FP
4867	if (isel.saved_registers.contains(.fp))
4868	try isel.emit(.ori(.fp, .sp, 0));
4869
4870	// save registers
4871	for (saves) |save| {
4872	switch (save.register.class()) {
4873	.int => switch (gpr_size) {
4874	4 => try isel.emit(.@"st.h"(save.register, .sp, -8 - @as(i12, save.offset))),
4875	8 => try isel.emit(.@"st.d"(save.register, .sp, -8 - @as(i12, save.offset))),
4876	else => unreachable,
4877	},
4878	.fp => try isel.emit(.@"fst.d"(save.register, .sp, -8 - @as(i12, save.offset))),
4879	.fcc => unreachable,
4880	}
4881	}
4882	wip_mir_log.debug("{f}<prologue>:", .{nav.fqn.fmt(ip)});
4883	}
4884
4885	// epilogue
4886	const epilogue = isel.instructions.items.len;
4887	if (isel.returns) {
4888	// return
4889	try isel.emit(.jirl(.zero, .ra, 0));
4890
4891	// restore registers
4892	for (saves) |save| {
4893	if (!save.needs_restore) continue;
4894	switch (save.register.class()) {
4895	.int => switch (gpr_size) {
4896	4 => try isel.emit(.@"ld.h"(save.register, .sp, -8 - @as(i12, save.offset))),
4897	8 => try isel.emit(.@"ld.d"(save.register, .sp, -8 - @as(i12, save.offset))),
4898	else => unreachable,
4899	},
4900	.fp => try isel.emit(.@"fld.d"(save.register, .sp, -8 - @as(i12, save.offset))),
4901	.fcc => unreachable,
4902	}
4903	}
4904
4905	// restore SP
4906	if (stack_frame_size == 0) {} else if (std.math.cast(i12, stack_frame_size)) |stack_size12| {
4907	switch (gpr_size) {
4908	4 => try isel.emit(.@"addi.w"(.sp, .sp, stack_size12)),
4909	8 => try isel.emit(.@"addi.d"(.sp, .sp, stack_size12)),
4910	else => unreachable,
4911	}
4912	} else {
4913	switch (gpr_size) {
4914	4 => try isel.emit(.@"add.w"(.sp, .sp, .t0)),
4915	8 => try isel.emit(.@"add.d"(.sp, .sp, .t0)),
4916	else => unreachable,
4917	}
4918	try isel.moveIntImm(.t0, @intCast(stack_frame_size));
4919	}
4920
4921	wip_mir_log.debug("{f}<epilogue>:\n", .{nav.fqn.fmt(ip)});
4922	}
4923	return epilogue;
4924	}
4925
4926	fn emit(isel: *Select, instruction: Instruction) !void {
4927	wip_mir_log.debug(" | {f}", .{(Disassemble{}).fmtInstruction(instruction)});
4928	try isel.instructions.append(isel.pt.zcu.gpa, instruction);
4929	}
4930
4931	pub fn verifyTargetFeatures(isel: *Select) !void {
4932	if (!verify_target_features) return;
4933
4934	for (isel.instructions.items) |inst| {
4935	if (Disassemble.decodeMnemonic(inst)) |decoded_mnemonic| {
4936	switch (decoded_mnemonic) {
4937	inline else => |mnemonic| {
4938	const expected_features = @field(@import("inst_formats.zon").instructions, @tagName(mnemonic)).features;
4939	inline for (@typeInfo(expected_features).@"struct".fields) |expected_feature_field| {
4940	const expected_feature = @tagName(@field(expected_features, expected_feature_field.name));
4941	const std_feature = @field(std.Target.loongarch.Feature, expected_feature);
4942	if (!isel.hasCpuFeature(std_feature)) {
4943	wip_mir_log.err("emitted instruction {t} requires feature {t} which is not available", .{ mnemonic, std_feature });
4944	unreachable;
4945	}
4946	}
4947	},
4948	}
4949	} else {
4950	wip_mir_log.err("invalid instruction was emitted in Select: {x}", .{inst.word});
4951	unreachable;
4952	}
4953	}
4954	}
4955
4956	fn hasCpuFeature(isel: *Select, feature: std.Target.loongarch.Feature) bool {
4957	return std.Target.loongarch.featureSetHas(isel.target.cpu.features, feature);
4958	}
4959
4960	fn block(
4961	isel: *Select,
4962	air_inst_index: Air.Inst.Index,
4963	res_ty: ZigType,
4964	air_body: []const Air.Inst.Index,
4965	) !void {
4966	if (res_ty.toIntern() != .noreturn_type) {
4967	const snapshot = try isel.takeLocationSnapshot();
4968	tracking_log.debug("block snapshot taken:\n{f}", .{snapshot});
4969	isel.active_blocks.putAssumeCapacityNoClobber(air_inst_index, .{
4970	.snapshot = snapshot,
4971	.target_label = @intCast(isel.instructions.items.len),
4972	});
4973	}
4974	try isel.body(air_body);
4975	if (res_ty.toIntern() != .noreturn_type) {
4976	var block_entry = isel.active_blocks.pop().?;
4977	assert(block_entry.key == air_inst_index);
4978	block_entry.value.deinit(isel);
4979	if (isel.live_values.fetchRemove(air_inst_index)) |result_vi| {
4980	var res_walk = result_vi.value.walk(isel, .{});
4981	while (res_walk.next()) |res_part_vi|
4982	_ = res_part_vi.takeLocationMarkWritten(isel);
4983	result_vi.value.deref(isel);
4984	}
4985	}
4986	}
4987
4988	fn initValue(isel: *Select, ty: ZigType) error{OutOfMemory}!Value.Index {
4989	const zcu = isel.pt.zcu;
4990	try isel.values.ensureUnusedCapacity(zcu.gpa, 1);
4991	try isel.value_types.ensureUnusedCapacity(zcu.gpa, 1);
4992	return isel.initValueAdvanced(ty.abiAlignment(zcu), 0, ty.abiSize(zcu), ty);
4993	}
4994
4995	fn initValueAssumeCapacity(isel: *Select, ty: ZigType) Value.Index {
4996	const zcu = isel.pt.zcu;
4997	return isel.initValueAdvanced(ty.abiAlignment(zcu), 0, ty.abiSize(zcu), ty);
4998	}
4999
5000	fn initValueAdvanced(
5001	isel: *Select,
5002	parent_alignment: InternPool.Alignment,
5003	offset_from_parent: u64,
5004	size: u64,
5005	ty: ?ZigType,
5006	) Value.Index {
5007	defer isel.values.addOneAssumeCapacity().* = .{
5008	.refs = 0,
5009	.flags = .{
5010	.alignment = .fromLog2Units(@min(parent_alignment.toLog2Units(), @ctz(offset_from_parent))),
5011	.parent_tag = .none,
5012	// TODO size < 32 when vectors are supported
5013	.location_tag = if (size <= 8)
5014	.small
5015	else if (std.math.cast(u32, size) != null)
5016	.large
5017	else
5018	.extreme,
5019	.parts_len_minus_one = 0,
5020	.splitted = false,
5021	},
5022	.offset_from_parent = offset_from_parent,
5023	.parent_payload = .{ .none = {} },
5024	// TODO ditto
5025	.location_payload = if (size <= 8) .{ .small = .{
5026	.flags = .{
5027	.size = @intCast(size),
5028	.extension = .garbage,
5029	.hint_modifier = .integer,
5030	.hint_register = .zero,
5031	.location_tag = .register,
5032	},
5033	.location_payload = .{ .register = .zero },
5034	} } else if (std.math.cast(u32, size)) |size32| .{ .large = .{
5035	.size = size32,
5036	.stack_slot = .unallocated,
5037	} } else .{ .extreme = .{ .size = size } },
5038	.parts = undefined,
5039	};
5040	defer isel.value_types.appendAssumeCapacity(ty orelse .{ .ip_index = .none });
5041	return @fromBackingInt(@intCast(isel.values.items.len));
5042	}
5043
5044	const WhichValues = enum { only_referenced, all };
5045	pub fn dumpValues(isel: *Select, which: WhichValues) void {
5046	dumpValuesInner(isel, which) catch |err| @panic(@errorName(err));
5047	}
5048	fn dumpValuesInner(isel: *Select, which: WhichValues) !void {
5049	const zcu = isel.pt.zcu;
5050	const gpa = zcu.gpa;
5051	const ip = &zcu.intern_pool;
5052	const nav = ip.getNav(isel.nav_index);
5053
5054	const locked_stderr = std.debug.lockStderr(&.{});
5055	defer std.debug.unlockStderr();
5056	const stderr = &locked_stderr.file_writer.interface;
5057
5058	var reverse_live_values: std.AutoArrayHashMapUnmanaged(Value.Index, std.ArrayList(Air.Inst.Index)) = .empty;
5059	defer {
5060	for (reverse_live_values.values()) |*list| list.deinit(gpa);
5061	reverse_live_values.deinit(gpa);
5062	}
5063	{
5064	try reverse_live_values.ensureTotalCapacity(gpa, isel.live_values.count());
5065	var live_val_it = isel.live_values.iterator();
5066	while (live_val_it.next()) |live_val_entry| switch (live_val_entry.value_ptr.*) {
5067	_ => {
5068	const gop = reverse_live_values.getOrPutAssumeCapacity(live_val_entry.value_ptr.*);
5069	if (!gop.found_existing) gop.value_ptr.* = .empty;
5070	try gop.value_ptr.append(gpa, live_val_entry.key_ptr.*);
5071	},
5072	.allocating, .free => unreachable,
5073	};
5074	}
5075
5076	var reverse_live_registers: std.AutoHashMapUnmanaged(Value.Index, Register) = .empty;
5077	defer reverse_live_registers.deinit(gpa);
5078	{
5079	try reverse_live_registers.ensureTotalCapacity(gpa, @typeInfo(Register).@"enum".field_names.len);
5080	var live_reg_it = isel.live_registers.iterator();
5081	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
5082	_ => reverse_live_registers.putAssumeCapacityNoClobber(live_reg_entry.value.*, live_reg_entry.key),
5083	.allocating, .free => {},
5084	};
5085	}
5086
5087	var roots: std.AutoArrayHashMapUnmanaged(Value.Index, u32) = .empty;
5088	defer roots.deinit(gpa);
5089	{
5090	try roots.ensureTotalCapacity(gpa, isel.values.items.len);
5091	var vi: Value.Index = @fromBackingInt(@intCast(isel.values.items.len));
5092	iter_values: while (@backingInt(vi) > 0) {
5093	vi = @fromBackingInt(@backingInt(vi) - 1);
5094	if (which == .only_referenced and vi.get(isel).refs == 0) continue;
5095	switch (vi.parent(isel)) {
5096	.none, .constant => {},
5097	.value => continue :iter_values,
5098	.address => |address_vi| roots.putAssumeCapacity(address_vi, 0),
5099	}
5100	roots.putAssumeCapacity(vi, 0);
5101	}
5102	}
5103
5104	try stderr.print("# Begin LA ISelect Value Dump: {f}:\n", .{nav.fqn.fmt(ip)});
5105	while (roots.pop()) |root_entry| {
5106	const vi = root_entry.key;
5107	try stderr.splatByteAll(' ', 2 * (@as(usize, 1) + root_entry.value));
5108	try vi.format(stderr);
5109	{
5110	var first = true;
5111	if (reverse_live_values.get(vi)) |aiis| for (aiis.items) |aii| {
5112	if (aii == Block.main) {
5113	try stderr.print("{s}%main", .{if (first) " <- " else ", "});
5114	} else {
5115	try stderr.print("{s}%{d}", .{ if (first) " <- " else ", ", @backingInt(aii) });
5116	}
5117	first = false;
5118	};
5119	if (reverse_live_registers.get(vi)) |ra| {
5120	try stderr.print("{s}{t}", .{ if (first) " <- " else ", ", ra });
5121	first = false;
5122	}
5123	}
5124	try stderr.writeByte(':');
5125	try isel.printValueInfo(stderr, vi);
5126	try stderr.writeByte('\n');
5127
5128	const value = vi.get(isel);
5129	var part_index = value.flags.parts_len_minus_one;
5130	if (part_index > 0) while (true) : (part_index -= 1) {
5131	try roots.put(
5132	gpa,
5133	@fromBackingInt(@backingInt(value.parts) + part_index),
5134	root_entry.value + 1,
5135	);
5136	if (part_index == 0) break;
5137	};
5138	}
5139	try stderr.print("# End LA ISelect Value Dump: {f}\n", .{nav.fqn.fmt(ip)});
5140	}
5141
5142	fn printValueAndParts(isel: *Select, writer: *std.Io.Writer, target_vi: Value.Index) !void {
5143	const zcu = isel.pt.zcu;
5144	const gpa = zcu.gpa;
5145
5146	var roots: std.AutoArrayHashMapUnmanaged(Value.Index, u32) = .empty;
5147	defer roots.deinit(gpa);
5148
5149	var root_vi = target_vi;
5150	while (true) switch (root_vi.parent(isel)) {
5151	.none, .constant => break,
5152	.value => |parent_vi| root_vi = parent_vi,
5153	.address => |address_vi| break try roots.put(gpa, address_vi, 0),
5154	};
5155	try roots.put(gpa, root_vi, 0);
5156
5157	while (roots.pop()) |root_entry| {
5158	const vi = root_entry.key;
5159	try writer.splatByteAll(' ', 2 * root_entry.value);
5160	try vi.format(writer);
5161	try writer.writeByte(':');
5162	try isel.printValueInfo(writer, vi);
5163
5164	const value = vi.get(isel);
5165	var part_index = value.flags.parts_len_minus_one;
5166	if (part_index > 0) while (true) : (part_index -= 1) {
5167	try roots.put(
5168	gpa,
5169	@fromBackingInt(@backingInt(value.parts) + part_index),
5170	root_entry.value + 1,
5171	);
5172	if (part_index == 0) break;
5173	};
5174
5175	if (roots.count() != 0)
5176	try writer.writeByte('\n');
5177	}
5178	}
5179
5180	fn printValueInfo(isel: *Select, writer: *std.Io.Writer, vi: Value.Index) !void {
5181	const zcu = isel.pt.zcu;
5182
5183	const value = vi.get(isel);
5184	switch (value.flags.parent_tag) {
5185	.none => {},
5186	.value => try writer.print(" {f}+0x{x}", .{ value.parent_payload.value, value.offset_from_parent }),
5187	.address => try writer.print(" {f}[0x{x}]", .{ value.parent_payload.address, value.offset_from_parent }),
5188	.constant => try writer.print(" <{f}, {f}>", .{
5189	isel.fmtType(value.parent_payload.constant.typeOf(zcu)),
5190	isel.fmtConstant(value.parent_payload.constant),
5191	}),
5192	}
5193	try writer.print(" align({s})", .{@tagName(value.flags.alignment)});
5194	switch (value.flags.location_tag) {
5195	.small => {
5196	const loc_info = value.location_payload.small;
5197	try writer.print(" {d}B", .{loc_info.flags.size});
5198	if (loc_info.flags.extension != .garbage) try writer.print(" {t}", .{loc_info.flags.extension});
5199
5200	var hints: u8 = 0;
5201	if (loc_info.flags.hint_modifier != .integer) hints += 1;
5202	if (loc_info.flags.hint_register != Register.zero) hints += 1;
5203	if (hints != 0) try writer.writeAll(" hint=");
5204	if (loc_info.flags.hint_modifier != .integer) try writer.print("{t}", .{loc_info.flags.hint_modifier});
5205	if (loc_info.flags.hint_register != Register.zero) try writer.print("{s}${t}", .{ if (hints != 1) "," else "", loc_info.flags.hint_register });
5206
5207	switch (loc_info.flags.location_tag) {
5208	.register => {
5209	if (loc_info.location_payload.register.reg != Register.zero) {
5210	try writer.print(" loc={f}", .{loc_info.location_payload.register});
5211	}
5212	},
5213	.stack_slot => try writer.print(" loc={f}", .{loc_info.location_payload.stack_slot}),
5214	}
5215	},
5216	.large => {
5217	try writer.print(" {d}B large", .{value.location_payload.large.size});
5218	if (value.location_payload.large.stack_slot != Value.Indirect.unallocated)
5219	try writer.print(" loc={f}", .{value.location_payload.large.stack_slot});
5220	},
5221	.extreme => try writer.print(" {d}B extreme", .{value.location_payload.large.size}),
5222	}
5223	if (value.flags.splitted)
5224	try writer.writeAll(" splitted");
5225	if (value.refs != 0)
5226	try writer.print(" refs={d}", .{value.refs});
5227	if (vi.typeOf(isel)) |ty| try writer.print(" {f}", .{isel.fmtType(ty)});
5228	}
5229
5230	fn fmtValue(isel: *Select, vi: Value.Index) struct {
5231	isel: *Select,
5232	vi: Value.Index,
5233	pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
5234	data.isel.printValueAndParts(writer, data.vi) catch |err| switch (err) {
5235	error.OutOfMemory => try writer.writeAll("OOM"),
5236	error.WriteFailed => return error.WriteFailed,
5237	};
5238	}
5239	} {
5240	return .{ .isel = isel, .vi = vi };
5241	}
5242
5243	fn fmtLoopLive(isel: *Select, loop_inst: Air.Inst.Index) struct {
5244	isel: *Select,
5245	inst: Air.Inst.Index,
5246	pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
5247	const loops = data.isel.loops.values();
5248	const loop_index = data.isel.loops.getIndex(data.inst).?;
5249	const live_insts =
5250	data.isel.loop_outer_live.list.items[loops[loop_index].outer_live..loops[loop_index + 1].outer_live];
5251
5252	try writer.print("%{d} <- {{", .{@backingInt(data.inst)});
5253	var first = true;
5254	for (live_insts) |live_inst| {
5255	if (first) first = false else try writer.writeByte(',');
5256	try writer.print(" %{d}", .{@backingInt(live_inst)});
5257	}
5258	if (!first) try writer.writeByte(' ');
5259	try writer.writeByte('}');
5260	}
5261	} {
5262	return .{ .isel = isel, .inst = loop_inst };
5263	}
5264
5265	fn fmtType(isel: *Select, ty: ZigType) ZigType.Formatter {
5266	return ty.fmt(isel.pt);
5267	}
5268
5269	fn fmtConstant(isel: *Select, constant: Constant) @typeInfo(@TypeOf(Constant.fmtValue)).@"fn".return_type.? {
5270	return constant.fmtValue(isel.pt);
5271	}
5272
5273	fn fmtRegisterSet(regs: RegisterSet) struct {
5274	regs: RegisterSet,
5275	pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
5276	var it = data.regs.iterator();
5277	var first = true;
5278	while (it.next()) |reg| {
5279	if (first) first = false else try writer.writeAll(", ");
5280	try writer.print("${t}", .{reg});
5281	}
5282	if (first) try writer.writeAll("(empty)");
5283	}
5284	} {
5285	return .{ .regs = regs };
5286	}
5287
5288	fn use(isel: *Select, air_ref: Air.Inst.Ref) !Value.Index {
5289	const zcu = isel.pt.zcu;
5290	const ip = &zcu.intern_pool;
5291	const vi, const ty = if (air_ref.toIndex()) |air_inst_index| vi_ty: {
5292	const live_gop = try isel.live_values.getOrPut(zcu.gpa, air_inst_index);
5293	if (live_gop.found_existing) return live_gop.value_ptr.*;
5294	const ty = isel.air.typeOf(air_ref, ip);
5295	const vi = try isel.initValue(ty);
5296	tracking_log.debug("{f} <- %{d}", .{ vi, @backingInt(air_inst_index) });
5297	live_gop.value_ptr.* = vi.ref(isel);
5298	break :vi_ty .{ vi, ty };
5299	} else vi_ty: {
5300	const constant: Constant = .fromInterned(air_ref.toInterned().?);
5301	const ty = constant.typeOf(zcu);
5302	const vi = try isel.initValue(ty);
5303	tracking_log.debug("{f} <- <{f}, {f}>", .{
5304	vi,
5305	isel.fmtType(ty),
5306	isel.fmtConstant(constant),
5307	});
5308	vi.setParent(isel, .{ .constant = constant });
5309	break :vi_ty .{ vi, ty };
5310	};
5311	if (ty.isAbiInt(zcu)) {
5312	const int_info = ty.intInfo(zcu);
5313	if (int_info.bits <= 16) vi.setExtension(isel, .fromSignedness(int_info.signedness));
5314	}
5315	return vi;
5316	}
5317
5318	// TODO: make r22 allocatable
5319	fn isRegisterAllocatable(rd: Register) bool {
5320	return switch (rd) {
5321	else => true,
5322	Register.zero, Register.tp, Register.sp, Register.fp, .r21 => false,
5323	};
5324	}
5325
5326	/// Frees a register by forgetting it.
5327	/// Returns true on success, false on failure (i.e. dst_reg is locked/allocated or unallocatable).
5328	fn forgetReg(isel: *Select, dst_reg: Register) error{ OutOfMemory, AlreadyReported }!bool {
5329	if (!isRegisterAllocatable(dst_reg)) return false;
5330	const dst_live_vi = isel.live_registers.getPtr(dst_reg);
5331	const dst_vi = switch (dst_live_vi.*) {
5332	_ => |dst_vi| dst_vi,
5333	.allocating => return false,
5334	.free => return true,
5335	};
5336	tracking_log.debug("{f} -> location forgotten", .{dst_vi});
5337	_ = dst_vi.takeLocation(isel);
5338	assert(dst_live_vi.* == .free);
5339	return true;
5340	}
5341
5342	/// Frees a register by moving it to another place.
5343	/// Returns true on success, false on failure (i.e. dst_reg is locked/allocated or unallocatable).
5344	fn fillReg(isel: *Select, dst_reg: Register) codegen.Error!bool {
5345	if (!isRegisterAllocatable(dst_reg)) return false;
5346	const dst_live_vi = isel.live_registers.getPtr(dst_reg);
5347	const dst_vi = switch (dst_live_vi.*) {
5348	_ => |dst_vi| dst_vi,
5349	.allocating => return false,
5350	.free => return true,
5351	};
5352	const src_loc: Value.Location = src: {
5353	if (dst_vi.hintRegister(isel)) |hint_reg| {
5354	dst_live_vi.* = .allocating;
5355	defer dst_live_vi.* = dst_vi;
5356	if (try isel.fillReg(hint_reg)) {
5357	isel.saved_registers.insert(hint_reg);
5358	break :src .{ .register = .{ .mod = dst_vi.hintModifier(isel), .reg = hint_reg } };
5359	}
5360	}
5361	if (dst_vi.isSmall(isel)) {
5362	switch (isel.tryAllocReg(dst_vi.hintModifier(isel).class())) {
5363	.allocated => |reg| {
5364	isel.freeReg(reg);
5365	break :src .{ .register = .{ .mod = dst_vi.hintModifier(isel), .reg = reg } };
5366	},
5367	.fill_candidate, .out_of_registers => {},
5368	}
5369	}
5370	break :src .{ .stack_slot = dst_vi.allocStackSlot(isel) };
5371	};
5372	try dst_vi.moveTo(isel, src_loc);
5373	assert(dst_live_vi.* == .free);
5374	return true;
5375	}
5376
5377	/// Frees a set of register. If locked is true, these registers are then locked.
5378	/// Requires all registers to be unlocked.
5379	/// Returns true on success.
5380	fn fillRegsBatch(isel: *Select, regs: RegisterSet, locking: bool) codegen.Error!void {
5381	tracking_log.debug("batch fill: {f}", .{fmtRegisterSet(regs)});
5382	// lock free registers
5383	var regs_it = regs.iterator();
5384	while (regs_it.next()) |reg| {
5385	const live_vi = isel.live_registers.getPtr(reg);
5386	switch (live_vi.*) {
5387	.allocating => unreachable,
5388	.free => live_vi.* = .allocating,
5389	_ => {}, // fill_candidate will be ignored by fillReg so there is no need to protect these values
5390	}
5391	}
5392
5393	// fill registers
5394	regs_it = regs.iterator();
5395	while (regs_it.next()) |reg| {
5396	const live_vi = isel.live_registers.getPtr(reg);
5397	switch (live_vi.*) {
5398	.free => unreachable,
5399	.allocating => {},
5400	_ => {
5401	assert(try isel.fillReg(reg));
5402	live_vi.* = .allocating;
5403	},
5404	}
5405	}
5406
5407	// unlock registers
5408	if (!locking) {
5409	regs_it = regs.iterator();
5410	while (regs_it.next()) |reg| {
5411	const live_vi = isel.live_registers.getPtr(reg);
5412	assert(live_vi.* == .allocating);
5413	live_vi.* = .free;
5414	}
5415	}
5416
5417	return;
5418	}
5419
5420	/// Frees a register by moving it to stack.
5421	/// Returns true on success, false on failure (i.e. dst_reg is locked/allocated or unallocatable).
5422	fn fillRegToMemory(isel: *Select, dst_reg: Register) codegen.Error!bool {
5423	if (!isRegisterAllocatable(dst_reg)) return false;
5424	const dst_live_vi = isel.live_registers.getPtr(dst_reg);
5425	const dst_vi = switch (dst_live_vi.*) {
5426	_ => |dst_vi| dst_vi,
5427	.allocating => return false,
5428	.free => return true,
5429	};
5430	try dst_vi.moveTo(isel, .{ .stack_slot = dst_vi.allocStackSlot(isel) });
5431	assert(dst_live_vi.* == .free);
5432	return true;
5433	}
5434
5435	const TryAllocRegResult = union(enum) {
5436	allocated: Register,
5437	fill_candidate: Register,
5438	out_of_registers,
5439	};
5440
5441	fn tryAllocReg(isel: *Select, class: Register.Class) TryAllocRegResult {
5442	return switch (class) {
5443	.int => isel.tryAllocRegInRanges(&.{
5444	.{ .r4, .r11 }, // argument registers
5445	.{ .r12, .r20 }, // temporary registers
5446	.{ .r23, .r31 }, // static registers
5447	.{ .r0, .r31 },
5448	}),
5449	.fp => isel.tryAllocRegInRange(.{ .f0, .f31 }),
5450	.fcc => isel.tryAllocRegInRange(.{ .fcc0, .fcc7 }),
5451	};
5452	}
5453
5454	fn tryAllocRegInRanges(isel: *Select, comptime ranges: []const struct { Register, Register }) TryAllocRegResult {
5455	inline for (ranges[0 .. ranges.len - 1]) |range| {
5456	switch (isel.tryAllocRegInRange(range)) {
5457	.allocated => |reg| return .{ .allocated = reg },
5458	else => {},
5459	}
5460	}
5461	return isel.tryAllocRegInRange(ranges[ranges.len - 1]);
5462	}
5463
5464	fn tryAllocRegInRange(isel: *Select, range: struct { Register, Register }) TryAllocRegResult {
5465	var failed_result: TryAllocRegResult = .out_of_registers;
5466	var reg, const last_reg = range;
5467	while (true) : (reg = @fromBackingInt(@backingInt(reg) + 1)) {
5468	if (!isRegisterAllocatable(reg)) continue;
5469	const live_vi = isel.live_registers.getPtr(reg);
5470	switch (live_vi.*) {
5471	_ => switch (failed_result) {
5472	.allocated => unreachable,
5473	.fill_candidate => {},
5474	.out_of_registers => failed_result = .{ .fill_candidate = reg },
5475	},
5476	.allocating => {},
5477	.free => {
5478	live_vi.* = .allocating;
5479	isel.saved_registers.insert(reg);
5480	return .{ .allocated = reg };
5481	},
5482	}
5483	if (reg == last_reg) return failed_result;
5484	}
5485	}
5486
5487	fn allocReg(isel: *Select, class: Register.Class) !Register {
5488	switch (isel.tryAllocReg(class)) {
5489	.allocated => |reg| return reg,
5490	.fill_candidate => |reg| {
5491	assert(try isel.fillRegToMemory(reg));
5492	const live_vi = isel.live_registers.getPtr(reg);
5493	assert(live_vi.* == .free);
5494	live_vi.* = .allocating;
5495	return reg;
5496	},
5497	.out_of_registers => return isel.fail("ran out of {t} registers", .{class}),
5498	}
5499	}
5500
5501	fn allocRegForWrite(isel: *Select, class: Register.Class) !Register {
5502	const reg = try isel.allocReg(class);
5503	isel.markRegWritten(reg);
5504	return reg;
5505	}
5506
5507	fn markRegWritten(isel: *Select, reg: Register) void {
5508	if (isel.active_loops.last()) |loop_index| {
5509	const loop = loop_index.get(isel);
5510	tracking_log.debug("${t} <- written", .{reg});
5511	loop.written_regs.insert(reg);
5512	}
5513	}
5514
5515	fn markRegsWritten(isel: *Select, regs: RegisterSet) void {
5516	if (isel.active_loops.last()) |loop_index| {
5517	const loop = loop_index.get(isel);
5518	tracking_log.debug("{f} <- written", .{fmtRegisterSet(regs)});
5519	loop.written_regs.setUnion(regs);
5520	}
5521	}
5522
5523	const RegLock = struct {
5524	reg: Register,
5525	const empty: RegLock = .{ .reg = .zero };
5526	fn unlock(lock: RegLock, isel: *Select) void {
5527	switch (lock.reg) {
5528	else => |reg| isel.freeReg(reg),
5529	Register.zero => {},
5530	}
5531	}
5532	};
5533
5534	fn lockReg(isel: *Select, reg: Register) RegLock {
5535	assert(reg != Register.zero);
5536	const live_vi = isel.live_registers.getPtr(reg);
5537	assert(live_vi.* == .free);
5538	live_vi.* = .allocating;
5539	return .{ .reg = reg };
5540	}
5541
5542	fn tryLockReg(isel: *Select, reg: Register) RegLock {
5543	assert(reg != Register.zero);
5544	const live_vi = isel.live_registers.getPtr(reg);
5545	switch (live_vi.*) {
5546	_ => {
5547	isel.dumpValues(.all);
5548	unreachable;
5549	},
5550	.allocating => return .empty,
5551	.free => {
5552	live_vi.* = .allocating;
5553	return .{ .reg = reg };
5554	},
5555	}
5556	}
5557
5558	fn freeReg(isel: *Select, reg: Register) void {
5559	assert(reg != Register.zero);
5560	const live_vi = isel.live_registers.getPtr(reg);
5561	assert(live_vi.* == .allocating);
5562	live_vi.* = .free;
5563	}
5564
5565	/// A snapshot of unresolved locations.
5566	const LocationSnapshot = struct {
5567	value_locs: std.MultiArrayList(Entry),
5568
5569	const Entry = union(enum(u2)) {
5570	none,
5571	register: Register.Alias,
5572	stack_slot: Value.Indirect,
5573	};
5574
5575	const empty: LocationSnapshot = .{ .value_locs = .empty };
5576
5577	fn deinit(snap: *LocationSnapshot, isel: *Select) void {
5578	const gpa = isel.pt.zcu.gpa;
5579	snap.value_locs.deinit(gpa);
5580	snap.* = undefined;
5581	}
5582
5583	/// Merges the captured locations and current expected locations.
5584	fn merge(snap: *const LocationSnapshot, isel: *Select) !void {
5585	const captured_locs = snap.value_locs.slice();
5586	for (0..snap.value_locs.len) |i| {
5587	const vi: Value.Index = @fromBackingInt(@intCast(i));
5588	const captured_loc: Value.Location = switch (captured_locs.get(i)) {
5589	.none => continue,
5590	.register => |captured_ra| .{ .register = captured_ra },
5591	.stack_slot => |captured_stack| .{ .stack_slot = captured_stack },
5592	};
5593	if (vi.location(isel)) |current_loc| {
5594	if (std.meta.eql(captured_loc, current_loc)) continue;
5595	}
5596	tracking_log.debug("{f} <- {f} (snapshot merge)", .{ vi, captured_loc });
5597
5598	if (captured_loc.asRegister()) |captured_reg| assert(try isel.fillReg(captured_reg));
5599	try vi.moveTo(isel, captured_loc);
5600	}
5601	}
5602
5603	pub fn format(snap: LocationSnapshot, w: *std.Io.Writer) std.Io.Writer.Error!void {
5604	const captured_locs = snap.value_locs.slice();
5605	var first = true;
5606	for (0..snap.value_locs.len) |i| {
5607	const vi: Value.Index = @fromBackingInt(@intCast(i));
5608	const captured_loc = captured_locs.get(i);
5609	if (captured_loc == .none) continue;
5610	if (first) first = false else try w.writeAll("\n");
5611	switch (captured_loc) {
5612	.none => unreachable,
5613	.register => |captured_ra| try w.print(" {f} <- {f}", .{ vi, captured_ra }),
5614	.stack_slot => |captured_stack| try w.print(" {f} <- {f}", .{ vi, captured_stack }),
5615	}
5616	}
5617	if (first) return w.writeAll("(empty)");
5618	}
5619	};
5620
5621	fn takeLocationSnapshot(isel: *Select) !LocationSnapshot {
5622	const gpa = isel.pt.zcu.gpa;
5623	var snapshot: LocationSnapshot = .empty;
5624	try snapshot.value_locs.resize(gpa, isel.values.items.len);
5625
5626	for (0..isel.values.items.len) |i| {
5627	const vi: Value.Index = @fromBackingInt(@intCast(i));
5628	if (vi.location(isel)) |vi_loc| {
5629	snapshot.value_locs.set(i, switch (vi_loc) {
5630	.register => |vi_ra| if (vi_ra.reg == Register.zero) .none else .{ .register = vi_ra },
5631	.stack_slot => |vi_stack| .{ .stack_slot = vi_stack },
5632	});
5633	} else {
5634	snapshot.value_locs.set(i, .none);
5635	}
5636	}
5637
5638	if (std.debug.runtime_safety) {
5639	var live_vi_it = isel.live_registers.iterator();
5640	while (live_vi_it.next()) |live_vi| {
5641	if (live_vi.value.* == .allocating) {
5642	tracking_log.debug("{t} is still locked when taking snapshot", .{live_vi.key});
5643	unreachable;
5644	}
5645	}
5646	}
5647
5648	return snapshot;
5649	}
5650
5651	/// Ways to treat bits in destination registers that may not be affected by an operation.
5652	const DestProtection = enum {
5653	/// Unrelated bits must be preserved.
5654	preserved,
5655	/// Unrelated bits may be destroyed.
5656	none,
5657	/// Unrelated bits must be filled with 0.
5658	wiped,
5659	};
5660
5661	fn fillUnusedBits(isel: *Select, rd: Register, rj: Register, dst_mode: Value.Extension, src_mode: Value.Extension, unused_bits: u9) !void {
5662	const gpr_bits = isel.gprBits();
5663	const used_bits = gpr_bits - unused_bits;
5664	wip_mir_log.debug(" | # fillUnusedBits {t}, {t}, {d} bits, {t} -> {t}", .{ rd, rj, used_bits, src_mode, dst_mode });
5665
5666	if (used_bits == gpr_bits or src_mode == dst_mode) {
5667	if (rd != rj) try isel.emit(.ori(rd, rj, 0));
5668	return;
5669	}
5670	switch (dst_mode) {
5671	.garbage => {},
5672	.sign_ext => {
5673	if (used_bits >= gpr_bits) return isel.fail("too many used bits", .{});
5674	switch (used_bits) {
5675	8 => try isel.emit(.@"sext.b"(rd, rj)),
5676	16 => try isel.emit(.@"sext.h"(rd, rj)),
5677	32 => try isel.emit(.@"addi.w"(rd, rj, 0)),
5678	0...7, 9...15, 17...31, 33...63 => {
5679	try isel.emit(.@"srai.d"(rd, rd, @intCast(gpr_bits - used_bits)));
5680	try isel.emit(.@"slli.d"(rd, rj, @intCast(gpr_bits - used_bits)));
5681	},
5682	else => unreachable,
5683	}
5684	},
5685	.zero_ext => {
5686	if (used_bits >= gpr_bits) return isel.fail("too many used bits", .{});
5687	switch (used_bits) {
5688	1...31 => try isel.emit(.@"bstrpick.w"(rd, rj, @intCast(used_bits - 1), 0)),
5689	32...63 => try isel.emit(.@"bstrpick.d"(rd, rj, @intCast(used_bits - 1), 0)),
5690	else => unreachable,
5691	}
5692	},
5693	}
5694	}
5695
5696	/// Loads from memory [base + offset] to register
5697	fn loadReg(
5698	isel: *Select,
5699	dst: Register,
5700	size: u64,
5701	signedness: std.builtin.Signedness,
5702	base: Register,
5703	offset: i65,
5704	) !void {
5705	if (dst.class() != .int) return isel.fail("TODO loadReg {t}", .{dst});
5706	switch (size) {
5707	0 => unreachable,
5708	1 => {
5709	if (std.math.cast(i12, offset)) |small_off| return isel.emit(switch (signedness) {
5710	.signed => .@"ld.b"(dst, base, small_off),
5711	.unsigned => .@"ld.bu"(dst, base, small_off),
5712	});
5713	},
5714	2 => {
5715	if (std.math.cast(i12, offset)) |small_off| return isel.emit(switch (signedness) {
5716	.signed => .@"ld.h"(dst, base, small_off),
5717	.unsigned => .@"ld.hu"(dst, base, small_off),
5718	});
5719	},
5720	4 => {
5721	if (std.math.cast(i12, offset)) |small_off| return isel.emit(switch (signedness) {
5722	.signed => .@"ld.w"(dst, base, small_off),
5723	.unsigned => .@"ld.wu"(dst, base, small_off),
5724	});
5725	if (signedness == .signed) if (std.math.cast(i16, offset)) |small_off| {
5726	if ((small_off & 0b11) == 0) {
5727	return isel.emit(.@"ldox4.w"(dst, base, @intCast(@divExact(small_off, 4))));
5728	}
5729	};
5730	},
5731	8 => {
5732	if (std.math.cast(i12, offset)) |small_off| return isel.emit(.@"ld.d"(dst, base, small_off));
5733	if (std.math.cast(i16, offset)) |small_off| {
5734	if ((small_off & 0b11) == 0) {
5735	return isel.emit(.@"ldox4.d"(dst, base, @intCast(@divExact(small_off, 4))));
5736	}
5737	}
5738	},
5739	else => return try isel.failUnimplemented("bad load size: {d}", .{size}),
5740	}
5741
5742	const ptr_reg = try isel.allocRegForWrite(.int);
5743	defer isel.freeReg(ptr_reg);
5744	switch (size) {
5745	1 => try isel.emit(switch (signedness) {
5746	.signed => .@"ldx.b"(dst, base, ptr_reg),
5747	.unsigned => .@"ldx.bu"(dst, base, ptr_reg),
5748	}),
5749	2 => try isel.emit(switch (signedness) {
5750	.signed => .@"ldx.h"(dst, base, ptr_reg),
5751	.unsigned => .@"ldx.hu"(dst, base, ptr_reg),
5752	}),
5753	4 => try isel.emit(switch (signedness) {
5754	.signed => .@"ldx.w"(dst, base, ptr_reg),
5755	.unsigned => .@"ldx.wu"(dst, base, ptr_reg),
5756	}),
5757	8 => try isel.emit(.@"ldx.d"(dst, base, ptr_reg)),
5758	else => {
5759	try isel.loadReg(dst, size, signedness, ptr_reg, 0);
5760	try isel.emit(.@"add.d"(ptr_reg, ptr_reg, base));
5761	},
5762	}
5763	try isel.moveIntImm(ptr_reg, std.math.cast(i64, offset) orelse return isel.fail("unimplemented load with large offset", .{}));
5764	}
5765
5766	/// Stores a register to memory [base + offset]
5767	fn storeReg(
5768	isel: *Select,
5769	src: Register,
5770	size: u64,
5771	base: Register,
5772	offset: i65,
5773	) !void {
5774	if (src.class() != .int) return isel.fail("TODO storeReg {t}", .{src});
5775	switch (size) {
5776	0 => unreachable,
5777	1 => {
5778	if (std.math.cast(i12, offset)) |small_off| return isel.emit(.@"st.b"(src, base, small_off));
5779	},
5780	2 => {
5781	if (std.math.cast(i12, offset)) |small_off| return isel.emit(.@"st.h"(src, base, small_off));
5782	},
5783	4 => {
5784	if (std.math.cast(i12, offset)) |small_off| return isel.emit(.@"st.w"(src, base, small_off));
5785	if (std.math.cast(i16, offset)) |small_off| {
5786	if ((small_off & 0b11) == 0) {
5787	return isel.emit(.@"stox4.w"(src, base, @intCast(@divExact(small_off, 4))));
5788	}
5789	}
5790	},
5791	8 => {
5792	if (std.math.cast(i12, offset)) |small_off| return isel.emit(.@"st.d"(src, base, small_off));
5793	if (std.math.cast(i16, offset)) |small_off| {
5794	if ((small_off & 0b11) == 0) {
5795	return isel.emit(.@"stox4.d"(src, base, @intCast(@divExact(small_off, 4))));
5796	}
5797	}
5798	},
5799	else => return try isel.failUnimplemented("bad store size: {d}", .{size}),
5800	}
5801
5802	if (std.math.cast(i64, offset)) |offset64| stx: {
5803	const ptr_reg = try isel.allocRegForWrite(.int);
5804	defer isel.freeReg(ptr_reg);
5805	switch (size) {
5806	1 => try isel.emit(.@"stx.b"(src, base, ptr_reg)),
5807	2 => try isel.emit(.@"stx.h"(src, base, ptr_reg)),
5808	4 => try isel.emit(.@"stx.w"(src, base, ptr_reg)),
5809	8 => try isel.emit(.@"stx.d"(src, base, ptr_reg)),
5810	else => break :stx,
5811	}
5812	try isel.moveIntImm(ptr_reg, offset64);
5813	}
5814
5815	const ptr_reg = try isel.allocRegForWrite(.int);
5816	defer isel.freeReg(ptr_reg);
5817	try isel.storeReg(src, size, ptr_reg, 0);
5818	try isel.emit(if (offset > 0) .@"add.d"(ptr_reg, ptr_reg, base) else .@"sub.d"(ptr_reg, ptr_reg, base));
5819	try isel.moveIntImm(ptr_reg, @intCast(@abs(offset)));
5820	}
5821
5822	/// Copies a part of a register to another.
5823	fn moveReg(
5824	isel: *Select,
5825	dst_ra: Register.Alias,
5826	dst_bit_off: u9,
5827	src_ra: Register.Alias,
5828	src_bit_off: u9,
5829	bit_size: u16,
5830	init_dst_prot: DestProtection,
5831	) !void {
5832	if (dst_ra.reg == src_ra.reg and dst_bit_off == src_bit_off) return;
5833	if (bit_size == 0) return;
5834	assert(init_dst_prot != .preserved or (isel.live_registers.get(dst_ra.reg) == .allocating));
5835	assert(isel.live_registers.get(src_ra.reg) == .allocating);
5836
5837	const dst_ra_bit_size = dst_ra.mod.bitSize(isel.target);
5838	const src_ra_bit_size = src_ra.mod.bitSize(isel.target);
5839	const dst_msb_plus_one = dst_bit_off + bit_size;
5840	const src_msb_plus_one = src_bit_off + bit_size;
5841	assert(dst_msb_plus_one <= dst_ra_bit_size and src_msb_plus_one <= src_ra_bit_size);
5842
5843	const dst_prot: DestProtection = switch (init_dst_prot) {
5844	.preserved => if (bit_size == dst_ra_bit_size) .none else .preserved,
5845	else => init_dst_prot,
5846	};
5847
5848	const dst_lock = isel.tryLockReg(dst_ra.reg);
5849	defer dst_lock.unlock(isel);
5850	const src_lock = isel.tryLockReg(src_ra.reg);
5851	defer src_lock.unlock(isel);
5852
5853	switch (dst_ra.mod) {
5854	.integer => switch (src_ra.mod) {
5855	.integer => {
5856	if (dst_bit_off == src_bit_off and dst_prot == .none)
5857	return try isel.emit(.ori(dst_ra.reg, src_ra.reg, 0));
5858	// bstrins
5859	const tmp_reg = tmp_reg: {
5860	if (dst_bit_off == 0 and dst_prot != .none) break :tmp_reg dst_ra.reg;
5861	const tmp_reg = if (src_bit_off == 0) src_ra.reg else try isel.allocRegForWrite(.int);
5862	const dst_msbw = dst_msb_plus_one - 1;
5863	try isel.emit(switch (dst_ra_bit_size) {
5864	32 => .@"bstrins.w"(dst_ra.reg, tmp_reg, @intCast(dst_msbw), @intCast(dst_bit_off)),
5865	64 => .@"bstrins.d"(dst_ra.reg, tmp_reg, @intCast(dst_msbw), @intCast(dst_bit_off)),
5866	else => unreachable,
5867	});
5868	break :tmp_reg tmp_reg;
5869	};
5870	defer if (tmp_reg != dst_ra.reg and tmp_reg != src_ra.reg) isel.freeReg(tmp_reg);
5871	// bstrpick
5872	const src_msbw = src_msb_plus_one - 1;
5873	try isel.emit(switch (dst_ra_bit_size) {
5874	32 => .@"bstrpick.w"(tmp_reg, src_ra.reg, @intCast(src_msbw), @intCast(src_bit_off)),
5875	64 => .@"bstrpick.d"(tmp_reg, src_ra.reg, @intCast(src_msbw), @intCast(src_bit_off)),
5876	else => unreachable,
5877	});
5878	},
5879	else => return isel.fail("unimplemented non-integral moveReg", .{}),
5880	},
5881	else => return isel.fail("unimplemented non-integral moveReg", .{}),
5882	}
5883	}
5884
5885	/// Moves an immediate to a register.
5886	fn moveIntImm(isel: *Select, rd: Register, si64: i64) !void {
5887	wip_mir_log.debug(" | # moveImm {t} <- 0x{x}", .{ rd, si64 });
5888	if (std.math.cast(u12, si64)) |imm12| return isel.emit(.ori(rd, .zero, imm12));
5889
5890	const ori12: u12 = @truncate(@as(u64, @bitCast(si64)));
5891	const lu12i20: i20 = @truncate(si64 >> 12);
5892	const use_lu12iw = lu12i20 != 0;
5893	const lu32i20: i20 = @truncate(si64 >> 32);
5894	const use_lu32id = lu32i20 != hi: {
5895	if (use_lu12iw) break :hi @as(i20, @intCast(@as(i1, @truncate(si64 >> 31))));
5896	break :hi 0;
5897	};
5898	const lu52i12: i12 = @truncate(si64 >> 52);
5899	const use_lu52id = lu52i12 != hi: {
5900	if (use_lu32id) break :hi @as(i12, @intCast(@as(i1, @truncate(si64 >> 51))));
5901	if (use_lu12iw) break :hi @as(i12, @intCast(@as(i1, @truncate(si64 >> 31))));
5902	break :hi 0;
5903	};
5904	const use_ori = (ori12 != 0) or (!use_lu12iw and use_lu32id) or si64 == 0;
5905	const ori_rj = if (use_lu12iw) rd else Register.zero;
5906	const lu52id_rj = if (use_ori or use_lu12iw) rd else Register.zero;
5907
5908	if (use_lu52id) try isel.emit(.@"cu52i.d"(rd, lu52id_rj, lu52i12));
5909	if (use_lu32id) try isel.emit(.@"cu32i.d"(rd, lu32i20));
5910	if (use_ori) try isel.emit(.ori(rd, ori_rj, ori12));
5911	if (use_lu12iw) try isel.emit(.@"lu12i.w"(rd, lu12i20));
5912	}
5913
5914	fn addImm(isel: *Select, rd: Register, rj: Register, si65: i65) !void {
5915	const gpr_size = isel.gprSize();
5916	if (si65 == 0) {
5917	try isel.emit(.ori(rd, rj, 0));
5918	} else if (std.math.cast(i12, si65)) |si12| {
5919	switch (gpr_size) {
5920	4 => try isel.emit(.@"addi.w"(rd, rj, si12)),
5921	8 => try isel.emit(.@"addi.d"(rd, rj, si12)),
5922	else => unreachable,
5923	}
5924	} else {
5925	if (si65 >= 0) switch (gpr_size) {
5926	4 => try isel.emit(.@"add.w"(rd, rd, rj)),
5927	8 => try isel.emit(.@"add.d"(rd, rd, rj)),
5928	else => unreachable,
5929	} else switch (gpr_size) {
5930	4 => try isel.emit(.@"sub.w"(rd, rd, rj)),
5931	8 => try isel.emit(.@"sub.d"(rd, rd, rj)),
5932	else => unreachable,
5933	}
5934	try isel.moveIntImm(rd, @bitCast(@as(u64, @truncate(@as(u65, @bitCast(si65))))));
5935	}
5936	}
5937
5938	/// Loads the incoming value of a register.
5939	fn ldIncoming(isel: *Select, rd: Register, rj: Register) !void {
5940	wip_mir_log.debug(" | # ldIncoming {t} <- {t}", .{ rd, rj });
5941	try isel.layout_relocs.append(isel.pt.zcu.gpa, @intCast(isel.instructions.items.len));
5942	try isel.emit(.ori(rd, rj, 0));
5943	}
5944
5945	fn cmp(
5946	isel: *Select,
5947	res_reg: Register,
5948	ty: ZigType,
5949	lhs_vi: Value.Index,
5950	op: std.math.CompareOperator,
5951	rhs_vi: Value.Index,
5952	) !void {
5953	wip_mir_log.debug(" | # cmp {f}, {t}, {f}, {t}, {f}", .{ isel.fmtType(ty), res_reg, lhs_vi, op, rhs_vi });
5954	const res_lock = isel.tryLockReg(res_reg);
5955	defer res_lock.unlock(isel);
5956
5957	if (!ty.isRuntimeFloat() and !ty.isArrayOrVector(isel.pt.zcu)) {
5958	// integeral comparison
5959	const int_info: std.builtin.Type.Int = if (ty.toIntern() == .bool_type)
5960	.{ .signedness = .unsigned, .bits = 1 }
5961	else if (ty.isAbiInt(isel.pt.zcu))
5962	ty.intInfo(isel.pt.zcu)
5963	else if (ty.isPtrAtRuntime(isel.pt.zcu))
5964	.{ .signedness = .unsigned, .bits = 64 }
5965	else
5966	return isel.fail("bad cmp_{t} {f}", .{ op, isel.fmtType(ty) });
5967
5968	var part_offset = lhs_vi.size(isel);
5969	while (part_offset > 0) {
5970	const part_size = @min(part_offset, isel.gprSize());
5971	part_offset -= part_size;
5972	// TODO optimize constant cmp
5973	// TODO relax LHS and RHS extension mode requirements to != .garbage
5974	const lhs_part_vi = try lhs_vi.partExact(isel, part_offset, part_size);
5975	const lhs_part_mat = try lhs_part_vi.matIntRegZeroExt(isel);
5976	const lhs_part_reg = lhs_part_mat.reg();
5977	const rhs_part_vi = try rhs_vi.partExact(isel, part_offset, part_size);
5978	const rhs_part_mat = try rhs_part_vi.matIntRegZeroExt(isel);
5979	const rhs_part_reg = rhs_part_mat.reg();
5980
5981	const res_part_reg = if (part_offset == 0) res_reg else res_part_reg: {
5982	const res_part_reg = try isel.allocRegForWrite(.int);
5983	try isel.emit(.@"or"(res_reg, res_reg, res_part_reg));
5984	break :res_part_reg res_part_reg;
5985	};
5986	defer if (res_part_reg != res_reg) isel.freeReg(res_part_reg);
5987
5988	switch (op) {
5989	.eq => {
5990	try isel.emit(.sltui(res_part_reg, res_part_reg, 1));
5991	try isel.emit(.xor(res_part_reg, lhs_part_reg, rhs_part_reg));
5992	},
5993	.neq => {
5994	try isel.emit(.sltu(res_part_reg, .zero, res_part_reg));
5995	try isel.emit(.xor(res_part_reg, lhs_part_reg, rhs_part_reg));
5996	},
5997	.lt, .lte, .gt, .gte => {
5998	var rj = lhs_part_reg;
5999	var rk = rhs_part_reg;
6000
6001	switch (op) {
6002	.lte, .gt => std.mem.swap(Register, &rj, &rk),
6003	else => {},
6004	}
6005	switch (op) {
6006	.lte, .gte => try isel.emit(.xori(res_part_reg, res_part_reg, 1)),
6007	else => {},
6008	}
6009
6010	try isel.emit(switch (int_info.signedness) {
6011	.signed => .slt(res_part_reg, rj, rk),
6012	.unsigned => .sltu(res_part_reg, rj, rk),
6013	});
6014	},
6015	}
6016	try rhs_part_mat.finish(isel);
6017	try lhs_part_mat.finish(isel);
6018	}
6019	} else return isel.fail("bad cmp_{t} {f}", .{ op, isel.fmtType(ty) });
6020	}
6021
6022	const AddOrSubtractOptions = struct {
6023	overflow: Overflow,
6024
6025	const Overflow = union(enum) {
6026	@"unreachable",
6027	panic: Zcu.SimplePanicId,
6028	wrap,
6029	overflow_ra: Register.Alias,
6030	};
6031	};
6032
6033	// TODO optimize constant add/sub
6034	fn addOrSubtract(
6035	isel: *Select,
6036	ty: ZigType,
6037	res_vi: Value.Index,
6038	op: enum { add, sub },
6039	lhs_vi: Value.Index,
6040	rhs_vi: Value.Index,
6041	opts: AddOrSubtractOptions,
6042	) !void {
6043	wip_mir_log.debug(" | # {t} ty = {f}, res = {f}, lhs = {f}, rhs = {f}, overflow = {t}", .{ op, isel.fmtType(ty), res_vi, lhs_vi, rhs_vi, opts.overflow });
6044	const zcu = isel.pt.zcu;
6045	assert(ty.isAbiInt(zcu));
6046	const int_info = ty.intInfo(zcu);
6047
6048	switch (opts.overflow) {
6049	.wrap, .@"unreachable" => {},
6050	.overflow_ra => |overflow_ra| {
6051	const overflow_reg = if (overflow_ra.mod == .integer) overflow_ra.reg else try isel.allocRegForWrite(.int);
6052	defer if (overflow_ra.mod != .integer) isel.freeReg(overflow_reg);
6053	switch (op) {
6054	.add => try isel.cmp(overflow_reg, ty, res_vi, .lt, lhs_vi),
6055	.sub => try isel.cmp(overflow_reg, ty, res_vi, .gt, lhs_vi),
6056	}
6057	},
6058	.panic => {
6059	try isel.failUnimplemented("unimplemented {t} with {t}", .{ op, opts.overflow });
6060	},
6061	}
6062
6063	if (int_info.bits <= 32) {
6064	try res_vi.reextendToGarbage(isel); // TODO optimize
6065	const res_reg = try res_vi.defRegMod(isel, .integer) orelse return;
6066	const lhs_mat = try lhs_vi.matIntRegZeroExt(isel);
6067	const rhs_mat = try rhs_vi.matIntRegZeroExt(isel);
6068
6069	switch (op) {
6070	.add => try isel.emit(.@"add.w"(res_reg, lhs_mat.reg(), rhs_mat.reg())),
6071	.sub => try isel.emit(.@"sub.w"(res_reg, lhs_mat.reg(), rhs_mat.reg())),
6072	}
6073
6074	try lhs_mat.finish(isel);
6075	try rhs_mat.finish(isel);
6076	} else if (int_info.bits <= 64) {
6077	try res_vi.reextendToGarbage(isel); // TODO optimize
6078	const res_reg = try res_vi.defRegMod(isel, .integer) orelse return;
6079	const lhs_mat = try lhs_vi.matIntRegZeroExt(isel);
6080	const rhs_mat = try rhs_vi.matIntRegZeroExt(isel);
6081
6082	switch (op) {
6083	.add => try isel.emit(.@"add.d"(res_reg, lhs_mat.reg(), rhs_mat.reg())),
6084	.sub => try isel.emit(.@"sub.d"(res_reg, lhs_mat.reg(), rhs_mat.reg())),
6085	}
6086
6087	try lhs_mat.finish(isel);
6088	try rhs_mat.finish(isel);
6089	} else {
6090	if (debug_trap_unimplemented_code) {
6091	isel.wipeLocationDfs(res_vi);
6092	}
6093	return try isel.failUnimplemented("unimplemented {t} {f}", .{ op, isel.fmtType(ty) });
6094	}
6095	}
6096
6097	/// elem_ptr = base +- elem_size * index
6098	/// elem_ptr, base, and index may alias. base_reg must be locked.
6099	fn elemPtr(
6100	isel: *Select,
6101	rd: Register,
6102	base_reg: Register,
6103	op: enum { add, sub },
6104	elem_size: u64,
6105	index_vi: Value.Index,
6106	) !void {
6107	assert(isel.live_registers.get(base_reg) == .allocating);
6108	wip_mir_log.debug(" | # elemPtr {t} = {t} {s} {f} * {d} (= 0b{b})", .{ rd, base_reg, switch (op) {
6109	.add => "+",
6110	.sub => "-",
6111	}, index_vi, elem_size, elem_size });
6112	switch (@popCount(elem_size)) {
6113	0 => unreachable, // Sema should optimize this
6114	1 => {
6115	const shift = @ctz(elem_size);
6116	if (shift == 0) {
6117	const index_mat = try index_vi.matIntRegZeroExt(isel);
6118	const index_reg = index_mat.reg();
6119	try isel.emit(switch (op) {
6120	.add => switch (isel.gprBits()) {
6121	else => unreachable,
6122	32 => .@"add.w"(rd, base_reg, index_reg),
6123	64 => .@"add.d"(rd, base_reg, index_reg),
6124	},
6125	.sub => switch (isel.gprBits()) {
6126	else => unreachable,
6127	32 => .@"sub.w"(rd, base_reg, index_reg),
6128	64 => .@"sub.d"(rd, base_reg, index_reg),
6129	},
6130	});
6131	try index_mat.finish(isel);
6132	return;
6133	} else if (std.math.cast(u2, shift - 1)) |sa2| {
6134	switch (op) {
6135	.add => {
6136	const index_mat = try index_vi.matIntRegZeroExt(isel);
6137	const index_reg = index_mat.reg();
6138	try isel.emit(switch (isel.gprBits()) {
6139	else => unreachable,
6140	32 => .@"sladd.w"(rd, index_reg, base_reg, sa2),
6141	64 => .@"sladd.d"(rd, index_reg, base_reg, sa2),
6142	});
6143	try index_mat.finish(isel);
6144	return;
6145	},
6146	.sub => {
6147	if (base_reg != rd) {
6148	const index_mat = try index_vi.matIntRegZeroExt(isel);
6149	const index_reg = index_mat.reg();
6150	switch (isel.gprBits()) {
6151	else => unreachable,
6152	32 => {
6153	try isel.emit(.@"sladd.w"(rd, rd, base_reg, sa2));
6154	try isel.emit(.@"sub.w"(rd, .zero, index_reg));
6155	},
6156	64 => {
6157	try isel.emit(.@"sladd.d"(rd, rd, base_reg, sa2));
6158	try isel.emit(.@"sub.d"(rd, .zero, index_reg));
6159	},
6160	}
6161	try index_mat.finish(isel);
6162	return;
6163	}
6164	},
6165	}
6166	}
6167	},
6168	2 => {
6169	const shift1 = @ctz(elem_size);
6170	const mask1 = @as(u64, 1) << @intCast(shift1);
6171	const mask2 = elem_size & ~mask1;
6172
6173	if ((op == .add or base_reg != rd) and mask1 <= 4 and mask2 <= 4) {
6174	try isel.elemPtr(rd, rd, op, mask2, index_vi);
6175	try isel.elemPtr(rd, base_reg, op, mask1, index_vi);
6176	return;
6177	}
6178	},
6179	else => {},
6180	}
6181
6182	const index_mat = try index_vi.matIntRegZeroExt(isel);
6183	const index_reg = index_mat.reg();
6184	const offset_reg = if (base_reg != rd) rd else try isel.allocRegForWrite(.int);
6185	defer if (offset_reg != rd) isel.freeReg(offset_reg);
6186	try isel.emit(switch (op) {
6187	.add => switch (isel.gprBits()) {
6188	else => unreachable,
6189	32 => .@"add.w"(rd, base_reg, offset_reg),
6190	64 => .@"add.d"(rd, base_reg, offset_reg),
6191	},
6192	.sub => switch (isel.gprBits()) {
6193	else => unreachable,
6194	32 => .@"sub.w"(rd, base_reg, offset_reg),
6195	64 => .@"sub.d"(rd, base_reg, offset_reg),
6196	},
6197	});
6198	try isel.emit(switch (isel.gprBits()) {
6199	else => unreachable,
6200	32 => .@"mul.w"(offset_reg, offset_reg, index_reg),
6201	64 => .@"mul.d"(offset_reg, offset_reg, index_reg),
6202	});
6203	try isel.moveIntImm(offset_reg, @bitCast(elem_size));
6204	try index_mat.finish(isel);
6205	}
6206
6207	fn moveLoc(
6208	isel: *Select,
6209	dst_loc: Value.Location,
6210	dst_off: u64,
6211	src_loc: Value.Location,
6212	src_off: u64,
6213	size: u64,
6214	dst_prot: DestProtection,
6215	) !void {
6216	if (dst_loc.isUnallocated()) return;
6217	if (std.meta.eql(dst_loc, src_loc) and dst_off == src_off) return;
6218	if (size == 0) return;
6219	assert(!src_loc.isUnallocated());
6220	wip_mir_log.debug(" | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}", .{
6221	dst_loc,
6222	dst_off,
6223	src_loc,
6224	src_off,
6225	size,
6226	dst_prot,
6227	});
6228
6229	const dst_lock: RegLock = if (dst_prot != .preserved) .empty else dst_loc.tryLock(isel);
6230	defer dst_lock.unlock(isel);
6231	const src_lock = src_loc.tryLock(isel);
6232	defer src_lock.unlock(isel);
6233
6234	switch (dst_loc) {
6235	.register => |dst_ra| switch (src_loc) {
6236	.register => |src_ra| try isel.moveReg(
6237	dst_ra,
6238	@intCast(dst_off * 8),
6239	src_ra,
6240	@intCast(src_off * 8),
6241	@intCast(size * 8),
6242	dst_prot,
6243	),
6244	.stack_slot => |src_stack| {
6245	const tmp_reg = if (dst_ra.mod == .integer and dst_off == 0)
6246	dst_ra.reg
6247	else
6248	try isel.allocRegForWrite(.int);
6249	defer if (tmp_reg != dst_ra.reg) isel.freeReg(tmp_reg);
6250	try isel.moveReg(
6251	dst_ra,
6252	@intCast(dst_off * 8),
6253	.{ .reg = tmp_reg, .mod = .integer },
6254	0,
6255	@intCast(size * 8),
6256	dst_prot,
6257	);
6258	try isel.loadReg(
6259	tmp_reg,
6260	memOpSizeFitting(size),
6261	.unsigned,
6262	src_stack.base,
6263	src_stack.offset + @as(i65, src_off),
6264	);
6265	},
6266	},
6267	.stack_slot => |dst_stack| {
6268	if (size > isel.gprSize()) {
6269	// large memory copies, src must be stack_slot
6270	const src_stack = src_loc.stack_slot;
6271	// TODO optimize to memmove call
6272
6273	const known_direction, const gen_low_to_high, const gen_high_to_low = move_dir: {
6274	if (dst_stack.base == src_stack.base) {
6275	if (dst_stack.offset == src_stack.offset) return;
6276	break :move_dir if (dst_stack.offset < src_stack.offset)
6277	.{ true, false, true }
6278	else
6279	.{ true, true, false };
6280	}
6281	// cannot determine direction
6282	if (assume_memmove_no_overlap)
6283	break :move_dir .{ true, true, false };
6284	break :move_dir .{ false, true, true };
6285	};
6286	if (!known_direction) {
6287	return isel.failUnimplemented("TODO moveLoc memmove", .{});
6288	}
6289
6290	const gpr_size = isel.gprSize();
6291	const steps = std.math.divCeil(u64, size, gpr_size) catch unreachable;
6292	if (gen_low_to_high) {
6293	var off: u64 = 0;
6294	for (0..@intCast(steps)) |_| {
6295	try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot);
6296	off += gpr_size;
6297	}
6298	}
6299	if (gen_high_to_low) {
6300	var off: u64 = steps * gpr_size;
6301	for (0..@intCast(steps)) |_| {
6302	off -= gpr_size;
6303	try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot);
6304	}
6305	}
6306
6307	return;
6308	}
6309
6310	// Move to a temp reg + store
6311	// If size is not direct mem op size and !kill_dst, old values have to
6312	// be loaded first.
6313	const memop_size = memOpSizeFitting(size);
6314	const need_load = memop_size != size;
6315	const tmp_reg, const tmp_allocated = tmp_reg: {
6316	if (src_off == 0 and !need_load) {
6317	switch (src_loc) {
6318	.register => |src_ra| if (src_ra.mod == .integer)
6319	break :tmp_reg .{ src_ra.reg, false },
6320	else => {},
6321	}
6322	}
6323	break :tmp_reg .{ try isel.allocRegForWrite(.int), true };
6324	};
6325	defer if (tmp_allocated) isel.freeReg(tmp_reg);
6326
6327	const dst_stack_off = dst_stack.offset + @as(i65, dst_off);
6328	try isel.storeReg(tmp_reg, memop_size, dst_stack.base, dst_stack_off);
6329	if (tmp_allocated)
6330	try isel.moveLoc(
6331	.{ .register = .{ .reg = tmp_reg, .mod = .integer } },
6332	0,
6333	src_loc,
6334	src_off,
6335	size,
6336	if (need_load) .preserved else .none,
6337	);
6338	if (need_load)
6339	try isel.loadReg(tmp_reg, memop_size, .unsigned, dst_stack.base, dst_stack_off);
6340	},
6341	}
6342	}
6343
6344	fn moveUndef(isel: *Select, dst_loc: Value.Location, size: u64) !void {
6345	if (isel.opt_mode == .fast or isel.opt_mode == .small) return;
6346	wip_mir_log.debug(" | # move {f} ({d}B) <- undef", .{ dst_loc, size });
6347	switch (dst_loc) {
6348	.register => |dst_ra| {
6349	assert(dst_ra.mod == .integer); // TODO
6350	try isel.moveIntImm(dst_ra.reg, switch (isel.gprBits()) {
6351	32 => 0xAAAAAAAA,
6352	64 => @bitCast(@as(u64, 0xAAAAAAAAAAAAAAAA)),
6353	else => unreachable,
6354	});
6355	},
6356	.stack_slot => {
6357	// TODO write undef to memory
6358	},
6359	}
6360	}
6361
6362	fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, init_offset: u64, size: u64) !void {
6363	wip_mir_log.debug(" | # move {f} <- {f} [{d}..{d}]", .{ dst, isel.fmtConstant(init_constant), init_offset, init_offset + size - 1 });
6364	var offset = init_offset;
6365	const zcu = isel.pt.zcu;
6366	const ip = &zcu.intern_pool;
6367	var constant = init_constant.toIntern();
6368	var constant_key = ip.indexToKey(constant);
6369	while (true) {
6370	// Try to coerce the constant value
6371	// also try better codegen
6372	constant_key: switch (constant_key) {
6373	else => {},
6374	.undef => return try isel.moveUndef(dst, size),
6375	.simple_value => |simple_value| switch (simple_value) {
6376	.void => {},
6377	.null, .@"unreachable" => unreachable,
6378	.true => continue :constant_key .{ .int = .{ .ty = .bool_type, .storage = .{ .u64 = 1 } } },
6379	.false => continue :constant_key .{ .int = .{ .ty = .bool_type, .storage = .{ .u64 = 0 } } },
6380	},
6381	.int => |int| if (dst.asRegisterAlias()) |dst_ra| {
6382	if (dst_ra.mod != .integer) break :constant_key;
6383	const dst_reg = dst_ra.reg;
6384	return switch (int.storage) {
6385	.u64 => |imm| try isel.moveIntImm(dst_reg, @bitCast(std.math.shr(u64, imm, 8 * offset))),
6386	.i64 => |imm| switch (size) {
6387	else => unreachable,
6388	1...4 => try isel.moveIntImm(dst_reg, @as(u32, @bitCast(@as(i32, @truncate(std.math.shr(i64, imm, 8 * offset)))))),
6389	5...8 => try isel.moveIntImm(dst_reg, @bitCast(std.math.shr(i64, imm, 8 * offset))),
6390	},
6391	.big_int => |big_int| {
6392	assert(size == isel.gprSize());
6393	var imm: u64 = 0;
6394	const limb_bits = @bitSizeOf(std.math.big.Limb);
6395	const limbs = @divExact(64, limb_bits);
6396	var limb_index: usize = @intCast(@divExact(offset, @divExact(limb_bits, 8)) + limbs);
6397	for (0..limbs) |_| {
6398	limb_index -= 1;
6399	if (limb_index >= big_int.limbs.len) continue;
6400	if (limb_bits < 64) imm <<= limb_bits;
6401	imm |= big_int.limbs[limb_index];
6402	}
6403	if (!big_int.positive) {
6404	limb_index = @min(limb_index, big_int.limbs.len);
6405	imm = while (limb_index > 0) {
6406	limb_index -= 1;
6407	if (big_int.limbs[limb_index] != 0) break ~imm;
6408	} else -%imm;
6409	}
6410	try isel.moveIntImm(dst_reg, @bitCast(imm));
6411	},
6412	};
6413	},
6414	.err => |err| continue :constant_key .{ .int = .{
6415	.ty = err.ty,
6416	.storage = .{ .u64 = ip.getErrorValueIfExists(err.name).? },
6417	} },
6418	.error_union => |error_union| {
6419	const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
6420	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
6421	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
6422	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
6423	const error_set_size = error_set_ty.abiSize(zcu);
6424	if (offset >= error_set_offset and offset + size <= error_set_offset + error_set_size) {
6425	offset -= error_set_offset;
6426	continue :constant_key switch (error_union.val) {
6427	.err_name => |err_name| .{ .err = .{
6428	.ty = error_union_type.error_set_type,
6429	.name = err_name,
6430	} },
6431	.payload => .{ .int = .{
6432	.ty = error_union_type.error_set_type,
6433	.storage = .{ .u64 = 0 },
6434	} },
6435	};
6436	}
6437	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
6438	const payload_size = payload_ty.abiSize(zcu);
6439	if (offset >= payload_offset and offset + size <= payload_offset + payload_size) {
6440	offset -= payload_offset;
6441	switch (error_union.val) {
6442	.err_name => continue :constant_key .{ .undef = error_union_type.payload_type },
6443	.payload => |payload| {
6444	constant = payload;
6445	constant_key = ip.indexToKey(constant);
6446	continue :constant_key constant_key;
6447	},
6448	}
6449	}
6450	},
6451	.enum_tag => |enum_tag| continue :constant_key .{ .int = ip.indexToKey(enum_tag.int).int },
6452	.float => return isel.fail("float unimplemented", .{}),
6453	.ptr => |ptr| {
6454	assert(offset == 0 and size == isel.gprSize());
6455	const dst_ra: Register.Alias, const use_tmp_reg = select_tmp: {
6456	if (dst.asRegisterAlias()) |dst_ra| {
6457	if (dst_ra.mod == .integer) break :select_tmp .{ dst_ra, false };
6458	}
6459	break :select_tmp .{ .{ .reg = try isel.allocRegForWrite(.int), .mod = .integer }, true };
6460	};
6461	const rd = dst_ra.reg;
6462	defer if (use_tmp_reg) isel.freeReg(rd);
6463
6464	if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .none);
6465	return switch (ptr.base_addr) {
6466	.nav => |nav| if (ZigType.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu)) {
6467	// TODO code model
6468	try isel.nav_relocs.append(zcu.gpa, .{
6469	.nav = nav,
6470	.reloc = .{
6471	.label = @intCast(isel.instructions.items.len),
6472	.addend = @intCast(ptr.byte_offset),
6473	.type = .PCALA_LO12,
6474	},
6475	});
6476	try isel.emit(.@"addi.d"(rd, rd, 0));
6477	try isel.nav_relocs.append(zcu.gpa, .{
6478	.nav = nav,
6479	.reloc = .{
6480	.label = @intCast(isel.instructions.items.len),
6481	.addend = @intCast(ptr.byte_offset),
6482	.type = .PCALA_HI20,
6483	},
6484	});
6485	try isel.emit(.pcalau12i(rd, 0));
6486	} else continue :constant_key .{ .int = .{
6487	.ty = .usize_type,
6488	.storage = .{ .u64 = isel.pt.zcu.navAlignment(nav).forward(0xaaaaaaaaaaaaaaaa) },
6489	} },
6490	.uav => |uav| if (ZigType.fromInterned(ip.typeOf(uav.val)).isRuntimeFnOrHasRuntimeBits(zcu)) {
6491	// TODO code model
6492	try isel.uav_relocs.append(zcu.gpa, .{
6493	.uav = uav,
6494	.reloc = .{
6495	.label = @intCast(isel.instructions.items.len),
6496	.addend = @intCast(ptr.byte_offset),
6497	.type = .PCALA_LO12,
6498	},
6499	});
6500	try isel.emit(.@"addi.d"(rd, rd, 0));
6501	try isel.uav_relocs.append(zcu.gpa, .{
6502	.uav = uav,
6503	.reloc = .{
6504	.label = @intCast(isel.instructions.items.len),
6505	.addend = @intCast(ptr.byte_offset),
6506	.type = .PCALA_HI20,
6507	},
6508	});
6509	try isel.emit(.pcalau12i(rd, 0));
6510	} else continue :constant_key .{ .int = .{
6511	.ty = .usize_type,
6512	.storage = .{ .u64 = ZigType.fromInterned(uav.orig_ty).ptrAlignment(zcu).forward(0xaaaaaaaaaaaaaaaa) },
6513	} },
6514	.int => continue :constant_key .{ .int = .{
6515	.ty = .usize_type,
6516	.storage = .{ .u64 = ptr.byte_offset },
6517	} },
6518	.eu_payload => |base| {
6519	var base_ptr = ip.indexToKey(base).ptr;
6520	const eu_ty = ip.indexToKey(base_ptr.ty).ptr_type.child;
6521	const payload_ty = ip.indexToKey(eu_ty).error_union_type.payload_type;
6522	base_ptr.byte_offset += codegen.errUnionPayloadOffset(.fromInterned(payload_ty), zcu) + ptr.byte_offset;
6523	continue :constant_key .{ .ptr = base_ptr };
6524	},
6525	.opt_payload => |base| {
6526	var base_ptr = ip.indexToKey(base).ptr;
6527	base_ptr.byte_offset += ptr.byte_offset;
6528	continue :constant_key .{ .ptr = base_ptr };
6529	},
6530	.field => |field_idx| {
6531	var base_ptr = ip.indexToKey(field_idx.base).ptr;
6532	const agg_ty: ZigType = .fromInterned(ip.indexToKey(base_ptr.ty).ptr_type.child);
6533	base_ptr.byte_offset += agg_ty.structFieldOffset(@intCast(field_idx.index), zcu) + ptr.byte_offset;
6534	continue :constant_key .{ .ptr = base_ptr };
6535	},
6536	.comptime_alloc, .comptime_field, .arr_elem => unreachable,
6537	};
6538	},
6539	.slice => |slice| {
6540	const ptr_size = isel.gprSize();
6541	if (offset == 0 and size == ptr_size) {
6542	constant = slice.ptr;
6543	continue :constant_key switch (ip.indexToKey(slice.ptr)) {
6544	else => unreachable,
6545	.undef => |undef| .{ .undef = undef },
6546	.ptr => |ptr| .{ .ptr = ptr },
6547	};
6548	} else if (offset == ptr_size) {
6549	offset = 0;
6550	constant = slice.len;
6551	continue :constant_key ip.indexToKey(slice.len);
6552	} else if (offset == 0 and size == (@as(u64, ptr_size) * 2)) {
6553	const dst_stack = dst.asStackSlot().?;
6554	try moveConstant(
6555	isel,
6556	.{ .stack_slot = dst_stack },
6557	.fromInterned(slice.ptr),
6558	0,
6559	ptr_size,
6560	);
6561	try moveConstant(
6562	isel,
6563	.{ .stack_slot = dst_stack.withOffset(ptr_size) },
6564	.fromInterned(slice.len),
6565	0,
6566	ptr_size,
6567	);
6568	return;
6569	}
6570	},
6571	.opt => |opt| {
6572	const child_ty = ip.indexToKey(opt.ty).opt_type;
6573	const child_size = ZigType.fromInterned(child_ty).abiSize(zcu);
6574	if (offset == child_size and size == 1) {
6575	offset = 0;
6576	continue :constant_key .{ .simple_value = switch (opt.val) {
6577	.none => .false,
6578	else => .true,
6579	} };
6580	}
6581	const opt_ty: ZigType = .fromInterned(opt.ty);
6582	if (offset + size <= child_size) continue :constant_key switch (opt.val) {
6583	.none => if (opt_ty.optionalReprIsPayload(zcu)) .{ .int = .{
6584	.ty = opt.ty,
6585	.storage = .{ .u64 = 0 },
6586	} } else .{ .undef = child_ty },
6587	else => |child| {
6588	constant = child;
6589	constant_key = ip.indexToKey(constant);
6590	continue :constant_key constant_key;
6591	},
6592	};
6593	},
6594	.aggregate => |aggregate| switch (ip.indexToKey(aggregate.ty)) {
6595	else => unreachable,
6596	.array_type => |array_type| {
6597	const elem_size = ZigType.fromInterned(array_type.child).abiSize(zcu);
6598	const elem_offset = @mod(offset, elem_size);
6599	if (size <= elem_size - elem_offset) {
6600	defer offset = elem_offset;
6601	continue :constant_key switch (aggregate.storage) {
6602	.bytes => |bytes| .{ .int = .{ .ty = .u8_type, .storage = .{
6603	.u64 = bytes.toSlice(array_type.lenIncludingSentinel(), ip)[@intCast(@divFloor(offset, elem_size))],
6604	} } },
6605	.elems => |elems| {
6606	constant = elems[@intCast(@divFloor(offset, elem_size))];
6607	constant_key = ip.indexToKey(constant);
6608	continue :constant_key constant_key;
6609	},
6610	.repeated_elem => |repeated_elem| {
6611	constant = repeated_elem;
6612	constant_key = ip.indexToKey(constant);
6613	continue :constant_key constant_key;
6614	},
6615	};
6616	}
6617	},
6618	.vector_type => {},
6619	.struct_type => {
6620	const loaded_struct = ip.loadStructType(aggregate.ty);
6621	switch (loaded_struct.layout) {
6622	.auto => {
6623	var field_it = loaded_struct.iterateRuntimeOrder(ip);
6624	while (field_it.next()) |field_index| {
6625	if (loaded_struct.field_is_comptime_bits.get(ip, field_index)) continue;
6626	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
6627	const field_offset = loaded_struct.field_offsets.get(ip)[field_index];
6628	const field_size = field_ty.abiSize(zcu);
6629	if (offset >= field_offset and offset + size <= field_offset + field_size) {
6630	offset -= field_offset;
6631	constant = switch (aggregate.storage) {
6632	.bytes => unreachable,
6633	.elems => |elems| elems[field_index],
6634	.repeated_elem => |repeated_elem| repeated_elem,
6635	};
6636	constant_key = ip.indexToKey(constant);
6637	continue :constant_key constant_key;
6638	}
6639	}
6640	},
6641	.@"extern", .@"packed" => {},
6642	}
6643	},
6644	.tuple_type => |tuple_type| {
6645	var field_offset: u64 = 0;
6646	for (tuple_type.types.get(ip), tuple_type.values.get(ip), 0..) |field_type, field_value, field_index| {
6647	if (field_value != .none) continue;
6648	const field_ty: ZigType = .fromInterned(field_type);
6649	field_offset = field_ty.abiAlignment(zcu).forward(field_offset);
6650	const field_size = field_ty.abiSize(zcu);
6651	if (offset >= field_offset and offset + size <= field_offset + field_size) {
6652	offset -= field_offset;
6653	constant = switch (aggregate.storage) {
6654	.bytes => unreachable,
6655	.elems => |elems| elems[field_index],
6656	.repeated_elem => |repeated_elem| repeated_elem,
6657	};
6658	constant_key = ip.indexToKey(constant);
6659	continue :constant_key constant_key;
6660	}
6661	field_offset += field_size;
6662	}
6663	},
6664	},
6665	.un => |un| {
6666	const loaded_union = ip.loadUnionType(un.ty);
6667	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
6668	if (loaded_union.has_runtime_tag) {
6669	const tag_offset = union_layout.tagOffset();
6670	if (offset >= tag_offset and offset + size <= tag_offset + union_layout.tag_size) {
6671	offset -= tag_offset;
6672	continue :constant_key switch (ip.indexToKey(un.tag)) {
6673	else => unreachable,
6674	.int => |int| .{ .int = int },
6675	.enum_tag => |enum_tag| .{ .enum_tag = enum_tag },
6676	};
6677	}
6678	}
6679	const payload_offset = union_layout.payloadOffset();
6680	if (offset >= payload_offset and offset + size <= payload_offset + union_layout.payload_size) {
6681	offset -= payload_offset;
6682	constant = un.val;
6683	constant_key = ip.indexToKey(constant);
6684	continue :constant_key constant_key;
6685	}
6686	},
6687	}
6688	const constant_size = ZigType.fromInterned(constant_key.typeOf()).abiSize(zcu);
6689	var buffer: [128]u8 align(8) = @splat(0);
6690	// Large constants should have been coerced to smaller ones, so use a buffer with fixed-size
6691	if (constant_size <= buffer.len and
6692	try isel.writeConstantToMemory(.fromInterned(constant), &buffer))
6693	{
6694	// TODO lower to literals or lazy symbols and memcpy for larger constants
6695	assert(offset + size <= buffer.len);
6696	const part_buffer = buffer[@intCast(offset)..];
6697	const gpr_size = isel.gprSize();
6698
6699	const tmp_reg, const tmp_lock: RegLock = tmp_reg: {
6700	if (dst.asRegisterAlias()) |dst_ra| {
6701	if (dst_ra.mod == .integer) break :tmp_reg .{ dst_ra.reg, isel.tryLockReg(dst_ra.reg) };
6702	}
6703	const tmp_reg = try isel.allocRegForWrite(.int);
6704	break :tmp_reg .{ tmp_reg, .{ .reg = tmp_reg } };
6705	};
6706	defer tmp_lock.unlock(isel);
6707	const tmp_ra: Register.Alias = .{ .mod = .integer, .reg = tmp_reg };
6708
6709	var part_offset = size & (0 -% gpr_size);
6710	while (true) {
6711	const part_size = @min(size - part_offset, gpr_size);
6712	const part_value: u64 = switch (part_size) {
6713	else => unreachable,
6714	0 => {
6715	part_offset -= gpr_size;
6716	continue;
6717	},
6718	inline 1...8 => |ct_size| std.mem.readInt(
6719	@Int(.unsigned, 8 * @as(u16, ct_size)),
6720	part_buffer[0..ct_size],
6721	.little,
6722	),
6723	};
6724
6725	try isel.moveLoc(dst, part_offset, .{ .register = tmp_ra }, 0, part_size, .preserved);
6726	try isel.moveIntImm(tmp_reg, @bitCast(part_value));
6727
6728	if (part_offset == 0) break else part_offset -= gpr_size;
6729	}
6730
6731	return;
6732	}
6733	if (ZigType.fromInterned(ip.typeOf(constant)).isRuntimeFnOrHasRuntimeBits(zcu)) {
6734	const ptr_ty = try isel.pt.singleConstPtrType(.fromInterned(ip.typeOf(constant)));
6735	const uav: InternPool.Key.Ptr.BaseAddr.Uav = .{
6736	.val = constant,
6737	.orig_ty = ptr_ty.ip_index,
6738	};
6739
6740	// allocate temporary register for pointers
6741	const tmp_reg, const allocated_tmp_reg = tmp_reg: {
6742	if (dst.asRegisterAlias()) |dst_ra| {
6743	if (dst_ra.mod == .integer) break :tmp_reg .{ dst_ra.reg, false };
6744	}
6745	break :tmp_reg .{ try isel.allocRegForWrite(.int), true };
6746	};
6747	defer if (allocated_tmp_reg) isel.freeReg(tmp_reg);
6748
6749	// load from the pointer
6750	try isel.moveLoc(
6751	dst,
6752	0,
6753	.{ .stack_slot = .{ .base = tmp_reg, .offset = 0 } },
6754	offset,
6755	size,
6756	.none,
6757	);
6758
6759	// load constant pointer
6760	try isel.uav_relocs.append(zcu.gpa, .{
6761	.uav = uav,
6762	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .PCALA_LO12 },
6763	});
6764	try isel.emit(.@"addi.d"(tmp_reg, tmp_reg, 0));
6765	try isel.uav_relocs.append(zcu.gpa, .{
6766	.uav = uav,
6767	.reloc = .{ .label = @intCast(isel.instructions.items.len), .type = .PCALA_HI20 },
6768	});
6769	try isel.emit(.pcalau12i(tmp_reg, 0));
6770
6771	return;
6772	}
6773	return isel.fail("unsupported value <{f}, {f}>[{d}..{d}] (full size={d}), from <{f}, {f}>[{d}..{d}]", .{
6774	isel.fmtType(.fromInterned(constant_key.typeOf())),
6775	isel.fmtConstant(.fromInterned(constant)),
6776	offset,
6777	offset + size - 1,
6778	constant_size,
6779	isel.fmtType(init_constant.typeOf(zcu)),
6780	isel.fmtConstant(init_constant),
6781	init_offset,
6782	init_offset + size - 1,
6783	});
6784	}
6785	}
6786
6787	/// Returns the minimum legal memory operation size that is equal or greater than the given size.
6788	fn memOpSizeFitting(size: u64) u64 {
6789	return switch (size) {
6790	0 => unreachable,
6791	1 => 1,
6792	2 => 2,
6793	3...4 => 4,
6794	5...8 => 8,
6795	9...16 => 16,
6796	17...32 => 32,
6797	else => unreachable,
6798	};
6799	}
6800
6801	pub const CallAbiIterator = struct {
6802	isel: *Select,
6803	cc: *const std.builtin.CallingConvention,
6804	next_reg: std.EnumArray(RegisterClass, Register) = .init(.{
6805	.gpr = .r4,
6806	.fpr = .f0,
6807	.ret_byref = .r4,
6808	}),
6809	next_stack: usize = 0,
6810	// TODO optimize, use SP to read incoming arguments when possible
6811	stack_pointer: Register = .sp,
6812
6813	const RegisterClass = enum {
6814	gpr,
6815	fpr,
6816	/// Virtual register class, for allocating GPRs for by-reference returning.
6817	ret_byref,
6818	};
6819
6820	fn allocReg(it: *CallAbiIterator, class: RegisterClass) ?Register {
6821	const last_reg: Register = switch (class) {
6822	.gpr, .ret_byref => .r11,
6823	.fpr => .f7,
6824	};
6825	const next = it.next_reg.getPtr(class);
6826	if (@backingInt(last_reg) >= @backingInt(next.*)) {
6827	const allocated = next.*;
6828	next.* = @fromBackingInt(@backingInt(allocated) + 1);
6829	return allocated;
6830	} else return null;
6831	}
6832
6833	/// Trys to allocate some registers, returning amount of allocated registers.
6834	fn allocRegs(it: *CallAbiIterator, class: RegisterClass, result: []Register) usize {
6835	const last_reg: Register = switch (class) {
6836	.gpr, .ret_byref => .r11,
6837	.fpr => .f7,
6838	};
6839	const next = it.next_reg.getPtr(class);
6840	const remaining = @backingInt(last_reg) - @backingInt(next.*) + 1;
6841	if (remaining >= result.len) {
6842	for (result, @backingInt(next.*)..) |*v, reg|
6843	v.* = @fromBackingInt(@intCast(reg));
6844	next.* = @fromBackingInt(@intCast(@backingInt(next.*) + result.len));
6845	return result.len;
6846	} else {
6847	for (@backingInt(next.*)..@backingInt(last_reg) + 1, result[0..remaining]) |reg, *v|
6848	v.* = @fromBackingInt(@intCast(reg));
6849	next.* = @fromBackingInt(@backingInt(last_reg) + 1);
6850	return remaining;
6851	}
6852	}
6853
6854	fn assignStack(it: *CallAbiIterator, wip_vi: Value.Index) void {
6855	const isel = it.isel;
6856	assert(wip_vi.stackSlot(isel) == null);
6857	it.next_stack = @intCast(wip_vi.alignment(isel).forward(it.next_stack));
6858	wip_vi.setStackSlot(isel, .{
6859	.base = it.stack_pointer,
6860	.offset = @intCast(it.next_stack),
6861	});
6862	it.next_stack += @intCast(wip_vi.size(isel));
6863	}
6864
6865	fn assignUsize(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
6866	if (it.allocReg(.gpr)) |reg| {
6867	wip_vi.setHintRegister(isel, reg);
6868	} else it.assignStack(wip_vi);
6869	}
6870
6871	fn assignGprPair(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index, part_sizes: [2]u64, part_bit_size: [2]u9) !void {
6872	const grsize: u8 = isel.gprSize();
6873	var regs: [2]Register = undefined;
6874	const allocated_regs = it.allocRegs(.gpr, &regs);
6875	switch (allocated_regs) {
6876	0 => it.assignStack(wip_vi),
6877	1 => {
6878	wip_vi.setParts(isel, 2);
6879	(try wip_vi.addIntPart(isel, 0, part_sizes[0], part_bit_size[0])).setHintRegister(isel, regs[0]);
6880	it.assignStack(try wip_vi.addIntPart(isel, grsize, part_sizes[1], part_bit_size[1]));
6881	},
6882	2 => {
6883	wip_vi.setParts(isel, 2);
6884	(try wip_vi.addIntPart(isel, 0, part_sizes[0], part_bit_size[0])).setHintRegister(isel, regs[0]);
6885	(try wip_vi.addIntPart(isel, grsize, part_sizes[1], part_bit_size[1])).setHintRegister(isel, regs[1]);
6886	},
6887	else => unreachable,
6888	}
6889	}
6890
6891	fn assignIndirect(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index, is_return: bool) void {
6892	const wip_address_vi = isel.initValueAssumeCapacity(.usize);
6893	wip_vi.setParent(isel, .{ .address = wip_address_vi });
6894
6895	if (it.allocReg(if (is_return) .ret_byref else .gpr)) |reg| {
6896	wip_address_vi.setHintRegister(isel, reg);
6897	} else it.assignStack(wip_address_vi);
6898	}
6899
6900	pub fn resolve(it: *CallAbiIterator, ty: ZigType, is_return: bool) !?Value.Index {
6901	const isel = it.isel;
6902	const zcu = isel.pt.zcu;
6903	const ip = &zcu.intern_pool;
6904
6905	if (!ty.hasRuntimeBits(zcu)) return null;
6906	try isel.values.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
6907	try isel.value_types.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
6908	const wip_vi = isel.initValueAssumeCapacity(ty);
6909	wip_vi.setExtension(isel, .pcsMode(isel, ty));
6910
6911	const grsize = isel.gprSize();
6912	const grlen: u8 = isel.gprBits();
6913
6914	type_key: switch (ip.indexToKey(ty.toIntern())) {
6915	else => return isel.fail("CallAbiIterator.resolve({f})", .{isel.fmtType(ty)}),
6916	.int_type => |int_ty| {
6917	if (int_ty.bits <= grlen) {
6918	it.assignUsize(isel, wip_vi);
6919	} else if (int_ty.bits <= 2 * grlen) {
6920	try it.assignGprPair(isel, wip_vi, .{ grsize, ty.abiSize(zcu) - grsize }, .{ grlen, @intCast(int_ty.bits - grlen) });
6921	} else it.assignStack(wip_vi);
6922	},
6923	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
6924	.one, .many, .c => it.assignUsize(isel, wip_vi),
6925	.slice => continue :type_key .{ .int_type = .{
6926	.signedness = .unsigned,
6927	.bits = 2 * grlen,
6928	} },
6929	},
6930	.opt_type => |child_type| if (ty.optionalReprIsPayload(zcu))
6931	continue :type_key ip.indexToKey(child_type)
6932	else switch (ZigType.fromInterned(child_type).abiSize(zcu)) {
6933	0 => continue :type_key .{ .simple_type = .bool },
6934	1...7 => it.assignUsize(isel, wip_vi),
6935	8...15 => |child_size| {
6936	try it.assignGprPair(isel, wip_vi, .{ child_size, 1 }, .{ @intCast(child_size * 8), 1 });
6937	},
6938	else => it.assignIndirect(isel, wip_vi, is_return),
6939	},
6940	.anyframe_type => unreachable,
6941	.error_union_type => switch (wip_vi.size(isel)) {
6942	0 => unreachable,
6943	1...8 => it.assignUsize(isel, wip_vi),
6944	// 9...16 => {}, TODO optimize
6945	else => it.assignIndirect(isel, wip_vi, is_return),
6946	},
6947	.simple_type => |simple_type| switch (simple_type) {
6948	.f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
6949	.f128 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 128 } },
6950	.usize,
6951	.isize,
6952	.c_char,
6953	.c_short,
6954	.c_ushort,
6955	.c_int,
6956	.c_uint,
6957	.c_long,
6958	.c_ulong,
6959	.c_longlong,
6960	.c_ulonglong,
6961	=> continue :type_key .{ .int_type = ty.intInfo(zcu) },
6962	.anyopaque, .bool => it.assignUsize(isel, wip_vi),
6963	.anyerror => continue :type_key .{ .int_type = .{
6964	.signedness = .unsigned,
6965	.bits = zcu.errorSetBits(),
6966	} },
6967	.f16, .f32, .f64 => {
6968	const bits = ty.floatBits(isel.target);
6969	if (isel.canUseFprForFloat(bits)) {
6970	if (it.allocReg(.fpr)) |reg| {
6971	wip_vi.setHintRegister(isel, reg);
6972	if (bits != 16) {
6973	wip_vi.setHintModifier(isel, .fromFloatBits(bits));
6974	} else {
6975	wip_vi.setHintModifier(isel, .floating32);
6976	}
6977	} else it.assignStack(wip_vi);
6978	} else {
6979	continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = bits } };
6980	}
6981	},
6982	.c_longdouble => return isel.fail("CallAbiIterator.resolve({t})", .{simple_type}),
6983	else => return isel.fail("CallAbiIterator.resolve({t})", .{simple_type}),
6984	},
6985	.struct_type => {
6986	// TODO: implement floating-point structures rules defined in lapcs
6987	const loaded_struct = ip.loadStructType(ty.toIntern());
6988	switch (loaded_struct.layout) {
6989	.auto, .@"extern" => {},
6990	.@"packed" => continue :type_key ip.indexToKey(loaded_struct.packed_backing_int_type),
6991	}
6992	const size = wip_vi.size(isel);
6993	if (size == 0)
6994	unreachable
6995	else if (size <= grsize)
6996	it.assignUsize(isel, wip_vi)
6997	else if (size <= 2 * @as(u64, grsize))
6998	try it.assignGprPair(isel, wip_vi, .{ grsize, size - grsize }, .{ grlen, @intCast((size * 8) - grlen) })
6999	else
7000	// TODO flatten single-field structs
7001	it.assignIndirect(isel, wip_vi, is_return);
7002	},
7003	.union_type => {
7004	const loaded_union = ip.loadUnionType(ty.toIntern());
7005	switch (loaded_union.layout) {
7006	.auto, .@"extern" => {},
7007	.@"packed" => continue :type_key .{ .int_type = .{
7008	.signedness = .unsigned,
7009	.bits = @intCast(ty.bitSize(zcu)),
7010	} },
7011	}
7012	const size = wip_vi.size(isel);
7013	if (size == 0)
7014	unreachable
7015	else if (size <= grsize)
7016	it.assignUsize(isel, wip_vi)
7017	else if (size <= 2 * @as(u64, grsize)) {
7018	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
7019	var sizes: [2]u64 = @splat(0);
7020	{
7021	const offset = union_layout.tagOffset();
7022	const end = offset % grsize + union_layout.tag_size;
7023	const part_index: usize = @intCast(offset / grsize);
7024	sizes[part_index] = @max(sizes[part_index], @min(end, grsize));
7025	if (end > grsize) sizes[part_index + 1] = @max(sizes[part_index + 1], end - grsize);
7026	}
7027	{
7028	const offset = union_layout.payloadOffset();
7029	const end = offset % grsize + union_layout.payload_size;
7030	const part_index: usize = @intCast(offset / grsize);
7031	sizes[part_index] = @max(sizes[part_index], @min(end, grsize));
7032	if (end > grsize) sizes[part_index + 1] = @max(sizes[part_index + 1], end - grsize);
7033	}
7034	try it.assignGprPair(isel, wip_vi, sizes, .{ @intCast(sizes[0] * 8), @intCast(sizes[1] * 8) });
7035	} else it.assignIndirect(isel, wip_vi, is_return);
7036	},
7037	.tuple_type => |tuple_ty| {
7038	assert(it.cc.* == .auto);
7039	const size = wip_vi.size(isel);
7040	switch (size) {
7041	0 => unreachable,
7042	1...8 => it.assignUsize(isel, wip_vi),
7043	9...16 => {
7044	var part_offset: u64 = 0;
7045	var part_sizes: [2]u64 = undefined;
7046	var parts_len: Value.PartsLen = 0;
7047	var next_field_end: u64 = 0;
7048	var field_index: usize = 0;
7049	while (part_offset < size) {
7050	const field_end = next_field_end;
7051	const next_field_begin = while (field_index < tuple_ty.types.len) {
7052	defer field_index += 1;
7053	if (tuple_ty.values.get(ip)[field_index] != .none) continue;
7054	const field_ty: ZigType = .fromInterned(tuple_ty.types.get(ip)[field_index]);
7055	const next_field_begin = field_ty.abiAlignment(zcu).forward(field_end);
7056	next_field_end = next_field_begin + field_ty.abiSize(zcu);
7057	break next_field_begin;
7058	} else std.mem.alignForward(u64, size, 8);
7059	while (next_field_begin - part_offset >= 8) {
7060	const part_size = @min(field_end - part_offset, 8);
7061	part_sizes[parts_len] = part_size;
7062	assert(part_offset + part_size <= size);
7063	parts_len += 1;
7064	part_offset += part_size;
7065	if (part_offset >= field_end) part_offset = next_field_begin;
7066	}
7067	}
7068	assert(parts_len == part_sizes.len);
7069	try it.assignGprPair(isel, wip_vi, part_sizes, .{ @intCast(part_sizes[0] * 8), @intCast(part_sizes[1] * 8) });
7070	},
7071	else => it.assignIndirect(isel, wip_vi, is_return),
7072	}
7073	},
7074	// TODO: optimize chance
7075	.array_type => it.assignIndirect(isel, wip_vi, is_return),
7076	.opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
7077	.enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).int_tag_type),
7078	.error_set_type,
7079	.inferred_error_set_type,
7080	=> continue :type_key .{ .simple_type = .anyerror },
7081	}
7082
7083	if (is_return) {
7084	it.next_reg = .init(.{
7085	.gpr = it.next_reg.get(.ret_byref), // skip registers for by-ref returning
7086	.fpr = .f0,
7087	.ret_byref = .zero,
7088	});
7089	it.next_stack = 0;
7090	abi_log.debug("| Return: {f} -> {f}", .{ isel.fmtType(ty), isel.fmtValue(wip_vi) });
7091	} else {
7092	abi_log.debug("| Param: {f} -> {f}", .{ isel.fmtType(ty), isel.fmtValue(wip_vi) });
7093	}
7094
7095	return wip_vi.ref(isel);
7096	}
7097	};
7098
7099	const call = struct {
7100	const param_reg: Value.Index = @fromBackingInt(@backingInt(Value.Index.allocating) - 2);
7101	const callee_clobbered_reg: Value.Index = @fromBackingInt(@backingInt(Value.Index.allocating) - 1);
7102	const caller_saved_regs: LiveRegisters = .init(.{
7103	.r0 = .free,
7104	.r1 = callee_clobbered_reg,
7105	.r2 = .free,
7106	.r3 = .free,
7107	.r4 = param_reg,
7108	.r5 = param_reg,
7109	.r6 = param_reg,
7110	.r7 = param_reg,
7111	.r8 = param_reg,
7112	.r9 = param_reg,
7113	.r10 = param_reg,
7114	.r11 = param_reg,
7115	.r12 = callee_clobbered_reg,
7116	.r13 = callee_clobbered_reg,
7117	.r14 = callee_clobbered_reg,
7118	.r15 = callee_clobbered_reg,
7119	.r16 = callee_clobbered_reg,
7120	.r17 = callee_clobbered_reg,
7121	.r18 = callee_clobbered_reg,
7122	.r19 = callee_clobbered_reg,
7123	.r20 = callee_clobbered_reg,
7124	.r21 = .free,
7125	.r22 = .free,
7126	.r23 = .free,
7127	.r24 = .free,
7128	.r25 = .free,
7129	.r26 = .free,
7130	.r27 = .free,
7131	.r28 = .free,
7132	.r29 = .free,
7133	.r30 = .free,
7134	.r31 = .free,
7135
7136	.f0 = param_reg,
7137	.f1 = param_reg,
7138	.f2 = param_reg,
7139	.f3 = param_reg,
7140	.f4 = param_reg,
7141	.f5 = param_reg,
7142	.f6 = param_reg,
7143	.f7 = param_reg,
7144	.f8 = callee_clobbered_reg,
7145	.f9 = callee_clobbered_reg,
7146	.f10 = callee_clobbered_reg,
7147	.f11 = callee_clobbered_reg,
7148	.f12 = callee_clobbered_reg,
7149	.f13 = callee_clobbered_reg,
7150	.f14 = callee_clobbered_reg,
7151	.f15 = callee_clobbered_reg,
7152	.f16 = callee_clobbered_reg,
7153	.f17 = callee_clobbered_reg,
7154	.f18 = callee_clobbered_reg,
7155	.f19 = callee_clobbered_reg,
7156	.f20 = callee_clobbered_reg,
7157	.f21 = callee_clobbered_reg,
7158	.f22 = callee_clobbered_reg,
7159	.f23 = callee_clobbered_reg,
7160	.f24 = .free,
7161	.f25 = .free,
7162	.f26 = .free,
7163	.f27 = .free,
7164	.f28 = .free,
7165	.f29 = .free,
7166	.f30 = .free,
7167	.f31 = .free,
7168
7169	.fcc0 = callee_clobbered_reg,
7170	.fcc1 = callee_clobbered_reg,
7171	.fcc2 = callee_clobbered_reg,
7172	.fcc3 = callee_clobbered_reg,
7173	.fcc4 = callee_clobbered_reg,
7174	.fcc5 = callee_clobbered_reg,
7175	.fcc6 = callee_clobbered_reg,
7176	.fcc7 = callee_clobbered_reg,
7177	});
7178
7179	fn prepareReturn(_: *Select) !void {}
7180
7181	fn finishReturn(isel: *Select) !void {
7182	// Lock remaining clobberred registers
7183	const locked_regs = comptime locked_regs: {
7184	var locked_regs: RegisterSet = .empty;
7185	for (std.enums.values(Register)) |reg| switch (caller_saved_regs.get(reg)) {
7186	else => unreachable,
7187	param_reg, callee_clobbered_reg => locked_regs.insert(reg),
7188	.free => {},
7189	};
7190	break :locked_regs locked_regs;
7191	};
7192	try isel.fillRegsBatch(locked_regs, true);
7193	isel.markRegsWritten(locked_regs);
7194	}
7195
7196	fn prepareCallee(isel: *Select) !void {
7197	// Free clobbered registers
7198	var live_reg_it = isel.live_registers.iterator();
7199	while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
7200	else => unreachable,
7201	param_reg => assert(live_reg_entry.value.* == .allocating),
7202	callee_clobbered_reg => isel.freeReg(live_reg_entry.key),
7203	.free => {},
7204	};
7205	}
7206	fn finishCallee(_: *Select) !void {}
7207
7208	fn prepareParams(_: *Select) !void {}
7209	fn paramLiveOut(isel: *Select, vi: Value.Index, layout_vi: Value.Index) !void {
7210	switch (layout_vi.parent(isel)) {
7211	else => return vi.matLiveOut(isel, layout_vi, .{ .mode = .param }),
7212	.address => |addr_vi| return call.paramIndirect(isel, vi, addr_vi),
7213	}
7214	}
7215	fn paramIndirect(isel: *Select, vi: Value.Index, addr_vi: Value.Index) !void {
7216	const val_mat = try vi.mat(isel, .{ .pref = .only_stack });
7217	try paramAddress(
7218	isel,
7219	val_mat.loc().asStackSlot().?,
7220	addr_vi,
7221	);
7222	try val_mat.finish(isel);
7223	}
7224	fn paramAddress(isel: *Select, stack: Value.Indirect, addr_vi: Value.Index) !void {
7225	if (addr_vi.hintRegister(isel)) |addr_reg| {
7226	assert(isel.live_registers.get(addr_reg) == .allocating);
7227	try isel.addImm(addr_reg, stack.base, stack.offset);
7228	} else if (addr_vi.location(isel)) |addr_loc| {
7229	const tmp_reg = try isel.allocRegForWrite(.int);
7230	defer isel.freeReg(tmp_reg);
7231	try isel.addImm(tmp_reg, stack.base, stack.offset);
7232	try isel.moveLoc(
7233	addr_loc,
7234	0,
7235	.{ .register = .{ .mod = .integer, .reg = tmp_reg } },
7236	0,
7237	isel.gprSize(),
7238	.preserved,
7239	);
7240	} else unreachable;
7241	}
7242	fn finishParams(isel: *Select) !void {
7243	// Free parameter registers
7244	var live_reg_it = isel.live_registers.iterator();
7245	while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
7246	else => unreachable,
7247	param_reg => switch (live_reg_entry.value.*) {
7248	_ => {},
7249	.allocating => live_reg_entry.value.* = .free,
7250	.free => unreachable,
7251	},
7252	callee_clobbered_reg, .free => {},
7253	};
7254	}
7255	};
7256
7257	fn gprSize(isel: *Select) u4 {
7258	return switch (isel.target.cpu.arch) {
7259	.loongarch32 => 4,
7260	.loongarch64 => 8,
7261	else => unreachable,
7262	};
7263	}
7264
7265	fn gprBits(isel: *Select) u7 {
7266	return switch (isel.target.cpu.arch) {
7267	.loongarch32 => 32,
7268	.loongarch64 => 64,
7269	else => unreachable,
7270	};
7271	}
7272
7273	fn gprAlignment(isel: *Select) std.mem.Alignment {
7274	return switch (isel.target.cpu.arch) {
7275	.loongarch32 => .@"4",
7276	.loongarch64 => .@"8",
7277	else => unreachable,
7278	};
7279	}
7280
7281	fn fprBits(isel: *Select) u7 {
7282	const cpu = &isel.target.cpu;
7283	if (cpu.has(.loongarch, .d)) {
7284	return 64;
7285	} else if (cpu.has(.loongarch, .f)) {
7286	return 32;
7287	} else {
7288	return 0;
7289	}
7290	}
7291
7292	fn vectorBits(isel: *Select) u7 {
7293	const cpu = &isel.target.cpu;
7294	if (cpu.has(.loongarch, .lasx)) {
7295	return 256;
7296	} else if (cpu.has(.loongarch, .lsx)) {
7297	return 128;
7298	} else {
7299	return isel.fprBits();
7300	}
7301	}
7302
7303	fn canUseFprForFloat(isel: *Select, bits: u16) bool {
7304	return bits <= isel.fprBits();
7305	}
7306
7307	fn typeOfField(isel: *Select, ty: ZigType, offset: u64) ?ZigType {
7308	const zcu = isel.pt.zcu;
7309	const ip = &zcu.intern_pool;
7310	type_key: switch (ip.indexToKey(ty.toIntern())) {
7311	else => {},
7312	// TODO large int splitting
7313	.int_type => {
7314	if (ty.abiSize(zcu) > isel.gprSize() and offset % isel.gprSize() == 0) return .usize;
7315	},
7316	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
7317	.one, .many, .c => {},
7318	.slice => if (offset == 0)
7319	return ty.elemPtrType(null, isel.pt) catch unreachable
7320	else if (offset == isel.gprSize())
7321	return .usize,
7322	},
7323	.opt_type => |child_type| if (ty.optionalReprIsPayload(zcu))
7324	continue :type_key ip.indexToKey(child_type)
7325	else {
7326	const child_ty: ZigType = .fromInterned(child_type);
7327	if (offset == 0)
7328	return child_ty
7329	else if (offset == child_ty.abiSize(zcu))
7330	return .usize;
7331	},
7332	.array_type => unreachable, // TODO
7333	.anyframe_type => unreachable,
7334	.error_union_type => |error_union_type| {
7335	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
7336	if (offset == codegen.errUnionErrorOffset(payload_ty, zcu))
7337	return .fromInterned(error_union_type.error_set_type)
7338	else if (offset == codegen.errUnionPayloadOffset(payload_ty, zcu))
7339	return payload_ty;
7340	},
7341	.simple_type => |simple_type| switch (simple_type) {
7342	else => {},
7343	.f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
7344	.usize,
7345	.isize,
7346	.c_char,
7347	.c_short,
7348	.c_ushort,
7349	.c_int,
7350	.c_uint,
7351	.c_long,
7352	.c_ulong,
7353	.c_longlong,
7354	.c_ulonglong,
7355	=> continue :type_key .{ .int_type = ty.intInfo(zcu) },
7356	.anyerror => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } },
7357	},
7358	.struct_type => {
7359	const loaded_struct = ip.loadStructType(ty.toIntern());
7360	switch (loaded_struct.layout) {
7361	.auto, .@"extern" => {},
7362	.@"packed" => continue :type_key ip.indexToKey(loaded_struct.backingIntTypeUnordered(ip)).int_type,
7363	}
7364	var field_end: u64 = 0;
7365	var field_it = loaded_struct.iterateRuntimeOrder(ip);
7366	while (field_it.next()) |field_index| {
7367	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
7368	const field_begin = switch (loaded_struct.fieldAlign(ip, field_index)) {
7369	.none => field_ty.abiAlignment(zcu),
7370	else => |field_align| field_align,
7371	}.forward(field_end);
7372	const field_size = field_ty.abiSize(zcu);
7373	field_end = field_begin + field_size;
7374	if (field_begin > offset) break;
7375	if (field_begin == offset)
7376	return field_ty
7377	else if (field_end > offset)
7378	return isel.typeOfField(field_ty, offset - field_begin);
7379	}
7380	},
7381	.tuple_type => |tuple_type| {
7382	var field_end: u64 = 0;
7383	for (tuple_type.types.get(ip), tuple_type.values.get(ip)) |field_type, field_value| {
7384	if (field_value != .none) continue;
7385	const field_ty: ZigType = .fromInterned(field_type);
7386	const field_begin = field_ty.abiAlignment(zcu).forward(field_end);
7387	const field_size = field_ty.abiSize(zcu);
7388	if (field_size == 0) continue;
7389	field_end = field_begin + field_size;
7390	if (field_begin > offset) break;
7391	if (field_begin == offset)
7392	return field_ty
7393	else if (field_end > offset)
7394	return isel.typeOfField(field_ty, offset - field_begin);
7395	}
7396	},
7397	.union_type => {
7398	const loaded_union = ip.loadUnionType(ty.toIntern());
7399	switch (loaded_union.flagsUnordered(ip).layout) {
7400	.auto, .@"extern" => {},
7401	.@"packed" => continue :type_key .{ .int_type = .{
7402	.signedness = .unsigned,
7403	.bits = @intCast(ty.bitSize(zcu)),
7404	} },
7405	}
7406	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
7407	if (offset == union_layout.tagOffset())
7408	return .fromInterned(loaded_union.enum_tag_ty);
7409	},
7410	.opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
7411	.enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty),
7412	.error_set_type,
7413	.inferred_error_set_type,
7414	=> continue :type_key .{ .simple_type = .anyerror },
7415	}
7416	tracking_log.debug("cannot split {f} at {d}", .{ isel.fmtType(ty), offset });
7417	return null;
7418	}
7419
7420	fn hasRepeatedByteRepr(isel: *Select, constant: Constant) error{OutOfMemory}!?u8 {
7421	const zcu = isel.pt.zcu;
7422	const ty = constant.typeOf(zcu);
7423	const abi_size = std.math.cast(usize, ty.abiSize(zcu)) orelse return null;
7424	const byte_buffer = try zcu.gpa.alloc(u8, abi_size);
7425	defer zcu.gpa.free(byte_buffer);
7426	return if (try isel.writeConstantToMemory(constant, byte_buffer) and
7427	std.mem.allEqual(u8, byte_buffer[1..], byte_buffer[0])) byte_buffer[0] else null;
7428	}
7429
7430	fn writeConstantToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMemory}!bool {
7431	const zcu = isel.pt.zcu;
7432	const ip = &zcu.intern_pool;
7433	if (try isel.writeConstantKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
7434	constant.writeToMemory(isel.pt.zcu, buffer) catch |err| switch (err) {
7435	error.OutOfMemory => return error.OutOfMemory,
7436	error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
7437	};
7438	return true;
7439	}
7440
7441	fn writeConstantKeyToMemory(isel: *Select, constant_key: InternPool.Key, buffer: []u8) error{OutOfMemory}!bool {
7442	const zcu = isel.pt.zcu;
7443	const ip = &zcu.intern_pool;
7444	switch (constant_key) {
7445	.int_type,
7446	.ptr_type,
7447	.array_type,
7448	.vector_type,
7449	.opt_type,
7450	.anyframe_type,
7451	.error_union_type,
7452	.simple_type,
7453	.struct_type,
7454	.tuple_type,
7455	.union_type,
7456	.opaque_type,
7457	.enum_type,
7458	.func_type,
7459	.error_set_type,
7460	.inferred_error_set_type,
7461
7462	.enum_literal,
7463	.memoized_call,
7464	=> unreachable, // not a runtime value
7465	.err => |err| {
7466	const error_int = ip.getErrorValueIfExists(err.name).?;
7467	switch (buffer.len) {
7468	else => unreachable,
7469	inline 1...4 => |size| std.mem.writeInt(
7470	@Int(.unsigned, 8 * size),
7471	buffer[0..size],
7472	@intCast(error_int),
7473	isel.target.cpu.arch.endian(),
7474	),
7475	}
7476	},
7477	.error_union => |error_union| {
7478	const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
7479	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
7480	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
7481	const error_set = buffer[@intCast(codegen.errUnionErrorOffset(payload_ty, zcu))..][0..@intCast(error_set_ty.abiSize(zcu))];
7482	switch (error_union.val) {
7483	.err_name => |err_name| if (!try isel.writeConstantKeyToMemory(.{ .err = .{
7484	.ty = error_set_ty.toIntern(),
7485	.name = err_name,
7486	} }, error_set)) return false,
7487	.payload => |payload| {
7488	if (!try isel.writeConstantToMemory(
7489	.fromInterned(payload),
7490	buffer[@intCast(codegen.errUnionPayloadOffset(payload_ty, zcu))..][0..@intCast(payload_ty.abiSize(zcu))],
7491	)) return false;
7492	@memset(error_set, 0);
7493	},
7494	}
7495	},
7496	.opt => |opt| {
7497	const child_size: usize = @intCast(ZigType.fromInterned(ip.indexToKey(opt.ty).opt_type).abiSize(zcu));
7498	switch (opt.val) {
7499	.none => if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) {
7500	buffer[child_size] = @intFromBool(false);
7501	} else @memset(buffer[0..child_size], 0x00),
7502	else => |child_constant| {
7503	if (!try isel.writeConstantToMemory(.fromInterned(child_constant), buffer[0..child_size])) return false;
7504	if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) buffer[child_size] = @intFromBool(true);
7505	},
7506	}
7507	},
7508	.aggregate => |aggregate| switch (ip.indexToKey(aggregate.ty)) {
7509	else => unreachable,
7510	.array_type => |array_type| {
7511	var elem_offset: usize = 0;
7512	const elem_size: usize = @intCast(ZigType.fromInterned(array_type.child).abiSize(zcu));
7513	const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
7514	switch (aggregate.storage) {
7515	.bytes => |bytes| @memcpy(buffer[0..len_including_sentinel], bytes.toSlice(len_including_sentinel, ip)),
7516	.elems => |elems| for (elems) |elem| {
7517	if (!try isel.writeConstantToMemory(.fromInterned(elem), buffer[elem_offset..][0..elem_size])) return false;
7518	elem_offset += elem_size;
7519	},
7520	.repeated_elem => |repeated_elem| for (0..len_including_sentinel) |_| {
7521	if (!try isel.writeConstantToMemory(.fromInterned(repeated_elem), buffer[elem_offset..][0..elem_size])) return false;
7522	elem_offset += elem_size;
7523	},
7524	}
7525	},
7526	.vector_type => return false,
7527	.struct_type => {
7528	const loaded_struct = ip.loadStructType(aggregate.ty);
7529	switch (loaded_struct.layout) {
7530	.auto => {
7531	var field_it = loaded_struct.iterateRuntimeOrder(ip);
7532	while (field_it.next()) |field_index| {
7533	if (loaded_struct.field_is_comptime_bits.get(ip, field_index)) continue;
7534	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
7535	const field_offset = loaded_struct.field_offsets.get(ip)[field_index];
7536	const field_size = field_ty.abiSize(zcu);
7537	if (!try isel.writeConstantToMemory(.fromInterned(switch (aggregate.storage) {
7538	.bytes => unreachable,
7539	.elems => |elems| elems[field_index],
7540	.repeated_elem => |repeated_elem| repeated_elem,
7541	}), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
7542	}
7543	},
7544	.@"extern", .@"packed" => return false,
7545	}
7546	},
7547	.tuple_type => |tuple_type| {
7548	var field_offset: u64 = 0;
7549	for (tuple_type.types.get(ip), tuple_type.values.get(ip), 0..) |field_type, field_value, field_index| {
7550	if (field_value != .none) continue;
7551	const field_ty: ZigType = .fromInterned(field_type);
7552	field_offset = field_ty.abiAlignment(zcu).forward(field_offset);
7553	const field_size = field_ty.abiSize(zcu);
7554	if (!try isel.writeConstantToMemory(.fromInterned(switch (aggregate.storage) {
7555	.bytes => unreachable,
7556	.elems => |elems| elems[field_index],
7557	.repeated_elem => |repeated_elem| repeated_elem,
7558	}), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
7559	field_offset += field_size;
7560	}
7561	},
7562	},
7563	.un => |union_val| {
7564	const loaded_union = ip.loadUnionType(union_val.ty);
7565	switch (loaded_union.layout) {
7566	.auto => {},
7567	.@"extern", .@"packed" => return false,
7568	}
7569	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
7570	if (loaded_union.has_runtime_tag)
7571	if (!try isel.writeConstantToMemory(
7572	.fromInterned(union_val.tag),
7573	buffer[@intCast(union_layout.tagOffset())..][0..@intCast(union_layout.tag_size)],
7574	)) return false;
7575	if (!try isel.writeConstantToMemory(
7576	.fromInterned(union_val.val),
7577	buffer[@intCast(union_layout.payloadOffset())..][0..@intCast(union_layout.payload_size)],
7578	)) return false;
7579	},
7580	else => return false,
7581	}
7582	return true;
7583	}
7584
7585	fn wipeLocationDfs(isel: *Select, vi: Value.Index) void {
7586	_ = vi.takeLocationMarkWritten(isel);
7587	var part_it = vi.parts(isel);
7588	while (part_it.next()) |part_vi| {
7589	if (part_vi != vi) isel.wipeLocationDfs(part_vi);
7590	}
7591	}
7592
7593	const Air = @import("../../Air.zig");
7594	const assert = std.debug.assert;
7595	const codegen = @import("../../codegen.zig");
7596	const Constant = @import("../../Value.zig");
7597	const InternPool = @import("../../InternPool.zig");
7598	const Module = @import("../../Module.zig");
7599	const Select = @This();
7600	const std = @import("std");
7601	const tracking_log = std.log.scoped(.tracking);
7602	const wip_mir_log = std.log.scoped(.@"wip-mir");
7603	const abi_log = std.log.scoped(.abi);
7604	const Zcu = @import("../../Zcu.zig");
7605	const ZigType = @import("../../Type.zig");